@foss.global/codefeed 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +43 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.js +131 -12
- package/dist_ts/classes.giteasourcetransfer.d.ts +55 -3
- package/dist_ts/classes.giteasourcetransfer.js +155 -4
- package/dist_ts/classes.lfsbatchclient.js +17 -118
- package/dist_ts/forgeinventory.transfer.d.ts +75 -0
- package/dist_ts/forgeinventory.transfer.js +174 -0
- package/dist_ts/giteasourceinventory.git.d.ts +16 -1
- package/dist_ts/giteasourceinventory.git.js +34 -7
- package/package.json +8 -11
- package/readme.md +15 -6
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +139 -13
- package/ts/classes.giteasourcetransfer.ts +182 -4
- package/ts/classes.lfsbatchclient.ts +15 -98
- package/ts/forgeinventory.transfer.ts +203 -0
- package/ts/giteasourceinventory.git.ts +44 -8
|
@@ -13,6 +13,9 @@ type IInstance = plugins.fossInterfaces.data.IForgeSourceInstanceInventory;
|
|
|
13
13
|
type TOwner = plugins.fossInterfaces.data.TForgeSourceOwnerInventory;
|
|
14
14
|
type IRepository = plugins.fossInterfaces.data.IForgeSourceRepositoryInventory;
|
|
15
15
|
type IPackageTypeCounts = plugins.fossInterfaces.data.IForgeSourcePackageTypeCounts;
|
|
16
|
+
type IRelease = plugins.fossInterfaces.data.IForgeSourceRelease;
|
|
17
|
+
type IBranchProtection = plugins.fossInterfaces.data.IForgeSourceBranchProtection;
|
|
18
|
+
type ITagProtection = plugins.fossInterfaces.data.IForgeSourceTagProtection;
|
|
16
19
|
type IInventory = plugins.fossInterfaces.data.IForgeSourceInventory;
|
|
17
20
|
|
|
18
21
|
export interface IGiteaSourceInventoryOptions {
|
|
@@ -119,6 +122,35 @@ const requiredString = (valueArg: unknown): string => {
|
|
|
119
122
|
return value;
|
|
120
123
|
};
|
|
121
124
|
|
|
125
|
+
/** A string the forge may leave empty, such as a release body. */
|
|
126
|
+
const text = (valueArg: unknown): string => {
|
|
127
|
+
if (typeof valueArg !== 'string') throw new validation.ForgeInventoryError('invalid-string');
|
|
128
|
+
return valueArg;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/** A list of names; Gitea answers an empty one as `null`. */
|
|
132
|
+
const texts = (valueArg: unknown): string[] => {
|
|
133
|
+
if (valueArg === null) return [];
|
|
134
|
+
if (!Array.isArray(valueArg)) throw new validation.ForgeInventoryError('invalid-string');
|
|
135
|
+
return valueArg.map(text);
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/** A signed account id; Gitea's placeholder for a deleted account is -1. */
|
|
139
|
+
const accountId = (valueArg: unknown): string => {
|
|
140
|
+
if (typeof valueArg === 'number' && Number.isSafeInteger(valueArg)) return String(valueArg);
|
|
141
|
+
throw new validation.ForgeInventoryError('invalid-or-imprecise-integer');
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
/** An RFC 3339 time as the forge reports it. */
|
|
145
|
+
const requiredTimestamp = (valueArg: unknown): string => {
|
|
146
|
+
const value = validation.timestamp(valueArg);
|
|
147
|
+
if (value === null) throw new validation.ForgeInventoryError('invalid-timestamp');
|
|
148
|
+
return value;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
/** Orders canonical nonnegative decimal strings numerically. */
|
|
152
|
+
const compareDecimal = (aArg: string, bArg: string): number => aArg.length - bArg.length || compareText(aArg, bArg);
|
|
153
|
+
|
|
122
154
|
/** One provider path segment, as an owner or repository name. */
|
|
123
155
|
const segment = (valueArg: unknown): string => {
|
|
124
156
|
const segments = validation.segments(valueArg);
|
|
@@ -142,6 +174,83 @@ const assets = (valueArg: unknown): { count: bigint; bytes: bigint } => {
|
|
|
142
174
|
}), { count: 0n, bytes: 0n });
|
|
143
175
|
};
|
|
144
176
|
|
|
177
|
+
/** A release as Gitea reads it, drafts included, with its assets by numeric id. */
|
|
178
|
+
const releaseRecord = (itemArg: unknown): IRelease => {
|
|
179
|
+
const item = validation.record(itemArg);
|
|
180
|
+
const author = validation.record(item.author);
|
|
181
|
+
const itemAssets = item.assets ?? [];
|
|
182
|
+
if (!Array.isArray(itemAssets)) throw new validation.ForgeInventoryError('invalid-assets');
|
|
183
|
+
return {
|
|
184
|
+
id: validation.decimal(item.id),
|
|
185
|
+
tagName: requiredString(item.tag_name),
|
|
186
|
+
targetCommitish: text(item.target_commitish),
|
|
187
|
+
name: text(item.name),
|
|
188
|
+
body: text(item.body),
|
|
189
|
+
draft: requiredBoolean(item.draft),
|
|
190
|
+
prerelease: requiredBoolean(item.prerelease),
|
|
191
|
+
createdAt: requiredTimestamp(item.created_at),
|
|
192
|
+
publishedAt: requiredTimestamp(item.published_at),
|
|
193
|
+
author: { id: accountId(author.id), login: requiredString(author.login) },
|
|
194
|
+
assets: itemAssets.map((assetArg: unknown) => {
|
|
195
|
+
const asset = validation.record(assetArg);
|
|
196
|
+
return {
|
|
197
|
+
id: validation.decimal(asset.id),
|
|
198
|
+
uuid: requiredString(asset.uuid),
|
|
199
|
+
name: requiredString(asset.name),
|
|
200
|
+
size: validation.decimal(asset.size),
|
|
201
|
+
downloadCount: validation.decimal(asset.download_count),
|
|
202
|
+
createdAt: requiredTimestamp(asset.created_at),
|
|
203
|
+
};
|
|
204
|
+
}).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
|
|
205
|
+
};
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const allowlist = (usernamesArg: unknown, teamsArg: unknown) => ({ usernames: texts(usernamesArg), teams: texts(teamsArg) });
|
|
209
|
+
|
|
210
|
+
/** A branch protection rule with every restriction Gitea reads back. */
|
|
211
|
+
const branchProtection = (itemArg: unknown): IBranchProtection => {
|
|
212
|
+
const item = validation.record(itemArg);
|
|
213
|
+
return {
|
|
214
|
+
ruleName: requiredString(item.rule_name),
|
|
215
|
+
priority: validation.decimal(item.priority),
|
|
216
|
+
enablePush: requiredBoolean(item.enable_push),
|
|
217
|
+
enablePushAllowlist: requiredBoolean(item.enable_push_whitelist),
|
|
218
|
+
pushAllowlist: allowlist(item.push_whitelist_usernames, item.push_whitelist_teams),
|
|
219
|
+
pushAllowlistDeployKeys: requiredBoolean(item.push_whitelist_deploy_keys),
|
|
220
|
+
enableForcePush: requiredBoolean(item.enable_force_push),
|
|
221
|
+
enableForcePushAllowlist: requiredBoolean(item.enable_force_push_allowlist),
|
|
222
|
+
forcePushAllowlist: allowlist(item.force_push_allowlist_usernames, item.force_push_allowlist_teams),
|
|
223
|
+
forcePushAllowlistDeployKeys: requiredBoolean(item.force_push_allowlist_deploy_keys),
|
|
224
|
+
enableMergeAllowlist: requiredBoolean(item.enable_merge_whitelist),
|
|
225
|
+
mergeAllowlist: allowlist(item.merge_whitelist_usernames, item.merge_whitelist_teams),
|
|
226
|
+
enableBypassAllowlist: requiredBoolean(item.enable_bypass_allowlist),
|
|
227
|
+
bypassAllowlist: allowlist(item.bypass_allowlist_usernames, item.bypass_allowlist_teams),
|
|
228
|
+
enableStatusCheck: requiredBoolean(item.enable_status_check),
|
|
229
|
+
statusCheckContexts: texts(item.status_check_contexts),
|
|
230
|
+
requiredApprovals: validation.decimal(item.required_approvals),
|
|
231
|
+
enableApprovalsAllowlist: requiredBoolean(item.enable_approvals_whitelist),
|
|
232
|
+
approvalsAllowlist: allowlist(item.approvals_whitelist_username, item.approvals_whitelist_teams),
|
|
233
|
+
blockOnRejectedReviews: requiredBoolean(item.block_on_rejected_reviews),
|
|
234
|
+
blockOnOfficialReviewRequests: requiredBoolean(item.block_on_official_review_requests),
|
|
235
|
+
blockOnOutdatedBranch: requiredBoolean(item.block_on_outdated_branch),
|
|
236
|
+
dismissStaleApprovals: requiredBoolean(item.dismiss_stale_approvals),
|
|
237
|
+
ignoreStaleApprovals: requiredBoolean(item.ignore_stale_approvals),
|
|
238
|
+
blockAdminMergeOverride: requiredBoolean(item.block_admin_merge_override),
|
|
239
|
+
requireSignedCommits: requiredBoolean(item.require_signed_commits),
|
|
240
|
+
protectedFilePatterns: text(item.protected_file_patterns),
|
|
241
|
+
unprotectedFilePatterns: text(item.unprotected_file_patterns),
|
|
242
|
+
};
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const tagProtection = (itemArg: unknown): ITagProtection => {
|
|
246
|
+
const item = validation.record(itemArg);
|
|
247
|
+
return {
|
|
248
|
+
id: validation.decimal(item.id),
|
|
249
|
+
namePattern: requiredString(item.name_pattern),
|
|
250
|
+
allowlist: allowlist(item.whitelist_usernames, item.whitelist_teams),
|
|
251
|
+
};
|
|
252
|
+
};
|
|
253
|
+
|
|
145
254
|
/** A `{ <rows>, total_count }` envelope whose rows must match its total. */
|
|
146
255
|
const countedEnvelope = (bodyArg: unknown, rowsKeyArg: string): { rows: unknown[]; failure: string | null } => {
|
|
147
256
|
const body = validation.record(bodyArg);
|
|
@@ -516,6 +625,8 @@ export class GiteaSourceInventory {
|
|
|
516
625
|
const parent = item.parent === undefined || item.parent === null ? null : requiredString(validation.record(item.parent).full_name);
|
|
517
626
|
if (fork !== (parent !== null)) throw new validation.ForgeInventoryError('inconsistent-fork-parent');
|
|
518
627
|
const options = { signal: signalArg };
|
|
628
|
+
const branchRules = await this.#unpaged(`${path}/branch_protections`, options);
|
|
629
|
+
const tagRules = await this.#unpaged(`${path}/tag_protections`, options);
|
|
519
630
|
return {
|
|
520
631
|
owner: ownerArg,
|
|
521
632
|
name: nameArg,
|
|
@@ -533,17 +644,21 @@ export class GiteaSourceInventory {
|
|
|
533
644
|
...await this.#git(ownerArg, nameArg, item, signalArg),
|
|
534
645
|
collaboration: await this.#collaboration(path, features, signalArg),
|
|
535
646
|
// Releases stay readable while their unit is disabled; a disabled wiki answers 404.
|
|
536
|
-
|
|
647
|
+
...await this.#releases(path, signalArg),
|
|
537
648
|
wikiPages: features.wiki ? await this.#count(`${path}/wiki/pages`, { ...options, missingIsEmpty: true }) : unitDisabled('wiki'),
|
|
538
649
|
access: {
|
|
539
650
|
collaborators: await this.#count(`${path}/collaborators`, options),
|
|
540
651
|
teams: await this.#repositoryTeams(path, options),
|
|
541
652
|
deployKeys: await this.#count(`${path}/keys`, options),
|
|
542
653
|
webhooks: await this.#count(`${path}/hooks`, options),
|
|
543
|
-
branchProtections:
|
|
544
|
-
tagProtections:
|
|
654
|
+
branchProtections: branchRules.measure,
|
|
655
|
+
tagProtections: tagRules.measure,
|
|
545
656
|
pushMirrors: await this.#count(`${path}/push_mirrors`, options),
|
|
546
657
|
},
|
|
658
|
+
protections: branchRules.rows === null || tagRules.rows === null ? null : {
|
|
659
|
+
branches: branchRules.rows.map(branchProtection).sort((aArg, bArg) => compareText(aArg.ruleName, bArg.ruleName)),
|
|
660
|
+
tags: tagRules.rows.map(tagProtection).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
|
|
661
|
+
},
|
|
547
662
|
actions: await this.#actions(path, empty, signalArg),
|
|
548
663
|
};
|
|
549
664
|
}
|
|
@@ -560,8 +675,8 @@ export class GiteaSourceInventory {
|
|
|
560
675
|
}
|
|
561
676
|
}
|
|
562
677
|
|
|
563
|
-
/** HEAD, refs, objects and LFS objects, read from a mirror clone; only the reported size comes from the API. */
|
|
564
|
-
async #git(ownerArg: string, nameArg: string, itemArg: Record<string, unknown>, signalArg?: AbortSignal): Promise<Pick<IRepository, 'git' | 'lfs' | 'head'>> {
|
|
678
|
+
/** HEAD, ref digests, refs, objects and LFS objects, read from a mirror clone; only the reported size comes from the API. */
|
|
679
|
+
async #git(ownerArg: string, nameArg: string, itemArg: Record<string, unknown>, signalArg?: AbortSignal): Promise<Pick<IRepository, 'git' | 'lfs' | 'head' | 'refDigests'>> {
|
|
565
680
|
const sizeKiB = BigInt(validation.decimal(itemArg.size));
|
|
566
681
|
const username = await this.#login(signalArg);
|
|
567
682
|
const measured = await measureGiteaMirror({
|
|
@@ -570,7 +685,7 @@ export class GiteaSourceInventory {
|
|
|
570
685
|
timeoutMs: this.#options.gitTimeoutMs,
|
|
571
686
|
lfsClient: this.#lfs(username), signal: signalArg,
|
|
572
687
|
});
|
|
573
|
-
return { git: { reportedSizeKiB: complete(sizeKiB), ...measured.git }, lfs: measured.lfs, head: measured.head };
|
|
688
|
+
return { git: { reportedSizeKiB: complete(sizeKiB), ...measured.git }, lfs: measured.lfs, head: measured.head, refDigests: measured.refDigests };
|
|
574
689
|
}
|
|
575
690
|
|
|
576
691
|
/** The reader's login, the user name of its Git and LFS credentials. */
|
|
@@ -655,7 +770,8 @@ export class GiteaSourceInventory {
|
|
|
655
770
|
};
|
|
656
771
|
}
|
|
657
772
|
|
|
658
|
-
|
|
773
|
+
/** The release counts, and the release records once the listing is complete. */
|
|
774
|
+
async #releases(pathArg: string, signalArg?: AbortSignal): Promise<Pick<IRepository, 'releases' | 'releaseRecords'>> {
|
|
659
775
|
// An administrator's listing includes drafts.
|
|
660
776
|
const releases = await this.#list(`${pathArg}/releases`, { signal: signalArg });
|
|
661
777
|
let drafts = 0n;
|
|
@@ -667,10 +783,15 @@ export class GiteaSourceInventory {
|
|
|
667
783
|
releaseAssets = { count: releaseAssets.count + itemAssets.count, bytes: releaseAssets.bytes + itemAssets.bytes };
|
|
668
784
|
}
|
|
669
785
|
return {
|
|
670
|
-
releases:
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
786
|
+
releases: {
|
|
787
|
+
releases: listingMeasure(releases),
|
|
788
|
+
drafts: listingMeasure(releases, drafts),
|
|
789
|
+
assets: listingMeasure(releases, releaseAssets.count),
|
|
790
|
+
assetBytes: listingMeasure(releases, releaseAssets.bytes),
|
|
791
|
+
},
|
|
792
|
+
releaseRecords: releases.failure === null
|
|
793
|
+
? releases.items.map(releaseRecord).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id))
|
|
794
|
+
: null,
|
|
674
795
|
};
|
|
675
796
|
}
|
|
676
797
|
|
|
@@ -725,13 +846,18 @@ export class GiteaSourceInventory {
|
|
|
725
846
|
}
|
|
726
847
|
|
|
727
848
|
async #unpagedCount(pathArg: string, optionsArg: IListOptions): Promise<TMeasure> {
|
|
849
|
+
return (await this.#unpaged(pathArg, optionsArg)).measure;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/** A listing Gitea does not page: its count, and its rows when the read succeeded. */
|
|
853
|
+
async #unpaged(pathArg: string, optionsArg: IListOptions): Promise<{ measure: TMeasure; rows: unknown[] | null }> {
|
|
728
854
|
try {
|
|
729
855
|
const response = (await this.#get(pathArg, optionsArg)).body;
|
|
730
856
|
const body = response === null && optionsArg.nullIsEmpty ? [] : response;
|
|
731
857
|
if (!Array.isArray(body)) throw new validation.ForgeInventoryError('invalid-page');
|
|
732
|
-
return complete(body.length);
|
|
858
|
+
return { measure: complete(body.length), rows: body };
|
|
733
859
|
} catch (error) {
|
|
734
|
-
return partial(0, reason(error));
|
|
860
|
+
return { measure: partial(0, reason(error)), rows: null };
|
|
735
861
|
}
|
|
736
862
|
}
|
|
737
863
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
6
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
|
|
|
27
28
|
maxResponseBytes?: number;
|
|
28
29
|
/** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
|
|
29
30
|
lfsTransferTimeoutMs?: number;
|
|
31
|
+
/** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
|
|
32
|
+
releaseAssetTimeoutMs?: number;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
35
|
/**
|
|
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
|
|
|
52
55
|
signal?: AbortSignal;
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
/**
|
|
59
|
+
* A release asset as the source inventory recorded it: what a download asks
|
|
60
|
+
* for and checks Gitea still holds. The download count is not part of it.
|
|
61
|
+
*/
|
|
62
|
+
export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
|
|
63
|
+
|
|
64
|
+
export interface IGiteaReleaseAssetDownloadOptions {
|
|
65
|
+
/** Aborts the download with `request-aborted`, also while the stream is read. */
|
|
66
|
+
signal?: AbortSignal;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface IGiteaReleaseAssetDownload {
|
|
70
|
+
releaseId: string;
|
|
71
|
+
asset: TGiteaReleaseAssetSelector;
|
|
72
|
+
/**
|
|
73
|
+
* The asset's bytes, never held whole. The last chunk is held back until
|
|
74
|
+
* exactly `asset.size` bytes have arrived and both identity reads have
|
|
75
|
+
* passed again. Destroying the stream ends the download.
|
|
76
|
+
*/
|
|
77
|
+
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Gitea names attachment content by a random UUID in canonical lowercase form. */
|
|
81
|
+
const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
82
|
+
|
|
55
83
|
export interface IGiteaMirrorImport {
|
|
56
84
|
repository: plugins.TGitManagerRepositoryInfo;
|
|
57
85
|
/** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
|
|
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
|
|
|
61
89
|
/**
|
|
62
90
|
* Reads a Gitea repository's Git data and LFS objects for its one-time
|
|
63
91
|
* migration, with the reader's login and token: a mirror clone into a
|
|
64
|
-
* GitManager the caller supplies,
|
|
65
|
-
* writes to Gitea: Git is read only through
|
|
66
|
-
* through batch `download` requests and their
|
|
92
|
+
* GitManager the caller supplies, verified LFS object downloads and release
|
|
93
|
+
* asset downloads. It never writes to Gitea: Git is read only through
|
|
94
|
+
* `git-upload-pack`, LFS only through batch `download` requests and their
|
|
95
|
+
* download actions, release assets only through GETs of their attachment.
|
|
67
96
|
*/
|
|
68
97
|
export class GiteaSourceTransfer {
|
|
69
98
|
readonly #options: {
|
|
70
99
|
baseUrl: string; login: string; token: string; caCertificates?: string[];
|
|
71
|
-
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
|
|
100
|
+
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
|
|
72
101
|
};
|
|
73
102
|
readonly #client: ForgeRequestClient;
|
|
74
103
|
|
|
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
|
|
|
83
112
|
requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
|
|
84
113
|
maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
|
|
85
114
|
lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
115
|
+
releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
|
|
86
116
|
};
|
|
87
117
|
for (const [name, maximum] of [
|
|
88
118
|
['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
119
|
+
['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
|
|
89
120
|
] as const) {
|
|
90
121
|
const value = limits[name];
|
|
91
122
|
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
|
|
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
|
|
|
223
254
|
return download;
|
|
224
255
|
}
|
|
225
256
|
|
|
257
|
+
/**
|
|
258
|
+
* Downloads one release asset by GET of `/attachments/{uuid}` with the
|
|
259
|
+
* reader's `token` authorization: no redirect is followed and no range is
|
|
260
|
+
* asked for, so only a 200 answer of the whole content is accepted, and a
|
|
261
|
+
* `content-length`, when sent, must equal `asset.size`. Before the request
|
|
262
|
+
* the repository is read by its id as for `importMirror`, then the release
|
|
263
|
+
* by its id: it must still carry an asset with the same `id`, `uuid`,
|
|
264
|
+
* `name` and `size`, otherwise the download rejects with
|
|
265
|
+
* `release-asset-identity-changed` (a replaced file carries another id and
|
|
266
|
+
* uuid). The bytes stream without a digest; the last chunk is held back
|
|
267
|
+
* until exactly `size` bytes have arrived and both identity reads have
|
|
268
|
+
* passed once more, so a consumer never receives a complete asset that is
|
|
269
|
+
* short, long or no longer the recorded one. Gitea exposes no content
|
|
270
|
+
* digest; a digest the caller needs comes from the content itself.
|
|
271
|
+
*
|
|
272
|
+
* Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
|
|
273
|
+
* `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
|
|
274
|
+
* code such as `http-404` (also for a deleted release),
|
|
275
|
+
* `repository-identity-changed`, `release-asset-identity-changed`,
|
|
276
|
+
* `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
|
|
277
|
+
* `release-asset-size-overrun`, `release-asset-short-body`,
|
|
278
|
+
* `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
|
|
279
|
+
* stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
|
|
280
|
+
* for a refused redirect, or `release-asset-transfer-failed` when body bytes
|
|
281
|
+
* still arrive after the download has ended; the download ends only once
|
|
282
|
+
* the stream is destroyed or finished, and a destroyed stream reports no
|
|
283
|
+
* error, so a reader does not see this code. A failed request is explained
|
|
284
|
+
* by the identity reads as for `downloadLfsObject`.
|
|
285
|
+
*/
|
|
286
|
+
public async downloadReleaseAsset(
|
|
287
|
+
repositoryArg: IGiteaSourceRepository,
|
|
288
|
+
releaseIdArg: string,
|
|
289
|
+
assetArg: TGiteaReleaseAssetSelector,
|
|
290
|
+
optionsArg: IGiteaReleaseAssetDownloadOptions = {},
|
|
291
|
+
): Promise<IGiteaReleaseAssetDownload> {
|
|
292
|
+
const repository = this.#repository(repositoryArg);
|
|
293
|
+
const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
|
|
294
|
+
const { signal } = optionsArg;
|
|
295
|
+
if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
296
|
+
await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
|
|
297
|
+
const agents = transport.createNodeAgents(this.#options.caCertificates);
|
|
298
|
+
const close = () => agents.https?.destroy();
|
|
299
|
+
const transfer = createTransferBounds(signal, {
|
|
300
|
+
limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
|
|
301
|
+
timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
|
|
302
|
+
});
|
|
303
|
+
let body: AsyncIterable<Uint8Array>;
|
|
304
|
+
try {
|
|
305
|
+
const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
|
|
306
|
+
method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
|
|
307
|
+
}));
|
|
308
|
+
if (response.status !== 200 || !response.body) {
|
|
309
|
+
await transfer.run(() => discardBody(response.body));
|
|
310
|
+
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
311
|
+
}
|
|
312
|
+
const length = response.headers.get('content-length');
|
|
313
|
+
if (length !== null && length.trim() !== asset.size) {
|
|
314
|
+
await transfer.run(() => discardBody(response.body));
|
|
315
|
+
throw new validation.ForgeInventoryError('release-asset-size-mismatch');
|
|
316
|
+
}
|
|
317
|
+
body = response.body;
|
|
318
|
+
} catch (error) {
|
|
319
|
+
const failure = transfer.failure(error, 'release-asset-download-failed');
|
|
320
|
+
transfer.end();
|
|
321
|
+
close();
|
|
322
|
+
if (signal?.aborted) throw failure;
|
|
323
|
+
return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
|
|
324
|
+
}
|
|
325
|
+
// After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
|
|
326
|
+
const identified = async () => {
|
|
327
|
+
transfer.touch();
|
|
328
|
+
await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
|
|
329
|
+
};
|
|
330
|
+
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
331
|
+
try {
|
|
332
|
+
yield* heldBackChunks(transfer.watch(body), size, {
|
|
333
|
+
invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
|
|
334
|
+
}, { complete: identified });
|
|
335
|
+
} catch (error) {
|
|
336
|
+
throw transfer.failure(error, 'release-asset-download-failed');
|
|
337
|
+
} finally {
|
|
338
|
+
transfer.end();
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
// Destroying the stream ends the download and closes its connections at once, also while a read is pending.
|
|
342
|
+
return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
|
|
343
|
+
}
|
|
344
|
+
|
|
226
345
|
/**
|
|
227
346
|
* Reads the repository by its id and checks that Gitea still names it
|
|
228
347
|
* `owner/name`, without case as Gitea looks names up: other names reject
|
|
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
|
|
|
259
378
|
throw errorArg;
|
|
260
379
|
}
|
|
261
380
|
|
|
381
|
+
/**
|
|
382
|
+
* Reads the repository by its id, then the release by its id under the
|
|
383
|
+
* repository's name: the release must still carry the asset with its id,
|
|
384
|
+
* uuid, name and size, else `release-asset-identity-changed`. A release
|
|
385
|
+
* read that fails is explained by the repository read as for `#explained`.
|
|
386
|
+
*/
|
|
387
|
+
async #releaseAssetIdentified(
|
|
388
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
|
|
389
|
+
): Promise<void> {
|
|
390
|
+
await this.#identified(repositoryArg, signalArg);
|
|
391
|
+
let release: Record<string, unknown>;
|
|
392
|
+
try {
|
|
393
|
+
release = validation.record((await this.#client.request(new URL(
|
|
394
|
+
`${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
|
|
395
|
+
), { signal: signalArg })).body);
|
|
396
|
+
} catch (error) {
|
|
397
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
398
|
+
return this.#explained(repositoryArg, error, signalArg);
|
|
399
|
+
}
|
|
400
|
+
const assets = release.assets ?? [];
|
|
401
|
+
if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
402
|
+
const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
|
|
403
|
+
if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
|
|
404
|
+
throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Rejects with `errorArg`, a failed asset download, unless the identity
|
|
410
|
+
* reads explain it: then with `repository-identity-changed`,
|
|
411
|
+
* `release-asset-identity-changed` or `http-404`.
|
|
412
|
+
*/
|
|
413
|
+
async #releaseAssetExplained(
|
|
414
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
|
|
415
|
+
): Promise<never> {
|
|
416
|
+
try {
|
|
417
|
+
await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
|
|
418
|
+
} catch (error) {
|
|
419
|
+
if (error instanceof validation.ForgeInventoryError
|
|
420
|
+
&& ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
|
|
421
|
+
}
|
|
422
|
+
throw errorArg;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
|
|
426
|
+
#releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
|
|
427
|
+
try {
|
|
428
|
+
if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
|
|
429
|
+
const releaseId = validation.decimal(releaseIdArg);
|
|
430
|
+
const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
|
|
431
|
+
const size = Number(asset.size);
|
|
432
|
+
if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
|
|
433
|
+
|| !Number.isSafeInteger(size)) throw new Error();
|
|
434
|
+
return { releaseId, asset, size };
|
|
435
|
+
} catch {
|
|
436
|
+
throw new validation.ForgeInventoryError('release-asset-invalid');
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
262
440
|
#repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
|
|
263
441
|
const single = (valueArg: unknown): string => {
|
|
264
442
|
const segments = validation.segments(valueArg);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
6
|
import type { IForgeRequestMetrics } from './forgeinventory.client.js';
|
|
6
7
|
|
|
@@ -82,8 +83,6 @@ type TBatchEntry = ILfsObjectId & { actions: Record<string, ILfsAction>; errorCo
|
|
|
82
83
|
const lfsMediaType = 'application/vnd.git-lfs+json';
|
|
83
84
|
/** Objects per batch request; the Git LFS reference server and Gitea accept 100. */
|
|
84
85
|
const maxBatchSize = 100;
|
|
85
|
-
/** Response bytes read from a PUT or verify answer before the rest is discarded. */
|
|
86
|
-
const maxDiscardedResponseBytes = 64 * 1024;
|
|
87
86
|
const maxActionHeaders = 32;
|
|
88
87
|
const maxHeaderValueLength = 8 * 1024;
|
|
89
88
|
const maxHrefLength = 8 * 1024;
|
|
@@ -179,44 +178,14 @@ const actionExpiry = (actionArg: Record<string, unknown>, receivedAtArg: number)
|
|
|
179
178
|
* wrong; otherwise it throws `lfs-size-overrun`, `lfs-short-body` or
|
|
180
179
|
* `lfs-digest-mismatch`.
|
|
181
180
|
*/
|
|
182
|
-
|
|
181
|
+
const verifiedChunks = (sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> => {
|
|
183
182
|
const hash = plugins.crypto.createHash('sha256');
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (bytes > sizeArg) throw new validation.ForgeInventoryError('lfs-size-overrun');
|
|
191
|
-
hash.update(chunk);
|
|
192
|
-
if (held) yield held;
|
|
193
|
-
held = chunk;
|
|
194
|
-
}
|
|
195
|
-
if (bytes !== sizeArg) throw new validation.ForgeInventoryError('lfs-short-body');
|
|
196
|
-
if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
|
|
197
|
-
if (held) yield held;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* Reads and drops at most `maxDiscardedResponseBytes` of a response body that
|
|
202
|
-
* carries nothing the client uses, so its connection can be reused; a longer
|
|
203
|
-
* or failing body is discarded with its connection.
|
|
204
|
-
*/
|
|
205
|
-
const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
|
|
206
|
-
if (!bodyArg) return;
|
|
207
|
-
const chunks = bodyArg[Symbol.asyncIterator]();
|
|
208
|
-
let bytes = 0;
|
|
209
|
-
try {
|
|
210
|
-
while (bytes <= maxDiscardedResponseBytes) {
|
|
211
|
-
const result = await chunks.next();
|
|
212
|
-
if (result.done) return;
|
|
213
|
-
bytes += result.value.byteLength;
|
|
214
|
-
}
|
|
215
|
-
} catch {
|
|
216
|
-
// The status already decided the outcome.
|
|
217
|
-
} finally {
|
|
218
|
-
try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
|
|
219
|
-
}
|
|
183
|
+
return heldBackChunks(sourceArg, sizeArg, { invalidChunk: 'invalid-lfs-chunk', overrun: 'lfs-size-overrun', short: 'lfs-short-body' }, {
|
|
184
|
+
inspect: (chunkArg) => hash.update(chunkArg),
|
|
185
|
+
complete: () => {
|
|
186
|
+
if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
|
|
187
|
+
},
|
|
188
|
+
});
|
|
220
189
|
};
|
|
221
190
|
|
|
222
191
|
/**
|
|
@@ -390,10 +359,8 @@ export class LfsBatchClient {
|
|
|
390
359
|
transfer.end();
|
|
391
360
|
}
|
|
392
361
|
};
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
stream.once('close', () => transfer.end());
|
|
396
|
-
return { oid, size, status: 'available', stream };
|
|
362
|
+
// Destroying the stream ends the download and closes its connection at once, also while a read is pending.
|
|
363
|
+
return { oid, size, status: 'available', stream: createTransferStream(transfer, chunks()) };
|
|
397
364
|
} catch (error) {
|
|
398
365
|
transfer.end();
|
|
399
366
|
throw transfer.failure(error, 'lfs-download-failed');
|
|
@@ -553,62 +520,12 @@ export class LfsBatchClient {
|
|
|
553
520
|
|
|
554
521
|
/**
|
|
555
522
|
* The bounds of one transfer: the caller's signal, a wall-clock limit and an
|
|
556
|
-
* idle limit
|
|
557
|
-
*
|
|
523
|
+
* idle limit of `requestTimeoutMs`; a passed limit fails with
|
|
524
|
+
* `lfs-transfer-timed-out`.
|
|
558
525
|
*/
|
|
559
526
|
#transfer(signalArg: AbortSignal | undefined, limitMsArg = this.#limits.transferTimeoutMs) {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
reason ??= reasonArg;
|
|
564
|
-
controller.abort();
|
|
565
|
-
};
|
|
566
|
-
const abort = () => stop('request-aborted');
|
|
567
|
-
if (signalArg?.aborted) abort();
|
|
568
|
-
signalArg?.addEventListener('abort', abort, { once: true });
|
|
569
|
-
const deadline = setTimeout(() => stop('lfs-transfer-timed-out'), limitMsArg);
|
|
570
|
-
let idle: ReturnType<typeof setTimeout> | undefined;
|
|
571
|
-
const rearm = () => {
|
|
572
|
-
clearTimeout(idle);
|
|
573
|
-
if (!controller.signal.aborted) idle = setTimeout(() => stop('lfs-transfer-timed-out'), this.#limits.requestTimeoutMs);
|
|
574
|
-
};
|
|
575
|
-
rearm();
|
|
576
|
-
const end = () => {
|
|
577
|
-
clearTimeout(deadline);
|
|
578
|
-
clearTimeout(idle);
|
|
579
|
-
signalArg?.removeEventListener('abort', abort);
|
|
580
|
-
stop('transfer-ended');
|
|
581
|
-
};
|
|
582
|
-
const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
|
|
583
|
-
if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
|
|
584
|
-
return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
|
|
585
|
-
};
|
|
586
|
-
return {
|
|
587
|
-
signal: controller.signal,
|
|
588
|
-
end,
|
|
589
|
-
failure,
|
|
590
|
-
/** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
|
|
591
|
-
run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
|
|
592
|
-
if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
|
|
593
|
-
let onAbort!: () => void;
|
|
594
|
-
const aborted = new Promise<never>((_resolve, reject) => {
|
|
595
|
-
onAbort = () => reject(failure(undefined, 'request-aborted'));
|
|
596
|
-
controller.signal.addEventListener('abort', onAbort, { once: true });
|
|
597
|
-
});
|
|
598
|
-
try {
|
|
599
|
-
return await Promise.race([stepArg(), aborted]);
|
|
600
|
-
} finally {
|
|
601
|
-
controller.signal.removeEventListener('abort', onAbort);
|
|
602
|
-
}
|
|
603
|
-
},
|
|
604
|
-
/** Passes chunks on and re-arms the idle limit on each one. */
|
|
605
|
-
watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
|
|
606
|
-
for await (const chunk of sourceArg) {
|
|
607
|
-
if (controller.signal.aborted) throw failure(undefined, 'lfs-transfer-failed');
|
|
608
|
-
rearm();
|
|
609
|
-
yield chunk;
|
|
610
|
-
}
|
|
611
|
-
},
|
|
612
|
-
};
|
|
527
|
+
return createTransferBounds(signalArg, {
|
|
528
|
+
limitMs: limitMsArg, idleMs: this.#limits.requestTimeoutMs, timedOutCode: 'lfs-transfer-timed-out', failedCode: 'lfs-transfer-failed',
|
|
529
|
+
});
|
|
613
530
|
}
|
|
614
531
|
}
|