@foss.global/codefeed 4.0.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +32 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.d.ts +30 -6
- package/dist_ts/classes.giteasourceinventory.js +179 -23
- package/dist_ts/classes.giteasourcetransfer.d.ts +55 -3
- package/dist_ts/classes.giteasourcetransfer.js +155 -4
- package/dist_ts/classes.lfsbatchclient.js +17 -118
- package/dist_ts/forgeinventory.transfer.d.ts +75 -0
- package/dist_ts/forgeinventory.transfer.js +174 -0
- package/package.json +3 -3
- package/readme.md +16 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +188 -23
- package/ts/classes.giteasourcetransfer.ts +182 -4
- package/ts/classes.lfsbatchclient.ts +15 -98
- package/ts/forgeinventory.transfer.ts +203 -0
|
@@ -13,6 +13,9 @@ type IInstance = plugins.fossInterfaces.data.IForgeSourceInstanceInventory;
|
|
|
13
13
|
type TOwner = plugins.fossInterfaces.data.TForgeSourceOwnerInventory;
|
|
14
14
|
type IRepository = plugins.fossInterfaces.data.IForgeSourceRepositoryInventory;
|
|
15
15
|
type IPackageTypeCounts = plugins.fossInterfaces.data.IForgeSourcePackageTypeCounts;
|
|
16
|
+
type IRelease = plugins.fossInterfaces.data.IForgeSourceRelease;
|
|
17
|
+
type IBranchProtection = plugins.fossInterfaces.data.IForgeSourceBranchProtection;
|
|
18
|
+
type ITagProtection = plugins.fossInterfaces.data.IForgeSourceTagProtection;
|
|
16
19
|
type IInventory = plugins.fossInterfaces.data.IForgeSourceInventory;
|
|
17
20
|
|
|
18
21
|
export interface IGiteaSourceInventoryOptions {
|
|
@@ -52,6 +55,11 @@ export interface IGiteaSourceMeasureOptions {
|
|
|
52
55
|
onResult?: (resultArg: TUnitResult) => void | Promise<void>;
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
/** A repository read by its id: the name Gitea gives it now, or `absent` once it is deleted. */
|
|
59
|
+
export type TGiteaSourceRepositoryLookup =
|
|
60
|
+
| { status: 'present'; id: string; owner: string; name: string }
|
|
61
|
+
| { status: 'absent'; id: string };
|
|
62
|
+
|
|
55
63
|
interface IListing {
|
|
56
64
|
items: unknown[];
|
|
57
65
|
/** A sanitized reason when the listing did not finish. */
|
|
@@ -114,6 +122,35 @@ const requiredString = (valueArg: unknown): string => {
|
|
|
114
122
|
return value;
|
|
115
123
|
};
|
|
116
124
|
|
|
125
|
+
/** A string the forge may leave empty, such as a release body. */
|
|
126
|
+
const text = (valueArg: unknown): string => {
|
|
127
|
+
if (typeof valueArg !== 'string') throw new validation.ForgeInventoryError('invalid-string');
|
|
128
|
+
return valueArg;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/** A list of names; Gitea answers an empty one as `null`. */
|
|
132
|
+
const texts = (valueArg: unknown): string[] => {
|
|
133
|
+
if (valueArg === null) return [];
|
|
134
|
+
if (!Array.isArray(valueArg)) throw new validation.ForgeInventoryError('invalid-string');
|
|
135
|
+
return valueArg.map(text);
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/** A signed account id; Gitea's placeholder for a deleted account is -1. */
|
|
139
|
+
const accountId = (valueArg: unknown): string => {
|
|
140
|
+
if (typeof valueArg === 'number' && Number.isSafeInteger(valueArg)) return String(valueArg);
|
|
141
|
+
throw new validation.ForgeInventoryError('invalid-or-imprecise-integer');
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
/** An RFC 3339 time as the forge reports it. */
|
|
145
|
+
const requiredTimestamp = (valueArg: unknown): string => {
|
|
146
|
+
const value = validation.timestamp(valueArg);
|
|
147
|
+
if (value === null) throw new validation.ForgeInventoryError('invalid-timestamp');
|
|
148
|
+
return value;
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
/** Orders canonical nonnegative decimal strings numerically. */
|
|
152
|
+
const compareDecimal = (aArg: string, bArg: string): number => aArg.length - bArg.length || compareText(aArg, bArg);
|
|
153
|
+
|
|
117
154
|
/** One provider path segment, as an owner or repository name. */
|
|
118
155
|
const segment = (valueArg: unknown): string => {
|
|
119
156
|
const segments = validation.segments(valueArg);
|
|
@@ -137,6 +174,83 @@ const assets = (valueArg: unknown): { count: bigint; bytes: bigint } => {
|
|
|
137
174
|
}), { count: 0n, bytes: 0n });
|
|
138
175
|
};
|
|
139
176
|
|
|
177
|
+
/** A release as Gitea reads it, drafts included, with its assets by numeric id. */
|
|
178
|
+
const releaseRecord = (itemArg: unknown): IRelease => {
|
|
179
|
+
const item = validation.record(itemArg);
|
|
180
|
+
const author = validation.record(item.author);
|
|
181
|
+
const itemAssets = item.assets ?? [];
|
|
182
|
+
if (!Array.isArray(itemAssets)) throw new validation.ForgeInventoryError('invalid-assets');
|
|
183
|
+
return {
|
|
184
|
+
id: validation.decimal(item.id),
|
|
185
|
+
tagName: requiredString(item.tag_name),
|
|
186
|
+
targetCommitish: text(item.target_commitish),
|
|
187
|
+
name: text(item.name),
|
|
188
|
+
body: text(item.body),
|
|
189
|
+
draft: requiredBoolean(item.draft),
|
|
190
|
+
prerelease: requiredBoolean(item.prerelease),
|
|
191
|
+
createdAt: requiredTimestamp(item.created_at),
|
|
192
|
+
publishedAt: requiredTimestamp(item.published_at),
|
|
193
|
+
author: { id: accountId(author.id), login: requiredString(author.login) },
|
|
194
|
+
assets: itemAssets.map((assetArg: unknown) => {
|
|
195
|
+
const asset = validation.record(assetArg);
|
|
196
|
+
return {
|
|
197
|
+
id: validation.decimal(asset.id),
|
|
198
|
+
uuid: requiredString(asset.uuid),
|
|
199
|
+
name: requiredString(asset.name),
|
|
200
|
+
size: validation.decimal(asset.size),
|
|
201
|
+
downloadCount: validation.decimal(asset.download_count),
|
|
202
|
+
createdAt: requiredTimestamp(asset.created_at),
|
|
203
|
+
};
|
|
204
|
+
}).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
|
|
205
|
+
};
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const allowlist = (usernamesArg: unknown, teamsArg: unknown) => ({ usernames: texts(usernamesArg), teams: texts(teamsArg) });
|
|
209
|
+
|
|
210
|
+
/** A branch protection rule with every restriction Gitea reads back. */
|
|
211
|
+
const branchProtection = (itemArg: unknown): IBranchProtection => {
|
|
212
|
+
const item = validation.record(itemArg);
|
|
213
|
+
return {
|
|
214
|
+
ruleName: requiredString(item.rule_name),
|
|
215
|
+
priority: validation.decimal(item.priority),
|
|
216
|
+
enablePush: requiredBoolean(item.enable_push),
|
|
217
|
+
enablePushAllowlist: requiredBoolean(item.enable_push_whitelist),
|
|
218
|
+
pushAllowlist: allowlist(item.push_whitelist_usernames, item.push_whitelist_teams),
|
|
219
|
+
pushAllowlistDeployKeys: requiredBoolean(item.push_whitelist_deploy_keys),
|
|
220
|
+
enableForcePush: requiredBoolean(item.enable_force_push),
|
|
221
|
+
enableForcePushAllowlist: requiredBoolean(item.enable_force_push_allowlist),
|
|
222
|
+
forcePushAllowlist: allowlist(item.force_push_allowlist_usernames, item.force_push_allowlist_teams),
|
|
223
|
+
forcePushAllowlistDeployKeys: requiredBoolean(item.force_push_allowlist_deploy_keys),
|
|
224
|
+
enableMergeAllowlist: requiredBoolean(item.enable_merge_whitelist),
|
|
225
|
+
mergeAllowlist: allowlist(item.merge_whitelist_usernames, item.merge_whitelist_teams),
|
|
226
|
+
enableBypassAllowlist: requiredBoolean(item.enable_bypass_allowlist),
|
|
227
|
+
bypassAllowlist: allowlist(item.bypass_allowlist_usernames, item.bypass_allowlist_teams),
|
|
228
|
+
enableStatusCheck: requiredBoolean(item.enable_status_check),
|
|
229
|
+
statusCheckContexts: texts(item.status_check_contexts),
|
|
230
|
+
requiredApprovals: validation.decimal(item.required_approvals),
|
|
231
|
+
enableApprovalsAllowlist: requiredBoolean(item.enable_approvals_whitelist),
|
|
232
|
+
approvalsAllowlist: allowlist(item.approvals_whitelist_username, item.approvals_whitelist_teams),
|
|
233
|
+
blockOnRejectedReviews: requiredBoolean(item.block_on_rejected_reviews),
|
|
234
|
+
blockOnOfficialReviewRequests: requiredBoolean(item.block_on_official_review_requests),
|
|
235
|
+
blockOnOutdatedBranch: requiredBoolean(item.block_on_outdated_branch),
|
|
236
|
+
dismissStaleApprovals: requiredBoolean(item.dismiss_stale_approvals),
|
|
237
|
+
ignoreStaleApprovals: requiredBoolean(item.ignore_stale_approvals),
|
|
238
|
+
blockAdminMergeOverride: requiredBoolean(item.block_admin_merge_override),
|
|
239
|
+
requireSignedCommits: requiredBoolean(item.require_signed_commits),
|
|
240
|
+
protectedFilePatterns: text(item.protected_file_patterns),
|
|
241
|
+
unprotectedFilePatterns: text(item.unprotected_file_patterns),
|
|
242
|
+
};
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const tagProtection = (itemArg: unknown): ITagProtection => {
|
|
246
|
+
const item = validation.record(itemArg);
|
|
247
|
+
return {
|
|
248
|
+
id: validation.decimal(item.id),
|
|
249
|
+
namePattern: requiredString(item.name_pattern),
|
|
250
|
+
allowlist: allowlist(item.whitelist_usernames, item.whitelist_teams),
|
|
251
|
+
};
|
|
252
|
+
};
|
|
253
|
+
|
|
140
254
|
/** A `{ <rows>, total_count }` envelope whose rows must match its total. */
|
|
141
255
|
const countedEnvelope = (bodyArg: unknown, rowsKeyArg: string): { rows: unknown[]; failure: string | null } => {
|
|
142
256
|
const body = validation.record(bodyArg);
|
|
@@ -217,8 +331,8 @@ export class GiteaSourceInventory {
|
|
|
217
331
|
}
|
|
218
332
|
|
|
219
333
|
/**
|
|
220
|
-
* Identifies the source and its reader and lists every owner and repository
|
|
221
|
-
* Rejects unless the token belongs to a site administrator; a listing that
|
|
334
|
+
* Identifies the source and its reader and lists every owner and repository
|
|
335
|
+
* with its id. Rejects unless the token belongs to a site administrator; a listing that
|
|
222
336
|
* does not finish, or that lists a repository id twice, is reported in
|
|
223
337
|
* `incompleteReasons`.
|
|
224
338
|
*/
|
|
@@ -255,7 +369,7 @@ export class GiteaSourceInventory {
|
|
|
255
369
|
const id = validation.decimal(item.id);
|
|
256
370
|
if (ids.has(id)) continue;
|
|
257
371
|
ids.add(id);
|
|
258
|
-
discovered.push({ owner: segment(validation.record(item.owner).login), name: segment(item.name) });
|
|
372
|
+
discovered.push({ owner: segment(validation.record(item.owner).login), name: segment(item.name), id });
|
|
259
373
|
}
|
|
260
374
|
if (ids.size !== repositories.items.length) incompleteReasons.push('discovery: repositories: duplicate-id');
|
|
261
375
|
discovered.sort((aArg, bArg) => compareText(aArg.owner, bArg.owner) || compareText(aArg.name, bArg.name));
|
|
@@ -273,18 +387,45 @@ export class GiteaSourceInventory {
|
|
|
273
387
|
return [
|
|
274
388
|
{ kind: 'instance' },
|
|
275
389
|
...discoveryArg.owners.map((ownerArg): TUnit => ({ kind: 'owner', ownerKind: ownerArg.kind, name: ownerArg.name })),
|
|
276
|
-
...discoveryArg.repositories.map((repositoryArg): TUnit => ({
|
|
390
|
+
...discoveryArg.repositories.map((repositoryArg): TUnit => ({
|
|
391
|
+
kind: 'repository', owner: repositoryArg.owner, name: repositoryArg.name, ...repositoryArg.id === undefined ? {} : { id: repositoryArg.id },
|
|
392
|
+
})),
|
|
277
393
|
];
|
|
278
394
|
}
|
|
279
395
|
|
|
280
|
-
/**
|
|
396
|
+
/**
|
|
397
|
+
* Reads a repository by its id through `GET /repositories/{id}`, which
|
|
398
|
+
* keeps answering after a rename or transfer: `present` with the name Gitea
|
|
399
|
+
* gives it now, or `absent` when Gitea answers 404, which a site
|
|
400
|
+
* administrator only gets for a deleted repository. Any other failed read
|
|
401
|
+
* rejects with its error, so it never counts as a deletion; an id that is not
|
|
402
|
+
* a canonical decimal rejects with `invalid-repository-id`.
|
|
403
|
+
*/
|
|
404
|
+
public async lookupRepository(idArg: string, signalArg?: AbortSignal): Promise<TGiteaSourceRepositoryLookup> {
|
|
405
|
+
if (typeof idArg !== 'string' || !/^[1-9][0-9]{0,99}$/.test(idArg)) throw new validation.ForgeInventoryError('invalid-repository-id');
|
|
406
|
+
let item: Record<string, unknown>;
|
|
407
|
+
try {
|
|
408
|
+
item = validation.record((await this.#get(`/repositories/${idArg}`, { signal: signalArg })).body);
|
|
409
|
+
} catch (error) {
|
|
410
|
+
if (error instanceof validation.ForgeInventoryError && error.code === 'http-404') return { status: 'absent', id: idArg };
|
|
411
|
+
throw error;
|
|
412
|
+
}
|
|
413
|
+
if (validation.decimal(item.id) !== idArg) throw new validation.ForgeInventoryError('repository-identity-changed');
|
|
414
|
+
return { status: 'present', id: idArg, owner: segment(validation.record(item.owner).login), name: segment(item.name) };
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Measures one unit. Never rejects: a unit whose primary record cannot be
|
|
419
|
+
* read is `failed`, and so is a repository unit carrying an id that its name
|
|
420
|
+
* no longer answers with (`repository-identity-changed`).
|
|
421
|
+
*/
|
|
281
422
|
public async measureUnit(unitArg: TUnit, signalArg?: AbortSignal): Promise<TUnitResult> {
|
|
282
423
|
try {
|
|
283
424
|
if (unitArg.kind === 'instance') return { kind: 'instance', instance: await this.#instance(signalArg) };
|
|
284
425
|
if (unitArg.kind === 'owner') {
|
|
285
426
|
return { kind: 'owner', owner: unitArg.ownerKind === 'organization' ? await this.#organization(unitArg.name, signalArg) : await this.#user(unitArg.name, signalArg) };
|
|
286
427
|
}
|
|
287
|
-
return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, signalArg) };
|
|
428
|
+
return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, unitArg.id, signalArg) };
|
|
288
429
|
} catch (error) {
|
|
289
430
|
return { kind: 'failed', unit: unitArg, reason: reason(error) };
|
|
290
431
|
}
|
|
@@ -460,11 +601,14 @@ export class GiteaSourceInventory {
|
|
|
460
601
|
return counts;
|
|
461
602
|
}
|
|
462
603
|
|
|
463
|
-
async #repository(ownerArg: string, nameArg: string, signalArg?: AbortSignal): Promise<IRepository> {
|
|
604
|
+
async #repository(ownerArg: string, nameArg: string, idArg: string | undefined, signalArg?: AbortSignal): Promise<IRepository> {
|
|
464
605
|
const path = `/repos/${encodeURIComponent(ownerArg)}/${encodeURIComponent(nameArg)}`;
|
|
465
606
|
const item = validation.record((await this.#get(path, { signal: signalArg })).body);
|
|
466
607
|
const owner = validation.record(item.owner);
|
|
467
|
-
|
|
608
|
+
// A name re-created after a rename answers with another id; without a discovered id only the name is checked.
|
|
609
|
+
if (segment(owner.login) !== ownerArg || segment(item.name) !== nameArg || (idArg !== undefined && validation.decimal(item.id) !== idArg)) {
|
|
610
|
+
throw new validation.ForgeInventoryError('repository-identity-changed');
|
|
611
|
+
}
|
|
468
612
|
const visibility = validation.giteaRepositoryVisibility(item.private, item.internal, owner.visibility);
|
|
469
613
|
if (visibility === null) throw new validation.ForgeInventoryError('invalid-visibility');
|
|
470
614
|
const empty = requiredBoolean(item.empty);
|
|
@@ -481,6 +625,8 @@ export class GiteaSourceInventory {
|
|
|
481
625
|
const parent = item.parent === undefined || item.parent === null ? null : requiredString(validation.record(item.parent).full_name);
|
|
482
626
|
if (fork !== (parent !== null)) throw new validation.ForgeInventoryError('inconsistent-fork-parent');
|
|
483
627
|
const options = { signal: signalArg };
|
|
628
|
+
const branchRules = await this.#unpaged(`${path}/branch_protections`, options);
|
|
629
|
+
const tagRules = await this.#unpaged(`${path}/tag_protections`, options);
|
|
484
630
|
return {
|
|
485
631
|
owner: ownerArg,
|
|
486
632
|
name: nameArg,
|
|
@@ -498,17 +644,21 @@ export class GiteaSourceInventory {
|
|
|
498
644
|
...await this.#git(ownerArg, nameArg, item, signalArg),
|
|
499
645
|
collaboration: await this.#collaboration(path, features, signalArg),
|
|
500
646
|
// Releases stay readable while their unit is disabled; a disabled wiki answers 404.
|
|
501
|
-
|
|
647
|
+
...await this.#releases(path, signalArg),
|
|
502
648
|
wikiPages: features.wiki ? await this.#count(`${path}/wiki/pages`, { ...options, missingIsEmpty: true }) : unitDisabled('wiki'),
|
|
503
649
|
access: {
|
|
504
650
|
collaborators: await this.#count(`${path}/collaborators`, options),
|
|
505
651
|
teams: await this.#repositoryTeams(path, options),
|
|
506
652
|
deployKeys: await this.#count(`${path}/keys`, options),
|
|
507
653
|
webhooks: await this.#count(`${path}/hooks`, options),
|
|
508
|
-
branchProtections:
|
|
509
|
-
tagProtections:
|
|
654
|
+
branchProtections: branchRules.measure,
|
|
655
|
+
tagProtections: tagRules.measure,
|
|
510
656
|
pushMirrors: await this.#count(`${path}/push_mirrors`, options),
|
|
511
657
|
},
|
|
658
|
+
protections: branchRules.rows === null || tagRules.rows === null ? null : {
|
|
659
|
+
branches: branchRules.rows.map(branchProtection).sort((aArg, bArg) => compareText(aArg.ruleName, bArg.ruleName)),
|
|
660
|
+
tags: tagRules.rows.map(tagProtection).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
|
|
661
|
+
},
|
|
512
662
|
actions: await this.#actions(path, empty, signalArg),
|
|
513
663
|
};
|
|
514
664
|
}
|
|
@@ -620,7 +770,8 @@ export class GiteaSourceInventory {
|
|
|
620
770
|
};
|
|
621
771
|
}
|
|
622
772
|
|
|
623
|
-
|
|
773
|
+
/** The release counts, and the release records once the listing is complete. */
|
|
774
|
+
async #releases(pathArg: string, signalArg?: AbortSignal): Promise<Pick<IRepository, 'releases' | 'releaseRecords'>> {
|
|
624
775
|
// An administrator's listing includes drafts.
|
|
625
776
|
const releases = await this.#list(`${pathArg}/releases`, { signal: signalArg });
|
|
626
777
|
let drafts = 0n;
|
|
@@ -632,10 +783,15 @@ export class GiteaSourceInventory {
|
|
|
632
783
|
releaseAssets = { count: releaseAssets.count + itemAssets.count, bytes: releaseAssets.bytes + itemAssets.bytes };
|
|
633
784
|
}
|
|
634
785
|
return {
|
|
635
|
-
releases:
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
786
|
+
releases: {
|
|
787
|
+
releases: listingMeasure(releases),
|
|
788
|
+
drafts: listingMeasure(releases, drafts),
|
|
789
|
+
assets: listingMeasure(releases, releaseAssets.count),
|
|
790
|
+
assetBytes: listingMeasure(releases, releaseAssets.bytes),
|
|
791
|
+
},
|
|
792
|
+
releaseRecords: releases.failure === null
|
|
793
|
+
? releases.items.map(releaseRecord).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id))
|
|
794
|
+
: null,
|
|
639
795
|
};
|
|
640
796
|
}
|
|
641
797
|
|
|
@@ -690,13 +846,18 @@ export class GiteaSourceInventory {
|
|
|
690
846
|
}
|
|
691
847
|
|
|
692
848
|
async #unpagedCount(pathArg: string, optionsArg: IListOptions): Promise<TMeasure> {
|
|
849
|
+
return (await this.#unpaged(pathArg, optionsArg)).measure;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/** A listing Gitea does not page: its count, and its rows when the read succeeded. */
|
|
853
|
+
async #unpaged(pathArg: string, optionsArg: IListOptions): Promise<{ measure: TMeasure; rows: unknown[] | null }> {
|
|
693
854
|
try {
|
|
694
855
|
const response = (await this.#get(pathArg, optionsArg)).body;
|
|
695
856
|
const body = response === null && optionsArg.nullIsEmpty ? [] : response;
|
|
696
857
|
if (!Array.isArray(body)) throw new validation.ForgeInventoryError('invalid-page');
|
|
697
|
-
return complete(body.length);
|
|
858
|
+
return { measure: complete(body.length), rows: body };
|
|
698
859
|
} catch (error) {
|
|
699
|
-
return partial(0, reason(error));
|
|
860
|
+
return { measure: partial(0, reason(error)), rows: null };
|
|
700
861
|
}
|
|
701
862
|
}
|
|
702
863
|
|
|
@@ -784,9 +945,10 @@ const measuresOf = (valueArg: unknown, pathArg: string, outArg: Array<{ path: st
|
|
|
784
945
|
/**
|
|
785
946
|
* Composes the deterministic inventory from a discovery and its unit results.
|
|
786
947
|
* Repeated results for one unit keep the last. Units without a successful
|
|
787
|
-
* result,
|
|
788
|
-
*
|
|
789
|
-
*
|
|
948
|
+
* result, a repository result for another id than discovery recorded, two
|
|
949
|
+
* repositories measured with one id, owner repository counts that disagree
|
|
950
|
+
* with discovery, and every partial or unmeasured value make the inventory
|
|
951
|
+
* incomplete, with reasons aggregated per subject kind, path and reason.
|
|
790
952
|
*/
|
|
791
953
|
export const composeGiteaSourceInventory = (
|
|
792
954
|
discoveryArg: IDiscovery,
|
|
@@ -830,12 +992,15 @@ export const composeGiteaSourceInventory = (
|
|
|
830
992
|
const failedRepositories: string[] = [];
|
|
831
993
|
for (const repository of discoveryArg.repositories) {
|
|
832
994
|
const result = results.get(unitKey({ kind: 'repository', ...repository }));
|
|
833
|
-
|
|
995
|
+
// A result for another id under the discovered name measured the repository that took the name.
|
|
996
|
+
if (result?.kind === 'repository' && (repository.id === undefined || result.repository.id === repository.id)) repositories.push(result.repository);
|
|
834
997
|
else {
|
|
835
998
|
failedRepositories.push(`${repository.owner}/${repository.name}`);
|
|
836
|
-
addReason(`repository: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : 'unit not measured'}`);
|
|
999
|
+
addReason(`repository: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : result?.kind === 'repository' ? 'unit failed: repository-identity-changed' : 'unit not measured'}`);
|
|
837
1000
|
}
|
|
838
1001
|
}
|
|
1002
|
+
// Without discovered ids, two names measured as one id show a rename between their units.
|
|
1003
|
+
if (new Set(repositories.map((repositoryArg) => repositoryArg.id)).size !== repositories.length) addReason('repository: duplicate-id');
|
|
839
1004
|
owners.sort((aArg, bArg) => compareText(aArg.kind, bArg.kind) || compareText(aArg.name, bArg.name));
|
|
840
1005
|
repositories.sort((aArg, bArg) => compareText(aArg.owner, bArg.owner) || compareText(aArg.name, bArg.name));
|
|
841
1006
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
6
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
|
|
|
27
28
|
maxResponseBytes?: number;
|
|
28
29
|
/** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
|
|
29
30
|
lfsTransferTimeoutMs?: number;
|
|
31
|
+
/** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
|
|
32
|
+
releaseAssetTimeoutMs?: number;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
35
|
/**
|
|
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
|
|
|
52
55
|
signal?: AbortSignal;
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
/**
|
|
59
|
+
* A release asset as the source inventory recorded it: what a download asks
|
|
60
|
+
* for and checks Gitea still holds. The download count is not part of it.
|
|
61
|
+
*/
|
|
62
|
+
export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
|
|
63
|
+
|
|
64
|
+
export interface IGiteaReleaseAssetDownloadOptions {
|
|
65
|
+
/** Aborts the download with `request-aborted`, also while the stream is read. */
|
|
66
|
+
signal?: AbortSignal;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface IGiteaReleaseAssetDownload {
|
|
70
|
+
releaseId: string;
|
|
71
|
+
asset: TGiteaReleaseAssetSelector;
|
|
72
|
+
/**
|
|
73
|
+
* The asset's bytes, never held whole. The last chunk is held back until
|
|
74
|
+
* exactly `asset.size` bytes have arrived and both identity reads have
|
|
75
|
+
* passed again. Destroying the stream ends the download.
|
|
76
|
+
*/
|
|
77
|
+
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Gitea names attachment content by a random UUID in canonical lowercase form. */
|
|
81
|
+
const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
82
|
+
|
|
55
83
|
export interface IGiteaMirrorImport {
|
|
56
84
|
repository: plugins.TGitManagerRepositoryInfo;
|
|
57
85
|
/** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
|
|
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
|
|
|
61
89
|
/**
|
|
62
90
|
* Reads a Gitea repository's Git data and LFS objects for its one-time
|
|
63
91
|
* migration, with the reader's login and token: a mirror clone into a
|
|
64
|
-
* GitManager the caller supplies,
|
|
65
|
-
* writes to Gitea: Git is read only through
|
|
66
|
-
* through batch `download` requests and their
|
|
92
|
+
* GitManager the caller supplies, verified LFS object downloads and release
|
|
93
|
+
* asset downloads. It never writes to Gitea: Git is read only through
|
|
94
|
+
* `git-upload-pack`, LFS only through batch `download` requests and their
|
|
95
|
+
* download actions, release assets only through GETs of their attachment.
|
|
67
96
|
*/
|
|
68
97
|
export class GiteaSourceTransfer {
|
|
69
98
|
readonly #options: {
|
|
70
99
|
baseUrl: string; login: string; token: string; caCertificates?: string[];
|
|
71
|
-
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
|
|
100
|
+
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
|
|
72
101
|
};
|
|
73
102
|
readonly #client: ForgeRequestClient;
|
|
74
103
|
|
|
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
|
|
|
83
112
|
requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
|
|
84
113
|
maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
|
|
85
114
|
lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
115
|
+
releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
|
|
86
116
|
};
|
|
87
117
|
for (const [name, maximum] of [
|
|
88
118
|
['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
119
|
+
['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
|
|
89
120
|
] as const) {
|
|
90
121
|
const value = limits[name];
|
|
91
122
|
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
|
|
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
|
|
|
223
254
|
return download;
|
|
224
255
|
}
|
|
225
256
|
|
|
257
|
+
/**
|
|
258
|
+
* Downloads one release asset by GET of `/attachments/{uuid}` with the
|
|
259
|
+
* reader's `token` authorization: no redirect is followed and no range is
|
|
260
|
+
* asked for, so only a 200 answer of the whole content is accepted, and a
|
|
261
|
+
* `content-length`, when sent, must equal `asset.size`. Before the request
|
|
262
|
+
* the repository is read by its id as for `importMirror`, then the release
|
|
263
|
+
* by its id: it must still carry an asset with the same `id`, `uuid`,
|
|
264
|
+
* `name` and `size`, otherwise the download rejects with
|
|
265
|
+
* `release-asset-identity-changed` (a replaced file carries another id and
|
|
266
|
+
* uuid). The bytes stream without a digest; the last chunk is held back
|
|
267
|
+
* until exactly `size` bytes have arrived and both identity reads have
|
|
268
|
+
* passed once more, so a consumer never receives a complete asset that is
|
|
269
|
+
* short, long or no longer the recorded one. Gitea exposes no content
|
|
270
|
+
* digest; a digest the caller needs comes from the content itself.
|
|
271
|
+
*
|
|
272
|
+
* Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
|
|
273
|
+
* `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
|
|
274
|
+
* code such as `http-404` (also for a deleted release),
|
|
275
|
+
* `repository-identity-changed`, `release-asset-identity-changed`,
|
|
276
|
+
* `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
|
|
277
|
+
* `release-asset-size-overrun`, `release-asset-short-body`,
|
|
278
|
+
* `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
|
|
279
|
+
* stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
|
|
280
|
+
* for a refused redirect, or `release-asset-transfer-failed` when body bytes
|
|
281
|
+
* still arrive after the download has ended; the download ends only once
|
|
282
|
+
* the stream is destroyed or finished, and a destroyed stream reports no
|
|
283
|
+
* error, so a reader does not see this code. A failed request is explained
|
|
284
|
+
* by the identity reads as for `downloadLfsObject`.
|
|
285
|
+
*/
|
|
286
|
+
public async downloadReleaseAsset(
|
|
287
|
+
repositoryArg: IGiteaSourceRepository,
|
|
288
|
+
releaseIdArg: string,
|
|
289
|
+
assetArg: TGiteaReleaseAssetSelector,
|
|
290
|
+
optionsArg: IGiteaReleaseAssetDownloadOptions = {},
|
|
291
|
+
): Promise<IGiteaReleaseAssetDownload> {
|
|
292
|
+
const repository = this.#repository(repositoryArg);
|
|
293
|
+
const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
|
|
294
|
+
const { signal } = optionsArg;
|
|
295
|
+
if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
296
|
+
await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
|
|
297
|
+
const agents = transport.createNodeAgents(this.#options.caCertificates);
|
|
298
|
+
const close = () => agents.https?.destroy();
|
|
299
|
+
const transfer = createTransferBounds(signal, {
|
|
300
|
+
limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
|
|
301
|
+
timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
|
|
302
|
+
});
|
|
303
|
+
let body: AsyncIterable<Uint8Array>;
|
|
304
|
+
try {
|
|
305
|
+
const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
|
|
306
|
+
method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
|
|
307
|
+
}));
|
|
308
|
+
if (response.status !== 200 || !response.body) {
|
|
309
|
+
await transfer.run(() => discardBody(response.body));
|
|
310
|
+
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
311
|
+
}
|
|
312
|
+
const length = response.headers.get('content-length');
|
|
313
|
+
if (length !== null && length.trim() !== asset.size) {
|
|
314
|
+
await transfer.run(() => discardBody(response.body));
|
|
315
|
+
throw new validation.ForgeInventoryError('release-asset-size-mismatch');
|
|
316
|
+
}
|
|
317
|
+
body = response.body;
|
|
318
|
+
} catch (error) {
|
|
319
|
+
const failure = transfer.failure(error, 'release-asset-download-failed');
|
|
320
|
+
transfer.end();
|
|
321
|
+
close();
|
|
322
|
+
if (signal?.aborted) throw failure;
|
|
323
|
+
return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
|
|
324
|
+
}
|
|
325
|
+
// After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
|
|
326
|
+
const identified = async () => {
|
|
327
|
+
transfer.touch();
|
|
328
|
+
await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
|
|
329
|
+
};
|
|
330
|
+
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
331
|
+
try {
|
|
332
|
+
yield* heldBackChunks(transfer.watch(body), size, {
|
|
333
|
+
invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
|
|
334
|
+
}, { complete: identified });
|
|
335
|
+
} catch (error) {
|
|
336
|
+
throw transfer.failure(error, 'release-asset-download-failed');
|
|
337
|
+
} finally {
|
|
338
|
+
transfer.end();
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
// Destroying the stream ends the download and closes its connections at once, also while a read is pending.
|
|
342
|
+
return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
|
|
343
|
+
}
|
|
344
|
+
|
|
226
345
|
/**
|
|
227
346
|
* Reads the repository by its id and checks that Gitea still names it
|
|
228
347
|
* `owner/name`, without case as Gitea looks names up: other names reject
|
|
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
|
|
|
259
378
|
throw errorArg;
|
|
260
379
|
}
|
|
261
380
|
|
|
381
|
+
/**
|
|
382
|
+
* Reads the repository by its id, then the release by its id under the
|
|
383
|
+
* repository's name: the release must still carry the asset with its id,
|
|
384
|
+
* uuid, name and size, else `release-asset-identity-changed`. A release
|
|
385
|
+
* read that fails is explained by the repository read as for `#explained`.
|
|
386
|
+
*/
|
|
387
|
+
async #releaseAssetIdentified(
|
|
388
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
|
|
389
|
+
): Promise<void> {
|
|
390
|
+
await this.#identified(repositoryArg, signalArg);
|
|
391
|
+
let release: Record<string, unknown>;
|
|
392
|
+
try {
|
|
393
|
+
release = validation.record((await this.#client.request(new URL(
|
|
394
|
+
`${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
|
|
395
|
+
), { signal: signalArg })).body);
|
|
396
|
+
} catch (error) {
|
|
397
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
398
|
+
return this.#explained(repositoryArg, error, signalArg);
|
|
399
|
+
}
|
|
400
|
+
const assets = release.assets ?? [];
|
|
401
|
+
if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
402
|
+
const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
|
|
403
|
+
if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
|
|
404
|
+
throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Rejects with `errorArg`, a failed asset download, unless the identity
|
|
410
|
+
* reads explain it: then with `repository-identity-changed`,
|
|
411
|
+
* `release-asset-identity-changed` or `http-404`.
|
|
412
|
+
*/
|
|
413
|
+
async #releaseAssetExplained(
|
|
414
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
|
|
415
|
+
): Promise<never> {
|
|
416
|
+
try {
|
|
417
|
+
await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
|
|
418
|
+
} catch (error) {
|
|
419
|
+
if (error instanceof validation.ForgeInventoryError
|
|
420
|
+
&& ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
|
|
421
|
+
}
|
|
422
|
+
throw errorArg;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
|
|
426
|
+
#releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
|
|
427
|
+
try {
|
|
428
|
+
if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
|
|
429
|
+
const releaseId = validation.decimal(releaseIdArg);
|
|
430
|
+
const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
|
|
431
|
+
const size = Number(asset.size);
|
|
432
|
+
if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
|
|
433
|
+
|| !Number.isSafeInteger(size)) throw new Error();
|
|
434
|
+
return { releaseId, asset, size };
|
|
435
|
+
} catch {
|
|
436
|
+
throw new validation.ForgeInventoryError('release-asset-invalid');
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
262
440
|
#repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
|
|
263
441
|
const single = (valueArg: unknown): string => {
|
|
264
442
|
const segments = validation.segments(valueArg);
|