@foss.global/codefeed 4.0.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,9 @@ type IInstance = plugins.fossInterfaces.data.IForgeSourceInstanceInventory;
13
13
  type TOwner = plugins.fossInterfaces.data.TForgeSourceOwnerInventory;
14
14
  type IRepository = plugins.fossInterfaces.data.IForgeSourceRepositoryInventory;
15
15
  type IPackageTypeCounts = plugins.fossInterfaces.data.IForgeSourcePackageTypeCounts;
16
+ type IRelease = plugins.fossInterfaces.data.IForgeSourceRelease;
17
+ type IBranchProtection = plugins.fossInterfaces.data.IForgeSourceBranchProtection;
18
+ type ITagProtection = plugins.fossInterfaces.data.IForgeSourceTagProtection;
16
19
  type IInventory = plugins.fossInterfaces.data.IForgeSourceInventory;
17
20
 
18
21
  export interface IGiteaSourceInventoryOptions {
@@ -52,6 +55,11 @@ export interface IGiteaSourceMeasureOptions {
52
55
  onResult?: (resultArg: TUnitResult) => void | Promise<void>;
53
56
  }
54
57
 
58
+ /** A repository read by its id: the name Gitea gives it now, or `absent` once it is deleted. */
59
+ export type TGiteaSourceRepositoryLookup =
60
+ | { status: 'present'; id: string; owner: string; name: string }
61
+ | { status: 'absent'; id: string };
62
+
55
63
  interface IListing {
56
64
  items: unknown[];
57
65
  /** A sanitized reason when the listing did not finish. */
@@ -114,6 +122,35 @@ const requiredString = (valueArg: unknown): string => {
114
122
  return value;
115
123
  };
116
124
 
125
+ /** A string the forge may leave empty, such as a release body. */
126
+ const text = (valueArg: unknown): string => {
127
+ if (typeof valueArg !== 'string') throw new validation.ForgeInventoryError('invalid-string');
128
+ return valueArg;
129
+ };
130
+
131
+ /** A list of names; Gitea answers an empty one as `null`. */
132
+ const texts = (valueArg: unknown): string[] => {
133
+ if (valueArg === null) return [];
134
+ if (!Array.isArray(valueArg)) throw new validation.ForgeInventoryError('invalid-string');
135
+ return valueArg.map(text);
136
+ };
137
+
138
+ /** A signed account id; Gitea's placeholder for a deleted account is -1. */
139
+ const accountId = (valueArg: unknown): string => {
140
+ if (typeof valueArg === 'number' && Number.isSafeInteger(valueArg)) return String(valueArg);
141
+ throw new validation.ForgeInventoryError('invalid-or-imprecise-integer');
142
+ };
143
+
144
+ /** An RFC 3339 time as the forge reports it. */
145
+ const requiredTimestamp = (valueArg: unknown): string => {
146
+ const value = validation.timestamp(valueArg);
147
+ if (value === null) throw new validation.ForgeInventoryError('invalid-timestamp');
148
+ return value;
149
+ };
150
+
151
+ /** Orders canonical nonnegative decimal strings numerically. */
152
+ const compareDecimal = (aArg: string, bArg: string): number => aArg.length - bArg.length || compareText(aArg, bArg);
153
+
117
154
  /** One provider path segment, as an owner or repository name. */
118
155
  const segment = (valueArg: unknown): string => {
119
156
  const segments = validation.segments(valueArg);
@@ -137,6 +174,83 @@ const assets = (valueArg: unknown): { count: bigint; bytes: bigint } => {
137
174
  }), { count: 0n, bytes: 0n });
138
175
  };
139
176
 
177
+ /** A release as Gitea reads it, drafts included, with its assets by numeric id. */
178
+ const releaseRecord = (itemArg: unknown): IRelease => {
179
+ const item = validation.record(itemArg);
180
+ const author = validation.record(item.author);
181
+ const itemAssets = item.assets ?? [];
182
+ if (!Array.isArray(itemAssets)) throw new validation.ForgeInventoryError('invalid-assets');
183
+ return {
184
+ id: validation.decimal(item.id),
185
+ tagName: requiredString(item.tag_name),
186
+ targetCommitish: text(item.target_commitish),
187
+ name: text(item.name),
188
+ body: text(item.body),
189
+ draft: requiredBoolean(item.draft),
190
+ prerelease: requiredBoolean(item.prerelease),
191
+ createdAt: requiredTimestamp(item.created_at),
192
+ publishedAt: requiredTimestamp(item.published_at),
193
+ author: { id: accountId(author.id), login: requiredString(author.login) },
194
+ assets: itemAssets.map((assetArg: unknown) => {
195
+ const asset = validation.record(assetArg);
196
+ return {
197
+ id: validation.decimal(asset.id),
198
+ uuid: requiredString(asset.uuid),
199
+ name: requiredString(asset.name),
200
+ size: validation.decimal(asset.size),
201
+ downloadCount: validation.decimal(asset.download_count),
202
+ createdAt: requiredTimestamp(asset.created_at),
203
+ };
204
+ }).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
205
+ };
206
+ };
207
+
208
+ const allowlist = (usernamesArg: unknown, teamsArg: unknown) => ({ usernames: texts(usernamesArg), teams: texts(teamsArg) });
209
+
210
+ /** A branch protection rule with every restriction Gitea reads back. */
211
+ const branchProtection = (itemArg: unknown): IBranchProtection => {
212
+ const item = validation.record(itemArg);
213
+ return {
214
+ ruleName: requiredString(item.rule_name),
215
+ priority: validation.decimal(item.priority),
216
+ enablePush: requiredBoolean(item.enable_push),
217
+ enablePushAllowlist: requiredBoolean(item.enable_push_whitelist),
218
+ pushAllowlist: allowlist(item.push_whitelist_usernames, item.push_whitelist_teams),
219
+ pushAllowlistDeployKeys: requiredBoolean(item.push_whitelist_deploy_keys),
220
+ enableForcePush: requiredBoolean(item.enable_force_push),
221
+ enableForcePushAllowlist: requiredBoolean(item.enable_force_push_allowlist),
222
+ forcePushAllowlist: allowlist(item.force_push_allowlist_usernames, item.force_push_allowlist_teams),
223
+ forcePushAllowlistDeployKeys: requiredBoolean(item.force_push_allowlist_deploy_keys),
224
+ enableMergeAllowlist: requiredBoolean(item.enable_merge_whitelist),
225
+ mergeAllowlist: allowlist(item.merge_whitelist_usernames, item.merge_whitelist_teams),
226
+ enableBypassAllowlist: requiredBoolean(item.enable_bypass_allowlist),
227
+ bypassAllowlist: allowlist(item.bypass_allowlist_usernames, item.bypass_allowlist_teams),
228
+ enableStatusCheck: requiredBoolean(item.enable_status_check),
229
+ statusCheckContexts: texts(item.status_check_contexts),
230
+ requiredApprovals: validation.decimal(item.required_approvals),
231
+ enableApprovalsAllowlist: requiredBoolean(item.enable_approvals_whitelist),
232
+ approvalsAllowlist: allowlist(item.approvals_whitelist_username, item.approvals_whitelist_teams),
233
+ blockOnRejectedReviews: requiredBoolean(item.block_on_rejected_reviews),
234
+ blockOnOfficialReviewRequests: requiredBoolean(item.block_on_official_review_requests),
235
+ blockOnOutdatedBranch: requiredBoolean(item.block_on_outdated_branch),
236
+ dismissStaleApprovals: requiredBoolean(item.dismiss_stale_approvals),
237
+ ignoreStaleApprovals: requiredBoolean(item.ignore_stale_approvals),
238
+ blockAdminMergeOverride: requiredBoolean(item.block_admin_merge_override),
239
+ requireSignedCommits: requiredBoolean(item.require_signed_commits),
240
+ protectedFilePatterns: text(item.protected_file_patterns),
241
+ unprotectedFilePatterns: text(item.unprotected_file_patterns),
242
+ };
243
+ };
244
+
245
+ const tagProtection = (itemArg: unknown): ITagProtection => {
246
+ const item = validation.record(itemArg);
247
+ return {
248
+ id: validation.decimal(item.id),
249
+ namePattern: requiredString(item.name_pattern),
250
+ allowlist: allowlist(item.whitelist_usernames, item.whitelist_teams),
251
+ };
252
+ };
253
+
140
254
  /** A `{ <rows>, total_count }` envelope whose rows must match its total. */
141
255
  const countedEnvelope = (bodyArg: unknown, rowsKeyArg: string): { rows: unknown[]; failure: string | null } => {
142
256
  const body = validation.record(bodyArg);
@@ -217,8 +331,8 @@ export class GiteaSourceInventory {
217
331
  }
218
332
 
219
333
  /**
220
- * Identifies the source and its reader and lists every owner and repository.
221
- * Rejects unless the token belongs to a site administrator; a listing that
334
+ * Identifies the source and its reader and lists every owner and repository
335
+ * with its id. Rejects unless the token belongs to a site administrator; a listing that
222
336
  * does not finish, or that lists a repository id twice, is reported in
223
337
  * `incompleteReasons`.
224
338
  */
@@ -255,7 +369,7 @@ export class GiteaSourceInventory {
255
369
  const id = validation.decimal(item.id);
256
370
  if (ids.has(id)) continue;
257
371
  ids.add(id);
258
- discovered.push({ owner: segment(validation.record(item.owner).login), name: segment(item.name) });
372
+ discovered.push({ owner: segment(validation.record(item.owner).login), name: segment(item.name), id });
259
373
  }
260
374
  if (ids.size !== repositories.items.length) incompleteReasons.push('discovery: repositories: duplicate-id');
261
375
  discovered.sort((aArg, bArg) => compareText(aArg.owner, bArg.owner) || compareText(aArg.name, bArg.name));
@@ -273,18 +387,45 @@ export class GiteaSourceInventory {
273
387
  return [
274
388
  { kind: 'instance' },
275
389
  ...discoveryArg.owners.map((ownerArg): TUnit => ({ kind: 'owner', ownerKind: ownerArg.kind, name: ownerArg.name })),
276
- ...discoveryArg.repositories.map((repositoryArg): TUnit => ({ kind: 'repository', owner: repositoryArg.owner, name: repositoryArg.name })),
390
+ ...discoveryArg.repositories.map((repositoryArg): TUnit => ({
391
+ kind: 'repository', owner: repositoryArg.owner, name: repositoryArg.name, ...repositoryArg.id === undefined ? {} : { id: repositoryArg.id },
392
+ })),
277
393
  ];
278
394
  }
279
395
 
280
- /** Measures one unit. Never rejects: a unit whose primary record cannot be read is `failed`. */
396
+ /**
397
+ * Reads a repository by its id through `GET /repositories/{id}`, which
398
+ * keeps answering after a rename or transfer: `present` with the name Gitea
399
+ * gives it now, or `absent` when Gitea answers 404, which a site
400
+ * administrator only gets for a deleted repository. Any other failed read
401
+ * rejects with its error, so it never counts as a deletion; an id that is not
402
+ * a canonical decimal rejects with `invalid-repository-id`.
403
+ */
404
+ public async lookupRepository(idArg: string, signalArg?: AbortSignal): Promise<TGiteaSourceRepositoryLookup> {
405
+ if (typeof idArg !== 'string' || !/^[1-9][0-9]{0,99}$/.test(idArg)) throw new validation.ForgeInventoryError('invalid-repository-id');
406
+ let item: Record<string, unknown>;
407
+ try {
408
+ item = validation.record((await this.#get(`/repositories/${idArg}`, { signal: signalArg })).body);
409
+ } catch (error) {
410
+ if (error instanceof validation.ForgeInventoryError && error.code === 'http-404') return { status: 'absent', id: idArg };
411
+ throw error;
412
+ }
413
+ if (validation.decimal(item.id) !== idArg) throw new validation.ForgeInventoryError('repository-identity-changed');
414
+ return { status: 'present', id: idArg, owner: segment(validation.record(item.owner).login), name: segment(item.name) };
415
+ }
416
+
417
+ /**
418
+ * Measures one unit. Never rejects: a unit whose primary record cannot be
419
+ * read is `failed`, and so is a repository unit carrying an id that its name
420
+ * no longer answers with (`repository-identity-changed`).
421
+ */
281
422
  public async measureUnit(unitArg: TUnit, signalArg?: AbortSignal): Promise<TUnitResult> {
282
423
  try {
283
424
  if (unitArg.kind === 'instance') return { kind: 'instance', instance: await this.#instance(signalArg) };
284
425
  if (unitArg.kind === 'owner') {
285
426
  return { kind: 'owner', owner: unitArg.ownerKind === 'organization' ? await this.#organization(unitArg.name, signalArg) : await this.#user(unitArg.name, signalArg) };
286
427
  }
287
- return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, signalArg) };
428
+ return { kind: 'repository', repository: await this.#repository(unitArg.owner, unitArg.name, unitArg.id, signalArg) };
288
429
  } catch (error) {
289
430
  return { kind: 'failed', unit: unitArg, reason: reason(error) };
290
431
  }
@@ -460,11 +601,14 @@ export class GiteaSourceInventory {
460
601
  return counts;
461
602
  }
462
603
 
463
- async #repository(ownerArg: string, nameArg: string, signalArg?: AbortSignal): Promise<IRepository> {
604
+ async #repository(ownerArg: string, nameArg: string, idArg: string | undefined, signalArg?: AbortSignal): Promise<IRepository> {
464
605
  const path = `/repos/${encodeURIComponent(ownerArg)}/${encodeURIComponent(nameArg)}`;
465
606
  const item = validation.record((await this.#get(path, { signal: signalArg })).body);
466
607
  const owner = validation.record(item.owner);
467
- if (segment(owner.login) !== ownerArg || segment(item.name) !== nameArg) throw new validation.ForgeInventoryError('repository-identity-changed');
608
+ // A name re-created after a rename answers with another id; without a discovered id only the name is checked.
609
+ if (segment(owner.login) !== ownerArg || segment(item.name) !== nameArg || (idArg !== undefined && validation.decimal(item.id) !== idArg)) {
610
+ throw new validation.ForgeInventoryError('repository-identity-changed');
611
+ }
468
612
  const visibility = validation.giteaRepositoryVisibility(item.private, item.internal, owner.visibility);
469
613
  if (visibility === null) throw new validation.ForgeInventoryError('invalid-visibility');
470
614
  const empty = requiredBoolean(item.empty);
@@ -481,6 +625,8 @@ export class GiteaSourceInventory {
481
625
  const parent = item.parent === undefined || item.parent === null ? null : requiredString(validation.record(item.parent).full_name);
482
626
  if (fork !== (parent !== null)) throw new validation.ForgeInventoryError('inconsistent-fork-parent');
483
627
  const options = { signal: signalArg };
628
+ const branchRules = await this.#unpaged(`${path}/branch_protections`, options);
629
+ const tagRules = await this.#unpaged(`${path}/tag_protections`, options);
484
630
  return {
485
631
  owner: ownerArg,
486
632
  name: nameArg,
@@ -498,17 +644,21 @@ export class GiteaSourceInventory {
498
644
  ...await this.#git(ownerArg, nameArg, item, signalArg),
499
645
  collaboration: await this.#collaboration(path, features, signalArg),
500
646
  // Releases stay readable while their unit is disabled; a disabled wiki answers 404.
501
- releases: await this.#releases(path, signalArg),
647
+ ...await this.#releases(path, signalArg),
502
648
  wikiPages: features.wiki ? await this.#count(`${path}/wiki/pages`, { ...options, missingIsEmpty: true }) : unitDisabled('wiki'),
503
649
  access: {
504
650
  collaborators: await this.#count(`${path}/collaborators`, options),
505
651
  teams: await this.#repositoryTeams(path, options),
506
652
  deployKeys: await this.#count(`${path}/keys`, options),
507
653
  webhooks: await this.#count(`${path}/hooks`, options),
508
- branchProtections: await this.#unpagedCount(`${path}/branch_protections`, options),
509
- tagProtections: await this.#unpagedCount(`${path}/tag_protections`, options),
654
+ branchProtections: branchRules.measure,
655
+ tagProtections: tagRules.measure,
510
656
  pushMirrors: await this.#count(`${path}/push_mirrors`, options),
511
657
  },
658
+ protections: branchRules.rows === null || tagRules.rows === null ? null : {
659
+ branches: branchRules.rows.map(branchProtection).sort((aArg, bArg) => compareText(aArg.ruleName, bArg.ruleName)),
660
+ tags: tagRules.rows.map(tagProtection).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
661
+ },
512
662
  actions: await this.#actions(path, empty, signalArg),
513
663
  };
514
664
  }
@@ -620,7 +770,8 @@ export class GiteaSourceInventory {
620
770
  };
621
771
  }
622
772
 
623
- async #releases(pathArg: string, signalArg?: AbortSignal): Promise<IRepository['releases']> {
773
+ /** The release counts, and the release records once the listing is complete. */
774
+ async #releases(pathArg: string, signalArg?: AbortSignal): Promise<Pick<IRepository, 'releases' | 'releaseRecords'>> {
624
775
  // An administrator's listing includes drafts.
625
776
  const releases = await this.#list(`${pathArg}/releases`, { signal: signalArg });
626
777
  let drafts = 0n;
@@ -632,10 +783,15 @@ export class GiteaSourceInventory {
632
783
  releaseAssets = { count: releaseAssets.count + itemAssets.count, bytes: releaseAssets.bytes + itemAssets.bytes };
633
784
  }
634
785
  return {
635
- releases: listingMeasure(releases),
636
- drafts: listingMeasure(releases, drafts),
637
- assets: listingMeasure(releases, releaseAssets.count),
638
- assetBytes: listingMeasure(releases, releaseAssets.bytes),
786
+ releases: {
787
+ releases: listingMeasure(releases),
788
+ drafts: listingMeasure(releases, drafts),
789
+ assets: listingMeasure(releases, releaseAssets.count),
790
+ assetBytes: listingMeasure(releases, releaseAssets.bytes),
791
+ },
792
+ releaseRecords: releases.failure === null
793
+ ? releases.items.map(releaseRecord).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id))
794
+ : null,
639
795
  };
640
796
  }
641
797
 
@@ -690,13 +846,18 @@ export class GiteaSourceInventory {
690
846
  }
691
847
 
692
848
  async #unpagedCount(pathArg: string, optionsArg: IListOptions): Promise<TMeasure> {
849
+ return (await this.#unpaged(pathArg, optionsArg)).measure;
850
+ }
851
+
852
+ /** A listing Gitea does not page: its count, and its rows when the read succeeded. */
853
+ async #unpaged(pathArg: string, optionsArg: IListOptions): Promise<{ measure: TMeasure; rows: unknown[] | null }> {
693
854
  try {
694
855
  const response = (await this.#get(pathArg, optionsArg)).body;
695
856
  const body = response === null && optionsArg.nullIsEmpty ? [] : response;
696
857
  if (!Array.isArray(body)) throw new validation.ForgeInventoryError('invalid-page');
697
- return complete(body.length);
858
+ return { measure: complete(body.length), rows: body };
698
859
  } catch (error) {
699
- return partial(0, reason(error));
860
+ return { measure: partial(0, reason(error)), rows: null };
700
861
  }
701
862
  }
702
863
 
@@ -784,9 +945,10 @@ const measuresOf = (valueArg: unknown, pathArg: string, outArg: Array<{ path: st
784
945
  /**
785
946
  * Composes the deterministic inventory from a discovery and its unit results.
786
947
  * Repeated results for one unit keep the last. Units without a successful
787
- * result, owner repository counts that disagree with discovery, and every
788
- * partial or unmeasured value make the inventory incomplete, with reasons
789
- * aggregated per subject kind, path and reason.
948
+ * result, a repository result for another id than discovery recorded, two
949
+ * repositories measured with one id, owner repository counts that disagree
950
+ * with discovery, and every partial or unmeasured value make the inventory
951
+ * incomplete, with reasons aggregated per subject kind, path and reason.
790
952
  */
791
953
  export const composeGiteaSourceInventory = (
792
954
  discoveryArg: IDiscovery,
@@ -830,12 +992,15 @@ export const composeGiteaSourceInventory = (
830
992
  const failedRepositories: string[] = [];
831
993
  for (const repository of discoveryArg.repositories) {
832
994
  const result = results.get(unitKey({ kind: 'repository', ...repository }));
833
- if (result?.kind === 'repository') repositories.push(result.repository);
995
+ // A result for another id under the discovered name measured the repository that took the name.
996
+ if (result?.kind === 'repository' && (repository.id === undefined || result.repository.id === repository.id)) repositories.push(result.repository);
834
997
  else {
835
998
  failedRepositories.push(`${repository.owner}/${repository.name}`);
836
- addReason(`repository: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : 'unit not measured'}`);
999
+ addReason(`repository: ${result?.kind === 'failed' ? `unit failed: ${result.reason}` : result?.kind === 'repository' ? 'unit failed: repository-identity-changed' : 'unit not measured'}`);
837
1000
  }
838
1001
  }
1002
+ // Without discovered ids, two names measured as one id show a rename between their units.
1003
+ if (new Set(repositories.map((repositoryArg) => repositoryArg.id)).size !== repositories.length) addReason('repository: duplicate-id');
839
1004
  owners.sort((aArg, bArg) => compareText(aArg.kind, bArg.kind) || compareText(aArg.name, bArg.name));
840
1005
  repositories.sort((aArg, bArg) => compareText(aArg.owner, bArg.owner) || compareText(aArg.name, bArg.name));
841
1006
 
@@ -1,6 +1,7 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
6
  import { LfsBatchClient } from './classes.lfsbatchclient.js';
6
7
  import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
27
28
  maxResponseBytes?: number;
28
29
  /** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
29
30
  lfsTransferTimeoutMs?: number;
31
+ /** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
32
+ releaseAssetTimeoutMs?: number;
30
33
  }
31
34
 
32
35
  /**
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
52
55
  signal?: AbortSignal;
53
56
  }
54
57
 
58
+ /**
59
+ * A release asset as the source inventory recorded it: what a download asks
60
+ * for and checks Gitea still holds. The download count is not part of it.
61
+ */
62
+ export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
63
+
64
+ export interface IGiteaReleaseAssetDownloadOptions {
65
+ /** Aborts the download with `request-aborted`, also while the stream is read. */
66
+ signal?: AbortSignal;
67
+ }
68
+
69
+ export interface IGiteaReleaseAssetDownload {
70
+ releaseId: string;
71
+ asset: TGiteaReleaseAssetSelector;
72
+ /**
73
+ * The asset's bytes, never held whole. The last chunk is held back until
74
+ * exactly `asset.size` bytes have arrived and both identity reads have
75
+ * passed again. Destroying the stream ends the download.
76
+ */
77
+ stream: InstanceType<typeof plugins.stream.Readable>;
78
+ }
79
+
80
+ /** Gitea names attachment content by a random UUID in canonical lowercase form. */
81
+ const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
82
+
55
83
  export interface IGiteaMirrorImport {
56
84
  repository: plugins.TGitManagerRepositoryInfo;
57
85
  /** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
61
89
  /**
62
90
  * Reads a Gitea repository's Git data and LFS objects for its one-time
63
91
  * migration, with the reader's login and token: a mirror clone into a
64
- * GitManager the caller supplies, and verified LFS object downloads. It never
65
- * writes to Gitea: Git is read only through `git-upload-pack`, LFS only
66
- * through batch `download` requests and their download actions.
92
+ * GitManager the caller supplies, verified LFS object downloads and release
93
+ * asset downloads. It never writes to Gitea: Git is read only through
94
+ * `git-upload-pack`, LFS only through batch `download` requests and their
95
+ * download actions, release assets only through GETs of their attachment.
67
96
  */
68
97
  export class GiteaSourceTransfer {
69
98
  readonly #options: {
70
99
  baseUrl: string; login: string; token: string; caCertificates?: string[];
71
- gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
100
+ gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
72
101
  };
73
102
  readonly #client: ForgeRequestClient;
74
103
 
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
83
112
  requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
84
113
  maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
85
114
  lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
115
+ releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
86
116
  };
87
117
  for (const [name, maximum] of [
88
118
  ['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
119
+ ['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
89
120
  ] as const) {
90
121
  const value = limits[name];
91
122
  if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
223
254
  return download;
224
255
  }
225
256
 
257
+ /**
258
+ * Downloads one release asset by GET of `/attachments/{uuid}` with the
259
+ * reader's `token` authorization: no redirect is followed and no range is
260
+ * asked for, so only a 200 answer of the whole content is accepted, and a
261
+ * `content-length`, when sent, must equal `asset.size`. Before the request
262
+ * the repository is read by its id as for `importMirror`, then the release
263
+ * by its id: it must still carry an asset with the same `id`, `uuid`,
264
+ * `name` and `size`, otherwise the download rejects with
265
+ * `release-asset-identity-changed` (a replaced file carries another id and
266
+ * uuid). The bytes stream without a digest; the last chunk is held back
267
+ * until exactly `size` bytes have arrived and both identity reads have
268
+ * passed once more, so a consumer never receives a complete asset that is
269
+ * short, long or no longer the recorded one. Gitea exposes no content
270
+ * digest; a digest the caller needs comes from the content itself.
271
+ *
272
+ * Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
273
+ * `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
274
+ * code such as `http-404` (also for a deleted release),
275
+ * `repository-identity-changed`, `release-asset-identity-changed`,
276
+ * `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
277
+ * `release-asset-size-overrun`, `release-asset-short-body`,
278
+ * `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
279
+ * stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
280
+ * for a refused redirect, or `release-asset-transfer-failed` when body bytes
281
+ * still arrive after the download has ended; the download ends only once
282
+ * the stream is destroyed or finished, and a destroyed stream reports no
283
+ * error, so a reader does not see this code. A failed request is explained
284
+ * by the identity reads as for `downloadLfsObject`.
285
+ */
286
+ public async downloadReleaseAsset(
287
+ repositoryArg: IGiteaSourceRepository,
288
+ releaseIdArg: string,
289
+ assetArg: TGiteaReleaseAssetSelector,
290
+ optionsArg: IGiteaReleaseAssetDownloadOptions = {},
291
+ ): Promise<IGiteaReleaseAssetDownload> {
292
+ const repository = this.#repository(repositoryArg);
293
+ const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
294
+ const { signal } = optionsArg;
295
+ if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
296
+ await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
297
+ const agents = transport.createNodeAgents(this.#options.caCertificates);
298
+ const close = () => agents.https?.destroy();
299
+ const transfer = createTransferBounds(signal, {
300
+ limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
301
+ timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
302
+ });
303
+ let body: AsyncIterable<Uint8Array>;
304
+ try {
305
+ const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
306
+ method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
307
+ }));
308
+ if (response.status !== 200 || !response.body) {
309
+ await transfer.run(() => discardBody(response.body));
310
+ throw new validation.ForgeInventoryError(`http-${response.status}`);
311
+ }
312
+ const length = response.headers.get('content-length');
313
+ if (length !== null && length.trim() !== asset.size) {
314
+ await transfer.run(() => discardBody(response.body));
315
+ throw new validation.ForgeInventoryError('release-asset-size-mismatch');
316
+ }
317
+ body = response.body;
318
+ } catch (error) {
319
+ const failure = transfer.failure(error, 'release-asset-download-failed');
320
+ transfer.end();
321
+ close();
322
+ if (signal?.aborted) throw failure;
323
+ return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
324
+ }
325
+ // After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
326
+ const identified = async () => {
327
+ transfer.touch();
328
+ await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
329
+ };
330
+ const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
331
+ try {
332
+ yield* heldBackChunks(transfer.watch(body), size, {
333
+ invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
334
+ }, { complete: identified });
335
+ } catch (error) {
336
+ throw transfer.failure(error, 'release-asset-download-failed');
337
+ } finally {
338
+ transfer.end();
339
+ }
340
+ };
341
+ // Destroying the stream ends the download and closes its connections at once, also while a read is pending.
342
+ return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
343
+ }
344
+
226
345
  /**
227
346
  * Reads the repository by its id and checks that Gitea still names it
228
347
  * `owner/name`, without case as Gitea looks names up: other names reject
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
259
378
  throw errorArg;
260
379
  }
261
380
 
381
+ /**
382
+ * Reads the repository by its id, then the release by its id under the
383
+ * repository's name: the release must still carry the asset with its id,
384
+ * uuid, name and size, else `release-asset-identity-changed`. A release
385
+ * read that fails is explained by the repository read as for `#explained`.
386
+ */
387
+ async #releaseAssetIdentified(
388
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
389
+ ): Promise<void> {
390
+ await this.#identified(repositoryArg, signalArg);
391
+ let release: Record<string, unknown>;
392
+ try {
393
+ release = validation.record((await this.#client.request(new URL(
394
+ `${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
395
+ ), { signal: signalArg })).body);
396
+ } catch (error) {
397
+ if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
398
+ return this.#explained(repositoryArg, error, signalArg);
399
+ }
400
+ const assets = release.assets ?? [];
401
+ if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
402
+ const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
403
+ if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
404
+ throw new validation.ForgeInventoryError('release-asset-identity-changed');
405
+ }
406
+ }
407
+
408
+ /**
409
+ * Rejects with `errorArg`, a failed asset download, unless the identity
410
+ * reads explain it: then with `repository-identity-changed`,
411
+ * `release-asset-identity-changed` or `http-404`.
412
+ */
413
+ async #releaseAssetExplained(
414
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
415
+ ): Promise<never> {
416
+ try {
417
+ await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
418
+ } catch (error) {
419
+ if (error instanceof validation.ForgeInventoryError
420
+ && ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
421
+ }
422
+ throw errorArg;
423
+ }
424
+
425
+ /** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
426
+ #releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
427
+ try {
428
+ if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
429
+ const releaseId = validation.decimal(releaseIdArg);
430
+ const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
431
+ const size = Number(asset.size);
432
+ if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
433
+ || !Number.isSafeInteger(size)) throw new Error();
434
+ return { releaseId, asset, size };
435
+ } catch {
436
+ throw new validation.ForgeInventoryError('release-asset-invalid');
437
+ }
438
+ }
439
+
262
440
  #repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
263
441
  const single = (valueArg: unknown): string => {
264
442
  const segments = validation.segments(valueArg);