@foss.global/codefeed 4.1.0 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,9 @@ type IInstance = plugins.fossInterfaces.data.IForgeSourceInstanceInventory;
13
13
  type TOwner = plugins.fossInterfaces.data.TForgeSourceOwnerInventory;
14
14
  type IRepository = plugins.fossInterfaces.data.IForgeSourceRepositoryInventory;
15
15
  type IPackageTypeCounts = plugins.fossInterfaces.data.IForgeSourcePackageTypeCounts;
16
+ type IRelease = plugins.fossInterfaces.data.IForgeSourceRelease;
17
+ type IBranchProtection = plugins.fossInterfaces.data.IForgeSourceBranchProtection;
18
+ type ITagProtection = plugins.fossInterfaces.data.IForgeSourceTagProtection;
16
19
  type IInventory = plugins.fossInterfaces.data.IForgeSourceInventory;
17
20
 
18
21
  export interface IGiteaSourceInventoryOptions {
@@ -119,6 +122,35 @@ const requiredString = (valueArg: unknown): string => {
119
122
  return value;
120
123
  };
121
124
 
125
+ /** A string the forge may leave empty, such as a release body. */
126
+ const text = (valueArg: unknown): string => {
127
+ if (typeof valueArg !== 'string') throw new validation.ForgeInventoryError('invalid-string');
128
+ return valueArg;
129
+ };
130
+
131
+ /** A list of names; Gitea answers an empty one as `null`. */
132
+ const texts = (valueArg: unknown): string[] => {
133
+ if (valueArg === null) return [];
134
+ if (!Array.isArray(valueArg)) throw new validation.ForgeInventoryError('invalid-string');
135
+ return valueArg.map(text);
136
+ };
137
+
138
+ /** A signed account id; Gitea's placeholder for a deleted account is -1. */
139
+ const accountId = (valueArg: unknown): string => {
140
+ if (typeof valueArg === 'number' && Number.isSafeInteger(valueArg)) return String(valueArg);
141
+ throw new validation.ForgeInventoryError('invalid-or-imprecise-integer');
142
+ };
143
+
144
+ /** An RFC 3339 time as the forge reports it. */
145
+ const requiredTimestamp = (valueArg: unknown): string => {
146
+ const value = validation.timestamp(valueArg);
147
+ if (value === null) throw new validation.ForgeInventoryError('invalid-timestamp');
148
+ return value;
149
+ };
150
+
151
+ /** Orders canonical nonnegative decimal strings numerically. */
152
+ const compareDecimal = (aArg: string, bArg: string): number => aArg.length - bArg.length || compareText(aArg, bArg);
153
+
122
154
  /** One provider path segment, as an owner or repository name. */
123
155
  const segment = (valueArg: unknown): string => {
124
156
  const segments = validation.segments(valueArg);
@@ -142,6 +174,83 @@ const assets = (valueArg: unknown): { count: bigint; bytes: bigint } => {
142
174
  }), { count: 0n, bytes: 0n });
143
175
  };
144
176
 
177
+ /** A release as Gitea reads it, drafts included, with its assets by numeric id. */
178
+ const releaseRecord = (itemArg: unknown): IRelease => {
179
+ const item = validation.record(itemArg);
180
+ const author = validation.record(item.author);
181
+ const itemAssets = item.assets ?? [];
182
+ if (!Array.isArray(itemAssets)) throw new validation.ForgeInventoryError('invalid-assets');
183
+ return {
184
+ id: validation.decimal(item.id),
185
+ tagName: requiredString(item.tag_name),
186
+ targetCommitish: text(item.target_commitish),
187
+ name: text(item.name),
188
+ body: text(item.body),
189
+ draft: requiredBoolean(item.draft),
190
+ prerelease: requiredBoolean(item.prerelease),
191
+ createdAt: requiredTimestamp(item.created_at),
192
+ publishedAt: requiredTimestamp(item.published_at),
193
+ author: { id: accountId(author.id), login: requiredString(author.login) },
194
+ assets: itemAssets.map((assetArg: unknown) => {
195
+ const asset = validation.record(assetArg);
196
+ return {
197
+ id: validation.decimal(asset.id),
198
+ uuid: requiredString(asset.uuid),
199
+ name: requiredString(asset.name),
200
+ size: validation.decimal(asset.size),
201
+ downloadCount: validation.decimal(asset.download_count),
202
+ createdAt: requiredTimestamp(asset.created_at),
203
+ };
204
+ }).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
205
+ };
206
+ };
207
+
208
+ const allowlist = (usernamesArg: unknown, teamsArg: unknown) => ({ usernames: texts(usernamesArg), teams: texts(teamsArg) });
209
+
210
+ /** A branch protection rule with every restriction Gitea reads back. */
211
+ const branchProtection = (itemArg: unknown): IBranchProtection => {
212
+ const item = validation.record(itemArg);
213
+ return {
214
+ ruleName: requiredString(item.rule_name),
215
+ priority: validation.decimal(item.priority),
216
+ enablePush: requiredBoolean(item.enable_push),
217
+ enablePushAllowlist: requiredBoolean(item.enable_push_whitelist),
218
+ pushAllowlist: allowlist(item.push_whitelist_usernames, item.push_whitelist_teams),
219
+ pushAllowlistDeployKeys: requiredBoolean(item.push_whitelist_deploy_keys),
220
+ enableForcePush: requiredBoolean(item.enable_force_push),
221
+ enableForcePushAllowlist: requiredBoolean(item.enable_force_push_allowlist),
222
+ forcePushAllowlist: allowlist(item.force_push_allowlist_usernames, item.force_push_allowlist_teams),
223
+ forcePushAllowlistDeployKeys: requiredBoolean(item.force_push_allowlist_deploy_keys),
224
+ enableMergeAllowlist: requiredBoolean(item.enable_merge_whitelist),
225
+ mergeAllowlist: allowlist(item.merge_whitelist_usernames, item.merge_whitelist_teams),
226
+ enableBypassAllowlist: requiredBoolean(item.enable_bypass_allowlist),
227
+ bypassAllowlist: allowlist(item.bypass_allowlist_usernames, item.bypass_allowlist_teams),
228
+ enableStatusCheck: requiredBoolean(item.enable_status_check),
229
+ statusCheckContexts: texts(item.status_check_contexts),
230
+ requiredApprovals: validation.decimal(item.required_approvals),
231
+ enableApprovalsAllowlist: requiredBoolean(item.enable_approvals_whitelist),
232
+ approvalsAllowlist: allowlist(item.approvals_whitelist_username, item.approvals_whitelist_teams),
233
+ blockOnRejectedReviews: requiredBoolean(item.block_on_rejected_reviews),
234
+ blockOnOfficialReviewRequests: requiredBoolean(item.block_on_official_review_requests),
235
+ blockOnOutdatedBranch: requiredBoolean(item.block_on_outdated_branch),
236
+ dismissStaleApprovals: requiredBoolean(item.dismiss_stale_approvals),
237
+ ignoreStaleApprovals: requiredBoolean(item.ignore_stale_approvals),
238
+ blockAdminMergeOverride: requiredBoolean(item.block_admin_merge_override),
239
+ requireSignedCommits: requiredBoolean(item.require_signed_commits),
240
+ protectedFilePatterns: text(item.protected_file_patterns),
241
+ unprotectedFilePatterns: text(item.unprotected_file_patterns),
242
+ };
243
+ };
244
+
245
+ const tagProtection = (itemArg: unknown): ITagProtection => {
246
+ const item = validation.record(itemArg);
247
+ return {
248
+ id: validation.decimal(item.id),
249
+ namePattern: requiredString(item.name_pattern),
250
+ allowlist: allowlist(item.whitelist_usernames, item.whitelist_teams),
251
+ };
252
+ };
253
+
145
254
  /** A `{ <rows>, total_count }` envelope whose rows must match its total. */
146
255
  const countedEnvelope = (bodyArg: unknown, rowsKeyArg: string): { rows: unknown[]; failure: string | null } => {
147
256
  const body = validation.record(bodyArg);
@@ -516,6 +625,8 @@ export class GiteaSourceInventory {
516
625
  const parent = item.parent === undefined || item.parent === null ? null : requiredString(validation.record(item.parent).full_name);
517
626
  if (fork !== (parent !== null)) throw new validation.ForgeInventoryError('inconsistent-fork-parent');
518
627
  const options = { signal: signalArg };
628
+ const branchRules = await this.#unpaged(`${path}/branch_protections`, options);
629
+ const tagRules = await this.#unpaged(`${path}/tag_protections`, options);
519
630
  return {
520
631
  owner: ownerArg,
521
632
  name: nameArg,
@@ -533,17 +644,21 @@ export class GiteaSourceInventory {
533
644
  ...await this.#git(ownerArg, nameArg, item, signalArg),
534
645
  collaboration: await this.#collaboration(path, features, signalArg),
535
646
  // Releases stay readable while their unit is disabled; a disabled wiki answers 404.
536
- releases: await this.#releases(path, signalArg),
647
+ ...await this.#releases(path, signalArg),
537
648
  wikiPages: features.wiki ? await this.#count(`${path}/wiki/pages`, { ...options, missingIsEmpty: true }) : unitDisabled('wiki'),
538
649
  access: {
539
650
  collaborators: await this.#count(`${path}/collaborators`, options),
540
651
  teams: await this.#repositoryTeams(path, options),
541
652
  deployKeys: await this.#count(`${path}/keys`, options),
542
653
  webhooks: await this.#count(`${path}/hooks`, options),
543
- branchProtections: await this.#unpagedCount(`${path}/branch_protections`, options),
544
- tagProtections: await this.#unpagedCount(`${path}/tag_protections`, options),
654
+ branchProtections: branchRules.measure,
655
+ tagProtections: tagRules.measure,
545
656
  pushMirrors: await this.#count(`${path}/push_mirrors`, options),
546
657
  },
658
+ protections: branchRules.rows === null || tagRules.rows === null ? null : {
659
+ branches: branchRules.rows.map(branchProtection).sort((aArg, bArg) => compareText(aArg.ruleName, bArg.ruleName)),
660
+ tags: tagRules.rows.map(tagProtection).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id)),
661
+ },
547
662
  actions: await this.#actions(path, empty, signalArg),
548
663
  };
549
664
  }
@@ -560,8 +675,8 @@ export class GiteaSourceInventory {
560
675
  }
561
676
  }
562
677
 
563
- /** HEAD, refs, objects and LFS objects, read from a mirror clone; only the reported size comes from the API. */
564
- async #git(ownerArg: string, nameArg: string, itemArg: Record<string, unknown>, signalArg?: AbortSignal): Promise<Pick<IRepository, 'git' | 'lfs' | 'head'>> {
678
+ /** HEAD, ref digests, refs, objects and LFS objects, read from a mirror clone; only the reported size comes from the API. */
679
+ async #git(ownerArg: string, nameArg: string, itemArg: Record<string, unknown>, signalArg?: AbortSignal): Promise<Pick<IRepository, 'git' | 'lfs' | 'head' | 'refDigests'>> {
565
680
  const sizeKiB = BigInt(validation.decimal(itemArg.size));
566
681
  const username = await this.#login(signalArg);
567
682
  const measured = await measureGiteaMirror({
@@ -570,7 +685,7 @@ export class GiteaSourceInventory {
570
685
  timeoutMs: this.#options.gitTimeoutMs,
571
686
  lfsClient: this.#lfs(username), signal: signalArg,
572
687
  });
573
- return { git: { reportedSizeKiB: complete(sizeKiB), ...measured.git }, lfs: measured.lfs, head: measured.head };
688
+ return { git: { reportedSizeKiB: complete(sizeKiB), ...measured.git }, lfs: measured.lfs, head: measured.head, refDigests: measured.refDigests };
574
689
  }
575
690
 
576
691
  /** The reader's login, the user name of its Git and LFS credentials. */
@@ -655,7 +770,8 @@ export class GiteaSourceInventory {
655
770
  };
656
771
  }
657
772
 
658
- async #releases(pathArg: string, signalArg?: AbortSignal): Promise<IRepository['releases']> {
773
+ /** The release counts, and the release records once the listing is complete. */
774
+ async #releases(pathArg: string, signalArg?: AbortSignal): Promise<Pick<IRepository, 'releases' | 'releaseRecords'>> {
659
775
  // An administrator's listing includes drafts.
660
776
  const releases = await this.#list(`${pathArg}/releases`, { signal: signalArg });
661
777
  let drafts = 0n;
@@ -667,10 +783,15 @@ export class GiteaSourceInventory {
667
783
  releaseAssets = { count: releaseAssets.count + itemAssets.count, bytes: releaseAssets.bytes + itemAssets.bytes };
668
784
  }
669
785
  return {
670
- releases: listingMeasure(releases),
671
- drafts: listingMeasure(releases, drafts),
672
- assets: listingMeasure(releases, releaseAssets.count),
673
- assetBytes: listingMeasure(releases, releaseAssets.bytes),
786
+ releases: {
787
+ releases: listingMeasure(releases),
788
+ drafts: listingMeasure(releases, drafts),
789
+ assets: listingMeasure(releases, releaseAssets.count),
790
+ assetBytes: listingMeasure(releases, releaseAssets.bytes),
791
+ },
792
+ releaseRecords: releases.failure === null
793
+ ? releases.items.map(releaseRecord).sort((aArg, bArg) => compareDecimal(aArg.id, bArg.id))
794
+ : null,
674
795
  };
675
796
  }
676
797
 
@@ -725,13 +846,18 @@ export class GiteaSourceInventory {
725
846
  }
726
847
 
727
848
  async #unpagedCount(pathArg: string, optionsArg: IListOptions): Promise<TMeasure> {
849
+ return (await this.#unpaged(pathArg, optionsArg)).measure;
850
+ }
851
+
852
+ /** A listing Gitea does not page: its count, and its rows when the read succeeded. */
853
+ async #unpaged(pathArg: string, optionsArg: IListOptions): Promise<{ measure: TMeasure; rows: unknown[] | null }> {
728
854
  try {
729
855
  const response = (await this.#get(pathArg, optionsArg)).body;
730
856
  const body = response === null && optionsArg.nullIsEmpty ? [] : response;
731
857
  if (!Array.isArray(body)) throw new validation.ForgeInventoryError('invalid-page');
732
- return complete(body.length);
858
+ return { measure: complete(body.length), rows: body };
733
859
  } catch (error) {
734
- return partial(0, reason(error));
860
+ return { measure: partial(0, reason(error)), rows: null };
735
861
  }
736
862
  }
737
863
 
@@ -1,6 +1,7 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
6
  import { LfsBatchClient } from './classes.lfsbatchclient.js';
6
7
  import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
27
28
  maxResponseBytes?: number;
28
29
  /** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
29
30
  lfsTransferTimeoutMs?: number;
31
+ /** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
32
+ releaseAssetTimeoutMs?: number;
30
33
  }
31
34
 
32
35
  /**
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
52
55
  signal?: AbortSignal;
53
56
  }
54
57
 
58
+ /**
59
+ * A release asset as the source inventory recorded it: what a download asks
60
+ * for and checks Gitea still holds. The download count is not part of it.
61
+ */
62
+ export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
63
+
64
+ export interface IGiteaReleaseAssetDownloadOptions {
65
+ /** Aborts the download with `request-aborted`, also while the stream is read. */
66
+ signal?: AbortSignal;
67
+ }
68
+
69
+ export interface IGiteaReleaseAssetDownload {
70
+ releaseId: string;
71
+ asset: TGiteaReleaseAssetSelector;
72
+ /**
73
+ * The asset's bytes, never held whole. The last chunk is held back until
74
+ * exactly `asset.size` bytes have arrived and both identity reads have
75
+ * passed again. Destroying the stream ends the download.
76
+ */
77
+ stream: InstanceType<typeof plugins.stream.Readable>;
78
+ }
79
+
80
+ /** Gitea names attachment content by a random UUID in canonical lowercase form. */
81
+ const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
82
+
55
83
  export interface IGiteaMirrorImport {
56
84
  repository: plugins.TGitManagerRepositoryInfo;
57
85
  /** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
61
89
  /**
62
90
  * Reads a Gitea repository's Git data and LFS objects for its one-time
63
91
  * migration, with the reader's login and token: a mirror clone into a
64
- * GitManager the caller supplies, and verified LFS object downloads. It never
65
- * writes to Gitea: Git is read only through `git-upload-pack`, LFS only
66
- * through batch `download` requests and their download actions.
92
+ * GitManager the caller supplies, verified LFS object downloads and release
93
+ * asset downloads. It never writes to Gitea: Git is read only through
94
+ * `git-upload-pack`, LFS only through batch `download` requests and their
95
+ * download actions, release assets only through GETs of their attachment.
67
96
  */
68
97
  export class GiteaSourceTransfer {
69
98
  readonly #options: {
70
99
  baseUrl: string; login: string; token: string; caCertificates?: string[];
71
- gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
100
+ gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
72
101
  };
73
102
  readonly #client: ForgeRequestClient;
74
103
 
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
83
112
  requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
84
113
  maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
85
114
  lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
115
+ releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
86
116
  };
87
117
  for (const [name, maximum] of [
88
118
  ['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
119
+ ['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
89
120
  ] as const) {
90
121
  const value = limits[name];
91
122
  if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
223
254
  return download;
224
255
  }
225
256
 
257
+ /**
258
+ * Downloads one release asset by GET of `/attachments/{uuid}` with the
259
+ * reader's `token` authorization: no redirect is followed and no range is
260
+ * asked for, so only a 200 answer of the whole content is accepted, and a
261
+ * `content-length`, when sent, must equal `asset.size`. Before the request
262
+ * the repository is read by its id as for `importMirror`, then the release
263
+ * by its id: it must still carry an asset with the same `id`, `uuid`,
264
+ * `name` and `size`, otherwise the download rejects with
265
+ * `release-asset-identity-changed` (a replaced file carries another id and
266
+ * uuid). The bytes stream without a digest; the last chunk is held back
267
+ * until exactly `size` bytes have arrived and both identity reads have
268
+ * passed once more, so a consumer never receives a complete asset that is
269
+ * short, long or no longer the recorded one. Gitea exposes no content
270
+ * digest; a digest the caller needs comes from the content itself.
271
+ *
272
+ * Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
273
+ * `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
274
+ * code such as `http-404` (also for a deleted release),
275
+ * `repository-identity-changed`, `release-asset-identity-changed`,
276
+ * `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
277
+ * `release-asset-size-overrun`, `release-asset-short-body`,
278
+ * `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
279
+ * stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
280
+ * for a refused redirect, or `release-asset-transfer-failed` when body bytes
281
+ * still arrive after the download has ended; the download ends only once
282
+ * the stream is destroyed or finished, and a destroyed stream reports no
283
+ * error, so a reader does not see this code. A failed request is explained
284
+ * by the identity reads as for `downloadLfsObject`.
285
+ */
286
+ public async downloadReleaseAsset(
287
+ repositoryArg: IGiteaSourceRepository,
288
+ releaseIdArg: string,
289
+ assetArg: TGiteaReleaseAssetSelector,
290
+ optionsArg: IGiteaReleaseAssetDownloadOptions = {},
291
+ ): Promise<IGiteaReleaseAssetDownload> {
292
+ const repository = this.#repository(repositoryArg);
293
+ const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
294
+ const { signal } = optionsArg;
295
+ if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
296
+ await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
297
+ const agents = transport.createNodeAgents(this.#options.caCertificates);
298
+ const close = () => agents.https?.destroy();
299
+ const transfer = createTransferBounds(signal, {
300
+ limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
301
+ timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
302
+ });
303
+ let body: AsyncIterable<Uint8Array>;
304
+ try {
305
+ const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
306
+ method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
307
+ }));
308
+ if (response.status !== 200 || !response.body) {
309
+ await transfer.run(() => discardBody(response.body));
310
+ throw new validation.ForgeInventoryError(`http-${response.status}`);
311
+ }
312
+ const length = response.headers.get('content-length');
313
+ if (length !== null && length.trim() !== asset.size) {
314
+ await transfer.run(() => discardBody(response.body));
315
+ throw new validation.ForgeInventoryError('release-asset-size-mismatch');
316
+ }
317
+ body = response.body;
318
+ } catch (error) {
319
+ const failure = transfer.failure(error, 'release-asset-download-failed');
320
+ transfer.end();
321
+ close();
322
+ if (signal?.aborted) throw failure;
323
+ return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
324
+ }
325
+ // After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
326
+ const identified = async () => {
327
+ transfer.touch();
328
+ await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
329
+ };
330
+ const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
331
+ try {
332
+ yield* heldBackChunks(transfer.watch(body), size, {
333
+ invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
334
+ }, { complete: identified });
335
+ } catch (error) {
336
+ throw transfer.failure(error, 'release-asset-download-failed');
337
+ } finally {
338
+ transfer.end();
339
+ }
340
+ };
341
+ // Destroying the stream ends the download and closes its connections at once, also while a read is pending.
342
+ return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
343
+ }
344
+
226
345
  /**
227
346
  * Reads the repository by its id and checks that Gitea still names it
228
347
  * `owner/name`, without case as Gitea looks names up: other names reject
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
259
378
  throw errorArg;
260
379
  }
261
380
 
381
+ /**
382
+ * Reads the repository by its id, then the release by its id under the
383
+ * repository's name: the release must still carry the asset with its id,
384
+ * uuid, name and size, else `release-asset-identity-changed`. A release
385
+ * read that fails is explained by the repository read as for `#explained`.
386
+ */
387
+ async #releaseAssetIdentified(
388
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
389
+ ): Promise<void> {
390
+ await this.#identified(repositoryArg, signalArg);
391
+ let release: Record<string, unknown>;
392
+ try {
393
+ release = validation.record((await this.#client.request(new URL(
394
+ `${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
395
+ ), { signal: signalArg })).body);
396
+ } catch (error) {
397
+ if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
398
+ return this.#explained(repositoryArg, error, signalArg);
399
+ }
400
+ const assets = release.assets ?? [];
401
+ if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
402
+ const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
403
+ if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
404
+ throw new validation.ForgeInventoryError('release-asset-identity-changed');
405
+ }
406
+ }
407
+
408
+ /**
409
+ * Rejects with `errorArg`, a failed asset download, unless the identity
410
+ * reads explain it: then with `repository-identity-changed`,
411
+ * `release-asset-identity-changed` or `http-404`.
412
+ */
413
+ async #releaseAssetExplained(
414
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
415
+ ): Promise<never> {
416
+ try {
417
+ await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
418
+ } catch (error) {
419
+ if (error instanceof validation.ForgeInventoryError
420
+ && ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
421
+ }
422
+ throw errorArg;
423
+ }
424
+
425
+ /** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
426
+ #releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
427
+ try {
428
+ if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
429
+ const releaseId = validation.decimal(releaseIdArg);
430
+ const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
431
+ const size = Number(asset.size);
432
+ if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
433
+ || !Number.isSafeInteger(size)) throw new Error();
434
+ return { releaseId, asset, size };
435
+ } catch {
436
+ throw new validation.ForgeInventoryError('release-asset-invalid');
437
+ }
438
+ }
439
+
262
440
  #repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
263
441
  const single = (valueArg: unknown): string => {
264
442
  const segments = validation.segments(valueArg);
@@ -1,6 +1,7 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
6
  import type { IForgeRequestMetrics } from './forgeinventory.client.js';
6
7
 
@@ -82,8 +83,6 @@ type TBatchEntry = ILfsObjectId & { actions: Record<string, ILfsAction>; errorCo
82
83
  const lfsMediaType = 'application/vnd.git-lfs+json';
83
84
  /** Objects per batch request; the Git LFS reference server and Gitea accept 100. */
84
85
  const maxBatchSize = 100;
85
- /** Response bytes read from a PUT or verify answer before the rest is discarded. */
86
- const maxDiscardedResponseBytes = 64 * 1024;
87
86
  const maxActionHeaders = 32;
88
87
  const maxHeaderValueLength = 8 * 1024;
89
88
  const maxHrefLength = 8 * 1024;
@@ -179,44 +178,14 @@ const actionExpiry = (actionArg: Record<string, unknown>, receivedAtArg: number)
179
178
  * wrong; otherwise it throws `lfs-size-overrun`, `lfs-short-body` or
180
179
  * `lfs-digest-mismatch`.
181
180
  */
182
- async function* verifiedChunks(sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> {
181
+ const verifiedChunks = (sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> => {
183
182
  const hash = plugins.crypto.createHash('sha256');
184
- let bytes = 0;
185
- let held: Uint8Array | null = null;
186
- for await (const chunk of sourceArg) {
187
- if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError('invalid-lfs-chunk');
188
- if (chunk.byteLength === 0) continue;
189
- bytes += chunk.byteLength;
190
- if (bytes > sizeArg) throw new validation.ForgeInventoryError('lfs-size-overrun');
191
- hash.update(chunk);
192
- if (held) yield held;
193
- held = chunk;
194
- }
195
- if (bytes !== sizeArg) throw new validation.ForgeInventoryError('lfs-short-body');
196
- if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
197
- if (held) yield held;
198
- }
199
-
200
- /**
201
- * Reads and drops at most `maxDiscardedResponseBytes` of a response body that
202
- * carries nothing the client uses, so its connection can be reused; a longer
203
- * or failing body is discarded with its connection.
204
- */
205
- const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
206
- if (!bodyArg) return;
207
- const chunks = bodyArg[Symbol.asyncIterator]();
208
- let bytes = 0;
209
- try {
210
- while (bytes <= maxDiscardedResponseBytes) {
211
- const result = await chunks.next();
212
- if (result.done) return;
213
- bytes += result.value.byteLength;
214
- }
215
- } catch {
216
- // The status already decided the outcome.
217
- } finally {
218
- try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
219
- }
183
+ return heldBackChunks(sourceArg, sizeArg, { invalidChunk: 'invalid-lfs-chunk', overrun: 'lfs-size-overrun', short: 'lfs-short-body' }, {
184
+ inspect: (chunkArg) => hash.update(chunkArg),
185
+ complete: () => {
186
+ if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
187
+ },
188
+ });
220
189
  };
221
190
 
222
191
  /**
@@ -390,10 +359,8 @@ export class LfsBatchClient {
390
359
  transfer.end();
391
360
  }
392
361
  };
393
- const stream = plugins.stream.Readable.from(chunks(), { objectMode: false });
394
- // A consumer that destroys the stream ends the download, also while a read is pending.
395
- stream.once('close', () => transfer.end());
396
- return { oid, size, status: 'available', stream };
362
+ // Destroying the stream ends the download and closes its connection at once, also while a read is pending.
363
+ return { oid, size, status: 'available', stream: createTransferStream(transfer, chunks()) };
397
364
  } catch (error) {
398
365
  transfer.end();
399
366
  throw transfer.failure(error, 'lfs-download-failed');
@@ -553,62 +520,12 @@ export class LfsBatchClient {
553
520
 
554
521
  /**
555
522
  * The bounds of one transfer: the caller's signal, a wall-clock limit and an
556
- * idle limit that `watch` re-arms on every chunk. Whichever ends it first
557
- * decides the failure code.
523
+ * idle limit of `requestTimeoutMs`; a passed limit fails with
524
+ * `lfs-transfer-timed-out`.
558
525
  */
559
526
  #transfer(signalArg: AbortSignal | undefined, limitMsArg = this.#limits.transferTimeoutMs) {
560
- const controller = new AbortController();
561
- let reason: string | null = null;
562
- const stop = (reasonArg: string) => {
563
- reason ??= reasonArg;
564
- controller.abort();
565
- };
566
- const abort = () => stop('request-aborted');
567
- if (signalArg?.aborted) abort();
568
- signalArg?.addEventListener('abort', abort, { once: true });
569
- const deadline = setTimeout(() => stop('lfs-transfer-timed-out'), limitMsArg);
570
- let idle: ReturnType<typeof setTimeout> | undefined;
571
- const rearm = () => {
572
- clearTimeout(idle);
573
- if (!controller.signal.aborted) idle = setTimeout(() => stop('lfs-transfer-timed-out'), this.#limits.requestTimeoutMs);
574
- };
575
- rearm();
576
- const end = () => {
577
- clearTimeout(deadline);
578
- clearTimeout(idle);
579
- signalArg?.removeEventListener('abort', abort);
580
- stop('transfer-ended');
581
- };
582
- const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
583
- if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
584
- return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
585
- };
586
- return {
587
- signal: controller.signal,
588
- end,
589
- failure,
590
- /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
591
- run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
592
- if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
593
- let onAbort!: () => void;
594
- const aborted = new Promise<never>((_resolve, reject) => {
595
- onAbort = () => reject(failure(undefined, 'request-aborted'));
596
- controller.signal.addEventListener('abort', onAbort, { once: true });
597
- });
598
- try {
599
- return await Promise.race([stepArg(), aborted]);
600
- } finally {
601
- controller.signal.removeEventListener('abort', onAbort);
602
- }
603
- },
604
- /** Passes chunks on and re-arms the idle limit on each one. */
605
- watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
606
- for await (const chunk of sourceArg) {
607
- if (controller.signal.aborted) throw failure(undefined, 'lfs-transfer-failed');
608
- rearm();
609
- yield chunk;
610
- }
611
- },
612
- };
527
+ return createTransferBounds(signalArg, {
528
+ limitMs: limitMsArg, idleMs: this.#limits.requestTimeoutMs, timedOutCode: 'lfs-transfer-timed-out', failedCode: 'lfs-transfer-failed',
529
+ });
613
530
  }
614
531
  }