@foss.global/codefeed 4.1.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
6
  import { LfsBatchClient } from './classes.lfsbatchclient.js';
6
7
  import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
27
28
  maxResponseBytes?: number;
28
29
  /** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
29
30
  lfsTransferTimeoutMs?: number;
31
+ /** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
32
+ releaseAssetTimeoutMs?: number;
30
33
  }
31
34
 
32
35
  /**
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
52
55
  signal?: AbortSignal;
53
56
  }
54
57
 
58
+ /**
59
+ * A release asset as the source inventory recorded it: what a download asks
60
+ * for and checks Gitea still holds. The download count is not part of it.
61
+ */
62
+ export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
63
+
64
+ export interface IGiteaReleaseAssetDownloadOptions {
65
+ /** Aborts the download with `request-aborted`, also while the stream is read. */
66
+ signal?: AbortSignal;
67
+ }
68
+
69
+ export interface IGiteaReleaseAssetDownload {
70
+ releaseId: string;
71
+ asset: TGiteaReleaseAssetSelector;
72
+ /**
73
+ * The asset's bytes, never held whole. The last chunk is held back until
74
+ * exactly `asset.size` bytes have arrived and both identity reads have
75
+ * passed again. Destroying the stream ends the download.
76
+ */
77
+ stream: InstanceType<typeof plugins.stream.Readable>;
78
+ }
79
+
80
+ /** Gitea names attachment content by a random UUID in canonical lowercase form. */
81
+ const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
82
+
55
83
  export interface IGiteaMirrorImport {
56
84
  repository: plugins.TGitManagerRepositoryInfo;
57
85
  /** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
61
89
  /**
62
90
  * Reads a Gitea repository's Git data and LFS objects for its one-time
63
91
  * migration, with the reader's login and token: a mirror clone into a
64
- * GitManager the caller supplies, and verified LFS object downloads. It never
65
- * writes to Gitea: Git is read only through `git-upload-pack`, LFS only
66
- * through batch `download` requests and their download actions.
92
+ * GitManager the caller supplies, verified LFS object downloads and release
93
+ * asset downloads. It never writes to Gitea: Git is read only through
94
+ * `git-upload-pack`, LFS only through batch `download` requests and their
95
+ * download actions, release assets only through GETs of their attachment.
67
96
  */
68
97
  export class GiteaSourceTransfer {
69
98
  readonly #options: {
70
99
  baseUrl: string; login: string; token: string; caCertificates?: string[];
71
- gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
100
+ gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
72
101
  };
73
102
  readonly #client: ForgeRequestClient;
74
103
 
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
83
112
  requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
84
113
  maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
85
114
  lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
115
+ releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
86
116
  };
87
117
  for (const [name, maximum] of [
88
118
  ['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
119
+ ['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
89
120
  ] as const) {
90
121
  const value = limits[name];
91
122
  if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
223
254
  return download;
224
255
  }
225
256
 
257
+ /**
258
+ * Downloads one release asset by GET of `/attachments/{uuid}` with the
259
+ * reader's `token` authorization: no redirect is followed and no range is
260
+ * asked for, so only a 200 answer of the whole content is accepted, and a
261
+ * `content-length`, when sent, must equal `asset.size`. Before the request
262
+ * the repository is read by its id as for `importMirror`, then the release
263
+ * by its id: it must still carry an asset with the same `id`, `uuid`,
264
+ * `name` and `size`, otherwise the download rejects with
265
+ * `release-asset-identity-changed` (a replaced file carries another id and
266
+ * uuid). The bytes stream without a digest; the last chunk is held back
267
+ * until exactly `size` bytes have arrived and both identity reads have
268
+ * passed once more, so a consumer never receives a complete asset that is
269
+ * short, long or no longer the recorded one. Gitea exposes no content
270
+ * digest; a digest the caller needs comes from the content itself.
271
+ *
272
+ * Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
273
+ * `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
274
+ * code such as `http-404` (also for a deleted release),
275
+ * `repository-identity-changed`, `release-asset-identity-changed`,
276
+ * `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
277
+ * `release-asset-size-overrun`, `release-asset-short-body`,
278
+ * `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
279
+ * stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
280
+ * for a refused redirect, or `release-asset-transfer-failed` when body bytes
281
+ * still arrive after the download has ended; the download ends only once
282
+ * the stream is destroyed or finished, and a destroyed stream reports no
283
+ * error, so a reader does not see this code. A failed request is explained
284
+ * by the identity reads as for `downloadLfsObject`.
285
+ */
286
+ public async downloadReleaseAsset(
287
+ repositoryArg: IGiteaSourceRepository,
288
+ releaseIdArg: string,
289
+ assetArg: TGiteaReleaseAssetSelector,
290
+ optionsArg: IGiteaReleaseAssetDownloadOptions = {},
291
+ ): Promise<IGiteaReleaseAssetDownload> {
292
+ const repository = this.#repository(repositoryArg);
293
+ const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
294
+ const { signal } = optionsArg;
295
+ if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
296
+ await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
297
+ const agents = transport.createNodeAgents(this.#options.caCertificates);
298
+ const close = () => agents.https?.destroy();
299
+ const transfer = createTransferBounds(signal, {
300
+ limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
301
+ timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
302
+ });
303
+ let body: AsyncIterable<Uint8Array>;
304
+ try {
305
+ const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
306
+ method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
307
+ }));
308
+ if (response.status !== 200 || !response.body) {
309
+ await transfer.run(() => discardBody(response.body));
310
+ throw new validation.ForgeInventoryError(`http-${response.status}`);
311
+ }
312
+ const length = response.headers.get('content-length');
313
+ if (length !== null && length.trim() !== asset.size) {
314
+ await transfer.run(() => discardBody(response.body));
315
+ throw new validation.ForgeInventoryError('release-asset-size-mismatch');
316
+ }
317
+ body = response.body;
318
+ } catch (error) {
319
+ const failure = transfer.failure(error, 'release-asset-download-failed');
320
+ transfer.end();
321
+ close();
322
+ if (signal?.aborted) throw failure;
323
+ return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
324
+ }
325
+ // After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
326
+ const identified = async () => {
327
+ transfer.touch();
328
+ await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
329
+ };
330
+ const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
331
+ try {
332
+ yield* heldBackChunks(transfer.watch(body), size, {
333
+ invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
334
+ }, { complete: identified });
335
+ } catch (error) {
336
+ throw transfer.failure(error, 'release-asset-download-failed');
337
+ } finally {
338
+ transfer.end();
339
+ }
340
+ };
341
+ // Destroying the stream ends the download and closes its connections at once, also while a read is pending.
342
+ return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
343
+ }
344
+
226
345
  /**
227
346
  * Reads the repository by its id and checks that Gitea still names it
228
347
  * `owner/name`, without case as Gitea looks names up: other names reject
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
259
378
  throw errorArg;
260
379
  }
261
380
 
381
+ /**
382
+ * Reads the repository by its id, then the release by its id under the
383
+ * repository's name: the release must still carry the asset with its id,
384
+ * uuid, name and size, else `release-asset-identity-changed`. A release
385
+ * read that fails is explained by the repository read as for `#explained`.
386
+ */
387
+ async #releaseAssetIdentified(
388
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
389
+ ): Promise<void> {
390
+ await this.#identified(repositoryArg, signalArg);
391
+ let release: Record<string, unknown>;
392
+ try {
393
+ release = validation.record((await this.#client.request(new URL(
394
+ `${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
395
+ ), { signal: signalArg })).body);
396
+ } catch (error) {
397
+ if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
398
+ return this.#explained(repositoryArg, error, signalArg);
399
+ }
400
+ const assets = release.assets ?? [];
401
+ if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
402
+ const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
403
+ if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
404
+ throw new validation.ForgeInventoryError('release-asset-identity-changed');
405
+ }
406
+ }
407
+
408
+ /**
409
+ * Rejects with `errorArg`, a failed asset download, unless the identity
410
+ * reads explain it: then with `repository-identity-changed`,
411
+ * `release-asset-identity-changed` or `http-404`.
412
+ */
413
+ async #releaseAssetExplained(
414
+ repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
415
+ ): Promise<never> {
416
+ try {
417
+ await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
418
+ } catch (error) {
419
+ if (error instanceof validation.ForgeInventoryError
420
+ && ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
421
+ }
422
+ throw errorArg;
423
+ }
424
+
425
+ /** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
426
+ #releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
427
+ try {
428
+ if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
429
+ const releaseId = validation.decimal(releaseIdArg);
430
+ const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
431
+ const size = Number(asset.size);
432
+ if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
433
+ || !Number.isSafeInteger(size)) throw new Error();
434
+ return { releaseId, asset, size };
435
+ } catch {
436
+ throw new validation.ForgeInventoryError('release-asset-invalid');
437
+ }
438
+ }
439
+
262
440
  #repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
263
441
  const single = (valueArg: unknown): string => {
264
442
  const segments = validation.segments(valueArg);
@@ -1,6 +1,7 @@
1
1
  import * as plugins from './plugins.js';
2
2
  import * as transport from './forgeinventory.transport.js';
3
3
  import * as validation from './forgeinventory.validation.js';
4
+ import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
4
5
  import { ForgeRequestClient } from './forgeinventory.client.js';
5
6
  import type { IForgeRequestMetrics } from './forgeinventory.client.js';
6
7
 
@@ -82,8 +83,6 @@ type TBatchEntry = ILfsObjectId & { actions: Record<string, ILfsAction>; errorCo
82
83
  const lfsMediaType = 'application/vnd.git-lfs+json';
83
84
  /** Objects per batch request; the Git LFS reference server and Gitea accept 100. */
84
85
  const maxBatchSize = 100;
85
- /** Response bytes read from a PUT or verify answer before the rest is discarded. */
86
- const maxDiscardedResponseBytes = 64 * 1024;
87
86
  const maxActionHeaders = 32;
88
87
  const maxHeaderValueLength = 8 * 1024;
89
88
  const maxHrefLength = 8 * 1024;
@@ -179,44 +178,14 @@ const actionExpiry = (actionArg: Record<string, unknown>, receivedAtArg: number)
179
178
  * wrong; otherwise it throws `lfs-size-overrun`, `lfs-short-body` or
180
179
  * `lfs-digest-mismatch`.
181
180
  */
182
- async function* verifiedChunks(sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> {
181
+ const verifiedChunks = (sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> => {
183
182
  const hash = plugins.crypto.createHash('sha256');
184
- let bytes = 0;
185
- let held: Uint8Array | null = null;
186
- for await (const chunk of sourceArg) {
187
- if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError('invalid-lfs-chunk');
188
- if (chunk.byteLength === 0) continue;
189
- bytes += chunk.byteLength;
190
- if (bytes > sizeArg) throw new validation.ForgeInventoryError('lfs-size-overrun');
191
- hash.update(chunk);
192
- if (held) yield held;
193
- held = chunk;
194
- }
195
- if (bytes !== sizeArg) throw new validation.ForgeInventoryError('lfs-short-body');
196
- if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
197
- if (held) yield held;
198
- }
199
-
200
- /**
201
- * Reads and drops at most `maxDiscardedResponseBytes` of a response body that
202
- * carries nothing the client uses, so its connection can be reused; a longer
203
- * or failing body is discarded with its connection.
204
- */
205
- const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
206
- if (!bodyArg) return;
207
- const chunks = bodyArg[Symbol.asyncIterator]();
208
- let bytes = 0;
209
- try {
210
- while (bytes <= maxDiscardedResponseBytes) {
211
- const result = await chunks.next();
212
- if (result.done) return;
213
- bytes += result.value.byteLength;
214
- }
215
- } catch {
216
- // The status already decided the outcome.
217
- } finally {
218
- try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
219
- }
183
+ return heldBackChunks(sourceArg, sizeArg, { invalidChunk: 'invalid-lfs-chunk', overrun: 'lfs-size-overrun', short: 'lfs-short-body' }, {
184
+ inspect: (chunkArg) => hash.update(chunkArg),
185
+ complete: () => {
186
+ if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
187
+ },
188
+ });
220
189
  };
221
190
 
222
191
  /**
@@ -390,10 +359,8 @@ export class LfsBatchClient {
390
359
  transfer.end();
391
360
  }
392
361
  };
393
- const stream = plugins.stream.Readable.from(chunks(), { objectMode: false });
394
- // A consumer that destroys the stream ends the download, also while a read is pending.
395
- stream.once('close', () => transfer.end());
396
- return { oid, size, status: 'available', stream };
362
+ // Destroying the stream ends the download and closes its connection at once, also while a read is pending.
363
+ return { oid, size, status: 'available', stream: createTransferStream(transfer, chunks()) };
397
364
  } catch (error) {
398
365
  transfer.end();
399
366
  throw transfer.failure(error, 'lfs-download-failed');
@@ -553,62 +520,12 @@ export class LfsBatchClient {
553
520
 
554
521
  /**
555
522
  * The bounds of one transfer: the caller's signal, a wall-clock limit and an
556
- * idle limit that `watch` re-arms on every chunk. Whichever ends it first
557
- * decides the failure code.
523
+ * idle limit of `requestTimeoutMs`; a passed limit fails with
524
+ * `lfs-transfer-timed-out`.
558
525
  */
559
526
  #transfer(signalArg: AbortSignal | undefined, limitMsArg = this.#limits.transferTimeoutMs) {
560
- const controller = new AbortController();
561
- let reason: string | null = null;
562
- const stop = (reasonArg: string) => {
563
- reason ??= reasonArg;
564
- controller.abort();
565
- };
566
- const abort = () => stop('request-aborted');
567
- if (signalArg?.aborted) abort();
568
- signalArg?.addEventListener('abort', abort, { once: true });
569
- const deadline = setTimeout(() => stop('lfs-transfer-timed-out'), limitMsArg);
570
- let idle: ReturnType<typeof setTimeout> | undefined;
571
- const rearm = () => {
572
- clearTimeout(idle);
573
- if (!controller.signal.aborted) idle = setTimeout(() => stop('lfs-transfer-timed-out'), this.#limits.requestTimeoutMs);
574
- };
575
- rearm();
576
- const end = () => {
577
- clearTimeout(deadline);
578
- clearTimeout(idle);
579
- signalArg?.removeEventListener('abort', abort);
580
- stop('transfer-ended');
581
- };
582
- const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
583
- if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
584
- return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
585
- };
586
- return {
587
- signal: controller.signal,
588
- end,
589
- failure,
590
- /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
591
- run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
592
- if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
593
- let onAbort!: () => void;
594
- const aborted = new Promise<never>((_resolve, reject) => {
595
- onAbort = () => reject(failure(undefined, 'request-aborted'));
596
- controller.signal.addEventListener('abort', onAbort, { once: true });
597
- });
598
- try {
599
- return await Promise.race([stepArg(), aborted]);
600
- } finally {
601
- controller.signal.removeEventListener('abort', onAbort);
602
- }
603
- },
604
- /** Passes chunks on and re-arms the idle limit on each one. */
605
- watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
606
- for await (const chunk of sourceArg) {
607
- if (controller.signal.aborted) throw failure(undefined, 'lfs-transfer-failed');
608
- rearm();
609
- yield chunk;
610
- }
611
- },
612
- };
527
+ return createTransferBounds(signalArg, {
528
+ limitMs: limitMsArg, idleMs: this.#limits.requestTimeoutMs, timedOutCode: 'lfs-transfer-timed-out', failedCode: 'lfs-transfer-failed',
529
+ });
613
530
  }
614
531
  }
@@ -0,0 +1,203 @@
1
+ import * as plugins from './plugins.js';
2
+ import * as validation from './forgeinventory.validation.js';
3
+
4
+ /** Response bytes read from an unused answer before the rest is discarded. */
5
+ const maxDiscardedResponseBytes = 64 * 1024;
6
+
7
+ /**
8
+ * Reads and drops at most 64 KiB of a response body that carries nothing the
9
+ * caller uses, so its connection can be reused; a longer or failing body is
10
+ * discarded with its connection.
11
+ */
12
+ export const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
13
+ if (!bodyArg) return;
14
+ const chunks = bodyArg[Symbol.asyncIterator]();
15
+ let bytes = 0;
16
+ try {
17
+ while (bytes <= maxDiscardedResponseBytes) {
18
+ const result = await chunks.next();
19
+ if (result.done) return;
20
+ bytes += result.value.byteLength;
21
+ }
22
+ } catch {
23
+ // The status already decided the outcome.
24
+ } finally {
25
+ try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
26
+ }
27
+ };
28
+
29
+ export interface ITransferBoundsOptions {
30
+ /** Wall-clock limit of the whole transfer. */
31
+ limitMs: number;
32
+ /** Longest gap between two chunks of a watched source. */
33
+ idleMs: number;
34
+ /** The code a passed wall-clock or idle limit fails with. */
35
+ timedOutCode: string;
36
+ /** The code a watched source that goes on after the transfer ended fails with. */
37
+ failedCode: string;
38
+ }
39
+
40
+ export interface ITransferBounds {
41
+ /** Aborts once the caller aborts, a limit passes or the transfer ends. */
42
+ signal: AbortSignal;
43
+ /** Ends the transfer: clears its timers and aborts `signal`. Idempotent. */
44
+ end: () => void;
45
+ /**
46
+ * The error the transfer fails with: the reason that ended it (`request-aborted`
47
+ * or the timeout code), else `errorArg` when it is a `ForgeInventoryError`,
48
+ * else `fallbackArg`.
49
+ */
50
+ failure: (errorArg: unknown, fallbackArg: string) => validation.ForgeInventoryError;
51
+ /** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
52
+ run: <T>(stepArg: () => Promise<T>) => Promise<T>;
53
+ /** Re-arms the idle limit, before a step that waits on the network without a watched source. */
54
+ touch: () => void;
55
+ /** Passes chunks on and re-arms the idle limit on each one. */
56
+ watch: (sourceArg: AsyncIterable<Uint8Array>) => AsyncGenerator<Uint8Array, void, undefined>;
57
+ }
58
+
59
+ /**
60
+ * The bounds of one streamed transfer: the caller's signal, a wall-clock limit
61
+ * and an idle limit that `watch` re-arms on every chunk. Whichever ends it
62
+ * first decides the failure code. Every transfer must be ended with `end`.
63
+ */
64
+ export const createTransferBounds = (signalArg: AbortSignal | undefined, optionsArg: ITransferBoundsOptions): ITransferBounds => {
65
+ const controller = new AbortController();
66
+ let reason: string | null = null;
67
+ const stop = (reasonArg: string) => {
68
+ reason ??= reasonArg;
69
+ controller.abort();
70
+ };
71
+ const abort = () => stop('request-aborted');
72
+ if (signalArg?.aborted) abort();
73
+ signalArg?.addEventListener('abort', abort, { once: true });
74
+ const deadline = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.limitMs);
75
+ let idle: ReturnType<typeof setTimeout> | undefined;
76
+ const rearm = () => {
77
+ clearTimeout(idle);
78
+ if (!controller.signal.aborted) idle = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.idleMs);
79
+ };
80
+ rearm();
81
+ const end = () => {
82
+ clearTimeout(deadline);
83
+ clearTimeout(idle);
84
+ signalArg?.removeEventListener('abort', abort);
85
+ stop('transfer-ended');
86
+ };
87
+ const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
88
+ if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
89
+ return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
90
+ };
91
+ return {
92
+ signal: controller.signal,
93
+ end,
94
+ failure,
95
+ touch: rearm,
96
+ run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
97
+ if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
98
+ let onAbort!: () => void;
99
+ const aborted = new Promise<never>((_resolve, reject) => {
100
+ onAbort = () => reject(failure(undefined, 'request-aborted'));
101
+ controller.signal.addEventListener('abort', onAbort, { once: true });
102
+ });
103
+ try {
104
+ return await Promise.race([stepArg(), aborted]);
105
+ } finally {
106
+ controller.signal.removeEventListener('abort', onAbort);
107
+ }
108
+ },
109
+ watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
110
+ for await (const chunk of sourceArg) {
111
+ if (controller.signal.aborted) throw failure(undefined, optionsArg.failedCode);
112
+ rearm();
113
+ yield chunk;
114
+ }
115
+ },
116
+ };
117
+ };
118
+
119
+ export interface IHeldBackCodes {
120
+ /** A chunk that is not a `Uint8Array`. */
121
+ invalidChunk: string;
122
+ /** More bytes than the declared size. */
123
+ overrun: string;
124
+ /** Fewer bytes than the declared size. */
125
+ short: string;
126
+ }
127
+
128
+ export interface IHeldBackChecks {
129
+ /** Sees every non-empty chunk in order, before it is passed on. */
130
+ inspect?: (chunkArg: Uint8Array) => void;
131
+ /** Runs once exactly `size` bytes have arrived, before the last chunk is passed on; a rejection keeps it back. */
132
+ complete?: () => void | Promise<void>;
133
+ }
134
+
135
+ /**
136
+ * Passes the bytes of `sourceArg` on while it counts them. The last chunk is
137
+ * held back until the source has ended with exactly `sizeArg` bytes and
138
+ * `checksArg.complete` has passed, so a consumer never receives a complete
139
+ * body that is wrong; otherwise it throws `codesArg.overrun` as soon as the
140
+ * size is exceeded, `codesArg.short` when the source ends early, or what
141
+ * `complete` threw.
142
+ */
143
+ export async function* heldBackChunks(
144
+ sourceArg: AsyncIterable<Uint8Array>, sizeArg: number, codesArg: IHeldBackCodes, checksArg: IHeldBackChecks = {},
145
+ ): AsyncGenerator<Uint8Array, void, undefined> {
146
+ let bytes = 0;
147
+ let held: Uint8Array | null = null;
148
+ for await (const chunk of sourceArg) {
149
+ if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError(codesArg.invalidChunk);
150
+ if (chunk.byteLength === 0) continue;
151
+ bytes += chunk.byteLength;
152
+ if (bytes > sizeArg) throw new validation.ForgeInventoryError(codesArg.overrun);
153
+ checksArg.inspect?.(chunk);
154
+ if (held) yield held;
155
+ held = chunk;
156
+ }
157
+ if (bytes !== sizeArg) throw new validation.ForgeInventoryError(codesArg.short);
158
+ await checksArg.complete?.();
159
+ if (held) yield held;
160
+ }
161
+
162
+ /**
163
+ * The readable stream of one transfer's bytes. It pulls `chunksArg` with
164
+ * backpressure and ends `boundsArg` and calls `onEndArg` as soon as it is
165
+ * destroyed, also while a pull is pending: ending the bounds aborts the
166
+ * request, which closes its connection, and the pending pull's result is
167
+ * dropped. (`Readable.from` waits for a pending pull before it finishes a
168
+ * destroy, so a stalled body would keep its connection until the idle limit.)
169
+ * The stream errors with what `chunksArg` throws.
170
+ */
171
+ export const createTransferStream = (
172
+ boundsArg: ITransferBounds, chunksArg: AsyncGenerator<Uint8Array, void, undefined>, onEndArg?: () => void,
173
+ ): InstanceType<typeof plugins.stream.Readable> => {
174
+ let pulling = false;
175
+ const stream: InstanceType<typeof plugins.stream.Readable> = new plugins.stream.Readable({
176
+ read() {
177
+ if (pulling) return;
178
+ pulling = true;
179
+ void (async () => {
180
+ try {
181
+ while (!stream.destroyed) {
182
+ const result = await chunksArg.next();
183
+ if (stream.destroyed) return;
184
+ if (result.done) return void stream.push(null);
185
+ if (!stream.push(result.value)) return;
186
+ }
187
+ } catch (error) {
188
+ stream.destroy(error as Error);
189
+ } finally {
190
+ pulling = false;
191
+ }
192
+ })();
193
+ },
194
+ destroy(errorArg, callbackArg) {
195
+ boundsArg.end();
196
+ onEndArg?.();
197
+ // Runs the generator's cleanup once a pending pull has settled; nothing waits for it.
198
+ chunksArg.return(undefined).catch(() => undefined);
199
+ callbackArg(errorArg);
200
+ },
201
+ });
202
+ return stream;
203
+ };