@foss.global/codefeed 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.md +20 -0
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.giteasourceinventory.js +129 -10
- package/dist_ts/classes.giteasourcetransfer.d.ts +55 -3
- package/dist_ts/classes.giteasourcetransfer.js +155 -4
- package/dist_ts/classes.lfsbatchclient.js +17 -118
- package/dist_ts/forgeinventory.transfer.d.ts +75 -0
- package/dist_ts/forgeinventory.transfer.js +174 -0
- package/package.json +3 -3
- package/readme.md +12 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.giteasourceinventory.ts +136 -10
- package/ts/classes.giteasourcetransfer.ts +182 -4
- package/ts/classes.lfsbatchclient.ts +15 -98
- package/ts/forgeinventory.transfer.ts +203 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
6
|
import { LfsBatchClient } from './classes.lfsbatchclient.js';
|
|
6
7
|
import type { ILfsRequestOptions, TLfsDownload } from './classes.lfsbatchclient.js';
|
|
@@ -27,6 +28,8 @@ export interface IGiteaSourceTransferOptions {
|
|
|
27
28
|
maxResponseBytes?: number;
|
|
28
29
|
/** Wall-clock limit of one LFS object download; defaults to 30 minutes, at most six hours. */
|
|
29
30
|
lfsTransferTimeoutMs?: number;
|
|
31
|
+
/** Wall-clock limit of one release asset download, its closing identity reads included; defaults to 30 minutes, at most six hours. */
|
|
32
|
+
releaseAssetTimeoutMs?: number;
|
|
30
33
|
}
|
|
31
34
|
|
|
32
35
|
/**
|
|
@@ -52,6 +55,31 @@ export interface IGiteaMirrorImportOptions {
|
|
|
52
55
|
signal?: AbortSignal;
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
/**
|
|
59
|
+
* A release asset as the source inventory recorded it: what a download asks
|
|
60
|
+
* for and checks Gitea still holds. The download count is not part of it.
|
|
61
|
+
*/
|
|
62
|
+
export type TGiteaReleaseAssetSelector = Pick<plugins.fossInterfaces.data.IForgeSourceReleaseAsset, 'id' | 'uuid' | 'name' | 'size'>;
|
|
63
|
+
|
|
64
|
+
export interface IGiteaReleaseAssetDownloadOptions {
|
|
65
|
+
/** Aborts the download with `request-aborted`, also while the stream is read. */
|
|
66
|
+
signal?: AbortSignal;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface IGiteaReleaseAssetDownload {
|
|
70
|
+
releaseId: string;
|
|
71
|
+
asset: TGiteaReleaseAssetSelector;
|
|
72
|
+
/**
|
|
73
|
+
* The asset's bytes, never held whole. The last chunk is held back until
|
|
74
|
+
* exactly `asset.size` bytes have arrived and both identity reads have
|
|
75
|
+
* passed again. Destroying the stream ends the download.
|
|
76
|
+
*/
|
|
77
|
+
stream: InstanceType<typeof plugins.stream.Readable>;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Gitea names attachment content by a random UUID in canonical lowercase form. */
|
|
81
|
+
const attachmentUuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
82
|
+
|
|
55
83
|
export interface IGiteaMirrorImport {
|
|
56
84
|
repository: plugins.TGitManagerRepositoryInfo;
|
|
57
85
|
/** The repository size Gitea reported, in KiB, which sized the scratch reservation. */
|
|
@@ -61,14 +89,15 @@ export interface IGiteaMirrorImport {
|
|
|
61
89
|
/**
|
|
62
90
|
* Reads a Gitea repository's Git data and LFS objects for its one-time
|
|
63
91
|
* migration, with the reader's login and token: a mirror clone into a
|
|
64
|
-
* GitManager the caller supplies,
|
|
65
|
-
* writes to Gitea: Git is read only through
|
|
66
|
-
* through batch `download` requests and their
|
|
92
|
+
* GitManager the caller supplies, verified LFS object downloads and release
|
|
93
|
+
* asset downloads. It never writes to Gitea: Git is read only through
|
|
94
|
+
* `git-upload-pack`, LFS only through batch `download` requests and their
|
|
95
|
+
* download actions, release assets only through GETs of their attachment.
|
|
67
96
|
*/
|
|
68
97
|
export class GiteaSourceTransfer {
|
|
69
98
|
readonly #options: {
|
|
70
99
|
baseUrl: string; login: string; token: string; caCertificates?: string[];
|
|
71
|
-
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number;
|
|
100
|
+
gitTimeoutMs: number; requestTimeoutMs: number; maxResponseBytes: number; lfsTransferTimeoutMs: number; releaseAssetTimeoutMs: number;
|
|
72
101
|
};
|
|
73
102
|
readonly #client: ForgeRequestClient;
|
|
74
103
|
|
|
@@ -83,9 +112,11 @@ export class GiteaSourceTransfer {
|
|
|
83
112
|
requestTimeoutMs: optionsArg.requestTimeoutMs ?? 60_000,
|
|
84
113
|
maxResponseBytes: optionsArg.maxResponseBytes ?? 16 * 1024 * 1024,
|
|
85
114
|
lfsTransferTimeoutMs: optionsArg.lfsTransferTimeoutMs ?? 30 * 60 * 1000,
|
|
115
|
+
releaseAssetTimeoutMs: optionsArg.releaseAssetTimeoutMs ?? 30 * 60 * 1000,
|
|
86
116
|
};
|
|
87
117
|
for (const [name, maximum] of [
|
|
88
118
|
['gitTimeoutMs', 6 * 60 * 60 * 1000], ['requestTimeoutMs', 300_000], ['maxResponseBytes', 64 * 1024 * 1024], ['lfsTransferTimeoutMs', 6 * 60 * 60 * 1000],
|
|
119
|
+
['releaseAssetTimeoutMs', 6 * 60 * 60 * 1000],
|
|
89
120
|
] as const) {
|
|
90
121
|
const value = limits[name];
|
|
91
122
|
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) throw new validation.ForgeInventoryError('invalid-limits');
|
|
@@ -223,6 +254,94 @@ export class GiteaSourceTransfer {
|
|
|
223
254
|
return download;
|
|
224
255
|
}
|
|
225
256
|
|
|
257
|
+
/**
|
|
258
|
+
* Downloads one release asset by GET of `/attachments/{uuid}` with the
|
|
259
|
+
* reader's `token` authorization: no redirect is followed and no range is
|
|
260
|
+
* asked for, so only a 200 answer of the whole content is accepted, and a
|
|
261
|
+
* `content-length`, when sent, must equal `asset.size`. Before the request
|
|
262
|
+
* the repository is read by its id as for `importMirror`, then the release
|
|
263
|
+
* by its id: it must still carry an asset with the same `id`, `uuid`,
|
|
264
|
+
* `name` and `size`, otherwise the download rejects with
|
|
265
|
+
* `release-asset-identity-changed` (a replaced file carries another id and
|
|
266
|
+
* uuid). The bytes stream without a digest; the last chunk is held back
|
|
267
|
+
* until exactly `size` bytes have arrived and both identity reads have
|
|
268
|
+
* passed once more, so a consumer never receives a complete asset that is
|
|
269
|
+
* short, long or no longer the recorded one. Gitea exposes no content
|
|
270
|
+
* digest; a digest the caller needs comes from the content itself.
|
|
271
|
+
*
|
|
272
|
+
* Rejects, or errors the stream, with a fixed `ForgeInventoryError` code:
|
|
273
|
+
* `invalid-repository`, `release-asset-invalid`, `request-aborted`, a REST
|
|
274
|
+
* code such as `http-404` (also for a deleted release),
|
|
275
|
+
* `repository-identity-changed`, `release-asset-identity-changed`,
|
|
276
|
+
* `http-<status>` for any answer but 200, `release-asset-size-mismatch`,
|
|
277
|
+
* `release-asset-size-overrun`, `release-asset-short-body`,
|
|
278
|
+
* `release-asset-transfer-timed-out` (after `releaseAssetTimeoutMs`, or a
|
|
279
|
+
* stall of `requestTimeoutMs`), `release-asset-download-failed`, such as
|
|
280
|
+
* for a refused redirect, or `release-asset-transfer-failed` when body bytes
|
|
281
|
+
* still arrive after the download has ended; the download ends only once
|
|
282
|
+
* the stream is destroyed or finished, and a destroyed stream reports no
|
|
283
|
+
* error, so a reader does not see this code. A failed request is explained
|
|
284
|
+
* by the identity reads as for `downloadLfsObject`.
|
|
285
|
+
*/
|
|
286
|
+
public async downloadReleaseAsset(
|
|
287
|
+
repositoryArg: IGiteaSourceRepository,
|
|
288
|
+
releaseIdArg: string,
|
|
289
|
+
assetArg: TGiteaReleaseAssetSelector,
|
|
290
|
+
optionsArg: IGiteaReleaseAssetDownloadOptions = {},
|
|
291
|
+
): Promise<IGiteaReleaseAssetDownload> {
|
|
292
|
+
const repository = this.#repository(repositoryArg);
|
|
293
|
+
const { releaseId, asset, size } = this.#releaseAssetSelector(releaseIdArg, assetArg);
|
|
294
|
+
const { signal } = optionsArg;
|
|
295
|
+
if (signal?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
296
|
+
await this.#releaseAssetIdentified(repository, releaseId, asset, signal);
|
|
297
|
+
const agents = transport.createNodeAgents(this.#options.caCertificates);
|
|
298
|
+
const close = () => agents.https?.destroy();
|
|
299
|
+
const transfer = createTransferBounds(signal, {
|
|
300
|
+
limitMs: this.#options.releaseAssetTimeoutMs, idleMs: this.#options.requestTimeoutMs,
|
|
301
|
+
timedOutCode: 'release-asset-transfer-timed-out', failedCode: 'release-asset-transfer-failed',
|
|
302
|
+
});
|
|
303
|
+
let body: AsyncIterable<Uint8Array>;
|
|
304
|
+
try {
|
|
305
|
+
const response = await transfer.run(() => transport.sendNodeRequest(new URL(`${this.#options.baseUrl}/attachments/${asset.uuid}`), {
|
|
306
|
+
method: 'GET', headers: { authorization: `token ${this.#options.token}` }, signal: transfer.signal, agents,
|
|
307
|
+
}));
|
|
308
|
+
if (response.status !== 200 || !response.body) {
|
|
309
|
+
await transfer.run(() => discardBody(response.body));
|
|
310
|
+
throw new validation.ForgeInventoryError(`http-${response.status}`);
|
|
311
|
+
}
|
|
312
|
+
const length = response.headers.get('content-length');
|
|
313
|
+
if (length !== null && length.trim() !== asset.size) {
|
|
314
|
+
await transfer.run(() => discardBody(response.body));
|
|
315
|
+
throw new validation.ForgeInventoryError('release-asset-size-mismatch');
|
|
316
|
+
}
|
|
317
|
+
body = response.body;
|
|
318
|
+
} catch (error) {
|
|
319
|
+
const failure = transfer.failure(error, 'release-asset-download-failed');
|
|
320
|
+
transfer.end();
|
|
321
|
+
close();
|
|
322
|
+
if (signal?.aborted) throw failure;
|
|
323
|
+
return this.#releaseAssetExplained(repository, releaseId, asset, failure, signal);
|
|
324
|
+
}
|
|
325
|
+
// After the last byte, before the held chunk: the repository and the asset are still the ones asked for.
|
|
326
|
+
const identified = async () => {
|
|
327
|
+
transfer.touch();
|
|
328
|
+
await transfer.run(() => this.#releaseAssetIdentified(repository, releaseId, asset, transfer.signal));
|
|
329
|
+
};
|
|
330
|
+
const chunks = async function* (): AsyncGenerator<Uint8Array, void, undefined> {
|
|
331
|
+
try {
|
|
332
|
+
yield* heldBackChunks(transfer.watch(body), size, {
|
|
333
|
+
invalidChunk: 'release-asset-invalid-chunk', overrun: 'release-asset-size-overrun', short: 'release-asset-short-body',
|
|
334
|
+
}, { complete: identified });
|
|
335
|
+
} catch (error) {
|
|
336
|
+
throw transfer.failure(error, 'release-asset-download-failed');
|
|
337
|
+
} finally {
|
|
338
|
+
transfer.end();
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
// Destroying the stream ends the download and closes its connections at once, also while a read is pending.
|
|
342
|
+
return { releaseId, asset, stream: createTransferStream(transfer, chunks(), close) };
|
|
343
|
+
}
|
|
344
|
+
|
|
226
345
|
/**
|
|
227
346
|
* Reads the repository by its id and checks that Gitea still names it
|
|
228
347
|
* `owner/name`, without case as Gitea looks names up: other names reject
|
|
@@ -259,6 +378,65 @@ export class GiteaSourceTransfer {
|
|
|
259
378
|
throw errorArg;
|
|
260
379
|
}
|
|
261
380
|
|
|
381
|
+
/**
|
|
382
|
+
* Reads the repository by its id, then the release by its id under the
|
|
383
|
+
* repository's name: the release must still carry the asset with its id,
|
|
384
|
+
* uuid, name and size, else `release-asset-identity-changed`. A release
|
|
385
|
+
* read that fails is explained by the repository read as for `#explained`.
|
|
386
|
+
*/
|
|
387
|
+
async #releaseAssetIdentified(
|
|
388
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, signalArg: AbortSignal | undefined,
|
|
389
|
+
): Promise<void> {
|
|
390
|
+
await this.#identified(repositoryArg, signalArg);
|
|
391
|
+
let release: Record<string, unknown>;
|
|
392
|
+
try {
|
|
393
|
+
release = validation.record((await this.#client.request(new URL(
|
|
394
|
+
`${this.#options.baseUrl}/api/v1/repos/${encodeURIComponent(repositoryArg.owner)}/${encodeURIComponent(repositoryArg.name)}/releases/${releaseIdArg}`,
|
|
395
|
+
), { signal: signalArg })).body);
|
|
396
|
+
} catch (error) {
|
|
397
|
+
if (signalArg?.aborted) throw new validation.ForgeInventoryError('request-aborted');
|
|
398
|
+
return this.#explained(repositoryArg, error, signalArg);
|
|
399
|
+
}
|
|
400
|
+
const assets = release.assets ?? [];
|
|
401
|
+
if (validation.decimal(release.id) !== releaseIdArg || !Array.isArray(assets)) throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
402
|
+
const found = assets.map((itemArg: unknown) => validation.record(itemArg)).find((itemArg) => validation.decimal(itemArg.id) === assetArg.id);
|
|
403
|
+
if (!found || found.uuid !== assetArg.uuid || found.name !== assetArg.name || validation.decimal(found.size) !== assetArg.size) {
|
|
404
|
+
throw new validation.ForgeInventoryError('release-asset-identity-changed');
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Rejects with `errorArg`, a failed asset download, unless the identity
|
|
410
|
+
* reads explain it: then with `repository-identity-changed`,
|
|
411
|
+
* `release-asset-identity-changed` or `http-404`.
|
|
412
|
+
*/
|
|
413
|
+
async #releaseAssetExplained(
|
|
414
|
+
repositoryArg: IGiteaSourceRepository, releaseIdArg: string, assetArg: TGiteaReleaseAssetSelector, errorArg: unknown, signalArg: AbortSignal | undefined,
|
|
415
|
+
): Promise<never> {
|
|
416
|
+
try {
|
|
417
|
+
await this.#releaseAssetIdentified(repositoryArg, releaseIdArg, assetArg, signalArg);
|
|
418
|
+
} catch (error) {
|
|
419
|
+
if (error instanceof validation.ForgeInventoryError
|
|
420
|
+
&& ['repository-identity-changed', 'release-asset-identity-changed', 'http-404'].includes(error.code)) throw error;
|
|
421
|
+
}
|
|
422
|
+
throw errorArg;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/** The release id and asset of a download request, validated; sizes beyond `Number.MAX_SAFE_INTEGER` are refused. */
|
|
426
|
+
#releaseAssetSelector(releaseIdArg: unknown, assetArg: TGiteaReleaseAssetSelector): { releaseId: string; asset: TGiteaReleaseAssetSelector; size: number } {
|
|
427
|
+
try {
|
|
428
|
+
if (typeof releaseIdArg !== 'string' || typeof assetArg?.id !== 'string' || typeof assetArg.size !== 'string') throw new Error();
|
|
429
|
+
const releaseId = validation.decimal(releaseIdArg);
|
|
430
|
+
const asset = { id: validation.decimal(assetArg.id), uuid: assetArg.uuid, name: assetArg.name, size: validation.decimal(assetArg.size) };
|
|
431
|
+
const size = Number(asset.size);
|
|
432
|
+
if (typeof asset.uuid !== 'string' || !attachmentUuidPattern.test(asset.uuid) || typeof asset.name !== 'string' || !asset.name
|
|
433
|
+
|| !Number.isSafeInteger(size)) throw new Error();
|
|
434
|
+
return { releaseId, asset, size };
|
|
435
|
+
} catch {
|
|
436
|
+
throw new validation.ForgeInventoryError('release-asset-invalid');
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
262
440
|
#repository(repositoryArg: IGiteaSourceRepository): IGiteaSourceRepository {
|
|
263
441
|
const single = (valueArg: unknown): string => {
|
|
264
442
|
const segments = validation.segments(valueArg);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
2
|
import * as transport from './forgeinventory.transport.js';
|
|
3
3
|
import * as validation from './forgeinventory.validation.js';
|
|
4
|
+
import { createTransferBounds, createTransferStream, discardBody, heldBackChunks } from './forgeinventory.transfer.js';
|
|
4
5
|
import { ForgeRequestClient } from './forgeinventory.client.js';
|
|
5
6
|
import type { IForgeRequestMetrics } from './forgeinventory.client.js';
|
|
6
7
|
|
|
@@ -82,8 +83,6 @@ type TBatchEntry = ILfsObjectId & { actions: Record<string, ILfsAction>; errorCo
|
|
|
82
83
|
const lfsMediaType = 'application/vnd.git-lfs+json';
|
|
83
84
|
/** Objects per batch request; the Git LFS reference server and Gitea accept 100. */
|
|
84
85
|
const maxBatchSize = 100;
|
|
85
|
-
/** Response bytes read from a PUT or verify answer before the rest is discarded. */
|
|
86
|
-
const maxDiscardedResponseBytes = 64 * 1024;
|
|
87
86
|
const maxActionHeaders = 32;
|
|
88
87
|
const maxHeaderValueLength = 8 * 1024;
|
|
89
88
|
const maxHrefLength = 8 * 1024;
|
|
@@ -179,44 +178,14 @@ const actionExpiry = (actionArg: Record<string, unknown>, receivedAtArg: number)
|
|
|
179
178
|
* wrong; otherwise it throws `lfs-size-overrun`, `lfs-short-body` or
|
|
180
179
|
* `lfs-digest-mismatch`.
|
|
181
180
|
*/
|
|
182
|
-
|
|
181
|
+
const verifiedChunks = (sourceArg: AsyncIterable<Uint8Array>, oidArg: string, sizeArg: number): AsyncGenerator<Uint8Array, void, undefined> => {
|
|
183
182
|
const hash = plugins.crypto.createHash('sha256');
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (bytes > sizeArg) throw new validation.ForgeInventoryError('lfs-size-overrun');
|
|
191
|
-
hash.update(chunk);
|
|
192
|
-
if (held) yield held;
|
|
193
|
-
held = chunk;
|
|
194
|
-
}
|
|
195
|
-
if (bytes !== sizeArg) throw new validation.ForgeInventoryError('lfs-short-body');
|
|
196
|
-
if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
|
|
197
|
-
if (held) yield held;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* Reads and drops at most `maxDiscardedResponseBytes` of a response body that
|
|
202
|
-
* carries nothing the client uses, so its connection can be reused; a longer
|
|
203
|
-
* or failing body is discarded with its connection.
|
|
204
|
-
*/
|
|
205
|
-
const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
|
|
206
|
-
if (!bodyArg) return;
|
|
207
|
-
const chunks = bodyArg[Symbol.asyncIterator]();
|
|
208
|
-
let bytes = 0;
|
|
209
|
-
try {
|
|
210
|
-
while (bytes <= maxDiscardedResponseBytes) {
|
|
211
|
-
const result = await chunks.next();
|
|
212
|
-
if (result.done) return;
|
|
213
|
-
bytes += result.value.byteLength;
|
|
214
|
-
}
|
|
215
|
-
} catch {
|
|
216
|
-
// The status already decided the outcome.
|
|
217
|
-
} finally {
|
|
218
|
-
try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
|
|
219
|
-
}
|
|
183
|
+
return heldBackChunks(sourceArg, sizeArg, { invalidChunk: 'invalid-lfs-chunk', overrun: 'lfs-size-overrun', short: 'lfs-short-body' }, {
|
|
184
|
+
inspect: (chunkArg) => hash.update(chunkArg),
|
|
185
|
+
complete: () => {
|
|
186
|
+
if (hash.digest('hex') !== oidArg) throw new validation.ForgeInventoryError('lfs-digest-mismatch');
|
|
187
|
+
},
|
|
188
|
+
});
|
|
220
189
|
};
|
|
221
190
|
|
|
222
191
|
/**
|
|
@@ -390,10 +359,8 @@ export class LfsBatchClient {
|
|
|
390
359
|
transfer.end();
|
|
391
360
|
}
|
|
392
361
|
};
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
stream.once('close', () => transfer.end());
|
|
396
|
-
return { oid, size, status: 'available', stream };
|
|
362
|
+
// Destroying the stream ends the download and closes its connection at once, also while a read is pending.
|
|
363
|
+
return { oid, size, status: 'available', stream: createTransferStream(transfer, chunks()) };
|
|
397
364
|
} catch (error) {
|
|
398
365
|
transfer.end();
|
|
399
366
|
throw transfer.failure(error, 'lfs-download-failed');
|
|
@@ -553,62 +520,12 @@ export class LfsBatchClient {
|
|
|
553
520
|
|
|
554
521
|
/**
|
|
555
522
|
* The bounds of one transfer: the caller's signal, a wall-clock limit and an
|
|
556
|
-
* idle limit
|
|
557
|
-
*
|
|
523
|
+
* idle limit of `requestTimeoutMs`; a passed limit fails with
|
|
524
|
+
* `lfs-transfer-timed-out`.
|
|
558
525
|
*/
|
|
559
526
|
#transfer(signalArg: AbortSignal | undefined, limitMsArg = this.#limits.transferTimeoutMs) {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
reason ??= reasonArg;
|
|
564
|
-
controller.abort();
|
|
565
|
-
};
|
|
566
|
-
const abort = () => stop('request-aborted');
|
|
567
|
-
if (signalArg?.aborted) abort();
|
|
568
|
-
signalArg?.addEventListener('abort', abort, { once: true });
|
|
569
|
-
const deadline = setTimeout(() => stop('lfs-transfer-timed-out'), limitMsArg);
|
|
570
|
-
let idle: ReturnType<typeof setTimeout> | undefined;
|
|
571
|
-
const rearm = () => {
|
|
572
|
-
clearTimeout(idle);
|
|
573
|
-
if (!controller.signal.aborted) idle = setTimeout(() => stop('lfs-transfer-timed-out'), this.#limits.requestTimeoutMs);
|
|
574
|
-
};
|
|
575
|
-
rearm();
|
|
576
|
-
const end = () => {
|
|
577
|
-
clearTimeout(deadline);
|
|
578
|
-
clearTimeout(idle);
|
|
579
|
-
signalArg?.removeEventListener('abort', abort);
|
|
580
|
-
stop('transfer-ended');
|
|
581
|
-
};
|
|
582
|
-
const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
|
|
583
|
-
if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
|
|
584
|
-
return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
|
|
585
|
-
};
|
|
586
|
-
return {
|
|
587
|
-
signal: controller.signal,
|
|
588
|
-
end,
|
|
589
|
-
failure,
|
|
590
|
-
/** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
|
|
591
|
-
run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
|
|
592
|
-
if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
|
|
593
|
-
let onAbort!: () => void;
|
|
594
|
-
const aborted = new Promise<never>((_resolve, reject) => {
|
|
595
|
-
onAbort = () => reject(failure(undefined, 'request-aborted'));
|
|
596
|
-
controller.signal.addEventListener('abort', onAbort, { once: true });
|
|
597
|
-
});
|
|
598
|
-
try {
|
|
599
|
-
return await Promise.race([stepArg(), aborted]);
|
|
600
|
-
} finally {
|
|
601
|
-
controller.signal.removeEventListener('abort', onAbort);
|
|
602
|
-
}
|
|
603
|
-
},
|
|
604
|
-
/** Passes chunks on and re-arms the idle limit on each one. */
|
|
605
|
-
watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
|
|
606
|
-
for await (const chunk of sourceArg) {
|
|
607
|
-
if (controller.signal.aborted) throw failure(undefined, 'lfs-transfer-failed');
|
|
608
|
-
rearm();
|
|
609
|
-
yield chunk;
|
|
610
|
-
}
|
|
611
|
-
},
|
|
612
|
-
};
|
|
527
|
+
return createTransferBounds(signalArg, {
|
|
528
|
+
limitMs: limitMsArg, idleMs: this.#limits.requestTimeoutMs, timedOutCode: 'lfs-transfer-timed-out', failedCode: 'lfs-transfer-failed',
|
|
529
|
+
});
|
|
613
530
|
}
|
|
614
531
|
}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
import * as validation from './forgeinventory.validation.js';
|
|
3
|
+
|
|
4
|
+
/** Response bytes read from an unused answer before the rest is discarded. */
|
|
5
|
+
const maxDiscardedResponseBytes = 64 * 1024;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Reads and drops at most 64 KiB of a response body that carries nothing the
|
|
9
|
+
* caller uses, so its connection can be reused; a longer or failing body is
|
|
10
|
+
* discarded with its connection.
|
|
11
|
+
*/
|
|
12
|
+
export const discardBody = async (bodyArg: AsyncIterable<Uint8Array> | null): Promise<void> => {
|
|
13
|
+
if (!bodyArg) return;
|
|
14
|
+
const chunks = bodyArg[Symbol.asyncIterator]();
|
|
15
|
+
let bytes = 0;
|
|
16
|
+
try {
|
|
17
|
+
while (bytes <= maxDiscardedResponseBytes) {
|
|
18
|
+
const result = await chunks.next();
|
|
19
|
+
if (result.done) return;
|
|
20
|
+
bytes += result.value.byteLength;
|
|
21
|
+
}
|
|
22
|
+
} catch {
|
|
23
|
+
// The status already decided the outcome.
|
|
24
|
+
} finally {
|
|
25
|
+
try { await chunks.return?.(); } catch { /* The connection is discarded. */ }
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export interface ITransferBoundsOptions {
|
|
30
|
+
/** Wall-clock limit of the whole transfer. */
|
|
31
|
+
limitMs: number;
|
|
32
|
+
/** Longest gap between two chunks of a watched source. */
|
|
33
|
+
idleMs: number;
|
|
34
|
+
/** The code a passed wall-clock or idle limit fails with. */
|
|
35
|
+
timedOutCode: string;
|
|
36
|
+
/** The code a watched source that goes on after the transfer ended fails with. */
|
|
37
|
+
failedCode: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ITransferBounds {
|
|
41
|
+
/** Aborts once the caller aborts, a limit passes or the transfer ends. */
|
|
42
|
+
signal: AbortSignal;
|
|
43
|
+
/** Ends the transfer: clears its timers and aborts `signal`. Idempotent. */
|
|
44
|
+
end: () => void;
|
|
45
|
+
/**
|
|
46
|
+
* The error the transfer fails with: the reason that ended it (`request-aborted`
|
|
47
|
+
* or the timeout code), else `errorArg` when it is a `ForgeInventoryError`,
|
|
48
|
+
* else `fallbackArg`.
|
|
49
|
+
*/
|
|
50
|
+
failure: (errorArg: unknown, fallbackArg: string) => validation.ForgeInventoryError;
|
|
51
|
+
/** Runs one step of the transfer; it rejects as soon as the transfer ends, even if the step does not. */
|
|
52
|
+
run: <T>(stepArg: () => Promise<T>) => Promise<T>;
|
|
53
|
+
/** Re-arms the idle limit, before a step that waits on the network without a watched source. */
|
|
54
|
+
touch: () => void;
|
|
55
|
+
/** Passes chunks on and re-arms the idle limit on each one. */
|
|
56
|
+
watch: (sourceArg: AsyncIterable<Uint8Array>) => AsyncGenerator<Uint8Array, void, undefined>;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The bounds of one streamed transfer: the caller's signal, a wall-clock limit
|
|
61
|
+
* and an idle limit that `watch` re-arms on every chunk. Whichever ends it
|
|
62
|
+
* first decides the failure code. Every transfer must be ended with `end`.
|
|
63
|
+
*/
|
|
64
|
+
export const createTransferBounds = (signalArg: AbortSignal | undefined, optionsArg: ITransferBoundsOptions): ITransferBounds => {
|
|
65
|
+
const controller = new AbortController();
|
|
66
|
+
let reason: string | null = null;
|
|
67
|
+
const stop = (reasonArg: string) => {
|
|
68
|
+
reason ??= reasonArg;
|
|
69
|
+
controller.abort();
|
|
70
|
+
};
|
|
71
|
+
const abort = () => stop('request-aborted');
|
|
72
|
+
if (signalArg?.aborted) abort();
|
|
73
|
+
signalArg?.addEventListener('abort', abort, { once: true });
|
|
74
|
+
const deadline = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.limitMs);
|
|
75
|
+
let idle: ReturnType<typeof setTimeout> | undefined;
|
|
76
|
+
const rearm = () => {
|
|
77
|
+
clearTimeout(idle);
|
|
78
|
+
if (!controller.signal.aborted) idle = setTimeout(() => stop(optionsArg.timedOutCode), optionsArg.idleMs);
|
|
79
|
+
};
|
|
80
|
+
rearm();
|
|
81
|
+
const end = () => {
|
|
82
|
+
clearTimeout(deadline);
|
|
83
|
+
clearTimeout(idle);
|
|
84
|
+
signalArg?.removeEventListener('abort', abort);
|
|
85
|
+
stop('transfer-ended');
|
|
86
|
+
};
|
|
87
|
+
const failure = (errorArg: unknown, fallbackArg: string): validation.ForgeInventoryError => {
|
|
88
|
+
if (reason !== null && reason !== 'transfer-ended') return new validation.ForgeInventoryError(reason);
|
|
89
|
+
return errorArg instanceof validation.ForgeInventoryError ? errorArg : new validation.ForgeInventoryError(fallbackArg);
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
signal: controller.signal,
|
|
93
|
+
end,
|
|
94
|
+
failure,
|
|
95
|
+
touch: rearm,
|
|
96
|
+
run: async <T>(stepArg: () => Promise<T>): Promise<T> => {
|
|
97
|
+
if (controller.signal.aborted) throw failure(undefined, 'request-aborted');
|
|
98
|
+
let onAbort!: () => void;
|
|
99
|
+
const aborted = new Promise<never>((_resolve, reject) => {
|
|
100
|
+
onAbort = () => reject(failure(undefined, 'request-aborted'));
|
|
101
|
+
controller.signal.addEventListener('abort', onAbort, { once: true });
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
return await Promise.race([stepArg(), aborted]);
|
|
105
|
+
} finally {
|
|
106
|
+
controller.signal.removeEventListener('abort', onAbort);
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
watch: async function* (sourceArg: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array, void, undefined> {
|
|
110
|
+
for await (const chunk of sourceArg) {
|
|
111
|
+
if (controller.signal.aborted) throw failure(undefined, optionsArg.failedCode);
|
|
112
|
+
rearm();
|
|
113
|
+
yield chunk;
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
export interface IHeldBackCodes {
|
|
120
|
+
/** A chunk that is not a `Uint8Array`. */
|
|
121
|
+
invalidChunk: string;
|
|
122
|
+
/** More bytes than the declared size. */
|
|
123
|
+
overrun: string;
|
|
124
|
+
/** Fewer bytes than the declared size. */
|
|
125
|
+
short: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface IHeldBackChecks {
|
|
129
|
+
/** Sees every non-empty chunk in order, before it is passed on. */
|
|
130
|
+
inspect?: (chunkArg: Uint8Array) => void;
|
|
131
|
+
/** Runs once exactly `size` bytes have arrived, before the last chunk is passed on; a rejection keeps it back. */
|
|
132
|
+
complete?: () => void | Promise<void>;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Passes the bytes of `sourceArg` on while it counts them. The last chunk is
|
|
137
|
+
* held back until the source has ended with exactly `sizeArg` bytes and
|
|
138
|
+
* `checksArg.complete` has passed, so a consumer never receives a complete
|
|
139
|
+
* body that is wrong; otherwise it throws `codesArg.overrun` as soon as the
|
|
140
|
+
* size is exceeded, `codesArg.short` when the source ends early, or what
|
|
141
|
+
* `complete` threw.
|
|
142
|
+
*/
|
|
143
|
+
export async function* heldBackChunks(
|
|
144
|
+
sourceArg: AsyncIterable<Uint8Array>, sizeArg: number, codesArg: IHeldBackCodes, checksArg: IHeldBackChecks = {},
|
|
145
|
+
): AsyncGenerator<Uint8Array, void, undefined> {
|
|
146
|
+
let bytes = 0;
|
|
147
|
+
let held: Uint8Array | null = null;
|
|
148
|
+
for await (const chunk of sourceArg) {
|
|
149
|
+
if (!(chunk instanceof Uint8Array)) throw new validation.ForgeInventoryError(codesArg.invalidChunk);
|
|
150
|
+
if (chunk.byteLength === 0) continue;
|
|
151
|
+
bytes += chunk.byteLength;
|
|
152
|
+
if (bytes > sizeArg) throw new validation.ForgeInventoryError(codesArg.overrun);
|
|
153
|
+
checksArg.inspect?.(chunk);
|
|
154
|
+
if (held) yield held;
|
|
155
|
+
held = chunk;
|
|
156
|
+
}
|
|
157
|
+
if (bytes !== sizeArg) throw new validation.ForgeInventoryError(codesArg.short);
|
|
158
|
+
await checksArg.complete?.();
|
|
159
|
+
if (held) yield held;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The readable stream of one transfer's bytes. It pulls `chunksArg` with
|
|
164
|
+
* backpressure and ends `boundsArg` and calls `onEndArg` as soon as it is
|
|
165
|
+
* destroyed, also while a pull is pending: ending the bounds aborts the
|
|
166
|
+
* request, which closes its connection, and the pending pull's result is
|
|
167
|
+
* dropped. (`Readable.from` waits for a pending pull before it finishes a
|
|
168
|
+
* destroy, so a stalled body would keep its connection until the idle limit.)
|
|
169
|
+
* The stream errors with what `chunksArg` throws.
|
|
170
|
+
*/
|
|
171
|
+
export const createTransferStream = (
|
|
172
|
+
boundsArg: ITransferBounds, chunksArg: AsyncGenerator<Uint8Array, void, undefined>, onEndArg?: () => void,
|
|
173
|
+
): InstanceType<typeof plugins.stream.Readable> => {
|
|
174
|
+
let pulling = false;
|
|
175
|
+
const stream: InstanceType<typeof plugins.stream.Readable> = new plugins.stream.Readable({
|
|
176
|
+
read() {
|
|
177
|
+
if (pulling) return;
|
|
178
|
+
pulling = true;
|
|
179
|
+
void (async () => {
|
|
180
|
+
try {
|
|
181
|
+
while (!stream.destroyed) {
|
|
182
|
+
const result = await chunksArg.next();
|
|
183
|
+
if (stream.destroyed) return;
|
|
184
|
+
if (result.done) return void stream.push(null);
|
|
185
|
+
if (!stream.push(result.value)) return;
|
|
186
|
+
}
|
|
187
|
+
} catch (error) {
|
|
188
|
+
stream.destroy(error as Error);
|
|
189
|
+
} finally {
|
|
190
|
+
pulling = false;
|
|
191
|
+
}
|
|
192
|
+
})();
|
|
193
|
+
},
|
|
194
|
+
destroy(errorArg, callbackArg) {
|
|
195
|
+
boundsArg.end();
|
|
196
|
+
onEndArg?.();
|
|
197
|
+
// Runs the generator's cleanup once a pending pull has settled; nothing waits for it.
|
|
198
|
+
chunksArg.return(undefined).catch(() => undefined);
|
|
199
|
+
callbackArg(errorArg);
|
|
200
|
+
},
|
|
201
|
+
});
|
|
202
|
+
return stream;
|
|
203
|
+
};
|