@giveitsmaller/sdk 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_audit.js +7 -0
- package/dist/client.d.ts +86 -2
- package/dist/client.js +942 -33
- package/dist/errors.d.ts +105 -1
- package/dist/errors.js +107 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +3 -1
- package/dist/sse.d.ts +20 -1
- package/dist/sse.js +62 -3
- package/dist/types.d.ts +132 -0
- package/package.json +2 -2
package/dist/client.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { open, stat } from 'node:fs/promises';
|
|
2
2
|
import { basename } from 'node:path';
|
|
3
|
-
import { AudioWatermarkDecodeRequestToJSON, AudioWatermarkDecodeResponseFromJSON, ExternalImportCreatedResponseFromJSON, ExternalImportRequestToJSON, LoginUser200ResponseDataFromJSON, CreditsBalanceResponseFromJSON, CreditsUsageResponseFromJSON, UploadResponseFromJSON, UploadProbeResponseFromJSON, MultipartInitiateResponseFromJSON, MultipartInitiateRequestMetadataHintToJSON, MultipartCompleteResponseFromJSON, MultipartCompleteRequestToJSON, WorkflowCancelResponseFromJSON, WorkflowCreateResponseFromJSON, WorkflowResumeResponseFromJSON, WorkflowStatusResponseFromJSON, WorkflowDownloadResponseFromJSON, MetadataResponseFromJSON, OperationsSchemaResponseFromJSON, RetryResponseFromJSON, WorkflowStatus, AuthErrorResponseFromJSON, AuthErrorType, BalanceExhaustedResponseFromJSON, BalanceExhaustedResponseRequiredActionEnum, FeatureNotAvailableResponseFromJSON, FeatureTierRestrictedResponseFromJSON, TierRestrictionKind, TierRestrictionResponseFromJSON, UserTier, WorkflowExpiredResponseFromJSON, UploadThresholdsSingleShotMaxBytesEnum, UploadThresholdsMultipartChunkSizeEnum, UploadThresholdsMultipartConcurrencyDefaultEnum, } from '@giveitsmaller/contracts/openapi';
|
|
4
|
-
import { GislAbortError, GislApiError, GislAuthError, GislBalanceExhaustedError, GislError, GislFeatureNotAvailableError, GislFeatureTierRestrictedError, GislTierRestrictedError, GislTimeoutError, GislValidationError, GislWorkflowExpiredError, } from './errors.js';
|
|
3
|
+
import { AudioWatermarkDecodeRequestToJSON, AudioWatermarkDecodeResponseFromJSON, ExternalImportCreatedResponseFromJSON, ExternalImportRequestToJSON, LoginUser200ResponseDataFromJSON, CreditsBalanceResponseFromJSON, CreditsUsageResponseFromJSON, UploadResponseFromJSON, UploadProbeResponseFromJSON, MultipartInitiateResponseFromJSON, MultipartInitiateRequestMetadataHintToJSON, MultipartCompleteResponseFromJSON, MultipartCompleteRequestToJSON, WorkflowCancelResponseFromJSON, WorkflowCreateResponseFromJSON, WorkflowResumeResponseFromJSON, WorkflowStatusResponseFromJSON, WorkflowDownloadResponseFromJSON, MetadataResponseFromJSON, OperationsSchemaResponseFromJSON, RetryResponseFromJSON, WorkflowStatus, AuthErrorResponseFromJSON, AuthErrorType, BalanceExhaustedResponseFromJSON, BalanceExhaustedResponseRequiredActionEnum, FeatureNotAvailableResponseFromJSON, FeatureTierRestrictedResponseFromJSON, TierRestrictionKind, TierRestrictionResponseFromJSON, UserTier, WorkflowExpiredResponseFromJSON, UploadSizeExceedsTierResponseFromJSON, UploadDurationExceedsTierResponseFromJSON, UploadConstraintsAppliedProcessingClassPreAssignmentEnum, UploadThresholdsSingleShotMaxBytesEnum, UploadThresholdsMultipartChunkSizeEnum, UploadThresholdsMultipartConcurrencyDefaultEnum, } from '@giveitsmaller/contracts/openapi';
|
|
4
|
+
import { GislAbortError, GislApiError, GislAuthError, GislBalanceExhaustedError, GislError, GislFeatureNotAvailableError, GislFeatureTierRestrictedError, GislMultipartPartCountError, GislMultipartPartError, GislMultipartSessionNotFoundError, GislMultipartSessionOwnershipError, GislMultipartSessionAuthRequiredError, GislTierRestrictedError, GislTimeoutError, GislUploadCapExceededError, GislValidationError, GislWorkflowExpiredError, } from './errors.js';
|
|
5
5
|
import { parseSseStream } from './sse.js';
|
|
6
6
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
7
7
|
// SDK-internal aliases derived from the contract-pinned UploadThresholds enums
|
|
@@ -11,7 +11,7 @@ const DEFAULT_TIMEOUT_MS = 30_000;
|
|
|
11
11
|
// release that regenerates the corresponding *Enum, plus updating the literal
|
|
12
12
|
// in the matching `_AssertTrue<>` line.
|
|
13
13
|
const SINGLE_SHOT_MAX_BYTES = UploadThresholdsSingleShotMaxBytesEnum.NUMBER_10000000;
|
|
14
|
-
const MULTIPART_CHUNK_SIZE = UploadThresholdsMultipartChunkSizeEnum.
|
|
14
|
+
const MULTIPART_CHUNK_SIZE = UploadThresholdsMultipartChunkSizeEnum.NUMBER_16777216;
|
|
15
15
|
export const MULTIPART_CONCURRENCY_DEFAULT = UploadThresholdsMultipartConcurrencyDefaultEnum.NUMBER_4;
|
|
16
16
|
const DEFAULT_MULTIPART_MAX_ATTEMPTS = 3;
|
|
17
17
|
const DEFAULT_MULTIPART_RETRY_BASE_MS = 500;
|
|
@@ -23,6 +23,33 @@ const DEFAULT_MULTIPART_RETRY_BASE_MS = 500;
|
|
|
23
23
|
// TODO(58nBQLWQ): replace with UploadThresholdsMultipartFirstChunkSizeEnum
|
|
24
24
|
// once contracts ticket promotes this to a typed const (v2.3.1 follow-up).
|
|
25
25
|
export const DEFAULT_MULTIPART_FIRST_CHUNK_SIZE = 8 * 1024 * 1024; // 8 MB
|
|
26
|
+
// The ~2 GB wall on a single Node file read is NOT a Buffer-size limit
|
|
27
|
+
// (modern 64-bit `buffer.constants.MAX_LENGTH` is ~8 PiB). It is libuv's
|
|
28
|
+
// hard-coded INT32_MAX (2 147 483 647) ceiling on one `uv_fs_read` — some
|
|
29
|
+
// platforms reject I/O larger than INT32_MAX bytes per call, so libuv caps
|
|
30
|
+
// every read at it (github.com/nodejs/node/issues/55864). The streaming
|
|
31
|
+
// upload path never approaches this (server chunk size is bounded to
|
|
32
|
+
// <=100 MiB and the first chunk is fixed 8 MiB), but `fileByteSource`
|
|
33
|
+
// asserts it per read so any future caller that requests an oversized range
|
|
34
|
+
// fails loudly here instead of getting a silently short read from libuv.
|
|
35
|
+
const LIBUV_MAX_SINGLE_READ_BYTES = 0x7fffffff; // INT32_MAX
|
|
36
|
+
// S3 hard limit: a multipart upload may have at most 10 000 parts. The
|
|
37
|
+
// server computes the part plan and returns `total_parts`; the SDK trusts
|
|
38
|
+
// that value (Model A) but guards the ceiling so an out-of-contract server
|
|
39
|
+
// response or a chunk-size regression surfaces as a typed error rather than
|
|
40
|
+
// a doomed run of presigned PUTs ending in a rejected /multipart/complete.
|
|
41
|
+
const S3_MAX_MULTIPART_PARTS = 10_000;
|
|
42
|
+
// Contract bound on `MultipartInitiateResponse.recommended_chunk_size`
|
|
43
|
+
// (compression_contracts/openapi api.yaml — `maximum: 104857600`). The
|
|
44
|
+
// minimum is `multipart_chunk_size` (== MULTIPART_CHUNK_SIZE, drift-guarded
|
|
45
|
+
// above). The generated TS `FromJSON` does NO runtime validation (unlike the
|
|
46
|
+
// strict PHP generated model, which rejects out-of-range values at
|
|
47
|
+
// deserialize), so the TS SDK must enforce this range itself — otherwise a
|
|
48
|
+
// malformed/hostile server `recommended_chunk_size` would pass the
|
|
49
|
+
// part-count guard and drive `fileByteSource` into an unbounded
|
|
50
|
+
// `Buffer.allocUnsafe(length)` (the exact memory-blowup class this SDK
|
|
51
|
+
// exists to prevent). codex review (high).
|
|
52
|
+
const RECOMMENDED_CHUNK_SIZE_MAX_BYTES = 104_857_600; // 100 MiB
|
|
26
53
|
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
|
27
54
|
const DEFAULT_POLL_TIMEOUT_MS = 300_000; // 5 min
|
|
28
55
|
// Statuses that waitForWorkflow() returns immediately on. Per ticket I24,
|
|
@@ -176,6 +203,69 @@ function bindAbortSignal(external, internal, onExternalAbort) {
|
|
|
176
203
|
external.addEventListener('abort', onAbort, { once: true });
|
|
177
204
|
return () => external.removeEventListener('abort', onAbort);
|
|
178
205
|
}
|
|
206
|
+
// Blob/File input is already lazy: `Blob.slice()` is a zero-copy view and a
|
|
207
|
+
// `File` from a browser picker is disk-backed, so this branch never buffered
|
|
208
|
+
// the whole file. Left structurally identical to the pre-streaming-rewrite
|
|
209
|
+
// behaviour.
|
|
210
|
+
function blobByteSource(blob) {
|
|
211
|
+
return {
|
|
212
|
+
size: blob.size,
|
|
213
|
+
// Pass `blob.type` as the 3rd arg: `Blob.slice()` defaults the slice's
|
|
214
|
+
// content-type to '' otherwise, which would strip the MIME type off the
|
|
215
|
+
// single-shot FormData part (the pre-streaming code appended the original
|
|
216
|
+
// typed Blob directly). Parity fixtures pin this content-type.
|
|
217
|
+
slice: (start, end) => Promise.resolve(blob.slice(start, end, blob.type)),
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
// File-path input. The pre-rewrite code did `readFileSync(path)` →
|
|
221
|
+
// `new Blob([whole file])`, which (a) OOMs on multi-GB files and (b) cannot
|
|
222
|
+
// even be attempted above ~2 GB because a single libuv `uv_fs_read` is capped
|
|
223
|
+
// at INT32_MAX (see LIBUV_MAX_SINGLE_READ_BYTES). This source instead does a
|
|
224
|
+
// positioned (POSIX pread-semantics) read of ONLY the requested range, with a
|
|
225
|
+
// fresh fd per call so concurrent multipart workers never share a FileHandle
|
|
226
|
+
// (overlapping reads on one handle are unsafe per the Node fs contract) and
|
|
227
|
+
// the fd is always closed in `finally`.
|
|
228
|
+
//
|
|
229
|
+
// Divergence from the old Blob-from-readFileSync behaviour (deliberate, in
|
|
230
|
+
// scope only for streaming): the old path snapshotted the whole file at t0,
|
|
231
|
+
// so every part was point-in-time consistent. Streaming reads each part at
|
|
232
|
+
// the time it is uploaded, so a file truncated/rewritten mid-upload now
|
|
233
|
+
// yields parts from different instants. Truncation is caught by the
|
|
234
|
+
// short-read guard below; full point-in-time snapshotting would require
|
|
235
|
+
// resumable/staged upload and is out of scope (SDK-3, Wb6ebOMM).
|
|
236
|
+
function fileByteSource(path, size) {
|
|
237
|
+
return {
|
|
238
|
+
size,
|
|
239
|
+
async slice(start, end) {
|
|
240
|
+
const length = end - start;
|
|
241
|
+
if (length <= 0)
|
|
242
|
+
return new Blob([]);
|
|
243
|
+
// Per-read tripwire for the libuv INT32_MAX ceiling. Unreachable on the
|
|
244
|
+
// normal path (chunk size <=100 MiB) — exists so a future oversized
|
|
245
|
+
// caller fails here loudly instead of getting a silent short read.
|
|
246
|
+
if (length > LIBUV_MAX_SINGLE_READ_BYTES) {
|
|
247
|
+
throw new GislError(`Refusing to read ${length} bytes in one operation: exceeds the ` +
|
|
248
|
+
`libuv single-read ceiling (${LIBUV_MAX_SINGLE_READ_BYTES}). ` +
|
|
249
|
+
'Reads must be chunked below INT32_MAX.');
|
|
250
|
+
}
|
|
251
|
+
const handle = await open(path, 'r');
|
|
252
|
+
try {
|
|
253
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
254
|
+
const { bytesRead } = await handle.read(buffer, 0, length, start);
|
|
255
|
+
if (bytesRead !== length) {
|
|
256
|
+
// Short read = the file shrank/was truncated under us. Mirrors the
|
|
257
|
+
// PHP SDK's readChunk short-read guard (GislClient.php readChunk).
|
|
258
|
+
throw new GislError(`Short read on ${path}: expected ${length} bytes at offset ` +
|
|
259
|
+
`${start}, got ${bytesRead}. File changed during upload.`);
|
|
260
|
+
}
|
|
261
|
+
return new Blob([buffer]);
|
|
262
|
+
}
|
|
263
|
+
finally {
|
|
264
|
+
await handle.close();
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
179
269
|
export class GislClient {
|
|
180
270
|
baseUrl;
|
|
181
271
|
headers;
|
|
@@ -373,6 +463,63 @@ export class GislClient {
|
|
|
373
463
|
if (status === 422 && errorType === 'workflow_expired') {
|
|
374
464
|
tryThrowStructured(WorkflowExpiredResponseFromJSON, GislWorkflowExpiredError, (p) => isValidDate(p.expiredAt));
|
|
375
465
|
}
|
|
466
|
+
// Upload cap errors. `GislUploadCapExceededError` takes an extra `kind`
|
|
467
|
+
// arg so it cannot use `tryThrowStructured` (whose ErrorClass signature
|
|
468
|
+
// is fixed) — this local helper applies the SAME defense-in-depth
|
|
469
|
+
// discipline: construct via FromJSON, validate required typed fields,
|
|
470
|
+
// fall through to the generic `GislApiError` on any malformed envelope.
|
|
471
|
+
const tryThrowCap = (construct, kind, validate) => {
|
|
472
|
+
let payload;
|
|
473
|
+
try {
|
|
474
|
+
payload = construct(json);
|
|
475
|
+
}
|
|
476
|
+
catch {
|
|
477
|
+
return undefined;
|
|
478
|
+
}
|
|
479
|
+
if (!validate(payload)) {
|
|
480
|
+
return undefined;
|
|
481
|
+
}
|
|
482
|
+
throw new GislUploadCapExceededError(status, errorMessage, kind, payload, path, i18n);
|
|
483
|
+
};
|
|
484
|
+
if (status === 422 && errorType === 'upload_size_exceeds_tier') {
|
|
485
|
+
tryThrowCap(UploadSizeExceedsTierResponseFromJSON, 'size_tier', (p) => isInEnum(p.currentTier, UserTier) &&
|
|
486
|
+
typeof p.maxSizeBytes === 'number');
|
|
487
|
+
}
|
|
488
|
+
if (status === 422 && errorType === 'upload_duration_exceeds_tier') {
|
|
489
|
+
tryThrowCap(UploadDurationExceedsTierResponseFromJSON, 'duration_tier', (p) => isInEnum(p.currentTier, UserTier) &&
|
|
490
|
+
typeof p.maxDurationSeconds === 'number');
|
|
491
|
+
}
|
|
492
|
+
// 413 = the absolute across-tier cap. The contract models 413 as a
|
|
493
|
+
// plain `ErrorEnvelope` (no `error_type` discriminator, no typed
|
|
494
|
+
// payload — api.yaml), so dispatch purely on status with no FromJSON
|
|
495
|
+
// and an undefined payload (the `absolute_413` kind tells the caller
|
|
496
|
+
// there is intentionally no structured envelope to read).
|
|
497
|
+
if (status === 413) {
|
|
498
|
+
throw new GislUploadCapExceededError(status, errorMessage, 'absolute_413', undefined, path, i18n);
|
|
499
|
+
}
|
|
500
|
+
// SDK-3 (Wb6ebOMM) resume-support endpoint error codes. API-2 / PR
|
|
501
|
+
// #283 specced these as plain `ErrorEnvelope` envelopes with the
|
|
502
|
+
// discriminating string on `error_type`. No typed payload to build —
|
|
503
|
+
// dispatch on the (status, error_type) tuple. The HxUmVr3Y contract
|
|
504
|
+
// regen will produce typed responses for these; today the 3 typed
|
|
505
|
+
// subclasses carry only the localisation triple + raw envelope.
|
|
506
|
+
if (status === 404 && errorType === 'MULTIPART_SESSION_NOT_FOUND') {
|
|
507
|
+
throw new GislMultipartSessionNotFoundError(status, errorMessage, path, i18n);
|
|
508
|
+
}
|
|
509
|
+
if (status === 403 && errorType === 'MULTIPART_SESSION_OWNERSHIP') {
|
|
510
|
+
throw new GislMultipartSessionOwnershipError(status, errorMessage, path, i18n);
|
|
511
|
+
}
|
|
512
|
+
if (status === 403 && errorType === 'MULTIPART_SESSION_AUTH_REQUIRED') {
|
|
513
|
+
throw new GislMultipartSessionAuthRequiredError(status, errorMessage, path, i18n);
|
|
514
|
+
}
|
|
515
|
+
// 422 `FILE_TOO_LARGE_FOR_MULTIPART` — pre-S3 capacity reject on the
|
|
516
|
+
// resume-support presign endpoint (more parts than the manifest can
|
|
517
|
+
// ever accept). No typed payload today (the contract carries no
|
|
518
|
+
// structured response for this code); `cap_v2_multipart` discriminant
|
|
519
|
+
// is documented on `GislUploadCapKind`.
|
|
520
|
+
if (status === 422 && errorType === 'FILE_TOO_LARGE_FOR_MULTIPART') {
|
|
521
|
+
throw new GislUploadCapExceededError(status, errorMessage, 'cap_v2_multipart', undefined, path, i18n);
|
|
522
|
+
}
|
|
376
523
|
throw new GislApiError(status, errorMessage, path, json.details, { ...i18n, payload: json });
|
|
377
524
|
}
|
|
378
525
|
const data = json.data ?? json;
|
|
@@ -395,34 +542,55 @@ export class GislClient {
|
|
|
395
542
|
* @param options Upload options including progress callback.
|
|
396
543
|
*/
|
|
397
544
|
async uploadFile(file, options) {
|
|
398
|
-
// Pre-abort check: bail before
|
|
399
|
-
//
|
|
545
|
+
// Pre-abort check: bail before touching the filesystem when the caller
|
|
546
|
+
// has already cancelled.
|
|
400
547
|
if (options?.signal?.aborted) {
|
|
401
548
|
throw new GislAbortError('Upload aborted before start');
|
|
402
549
|
}
|
|
403
|
-
let
|
|
550
|
+
let source;
|
|
404
551
|
let fileName;
|
|
405
|
-
let fileSize;
|
|
406
552
|
if (typeof file === 'string') {
|
|
407
|
-
|
|
408
|
-
|
|
553
|
+
// `stat` for the size only — the bytes are NEVER read up front. The old
|
|
554
|
+
// path did `readFileSync(file)` which OOMs on multi-GB files and is
|
|
555
|
+
// impossible above the libuv INT32_MAX single-read ceiling regardless
|
|
556
|
+
// of available memory (see fileByteSource / LIBUV_MAX_SINGLE_READ_BYTES).
|
|
557
|
+
const stats = await stat(file);
|
|
409
558
|
fileName = basename(file);
|
|
410
|
-
|
|
411
|
-
blob = new Blob([content]);
|
|
559
|
+
source = fileByteSource(file, stats.size);
|
|
412
560
|
}
|
|
413
561
|
else {
|
|
414
|
-
blob = file;
|
|
415
562
|
fileName = file.name ?? 'upload';
|
|
416
|
-
|
|
563
|
+
source = blobByteSource(file);
|
|
417
564
|
}
|
|
418
|
-
if (
|
|
419
|
-
|
|
565
|
+
if (typeof options?.resumeUploadId === 'string' && options.resumeUploadId !== '') {
|
|
566
|
+
// SDK-3 (Wb6ebOMM): resume path takes the durable session's
|
|
567
|
+
// `recommended_chunk_size` from the /status envelope rather than
|
|
568
|
+
// the initiate envelope (initiate is skipped). Below the multipart
|
|
569
|
+
// threshold a resume is still meaningful — the original session was
|
|
570
|
+
// started as multipart, so a sub-threshold file CAN'T be a "resume
|
|
571
|
+
// target" in practice. Guard explicitly so a confused caller gets a
|
|
572
|
+
// clear error rather than a 404 on /status.
|
|
573
|
+
if (source.size <= this.multipartThreshold) {
|
|
574
|
+
throw new GislError('uploadFile: resumeUploadId set but file size is at-or-below the multipart ' +
|
|
575
|
+
`threshold (${this.multipartThreshold} bytes); resume targets must be multipart sessions.`);
|
|
576
|
+
}
|
|
577
|
+
return this.multipartResume(source, fileName, source.size, options.resumeUploadId, options);
|
|
578
|
+
}
|
|
579
|
+
if (source.size > this.multipartThreshold) {
|
|
580
|
+
return this.multipartUpload(source, fileName, source.size, options);
|
|
420
581
|
}
|
|
421
|
-
return this.singleUpload(
|
|
582
|
+
return this.singleUpload(source, fileName, options);
|
|
422
583
|
}
|
|
423
|
-
async singleUpload(
|
|
584
|
+
async singleUpload(source, fileName, options) {
|
|
424
585
|
const form = new FormData();
|
|
425
|
-
|
|
586
|
+
// Single-shot is gated to <= single_shot_max_bytes (10 MB) by the router
|
|
587
|
+
// above, so this one bounded read is trivially under the libuv ceiling
|
|
588
|
+
// and a non-issue for memory. `slice` returns a Blob (the file-path
|
|
589
|
+
// source wraps the bounded Buffer) so FormData.append is unchanged —
|
|
590
|
+
// multipart never wraps a whole-file Blob, only this <=10 MB single-shot
|
|
591
|
+
// path ever holds a full payload Blob.
|
|
592
|
+
const body = await source.slice(0, source.size);
|
|
593
|
+
form.append('file', body, fileName);
|
|
426
594
|
return this.request('POST', '/api/uploads', {
|
|
427
595
|
body: form,
|
|
428
596
|
json: false,
|
|
@@ -441,10 +609,10 @@ export class GislClient {
|
|
|
441
609
|
* comes from the initiate response's first-chunk detection; for authoritative
|
|
442
610
|
* post-upload metadata callers should use getMetadata(fileId).
|
|
443
611
|
*/
|
|
444
|
-
async multipartUpload(
|
|
612
|
+
async multipartUpload(source, fileName, totalSize, options) {
|
|
445
613
|
// Step 1: Initiate with first chunk
|
|
446
614
|
const firstChunkSize = Math.min(totalSize, DEFAULT_MULTIPART_FIRST_CHUNK_SIZE);
|
|
447
|
-
const firstChunk =
|
|
615
|
+
const firstChunk = await source.slice(0, firstChunkSize);
|
|
448
616
|
const initiateForm = new FormData();
|
|
449
617
|
initiateForm.append('file', firstChunk, fileName);
|
|
450
618
|
initiateForm.append('filename', fileName);
|
|
@@ -473,6 +641,88 @@ export class GislClient {
|
|
|
473
641
|
const etags = [];
|
|
474
642
|
const presignedUrls = initResponse.presignedUrls;
|
|
475
643
|
const chunkSize = initResponse.recommendedChunkSize;
|
|
644
|
+
// MultipartInitiateResponseFromJSON does NO runtime validation (unlike
|
|
645
|
+
// the strict PHP generated model, which rejects these at deserialize —
|
|
646
|
+
// the documented lax-TS-vs-strict-PHP divergence). The TS SDK must
|
|
647
|
+
// therefore enforce, before any chunk read/PUT, what PHP gets for free
|
|
648
|
+
// from its generated model + its explicit pre-loop guards (codex review):
|
|
649
|
+
//
|
|
650
|
+
// (a) uploadId must be a non-empty string. `FromJSON` assigns
|
|
651
|
+
// `json['upload_id']` directly, so a malformed initiate could
|
|
652
|
+
// otherwise produce a typed GislMultipartPartError whose `uploadId`
|
|
653
|
+
// is `undefined` and synthesise a bogus UploadResponse.fileId —
|
|
654
|
+
// mirrors the PHP pre-loop `is_string && !== ''` guard.
|
|
655
|
+
if (typeof initResponse.uploadId !== 'string' ||
|
|
656
|
+
initResponse.uploadId === '') {
|
|
657
|
+
throw new GislError('Multipart initiate response missing or empty upload_id.');
|
|
658
|
+
}
|
|
659
|
+
// (b) recommendedChunkSize must be a finite number INSIDE the contract
|
|
660
|
+
// range [MULTIPART_CHUNK_SIZE, RECOMMENDED_CHUNK_SIZE_MAX_BYTES]. The
|
|
661
|
+
// old `>= 1` check let a malformed/hostile huge value pass the
|
|
662
|
+
// part-count guard and drive `fileByteSource` into an unbounded
|
|
663
|
+
// `Buffer.allocUnsafe(length)` — the memory-blowup class this SDK
|
|
664
|
+
// exists to prevent. PHP's strict generated model already rejects
|
|
665
|
+
// out-of-range values at deserialize; this is the TS equivalent.
|
|
666
|
+
if (typeof chunkSize !== 'number' ||
|
|
667
|
+
!Number.isInteger(chunkSize) ||
|
|
668
|
+
chunkSize < MULTIPART_CHUNK_SIZE ||
|
|
669
|
+
chunkSize > RECOMMENDED_CHUNK_SIZE_MAX_BYTES) {
|
|
670
|
+
// `Number.isInteger` also rejects NaN/Infinity and a fractional
|
|
671
|
+
// `recommended_chunk_size` (e.g. 5242880.5) that would otherwise
|
|
672
|
+
// reach `Buffer.allocUnsafe(fractional)` and fail later as a
|
|
673
|
+
// misleading part-read error (codex review).
|
|
674
|
+
throw new GislError('Multipart initiate response recommendedChunkSize is missing or ' +
|
|
675
|
+
`outside the contract range [${MULTIPART_CHUNK_SIZE}, ` +
|
|
676
|
+
`${RECOMMENDED_CHUNK_SIZE_MAX_BYTES}]: got ${String(chunkSize)}.`);
|
|
677
|
+
}
|
|
678
|
+
// S3 <=10 000-part ceiling guard (Model A). The server computes and
|
|
679
|
+
// returns `totalParts`; we trust it (consistent with how the SDK already
|
|
680
|
+
// trusts `recommendedChunkSize`/`presignedUrls` from the same envelope)
|
|
681
|
+
// but assert the ceiling, cross-checked against a client-side recompute
|
|
682
|
+
// from the same `chunkSize`. This necessarily fires AFTER the initiate
|
|
683
|
+
// round-trip + 8 MiB first-chunk upload — `totalParts` and `chunkSize`
|
|
684
|
+
// only exist on the initiate response, so a pure pre-flight check is
|
|
685
|
+
// impossible under Model A (this is the card-mandated trade-off).
|
|
686
|
+
const remainingBytes = Math.max(0, totalSize - firstChunkSize);
|
|
687
|
+
const computedParts = 1 + Math.ceil(remainingBytes / chunkSize);
|
|
688
|
+
const serverParts = initResponse.totalParts;
|
|
689
|
+
// `FromJSON` passes `total_parts` through unvalidated. Reject a
|
|
690
|
+
// missing/non-integer value here so the ≤10k guard's
|
|
691
|
+
// `Math.max(serverParts, computedParts)` cannot surface `NaN` in the
|
|
692
|
+
// GislMultipartPartCountError (codex review). Mirrors the uploadId /
|
|
693
|
+
// chunkSize guards above (the lax-TS-vs-strict-PHP-model divergence).
|
|
694
|
+
if (typeof serverParts !== 'number' ||
|
|
695
|
+
!Number.isInteger(serverParts) ||
|
|
696
|
+
serverParts < 1) {
|
|
697
|
+
throw new GislError('Multipart initiate response missing or invalid total_parts: ' +
|
|
698
|
+
`got ${String(serverParts)}.`);
|
|
699
|
+
}
|
|
700
|
+
if (serverParts > S3_MAX_MULTIPART_PARTS ||
|
|
701
|
+
computedParts > S3_MAX_MULTIPART_PARTS) {
|
|
702
|
+
throw new GislMultipartPartCountError(`Upload requires ${Math.max(serverParts, computedParts)} parts, ` +
|
|
703
|
+
`exceeding the S3 ${S3_MAX_MULTIPART_PARTS}-part multipart limit ` +
|
|
704
|
+
`(server reported ${serverParts}, client computed ${computedParts} ` +
|
|
705
|
+
`at ${chunkSize}-byte chunks). A larger chunk size is required ` +
|
|
706
|
+
'server-side to upload a file this large.', Math.max(serverParts, computedParts), S3_MAX_MULTIPART_PARTS);
|
|
707
|
+
}
|
|
708
|
+
// Plan-consistency guard (codex review). The ≤10k ceiling above only
|
|
709
|
+
// bounds the count; it does NOT catch an initiate plan that is internally
|
|
710
|
+
// inconsistent BELOW the cap. Under Model A a contract-compliant server
|
|
711
|
+
// computes `total_parts` from the same `recommended_chunk_size` it
|
|
712
|
+
// returns, and emits exactly one presigned URL per remaining part (part 1
|
|
713
|
+
// is the initiate first chunk). If `total_parts`, the client recompute,
|
|
714
|
+
// and `presigned_urls.length` disagree, proceeding would PUT the wrong
|
|
715
|
+
// number of byte ranges (or wrong offsets) and only fail opaquely at
|
|
716
|
+
// /multipart/complete. Fail fast here with the discrepancy instead.
|
|
717
|
+
if (!Number.isFinite(serverParts) ||
|
|
718
|
+
serverParts !== computedParts ||
|
|
719
|
+
presignedUrls.length !== computedParts - 1) {
|
|
720
|
+
throw new GislError('Multipart initiate plan is internally inconsistent: server ' +
|
|
721
|
+
`total_parts=${serverParts}, client computed ${computedParts} ` +
|
|
722
|
+
`from ${chunkSize}-byte chunks, presigned_urls.length=` +
|
|
723
|
+
`${presignedUrls.length} (expected ${computedParts - 1}). ` +
|
|
724
|
+
'Refusing to upload a mismatched part plan.');
|
|
725
|
+
}
|
|
476
726
|
// Internal abort signal that workers use to short-circuit each others'
|
|
477
727
|
// backoff sleeps. When any worker hits a terminal failure it aborts this
|
|
478
728
|
// controller, which races the caller's signal inside sleepWithSignal so
|
|
@@ -536,11 +786,33 @@ export class GislClient {
|
|
|
536
786
|
const part = presignedUrls[index];
|
|
537
787
|
const start = firstChunkSize + index * chunkSize;
|
|
538
788
|
const end = Math.min(start + chunkSize, totalSize);
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
//
|
|
542
|
-
//
|
|
543
|
-
|
|
789
|
+
// Read this part's bytes ONCE here, then reuse the captured chunk
|
|
790
|
+
// across every retry attempt below — so a retry never re-reads the
|
|
791
|
+
// file and the re-PUT is byte-identical (S3 parts are idempotent by
|
|
792
|
+
// partNumber; a re-PUT overwrites, no duplicate-data risk).
|
|
793
|
+
//
|
|
794
|
+
// Live-file caveat (streaming divergence from the old
|
|
795
|
+
// readFileSync→Blob path): the old code snapshotted the whole file at
|
|
796
|
+
// t0 so every part was point-in-time consistent. Streaming reads each
|
|
797
|
+
// part at the instant it is first uploaded, so a file mutated
|
|
798
|
+
// mid-upload yields parts from different instants. Truncation is
|
|
799
|
+
// caught by fileByteSource's short-read guard; full point-in-time
|
|
800
|
+
// snapshotting is resumable/staged-upload territory (SDK-3, Wb6ebOMM).
|
|
801
|
+
// Surface a read failure for THIS part as the typed
|
|
802
|
+
// GislMultipartPartError (with partNumber + uploadId), consistent with
|
|
803
|
+
// the PUT-failure path below — a bare GislError from fileByteSource
|
|
804
|
+
// (short read / libuv ceiling) would otherwise lose the per-part
|
|
805
|
+
// context (codex review). An abort must stay GislAbortError.
|
|
806
|
+
let chunk;
|
|
807
|
+
try {
|
|
808
|
+
chunk = await source.slice(start, end);
|
|
809
|
+
}
|
|
810
|
+
catch (err) {
|
|
811
|
+
if (err instanceof GislAbortError)
|
|
812
|
+
throw err;
|
|
813
|
+
throw new GislMultipartPartError(`Failed to read bytes for part ${part.partNumber}: ` +
|
|
814
|
+
(err instanceof Error ? err.message : String(err)), part.partNumber, initResponse.uploadId);
|
|
815
|
+
}
|
|
544
816
|
const contentLength = end - start;
|
|
545
817
|
let lastErr = null;
|
|
546
818
|
for (let attempt = 0; attempt < this.multipartMaxAttempts; attempt++) {
|
|
@@ -576,8 +848,8 @@ export class GislClient {
|
|
|
576
848
|
// forcing them to wait out their backoff timer.
|
|
577
849
|
await sleepWithEitherSignal(delay, options?.signal, failureController.signal);
|
|
578
850
|
}
|
|
579
|
-
throw new
|
|
580
|
-
(lastErr instanceof Error ? lastErr.message : String(lastErr)));
|
|
851
|
+
throw new GislMultipartPartError(`S3 chunk upload failed for part ${part.partNumber} after ${this.multipartMaxAttempts} attempts: ` +
|
|
852
|
+
(lastErr instanceof Error ? lastErr.message : String(lastErr)), part.partNumber, initResponse.uploadId);
|
|
581
853
|
};
|
|
582
854
|
// Upload with concurrency limit. Workers check the signal before pulling
|
|
583
855
|
// the next queue item so a mid-upload abort drains fast without
|
|
@@ -639,7 +911,9 @@ export class GislClient {
|
|
|
639
911
|
fileId: completeResp.uploadId,
|
|
640
912
|
originalName: fileName,
|
|
641
913
|
mimeType: initResponse.mimeType,
|
|
642
|
-
|
|
914
|
+
// `totalSize` (from fs.stat / Blob.size) — the streaming path no longer
|
|
915
|
+
// holds a whole-file Blob to read `.size` off.
|
|
916
|
+
sizeBytes: totalSize,
|
|
643
917
|
// Preserved from the initiate response: v2 contract makes
|
|
644
918
|
// `constraintsApplied` a REQUIRED field on UploadResponse, and the
|
|
645
919
|
// multipart/complete endpoint does not re-emit it. The first-chunk probe
|
|
@@ -647,6 +921,549 @@ export class GislClient {
|
|
|
647
921
|
constraintsApplied: initResponse.constraintsApplied,
|
|
648
922
|
};
|
|
649
923
|
}
|
|
924
|
+
/**
|
|
925
|
+
* SDK-3 (Wb6ebOMM): resume an in-progress multipart upload.
|
|
926
|
+
*
|
|
927
|
+
* Skips `/multipart/initiate` entirely (the original initiate happened in a
|
|
928
|
+
* prior process). Walks `/status` for the authoritative list of recorded
|
|
929
|
+
* parts, re-presigns the missing ones in batches of <=100, PUTs only those,
|
|
930
|
+
* and finalises with `/complete`. Caller's `source` MUST be byte-identical
|
|
931
|
+
* to the originally-uploaded file at the same offsets (parts whose etags
|
|
932
|
+
* don't match server state will fail `/complete`).
|
|
933
|
+
*
|
|
934
|
+
* Re-runs the same `uploadId` / `chunkSize` / `totalParts` / plan-consistency
|
|
935
|
+
* guards as the fresh-upload path (`multipartUpload`), using the /status
|
|
936
|
+
* envelope as the equivalent of the initiate envelope. Reuses the same
|
|
937
|
+
* `failureController` sibling-wake + `drainResponseBody` cleanup discipline
|
|
938
|
+
* as the fresh-upload PUT loop. `onProgress` fires on entry seeded from
|
|
939
|
+
* (uploadedPartNumbers.length * chunkSize) and again after every successful
|
|
940
|
+
* PUT. `onCheckpoint` fires OUTSIDE the retry-scoped path after every
|
|
941
|
+
* successful PUT — a callback-throw must not trigger a duplicate PUT.
|
|
942
|
+
*
|
|
943
|
+
* TODO(HxUmVr3Y): replace inline hand-coded request body marshalling on regen.
|
|
944
|
+
*/
|
|
945
|
+
async multipartResume(source, fileName, totalSize, resumeUploadId, options) {
|
|
946
|
+
// Step 1: Walk /status for the authoritative session state.
|
|
947
|
+
const status = await this.walkUploadStatus(resumeUploadId, {
|
|
948
|
+
signal: options?.signal,
|
|
949
|
+
});
|
|
950
|
+
// Validate the /status envelope shape, mirroring the fresh-upload
|
|
951
|
+
// post-initiate guards (`multipartUpload` lines around the
|
|
952
|
+
// total_parts / recommendedChunkSize / uploadId validation block).
|
|
953
|
+
if (typeof status.uploadId !== 'string' ||
|
|
954
|
+
status.uploadId === '' ||
|
|
955
|
+
status.uploadId !== resumeUploadId) {
|
|
956
|
+
throw new GislError('multipartResume: /status response uploadId does not match resumeUploadId.');
|
|
957
|
+
}
|
|
958
|
+
const chunkSize = status.recommendedChunkSize;
|
|
959
|
+
if (typeof chunkSize !== 'number' ||
|
|
960
|
+
!Number.isInteger(chunkSize) ||
|
|
961
|
+
chunkSize < MULTIPART_CHUNK_SIZE ||
|
|
962
|
+
chunkSize > RECOMMENDED_CHUNK_SIZE_MAX_BYTES) {
|
|
963
|
+
throw new GislError('multipartResume: /status recommendedChunkSize missing or outside the contract ' +
|
|
964
|
+
`range [${MULTIPART_CHUNK_SIZE}, ${RECOMMENDED_CHUNK_SIZE_MAX_BYTES}]: got ${String(chunkSize)}.`);
|
|
965
|
+
}
|
|
966
|
+
if (typeof status.totalParts !== 'number' ||
|
|
967
|
+
!Number.isInteger(status.totalParts) ||
|
|
968
|
+
status.totalParts < 1) {
|
|
969
|
+
throw new GislError(`multipartResume: /status totalParts missing or invalid: got ${String(status.totalParts)}.`);
|
|
970
|
+
}
|
|
971
|
+
if (status.totalParts > S3_MAX_MULTIPART_PARTS) {
|
|
972
|
+
throw new GislMultipartPartCountError(`multipartResume: /status totalParts=${status.totalParts} exceeds the S3 ` +
|
|
973
|
+
`${S3_MAX_MULTIPART_PARTS}-part multipart limit.`, status.totalParts, S3_MAX_MULTIPART_PARTS);
|
|
974
|
+
}
|
|
975
|
+
// Sanity-check the caller's byte source against the server's recorded
|
|
976
|
+
// plan. Mirrors the fresh-upload chunk-plan: part 1 = firstChunkSize
|
|
977
|
+
// (8 MiB), parts 2..totalParts each consume chunkSize bytes (last part
|
|
978
|
+
// may be a short tail). Reject a wrong-file resume here — /complete
|
|
979
|
+
// would otherwise fail on etag mismatch.
|
|
980
|
+
const firstChunkSize = Math.min(totalSize, DEFAULT_MULTIPART_FIRST_CHUNK_SIZE);
|
|
981
|
+
const expectedMinBytes = firstChunkSize + Math.max(0, status.totalParts - 2) * chunkSize + (status.totalParts > 1 ? 1 : 0);
|
|
982
|
+
const expectedMaxBytes = firstChunkSize + Math.max(0, status.totalParts - 1) * chunkSize;
|
|
983
|
+
if (totalSize < expectedMinBytes || totalSize > expectedMaxBytes) {
|
|
984
|
+
throw new GislError(`multipartResume: caller file size (${totalSize}) does not match the resumed ` +
|
|
985
|
+
`session's recorded plan (totalParts=${status.totalParts}, chunkSize=${chunkSize}, ` +
|
|
986
|
+
`expected ${expectedMinBytes}-${expectedMaxBytes} bytes). Wrong file for this uploadId?`);
|
|
987
|
+
}
|
|
988
|
+
// Step 2: Compute missing parts. Server records `uploadedParts` as the
|
|
989
|
+
// authoritative set; everything in [1, totalParts] not in that set is
|
|
990
|
+
// still-to-upload. Part 1 was uploaded inline at initiate — if it is
|
|
991
|
+
// missing from /status the session is unrecoverable (the server rejects
|
|
992
|
+
// re-presigning part 1 to preserve the recorded etag for /complete).
|
|
993
|
+
const uploaded = new Map();
|
|
994
|
+
for (const p of status.uploadedParts) {
|
|
995
|
+
uploaded.set(p.partNumber, p);
|
|
996
|
+
}
|
|
997
|
+
if (!uploaded.has(1)) {
|
|
998
|
+
throw new GislError('multipartResume: part 1 (initiate first chunk) is missing from /status. ' +
|
|
999
|
+
'Part 1 is sealed at initiate and cannot be re-presigned; this session is unrecoverable. ' +
|
|
1000
|
+
'Start a fresh upload (call uploadFile without resumeUploadId).');
|
|
1001
|
+
}
|
|
1002
|
+
const missingParts = [];
|
|
1003
|
+
for (let n = 2; n <= status.totalParts; n++) {
|
|
1004
|
+
if (!uploaded.has(n))
|
|
1005
|
+
missingParts.push(n);
|
|
1006
|
+
}
|
|
1007
|
+
// Seed uploadedBytes from already-uploaded parts so onProgress reflects
|
|
1008
|
+
// the true resumption point. Server reports authoritative part sizes
|
|
1009
|
+
// via `sizeBytes`; sum those rather than guessing chunkSize * count
|
|
1010
|
+
// (the last part may be a short tail).
|
|
1011
|
+
let uploadedBytes = 0;
|
|
1012
|
+
for (const p of status.uploadedParts) {
|
|
1013
|
+
uploadedBytes += p.sizeBytes;
|
|
1014
|
+
}
|
|
1015
|
+
options?.onProgress?.(uploadedBytes, totalSize);
|
|
1016
|
+
const fireCheckpoint = (extraPartNumber) => {
|
|
1017
|
+
const all = [...uploaded.keys()];
|
|
1018
|
+
if (extraPartNumber !== undefined)
|
|
1019
|
+
all.push(extraPartNumber);
|
|
1020
|
+
all.sort((a, b) => a - b);
|
|
1021
|
+
const state = {
|
|
1022
|
+
uploadId: status.uploadId,
|
|
1023
|
+
totalParts: status.totalParts,
|
|
1024
|
+
uploadedPartNumbers: all,
|
|
1025
|
+
manifestExpiresAt: status.manifestExpiresAt,
|
|
1026
|
+
};
|
|
1027
|
+
// Callback fires OUTSIDE retry-scope. A throw here propagates and
|
|
1028
|
+
// fails the upload but cannot trigger a duplicate PUT.
|
|
1029
|
+
options?.onCheckpoint?.(state);
|
|
1030
|
+
};
|
|
1031
|
+
// Fire an entry checkpoint so callers can persist the resumed state
|
|
1032
|
+
// even before any new PUT lands. Useful when the missing-parts list is
|
|
1033
|
+
// empty (everything already uploaded except /complete) — see below.
|
|
1034
|
+
fireCheckpoint();
|
|
1035
|
+
// Short-circuit: every part is already uploaded. Skip presign + PUT
|
|
1036
|
+
// and go straight to /complete with the etags the server has on file.
|
|
1037
|
+
const newEtags = [];
|
|
1038
|
+
if (missingParts.length === 0) {
|
|
1039
|
+
// No PUTs to run; proceed to /complete below with just the recorded parts.
|
|
1040
|
+
}
|
|
1041
|
+
else {
|
|
1042
|
+
// Step 3: For each batch of <=100 missing parts, re-presign + PUT.
|
|
1043
|
+
// We process batches sequentially (presign call) but PUTs within each
|
|
1044
|
+
// batch run concurrently up to multipartConcurrency, mirroring the
|
|
1045
|
+
// fresh-upload worker-pool semantics.
|
|
1046
|
+
const failureController = new AbortController();
|
|
1047
|
+
const putOne = async (part) => {
|
|
1048
|
+
// Offset math mirrors the fresh-upload path
|
|
1049
|
+
// (`multipartUpload`'s `uploadChunk`): part 1 is the initiate's 8 MiB
|
|
1050
|
+
// first chunk, parts 2..N each consume chunkSize bytes starting at
|
|
1051
|
+
// firstChunkSize. The resume path never PUTs part 1 (rejected
|
|
1052
|
+
// earlier as unrecoverable), so partNumber here is always >= 2.
|
|
1053
|
+
const firstChunkSize = Math.min(totalSize, DEFAULT_MULTIPART_FIRST_CHUNK_SIZE);
|
|
1054
|
+
const start = firstChunkSize + (part.partNumber - 2) * chunkSize;
|
|
1055
|
+
const end = Math.min(start + chunkSize, totalSize);
|
|
1056
|
+
const contentLength = end - start;
|
|
1057
|
+
let chunk;
|
|
1058
|
+
try {
|
|
1059
|
+
chunk = await source.slice(start, end);
|
|
1060
|
+
}
|
|
1061
|
+
catch (err) {
|
|
1062
|
+
if (err instanceof GislAbortError)
|
|
1063
|
+
throw err;
|
|
1064
|
+
throw new GislMultipartPartError(`multipartResume: failed to read bytes for part ${part.partNumber}: ` +
|
|
1065
|
+
(err instanceof Error ? err.message : String(err)), part.partNumber, status.uploadId);
|
|
1066
|
+
}
|
|
1067
|
+
let lastErr = null;
|
|
1068
|
+
for (let attempt = 0; attempt < this.multipartMaxAttempts; attempt++) {
|
|
1069
|
+
if (options?.signal?.aborted) {
|
|
1070
|
+
throw new GislAbortError(`multipartResume: S3 part ${part.partNumber} upload aborted`);
|
|
1071
|
+
}
|
|
1072
|
+
if (failureController.signal.aborted) {
|
|
1073
|
+
throw new GislError(`multipartResume: S3 part ${part.partNumber} upload abandoned after sibling failure`);
|
|
1074
|
+
}
|
|
1075
|
+
let s3Response;
|
|
1076
|
+
try {
|
|
1077
|
+
s3Response = await fetch(part.url, {
|
|
1078
|
+
method: 'PUT',
|
|
1079
|
+
body: chunk,
|
|
1080
|
+
headers: { 'Content-Length': contentLength.toString() },
|
|
1081
|
+
signal: options?.signal,
|
|
1082
|
+
});
|
|
1083
|
+
}
|
|
1084
|
+
catch (err) {
|
|
1085
|
+
if (isAbortError(err) && options?.signal?.aborted) {
|
|
1086
|
+
throw new GislAbortError(`multipartResume: S3 part ${part.partNumber} upload aborted`);
|
|
1087
|
+
}
|
|
1088
|
+
// Non-user-abort AbortError (e.g. transport cleanup) must still
|
|
1089
|
+
// surface as a typed GislError subclass — never as a raw
|
|
1090
|
+
// DOMException — to preserve the "every multipart failure is a
|
|
1091
|
+
// typed GislError" contract (code-reviewer P7).
|
|
1092
|
+
if (isAbortError(err)) {
|
|
1093
|
+
throw new GislMultipartPartError(`multipartResume: S3 part ${part.partNumber} aborted by transport: ` +
|
|
1094
|
+
(err instanceof Error ? err.message : String(err)), part.partNumber, status.uploadId);
|
|
1095
|
+
}
|
|
1096
|
+
if (isRetryableNetworkError(err)) {
|
|
1097
|
+
lastErr = err;
|
|
1098
|
+
if (attempt + 1 >= this.multipartMaxAttempts)
|
|
1099
|
+
break;
|
|
1100
|
+
const delay = fullJitterDelay(this.multipartRetryBaseMs, attempt);
|
|
1101
|
+
await sleepWithEitherSignal(delay, options?.signal, failureController.signal);
|
|
1102
|
+
continue;
|
|
1103
|
+
}
|
|
1104
|
+
throw err;
|
|
1105
|
+
}
|
|
1106
|
+
if (s3Response.ok) {
|
|
1107
|
+
const etag = s3Response.headers.get('etag');
|
|
1108
|
+
if (!etag) {
|
|
1109
|
+
await drainResponseBody(s3Response);
|
|
1110
|
+
throw new GislError(`multipartResume: S3 response missing ETag for part ${part.partNumber}`);
|
|
1111
|
+
}
|
|
1112
|
+
// Successful PUT — record etag, apply progress + checkpoint side
|
|
1113
|
+
// effects OUTSIDE the retry-scoped path (mirrors the fresh-upload
|
|
1114
|
+
// discipline at multipartUpload's ok-branch).
|
|
1115
|
+
newEtags.push({ partNumber: part.partNumber, etag });
|
|
1116
|
+
uploaded.set(part.partNumber, {
|
|
1117
|
+
partNumber: part.partNumber,
|
|
1118
|
+
etag,
|
|
1119
|
+
sizeBytes: contentLength,
|
|
1120
|
+
lastModified: new Date().toISOString(),
|
|
1121
|
+
});
|
|
1122
|
+
uploadedBytes = Math.min(uploadedBytes + contentLength, totalSize);
|
|
1123
|
+
options?.onProgress?.(uploadedBytes, totalSize);
|
|
1124
|
+
fireCheckpoint();
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1127
|
+
await drainResponseBody(s3Response);
|
|
1128
|
+
if (!isRetryableStatus(s3Response.status)) {
|
|
1129
|
+
throw new GislError(`multipartResume: S3 chunk upload failed for part ${part.partNumber}: HTTP ${s3Response.status} (non-retryable)`);
|
|
1130
|
+
}
|
|
1131
|
+
lastErr = new GislError(`multipartResume: S3 chunk upload failed for part ${part.partNumber}: HTTP ${s3Response.status}`);
|
|
1132
|
+
if (attempt + 1 >= this.multipartMaxAttempts)
|
|
1133
|
+
break;
|
|
1134
|
+
const delay = fullJitterDelay(this.multipartRetryBaseMs, attempt);
|
|
1135
|
+
await sleepWithEitherSignal(delay, options?.signal, failureController.signal);
|
|
1136
|
+
}
|
|
1137
|
+
throw new GislMultipartPartError(`multipartResume: S3 chunk upload failed for part ${part.partNumber} after ${this.multipartMaxAttempts} attempts: ` +
|
|
1138
|
+
(lastErr instanceof Error ? lastErr.message : String(lastErr)), part.partNumber, status.uploadId);
|
|
1139
|
+
};
|
|
1140
|
+
// Drive batches of <=100 part numbers.
|
|
1141
|
+
const PRESIGN_BATCH_SIZE = 100;
|
|
1142
|
+
for (let i = 0; i < missingParts.length; i += PRESIGN_BATCH_SIZE) {
|
|
1143
|
+
if (options?.signal?.aborted) {
|
|
1144
|
+
throw new GislAbortError('multipartResume aborted');
|
|
1145
|
+
}
|
|
1146
|
+
const batch = missingParts.slice(i, i + PRESIGN_BATCH_SIZE);
|
|
1147
|
+
const presigned = await this.presignParts(status.uploadId, batch, status.totalParts, { signal: options?.signal });
|
|
1148
|
+
// Concurrent PUTs within the batch.
|
|
1149
|
+
const queue = [...presigned.presignedUrls];
|
|
1150
|
+
const workers = Array.from({ length: Math.min(this.multipartConcurrency, queue.length) }, async () => {
|
|
1151
|
+
while (queue.length > 0 && !failureController.signal.aborted) {
|
|
1152
|
+
if (options?.signal?.aborted) {
|
|
1153
|
+
throw new GislAbortError('multipartResume aborted');
|
|
1154
|
+
}
|
|
1155
|
+
const part = queue.shift();
|
|
1156
|
+
try {
|
|
1157
|
+
await putOne(part);
|
|
1158
|
+
}
|
|
1159
|
+
catch (err) {
|
|
1160
|
+
failureController.abort();
|
|
1161
|
+
throw err;
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
});
|
|
1165
|
+
await Promise.all(workers);
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
// Step 4: /complete with the FULL parts list = (server-recorded etags
|
|
1169
|
+
// from /status) ∪ (newly-PUT etags this run). Sort ascending by
|
|
1170
|
+
// partNumber (the wire shape pin in fresh-upload mirrors this).
|
|
1171
|
+
const allParts = [];
|
|
1172
|
+
for (const p of status.uploadedParts) {
|
|
1173
|
+
allParts.push({ partNumber: p.partNumber, etag: p.etag });
|
|
1174
|
+
}
|
|
1175
|
+
for (const e of newEtags)
|
|
1176
|
+
allParts.push(e);
|
|
1177
|
+
allParts.sort((a, b) => a.partNumber - b.partNumber);
|
|
1178
|
+
if (allParts.length !== status.totalParts) {
|
|
1179
|
+
throw new GislError(`multipartResume: assembled parts list has ${allParts.length} entries, ` +
|
|
1180
|
+
`expected ${status.totalParts}. Refusing to /complete with an incomplete part set.`);
|
|
1181
|
+
}
|
|
1182
|
+
// Marshal via the generator's `*ToJSON` helper so the contracts-drift
|
|
1183
|
+
// guard test (`contract-drift-fields.test.ts`) covers BOTH the fresh and
|
|
1184
|
+
// resume paths uniformly (code-reviewer P7). If a future regen adds a
|
|
1185
|
+
// required field to `MultipartCompleteRequest`, tsc fails here at the
|
|
1186
|
+
// typed object literal — same as the fresh path.
|
|
1187
|
+
const completeRequest = {
|
|
1188
|
+
uploadId: status.uploadId,
|
|
1189
|
+
parts: allParts.map((p) => ({ partNumber: p.partNumber, etag: p.etag })),
|
|
1190
|
+
};
|
|
1191
|
+
const wireCompleteBody = MultipartCompleteRequestToJSON(completeRequest);
|
|
1192
|
+
if (typeof wireCompleteBody?.upload_id !== 'string' ||
|
|
1193
|
+
!Array.isArray(wireCompleteBody?.parts)) {
|
|
1194
|
+
throw new GislError('multipartResume: MultipartCompleteRequestToJSON returned an unexpected shape.');
|
|
1195
|
+
}
|
|
1196
|
+
const completeResp = await this.request('POST', '/api/uploads/multipart/complete', {
|
|
1197
|
+
body: wireCompleteBody,
|
|
1198
|
+
deserialize: MultipartCompleteResponseFromJSON,
|
|
1199
|
+
signal: options?.signal,
|
|
1200
|
+
});
|
|
1201
|
+
if (completeResp.status !== 'completed') {
|
|
1202
|
+
throw new GislError(`multipartResume: completed with unexpected status: ${completeResp.status}`);
|
|
1203
|
+
}
|
|
1204
|
+
// Resume-path information loss: the /status envelope (and /complete)
|
|
1205
|
+
// do NOT carry `mime_type` or `constraints_applied` — those were
|
|
1206
|
+
// emitted on the original initiate envelope, which the resume path
|
|
1207
|
+
// skipped. Fall back to caller-supplied `fileName` for `originalName`;
|
|
1208
|
+
// emit `mimeType` as `''` and `constraintsApplied` as a sentinel
|
|
1209
|
+
// populated with the only fact we DO know on resume: `maxSizeBytes =
|
|
1210
|
+
// totalSize` (the upload was permitted at this size when initiated),
|
|
1211
|
+
// `processingClassPreAssignment = 'unknown'`. Consumers needing
|
|
1212
|
+
// authoritative post-upload metadata SHOULD call `getMetadata(fileId)`
|
|
1213
|
+
// (the fresh-upload path's docblock already says the same).
|
|
1214
|
+
// TODO(HxUmVr3Y): when contracts ships the resume-support schemas,
|
|
1215
|
+
// extend `/status` (or add `/multipart/{id}/manifest`) to carry
|
|
1216
|
+
// mime_type + constraints_applied so this sentinel can go away.
|
|
1217
|
+
return {
|
|
1218
|
+
fileId: completeResp.uploadId,
|
|
1219
|
+
originalName: fileName,
|
|
1220
|
+
mimeType: '',
|
|
1221
|
+
sizeBytes: totalSize,
|
|
1222
|
+
constraintsApplied: {
|
|
1223
|
+
maxSizeBytes: totalSize,
|
|
1224
|
+
// `maxDurationSeconds` deliberately omitted (not `null`): parity
|
|
1225
|
+
// comparator filters `undefined` keys from both sides; cross-SDK
|
|
1226
|
+
// upload_small precedent.
|
|
1227
|
+
processingClassPreAssignment: UploadConstraintsAppliedProcessingClassPreAssignmentEnum.unknown,
|
|
1228
|
+
},
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1231
|
+
// -----------------------------------------------------------------------
|
|
1232
|
+
// SDK-3 (Wb6ebOMM) — resume-support endpoints
|
|
1233
|
+
// -----------------------------------------------------------------------
|
|
1234
|
+
/**
|
|
1235
|
+
* Fetch the durable status of an in-progress multipart upload session.
|
|
1236
|
+
*
|
|
1237
|
+
* Walks every page of `GET /api/uploads/multipart/{uploadId}/status`
|
|
1238
|
+
* (paginated via `next_part_number_marker` + `is_truncated`) and returns
|
|
1239
|
+
* the aggregated state. Callers see the complete set of recorded parts
|
|
1240
|
+
* across pages without driving the cursor themselves.
|
|
1241
|
+
*
|
|
1242
|
+
* Anonymous-initiated sessions return 403 → `GislMultipartSessionAuthRequiredError`.
|
|
1243
|
+
* Non-existent / expired sessions return 404 → `GislMultipartSessionNotFoundError`.
|
|
1244
|
+
* Authed-but-non-owning callers return 403 → `GislMultipartSessionOwnershipError`.
|
|
1245
|
+
*
|
|
1246
|
+
* TODO(HxUmVr3Y): replace hand-coded response shape on regen.
|
|
1247
|
+
*/
|
|
1248
|
+
async getUploadStatus(uploadId, opts = {}) {
|
|
1249
|
+
if (typeof uploadId !== 'string' || uploadId === '') {
|
|
1250
|
+
throw new GislError('getUploadStatus: uploadId must be a non-empty string.');
|
|
1251
|
+
}
|
|
1252
|
+
return this.walkUploadStatus(uploadId, opts);
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Re-presign a batch of missing part numbers on an in-progress multipart
|
|
1256
|
+
* session.
|
|
1257
|
+
*
|
|
1258
|
+
* Validates client-side BEFORE the HTTP round-trip:
|
|
1259
|
+
* - `partNumbers` non-empty
|
|
1260
|
+
* - length <=100 (server raw-body cap is 8 KiB before json_decode)
|
|
1261
|
+
* - every entry an integer in `[2, totalParts]` — part 1 is sealed at
|
|
1262
|
+
* initiate (re-presigning it would break the etag recorded server-side
|
|
1263
|
+
* for /complete)
|
|
1264
|
+
* - entries unique
|
|
1265
|
+
* - `totalParts` <=10 000 (S3 hard limit; mirrors the SDK-1 ceiling guard)
|
|
1266
|
+
*
|
|
1267
|
+
* TODO(HxUmVr3Y): replace hand-coded request/response shapes on regen.
|
|
1268
|
+
*/
|
|
1269
|
+
async presignParts(uploadId, partNumbers, totalParts, opts = {}) {
|
|
1270
|
+
if (typeof uploadId !== 'string' || uploadId === '') {
|
|
1271
|
+
throw new GislError('presignParts: uploadId must be a non-empty string.');
|
|
1272
|
+
}
|
|
1273
|
+
if (typeof totalParts !== 'number' ||
|
|
1274
|
+
!Number.isInteger(totalParts) ||
|
|
1275
|
+
totalParts < 1) {
|
|
1276
|
+
throw new GislError(`presignParts: totalParts must be a positive integer, got ${String(totalParts)}.`);
|
|
1277
|
+
}
|
|
1278
|
+
if (totalParts > S3_MAX_MULTIPART_PARTS) {
|
|
1279
|
+
throw new GislMultipartPartCountError(`presignParts: totalParts=${totalParts} exceeds the S3 ${S3_MAX_MULTIPART_PARTS}-part ` +
|
|
1280
|
+
'multipart limit. Refusing to re-presign on a session that cannot complete.', totalParts, S3_MAX_MULTIPART_PARTS);
|
|
1281
|
+
}
|
|
1282
|
+
if (!Array.isArray(partNumbers) || partNumbers.length === 0) {
|
|
1283
|
+
throw new GislError('presignParts: partNumbers must be a non-empty array.');
|
|
1284
|
+
}
|
|
1285
|
+
if (partNumbers.length > 100) {
|
|
1286
|
+
throw new GislError(`presignParts: partNumbers has ${partNumbers.length} entries — server caps batches at 100.`);
|
|
1287
|
+
}
|
|
1288
|
+
const seen = new Set();
|
|
1289
|
+
for (const n of partNumbers) {
|
|
1290
|
+
if (typeof n !== 'number' ||
|
|
1291
|
+
!Number.isInteger(n) ||
|
|
1292
|
+
n < 2 ||
|
|
1293
|
+
n > totalParts) {
|
|
1294
|
+
throw new GislError(`presignParts: partNumbers entry ${String(n)} is not an integer in [2, ${totalParts}]. ` +
|
|
1295
|
+
'Part 1 is sealed at initiate; re-presigning it would invalidate the recorded etag for /complete.');
|
|
1296
|
+
}
|
|
1297
|
+
if (seen.has(n)) {
|
|
1298
|
+
throw new GislError(`presignParts: partNumbers contains duplicate ${n}.`);
|
|
1299
|
+
}
|
|
1300
|
+
seen.add(n);
|
|
1301
|
+
}
|
|
1302
|
+
const path = `/api/uploads/multipart/${encodeURIComponent(uploadId)}/presign`;
|
|
1303
|
+
return this.request('POST', path, {
|
|
1304
|
+
// Hand-coded snake_case wire body. TODO(HxUmVr3Y): replace with
|
|
1305
|
+
// generated `*RequestToJSON` helper on regen.
|
|
1306
|
+
body: { part_numbers: [...partNumbers] },
|
|
1307
|
+
deserialize: (raw) => {
|
|
1308
|
+
// Hand-coded snake_case -> camelCase. TODO(HxUmVr3Y): replace with
|
|
1309
|
+
// generated FromJSON helper on regen.
|
|
1310
|
+
const r = raw;
|
|
1311
|
+
if (typeof r.upload_id !== 'string' || !Array.isArray(r.presigned_urls)) {
|
|
1312
|
+
throw new GislError('presignParts: malformed response envelope.');
|
|
1313
|
+
}
|
|
1314
|
+
return {
|
|
1315
|
+
uploadId: r.upload_id,
|
|
1316
|
+
presignedUrls: r.presigned_urls.map((p) => ({
|
|
1317
|
+
partNumber: p.part_number,
|
|
1318
|
+
url: p.url,
|
|
1319
|
+
expiresAt: p.expires_at,
|
|
1320
|
+
})),
|
|
1321
|
+
};
|
|
1322
|
+
},
|
|
1323
|
+
signal: opts.signal,
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
/**
|
|
1327
|
+
* Extend the manifest TTL of an in-progress multipart upload session.
|
|
1328
|
+
*
|
|
1329
|
+
* The durable session manifest defaults to a 48 h TTL (decoupled from the
|
|
1330
|
+
* shorter presigned-URL TTL). For a long-running resume that spans days
|
|
1331
|
+
* (e.g. an upload paused overnight on flaky Wi-Fi), callers SHOULD invoke
|
|
1332
|
+
* `keepaliveUpload` every **12-24 h** while resuming — the 12-24 h band
|
|
1333
|
+
* leaves >=24 h of slack against the 48 h ceiling even with worst-case
|
|
1334
|
+
* clock skew between client and server. The server atomically refreshes
|
|
1335
|
+
* the Redis EXPIRE for the manifest key; the call is idempotent.
|
|
1336
|
+
*
|
|
1337
|
+
* TODO(HxUmVr3Y): replace hand-coded response shape on regen.
|
|
1338
|
+
*/
|
|
1339
|
+
async keepaliveUpload(uploadId, opts = {}) {
|
|
1340
|
+
if (typeof uploadId !== 'string' || uploadId === '') {
|
|
1341
|
+
throw new GislError('keepaliveUpload: uploadId must be a non-empty string.');
|
|
1342
|
+
}
|
|
1343
|
+
const path = `/api/uploads/multipart/${encodeURIComponent(uploadId)}/keepalive`;
|
|
1344
|
+
return this.request('POST', path, {
|
|
1345
|
+
// Server expects an empty body; pass an empty object so the `request`
|
|
1346
|
+
// helper sets `Content-Type: application/json` for symmetry with the
|
|
1347
|
+
// other JSON-bodied POSTs. The endpoint ignores any fields if present.
|
|
1348
|
+
body: {},
|
|
1349
|
+
deserialize: (raw) => {
|
|
1350
|
+
// Hand-coded snake_case -> camelCase. TODO(HxUmVr3Y): replace with
|
|
1351
|
+
// generated FromJSON helper on regen.
|
|
1352
|
+
const r = raw;
|
|
1353
|
+
if (typeof r.upload_id !== 'string' ||
|
|
1354
|
+
typeof r.manifest_expires_at !== 'string') {
|
|
1355
|
+
throw new GislError('keepaliveUpload: malformed response envelope.');
|
|
1356
|
+
}
|
|
1357
|
+
return {
|
|
1358
|
+
uploadId: r.upload_id,
|
|
1359
|
+
manifestExpiresAt: r.manifest_expires_at,
|
|
1360
|
+
};
|
|
1361
|
+
},
|
|
1362
|
+
signal: opts.signal,
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
/**
|
|
1366
|
+
* Private walk-pagination helper for /status. Aggregates every page into
|
|
1367
|
+
* a single `_Sdk3HandCodedMultipartStatusResult`. AbortSignal short-circuits
|
|
1368
|
+
* the loop between page fetches AND propagates into each fetch.
|
|
1369
|
+
*
|
|
1370
|
+
* Limit pinned to 1000 (max per page) so we make the minimum number of
|
|
1371
|
+
* round-trips even for the worst-case ~10 pages on a 10 000-part upload.
|
|
1372
|
+
*/
|
|
1373
|
+
async walkUploadStatus(uploadId, opts) {
|
|
1374
|
+
const PAGE_LIMIT = 1000;
|
|
1375
|
+
// Slow-path DoS guard (code-reviewer minor 6). The cursor-advance check
|
|
1376
|
+
// already prevents an infinite loop; this cap additionally prevents a
|
|
1377
|
+
// pathological server that advances by 1 each page from forcing
|
|
1378
|
+
// O(totalParts) round-trips for a 10 000-part upload. PAGE_LIMIT=1000
|
|
1379
|
+
// means a healthy server completes in <=10 round-trips; 50 leaves
|
|
1380
|
+
// generous slack.
|
|
1381
|
+
const MAX_PAGES = 50;
|
|
1382
|
+
const collected = [];
|
|
1383
|
+
let cursor = 0;
|
|
1384
|
+
let totalParts = 0;
|
|
1385
|
+
let multipartUploadId = '';
|
|
1386
|
+
let cloudKey = '';
|
|
1387
|
+
let manifestExpiresAt = '';
|
|
1388
|
+
let recommendedChunkSize = 0;
|
|
1389
|
+
let pageCount = 0;
|
|
1390
|
+
while (true) {
|
|
1391
|
+
if (opts.signal?.aborted) {
|
|
1392
|
+
throw new GislAbortError('getUploadStatus aborted');
|
|
1393
|
+
}
|
|
1394
|
+
if (pageCount >= MAX_PAGES) {
|
|
1395
|
+
throw new GislError(`getUploadStatus: server returned more than ${MAX_PAGES} pages — refusing to ` +
|
|
1396
|
+
'continue. The /status endpoint should advance the cursor in 1000-part strides.');
|
|
1397
|
+
}
|
|
1398
|
+
pageCount += 1;
|
|
1399
|
+
const query = `?cursor=${cursor}&limit=${PAGE_LIMIT}`;
|
|
1400
|
+
// String-concat the query OUTSIDE the path backtick so the contract-drift
|
|
1401
|
+
// scanner (tests/unit/contract-drift.test.ts) sees the bare path
|
|
1402
|
+
// `/api/uploads/multipart/{id}/status`. Embedding `${query}` in the
|
|
1403
|
+
// template collapses to `/status{id}` and false-drifts — same reason
|
|
1404
|
+
// getSchema and getCreditsUsage concatenate their querystrings.
|
|
1405
|
+
const path = `/api/uploads/multipart/${encodeURIComponent(uploadId)}/status` + query;
|
|
1406
|
+
const page = await this.request('GET', path, { signal: opts.signal });
|
|
1407
|
+
// Defensive: server contract pins these fields. Strict-validate every
|
|
1408
|
+
// top-level field on each page (code-reviewer P7) so a malformed wire
|
|
1409
|
+
// envelope cannot silently coerce a missing key to '' / 0 / NaN and
|
|
1410
|
+
// flow it into MultipartCheckpointState.manifestExpiresAt or downstream
|
|
1411
|
+
// chunkSize guards.
|
|
1412
|
+
if (typeof page.total_parts !== 'number' || page.total_parts < 1) {
|
|
1413
|
+
throw new GislError('getUploadStatus: server page missing or invalid total_parts.');
|
|
1414
|
+
}
|
|
1415
|
+
if (typeof page.upload_id !== 'string' ||
|
|
1416
|
+
page.upload_id !== uploadId ||
|
|
1417
|
+
typeof page.multipart_upload_id !== 'string' ||
|
|
1418
|
+
page.multipart_upload_id === '' ||
|
|
1419
|
+
typeof page.cloud_key !== 'string' ||
|
|
1420
|
+
page.cloud_key === '' ||
|
|
1421
|
+
typeof page.manifest_expires_at !== 'string' ||
|
|
1422
|
+
page.manifest_expires_at === '' ||
|
|
1423
|
+
typeof page.recommended_chunk_size !== 'number') {
|
|
1424
|
+
throw new GislError('getUploadStatus: server page missing required fields or returned a ' +
|
|
1425
|
+
`mismatching upload_id (expected ${uploadId}, got ` +
|
|
1426
|
+
`${String(page.upload_id)}).`);
|
|
1427
|
+
}
|
|
1428
|
+
totalParts = page.total_parts;
|
|
1429
|
+
multipartUploadId = page.multipart_upload_id;
|
|
1430
|
+
cloudKey = page.cloud_key;
|
|
1431
|
+
manifestExpiresAt = page.manifest_expires_at;
|
|
1432
|
+
recommendedChunkSize = page.recommended_chunk_size;
|
|
1433
|
+
for (const p of page.uploaded_parts ?? []) {
|
|
1434
|
+
collected.push({
|
|
1435
|
+
partNumber: p.part_number,
|
|
1436
|
+
etag: p.etag,
|
|
1437
|
+
sizeBytes: p.size_bytes,
|
|
1438
|
+
lastModified: p.last_modified,
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
if (!page.is_truncated)
|
|
1442
|
+
break;
|
|
1443
|
+
// Advance cursor; guard against a contract-violating non-advancing
|
|
1444
|
+
// marker that would loop forever.
|
|
1445
|
+
if (typeof page.next_part_number_marker !== 'number' ||
|
|
1446
|
+
page.next_part_number_marker <= cursor) {
|
|
1447
|
+
throw new GislError('getUploadStatus: server is_truncated=true but next_part_number_marker ' +
|
|
1448
|
+
`did not advance (was ${cursor}, got ${String(page.next_part_number_marker)}).`);
|
|
1449
|
+
}
|
|
1450
|
+
cursor = page.next_part_number_marker;
|
|
1451
|
+
}
|
|
1452
|
+
// Sort ascending by partNumber — server SHOULD already deliver in order
|
|
1453
|
+
// page-by-page, but a defensive sort keeps the aggregated shape's
|
|
1454
|
+
// contract simple to consume (resume-branch missing-parts compute scans
|
|
1455
|
+
// it linearly).
|
|
1456
|
+
collected.sort((a, b) => a.partNumber - b.partNumber);
|
|
1457
|
+
return {
|
|
1458
|
+
uploadId,
|
|
1459
|
+
multipartUploadId,
|
|
1460
|
+
cloudKey,
|
|
1461
|
+
totalParts,
|
|
1462
|
+
uploadedParts: collected,
|
|
1463
|
+
manifestExpiresAt,
|
|
1464
|
+
recommendedChunkSize,
|
|
1465
|
+
};
|
|
1466
|
+
}
|
|
650
1467
|
// -----------------------------------------------------------------------
|
|
651
1468
|
// Workflows
|
|
652
1469
|
// -----------------------------------------------------------------------
|
|
@@ -740,13 +1557,105 @@ export class GislClient {
|
|
|
740
1557
|
/**
|
|
741
1558
|
* Stream SSE events for a workflow. Returns an async iterable.
|
|
742
1559
|
*/
|
|
743
|
-
async streamEvents(workflowId) {
|
|
1560
|
+
async streamEvents(workflowId, opts = {}) {
|
|
744
1561
|
const eventsPath = `/api/workflows/${encodeURIComponent(workflowId)}/events`;
|
|
745
|
-
|
|
1562
|
+
// SSE-lifetime AbortController. `request()` builds its own controller
|
|
1563
|
+
// and tears it down (`clearTimeout(timer); unbind()`) in its `finally`
|
|
1564
|
+
// the instant the response headers arrive — BEFORE the SSE body
|
|
1565
|
+
// streams — so that controller cannot cancel a long-lived stream.
|
|
1566
|
+
// `streamEvents` must own a controller for the stream's whole lifetime.
|
|
1567
|
+
// We pass its signal to `request()` too, so a pre-aborted signal /
|
|
1568
|
+
// connect-phase abort still fast-fails. After headers, the live socket
|
|
1569
|
+
// is freed only by `reader.cancel()` inside `parseSseStream` — driven
|
|
1570
|
+
// by aborting this controller from the iterator wrapper's
|
|
1571
|
+
// `return()`/`throw()` (a generator's own `return()` is unreachable
|
|
1572
|
+
// while suspended at `await reader.read()`; canonical pattern:
|
|
1573
|
+
// openai-node `Stream[Symbol.asyncIterator]` + PR #1314).
|
|
1574
|
+
const controller = new AbortController();
|
|
1575
|
+
// Compose an optional consumer-supplied signal onto our controller.
|
|
1576
|
+
// The teardown MUST run (normal completion, error, OR early return)
|
|
1577
|
+
// or a long-lived consumer AbortController leaks listeners.
|
|
1578
|
+
const releaseConsumerSignal = bindAbortSignal(opts.signal, controller);
|
|
1579
|
+
let response;
|
|
1580
|
+
try {
|
|
1581
|
+
response = await this.request('GET', eventsPath, {
|
|
1582
|
+
rawResponse: true,
|
|
1583
|
+
signal: controller.signal,
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
catch (err) {
|
|
1587
|
+
releaseConsumerSignal();
|
|
1588
|
+
throw err;
|
|
1589
|
+
}
|
|
746
1590
|
if (!response.ok) {
|
|
747
|
-
|
|
1591
|
+
try {
|
|
1592
|
+
await this.handleResponse(response, eventsPath); // always throws
|
|
1593
|
+
}
|
|
1594
|
+
finally {
|
|
1595
|
+
releaseConsumerSignal();
|
|
1596
|
+
}
|
|
748
1597
|
}
|
|
749
|
-
|
|
1598
|
+
const inner = parseSseStream(response, { signal: controller.signal });
|
|
1599
|
+
let started = false;
|
|
1600
|
+
let settled = false;
|
|
1601
|
+
// Idempotent teardown. `abort` only on consumer-driven early
|
|
1602
|
+
// termination (return/throw) — NOT on normal completion or stream
|
|
1603
|
+
// error, where aborting would be a spurious "aborted though it
|
|
1604
|
+
// wasn't" signal (openai-node#194). If the consumer disposes the
|
|
1605
|
+
// iterator before ever pulling an event, the inner generator never
|
|
1606
|
+
// ran, so its `finally` won't cancel the body — cancel it here as a
|
|
1607
|
+
// backstop (the body is still unlocked: no reader was acquired).
|
|
1608
|
+
const cleanup = (abort) => {
|
|
1609
|
+
if (settled)
|
|
1610
|
+
return;
|
|
1611
|
+
settled = true;
|
|
1612
|
+
if (abort)
|
|
1613
|
+
controller.abort();
|
|
1614
|
+
if (!started)
|
|
1615
|
+
void response.body?.cancel().catch(() => { });
|
|
1616
|
+
releaseConsumerSignal();
|
|
1617
|
+
};
|
|
1618
|
+
// Abort-before-first-pull backstop. If the consumer aborts (their
|
|
1619
|
+
// signal, composed onto `controller`) and then drops the iterator
|
|
1620
|
+
// WITHOUT ever calling next()/return()/throw(), nothing else frees the
|
|
1621
|
+
// already-fetched body: `request()` unbound its fetch controller at
|
|
1622
|
+
// header receipt, and `parseSseStream` only attaches its reader +
|
|
1623
|
+
// abort listener once iteration starts. `cleanup`'s `!started` branch
|
|
1624
|
+
// only runs from the wrapper methods, so it never fires on a pure
|
|
1625
|
+
// abort-and-drop. Cancel the (still-unlocked) body directly here.
|
|
1626
|
+
// Once started, `parseSseStream` owns the locked reader and cancels
|
|
1627
|
+
// via its own abort listener, so this no-ops.
|
|
1628
|
+
controller.signal.addEventListener('abort', () => {
|
|
1629
|
+
if (!started)
|
|
1630
|
+
void response.body?.cancel().catch(() => { });
|
|
1631
|
+
}, { once: true });
|
|
1632
|
+
const wrapper = {
|
|
1633
|
+
async next(...args) {
|
|
1634
|
+
started = true;
|
|
1635
|
+
try {
|
|
1636
|
+
const result = await inner.next(...args);
|
|
1637
|
+
if (result.done)
|
|
1638
|
+
cleanup(false);
|
|
1639
|
+
return result;
|
|
1640
|
+
}
|
|
1641
|
+
catch (err) {
|
|
1642
|
+
cleanup(false);
|
|
1643
|
+
throw err;
|
|
1644
|
+
}
|
|
1645
|
+
},
|
|
1646
|
+
async return(value) {
|
|
1647
|
+
cleanup(true);
|
|
1648
|
+
return inner.return(value);
|
|
1649
|
+
},
|
|
1650
|
+
async throw(err) {
|
|
1651
|
+
cleanup(true);
|
|
1652
|
+
return inner.throw(err);
|
|
1653
|
+
},
|
|
1654
|
+
[Symbol.asyncIterator]() {
|
|
1655
|
+
return this;
|
|
1656
|
+
},
|
|
1657
|
+
};
|
|
1658
|
+
return wrapper;
|
|
750
1659
|
}
|
|
751
1660
|
// -----------------------------------------------------------------------
|
|
752
1661
|
// File metadata
|