@gigamusic/checkout 4.8.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/index.d.ts +67 -1
- package/dist/index.js +327 -18
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/src/checkout.ts +17 -0
- package/src/types.ts +27 -0
- package/src/zip-length.ts +128 -0
- package/src/zip-stream.ts +501 -18
- package/src/zip.ts +19 -0
package/README.md
CHANGED
|
@@ -66,6 +66,30 @@ export const GET = createDownloadZipStreamHandler({
|
|
|
66
66
|
Failures are also logged individually plus an `n/total entries failed` summary
|
|
67
67
|
line per request, so they're greppable without wiring anything up.
|
|
68
68
|
|
|
69
|
+
### Truncated archives
|
|
70
|
+
|
|
71
|
+
The server-side stream sends a real `Content-Length` whenever it can work out
|
|
72
|
+
every entry's exact size — from `track_files.fileSize`, falling back to a
|
|
73
|
+
ranged-GET probe for the nullable rows and for cover art. That matters because
|
|
74
|
+
a chunked zip response that dies mid-body (function timeout, OOM) is saved by
|
|
75
|
+
the browser as a *complete* file: valid audio bytes with no
|
|
76
|
+
End-of-Central-Directory record, which macOS Archive Utility reports as
|
|
77
|
+
"Error 79 – Inappropriate file type or format". With a length declared, the
|
|
78
|
+
browser flags the download as failed instead.
|
|
79
|
+
|
|
80
|
+
A declared length is a promise, so once it is out a failed entry can no longer
|
|
81
|
+
become a `_FAILED_*.txt` placeholder — the transfer is torn down and reported
|
|
82
|
+
instead. If your catalog's recorded sizes have drifted from what's in storage,
|
|
83
|
+
`zipStreamContentLength: false` restores the old chunked behaviour while you
|
|
84
|
+
repair the rows.
|
|
85
|
+
|
|
86
|
+
Downloads whose consumer stops accepting bytes are treated as abandoned after
|
|
87
|
+
`zipStreamIdleTimeoutMs` (default 30s) and released, and a second concurrent
|
|
88
|
+
stream for the same token supersedes the first — a customer retrying a stuck
|
|
89
|
+
download shouldn't leave the old one running until the function times out.
|
|
90
|
+
Every request ends with a `[zip-stream] completed …` line carrying the entry
|
|
91
|
+
count, bytes sent, bytes promised and duration.
|
|
92
|
+
|
|
69
93
|
## Bundles — discounted subsets of the catalog
|
|
70
94
|
|
|
71
95
|
`catalogPurchase` is all-or-nothing. For a curated pack ("the remix EPs", "2024
|
package/dist/index.d.ts
CHANGED
|
@@ -201,6 +201,33 @@ interface DownloadDeps {
|
|
|
201
201
|
* download that is otherwise still producing bytes.
|
|
202
202
|
*/
|
|
203
203
|
onZipEntryFailure?: (failure: ZipEntryFailure) => void;
|
|
204
|
+
/**
|
|
205
|
+
* How long `createDownloadZipStreamHandler` lets the customer's connection
|
|
206
|
+
* go without accepting a byte before it treats the download as abandoned,
|
|
207
|
+
* tears the archive down and stops paying for R2 egress. Defaults to 30s.
|
|
208
|
+
*
|
|
209
|
+
* This exists because `req.signal` cannot be relied on to fire when a
|
|
210
|
+
* customer closes a bulk download on Vercel — abandoned streams were
|
|
211
|
+
* observed running the function's full `maxDuration` and stacking up on one
|
|
212
|
+
* instance until it ran out of memory. Raise it only if you serve customers
|
|
213
|
+
* slow enough to spend that long on a single 256 KB window; lower it to
|
|
214
|
+
* reclaim capacity faster.
|
|
215
|
+
*/
|
|
216
|
+
zipStreamIdleTimeoutMs?: number;
|
|
217
|
+
/**
|
|
218
|
+
* Set `false` to make `createDownloadZipStreamHandler` stream without a
|
|
219
|
+
* `Content-Length`, as it did before the header was introduced.
|
|
220
|
+
*
|
|
221
|
+
* The header is what lets a browser notice that a zip arrived truncated
|
|
222
|
+
* instead of filing a corrupt archive as a finished download, so leaving it
|
|
223
|
+
* on is strongly preferred. The escape hatch is here for one scenario: the
|
|
224
|
+
* declared length is computed from `track_files.fileSize`, so a catalog
|
|
225
|
+
* whose recorded sizes have drifted from the objects in storage would fail
|
|
226
|
+
* every bulk download rather than silently shipping a mismatched body.
|
|
227
|
+
* Turning this off restores the old behaviour without a package downgrade
|
|
228
|
+
* while the rows are repaired. Defaults to `true`.
|
|
229
|
+
*/
|
|
230
|
+
zipStreamContentLength?: boolean;
|
|
204
231
|
/**
|
|
205
232
|
* Optional artist-name prefix prepended to zip filenames — both the SW
|
|
206
233
|
* manifest and the server-side stream output use it. Example: passing
|
|
@@ -412,7 +439,8 @@ interface RouteContext {
|
|
|
412
439
|
* whole archive. Because that turns a broken purchase into an
|
|
413
440
|
* apparently-successful download, every failure is reported through
|
|
414
441
|
* `deps.onZipEntryFailure` and summarised in a single log line at the end of
|
|
415
|
-
* the request.
|
|
442
|
+
* the request. The one exception is a failure that lands *after* a
|
|
443
|
+
* `Content-Length` has been promised — see below.
|
|
416
444
|
*
|
|
417
445
|
* Unlike the manifest path, this handler presigns each file *inside* the loop,
|
|
418
446
|
* immediately before fetching it, so signature expiry can't bite. Its own
|
|
@@ -421,6 +449,44 @@ interface RouteContext {
|
|
|
421
449
|
* (Vercel: 300s by default, up to 800s on Fluid compute). Multi-gigabyte
|
|
422
450
|
* bundles need that raised in the consuming route — `export const maxDuration`
|
|
423
451
|
* — or they'll be cut off mid-stream.
|
|
452
|
+
*
|
|
453
|
+
* ## Surviving a stream that dies mid-body
|
|
454
|
+
*
|
|
455
|
+
* A function that hits its timeout or its memory ceiling after the response
|
|
456
|
+
* headers are already out produces a *silent* truncation: the customer's
|
|
457
|
+
* browser sees a connection close on a chunked response and marks the download
|
|
458
|
+
* complete. The file is megabytes of perfectly good MP3 with no
|
|
459
|
+
* End-of-Central-Directory record, which macOS Archive Utility reports as the
|
|
460
|
+
* famously unhelpful "Error 79 – Inappropriate file type or format". Four
|
|
461
|
+
* things here exist to keep that from happening again:
|
|
462
|
+
*
|
|
463
|
+
* 1. **`Content-Length`.** Entries are stored uncompressed, so the archive's
|
|
464
|
+
* byte count is predictable from the entry sizes (`predictStoredZipLength`).
|
|
465
|
+
* Sizes come from `track_files.fileSize`, with a ranged-GET probe covering
|
|
466
|
+
* the rows where it's null and the cover art that has no row at all. If
|
|
467
|
+
* any size stays unknown, the header is omitted rather than guessed — a
|
|
468
|
+
* wrong `Content-Length` is worse than none. Once the header is out, a
|
|
469
|
+
* failed entry can no longer be papered over with a placeholder (the byte
|
|
470
|
+
* count would no longer match), so the transfer is torn down instead: the
|
|
471
|
+
* browser reports a failed download, which is the honest outcome.
|
|
472
|
+
* 2. **Demand-driven output, on small buffers.** The response body pulls from
|
|
473
|
+
* archiver only when the customer's connection has drained what came
|
|
474
|
+
* before, so a slow client throttles the R2 fetch rather than filling the
|
|
475
|
+
* heap — and archiver's 1 MiB default high-water mark, which it silently
|
|
476
|
+
* applies to four separate stream buffers, is cut to 64 KiB (see
|
|
477
|
+
* `ARCHIVER_HIGH_WATER_MARK_BYTES`).
|
|
478
|
+
* 3. **An idle watchdog.** `req.signal` is the documented disconnect signal
|
|
479
|
+
* but doesn't reliably fire on Vercel, so an abandoned download is instead
|
|
480
|
+
* detected by the consumer going quiet (`zipStreamIdleTimeoutMs`, 30s by
|
|
481
|
+
* default) and torn down within seconds rather than burning the full
|
|
482
|
+
* `maxDuration` of egress and memory.
|
|
483
|
+
* 4. **One stream per token.** A customer retrying a stuck download would
|
|
484
|
+
* otherwise stack zombie archives on a single instance; a second request
|
|
485
|
+
* supersedes the first (see `activeStreams`).
|
|
486
|
+
*
|
|
487
|
+
* Every request ends with a one-line summary — outcome, entries, bytes sent
|
|
488
|
+
* versus bytes promised, duration — so a future truncation shows up in the
|
|
489
|
+
* logs instead of only in a customer's inbox.
|
|
424
490
|
*/
|
|
425
491
|
declare function createDownloadZipStreamHandler(deps: DownloadDeps): (req: NextRequest, ctx: RouteContext) => Promise<Response>;
|
|
426
492
|
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import archiver from 'archiver';
|
|
|
9
9
|
|
|
10
10
|
// src/checkout.ts
|
|
11
11
|
var STRIPE_MIN_CHARGE_CENTS = 50;
|
|
12
|
+
var MANAGED_PAYMENTS_DISABLED = { enabled: false };
|
|
12
13
|
function resolveUnitAmount(overrideCents, catalogCents) {
|
|
13
14
|
if (typeof overrideCents === "number" && Number.isFinite(overrideCents) && overrideCents > 0) {
|
|
14
15
|
return Math.max(STRIPE_MIN_CHARGE_CENTS, Math.round(overrideCents));
|
|
@@ -93,6 +94,7 @@ function createCheckoutHandler(deps) {
|
|
|
93
94
|
release_ids: JSON.stringify(releases.map((r) => r.id)),
|
|
94
95
|
track_ids: "[]"
|
|
95
96
|
},
|
|
97
|
+
managed_payments: MANAGED_PAYMENTS_DISABLED,
|
|
96
98
|
success_url: successUrl,
|
|
97
99
|
cancel_url: cancelUrl
|
|
98
100
|
});
|
|
@@ -233,6 +235,7 @@ function createCheckoutHandler(deps) {
|
|
|
233
235
|
// items — but invaluable when supporting "what did I actually buy?".
|
|
234
236
|
...bundleIds.length > 0 ? { bundle_ids: JSON.stringify(bundleIds) } : {}
|
|
235
237
|
},
|
|
238
|
+
managed_payments: MANAGED_PAYMENTS_DISABLED,
|
|
236
239
|
success_url: successUrl,
|
|
237
240
|
cancel_url: cancelUrl
|
|
238
241
|
});
|
|
@@ -512,6 +515,9 @@ async function handleCoverArt(deps, token, releaseIdRaw) {
|
|
|
512
515
|
}
|
|
513
516
|
|
|
514
517
|
// src/zip.ts
|
|
518
|
+
function knownByteSize(fileSize) {
|
|
519
|
+
return typeof fileSize === "number" && Number.isSafeInteger(fileSize) && fileSize > 0 ? fileSize : void 0;
|
|
520
|
+
}
|
|
515
521
|
var ZIP_MANIFEST_EXPIRES_IN_SECONDS = 12 * 60 * 60;
|
|
516
522
|
function applyZipNamePrefix(prefix, base) {
|
|
517
523
|
const trimmed = prefix?.trim();
|
|
@@ -599,7 +605,8 @@ function resolveTrackList(order, trackIdsParam, format, audioContentType, fmt, z
|
|
|
599
605
|
fileName: zipEntryPath(ctx.release?.name ?? null, file.fileName),
|
|
600
606
|
storageKey: file.storageKey,
|
|
601
607
|
contentType: audioContentType,
|
|
602
|
-
source: { kind: "track", trackId: id, format }
|
|
608
|
+
source: { kind: "track", trackId: id, format },
|
|
609
|
+
byteSize: knownByteSize(file.fileSize)
|
|
603
610
|
});
|
|
604
611
|
}
|
|
605
612
|
if (files.length === 0) {
|
|
@@ -631,7 +638,8 @@ function resolveSingleRelease(order, releaseId, format, audioContentType, fmt, z
|
|
|
631
638
|
fileName: zipEntryPath(null, file.fileName),
|
|
632
639
|
storageKey: file.storageKey,
|
|
633
640
|
contentType: audioContentType,
|
|
634
|
-
source: { kind: "track", trackId: track.id, format }
|
|
641
|
+
source: { kind: "track", trackId: track.id, format },
|
|
642
|
+
byteSize: knownByteSize(file.fileSize)
|
|
635
643
|
};
|
|
636
644
|
}).filter((f) => f !== null);
|
|
637
645
|
if (files.length > 0 && release.coverImageUrl) {
|
|
@@ -664,7 +672,8 @@ function resolveWholeOrder(order, format, audioContentType, fmt, zipNamePrefix)
|
|
|
664
672
|
fileName: zipEntryPath(release.name, file.fileName),
|
|
665
673
|
storageKey: file.storageKey,
|
|
666
674
|
contentType: audioContentType,
|
|
667
|
-
source: { kind: "track", trackId: track.id, format }
|
|
675
|
+
source: { kind: "track", trackId: track.id, format },
|
|
676
|
+
byteSize: knownByteSize(file.fileSize)
|
|
668
677
|
});
|
|
669
678
|
}
|
|
670
679
|
if (entries.length > 0 && release.coverImageUrl) {
|
|
@@ -687,7 +696,8 @@ function resolveWholeOrder(order, format, audioContentType, fmt, zipNamePrefix)
|
|
|
687
696
|
fileName: zipEntryPath(release?.name ?? null, file.fileName),
|
|
688
697
|
storageKey: file.storageKey,
|
|
689
698
|
contentType: audioContentType,
|
|
690
|
-
source: { kind: "track", trackId: track.id, format }
|
|
699
|
+
source: { kind: "track", trackId: track.id, format },
|
|
700
|
+
byteSize: knownByteSize(file.fileSize)
|
|
691
701
|
});
|
|
692
702
|
}
|
|
693
703
|
}
|
|
@@ -792,9 +802,58 @@ function sourceUrl(downloadUrl, source) {
|
|
|
792
802
|
}
|
|
793
803
|
return url.toString();
|
|
794
804
|
}
|
|
805
|
+
|
|
806
|
+
// src/zip-length.ts
|
|
807
|
+
var LOCAL_FILE_HEADER_BYTES = 30;
|
|
808
|
+
var CENTRAL_FILE_HEADER_BYTES = 46;
|
|
809
|
+
var DATA_DESCRIPTOR_BYTES = 16;
|
|
810
|
+
var ZIP64_DATA_DESCRIPTOR_BYTES = 24;
|
|
811
|
+
var ZIP64_EXTRA_FIELD_BYTES = 28;
|
|
812
|
+
var END_OF_CENTRAL_DIRECTORY_BYTES = 22;
|
|
813
|
+
var ZIP64_END_OF_CENTRAL_DIRECTORY_BYTES = 56 + 20;
|
|
814
|
+
var ZIP64_MAGIC = 4294967295;
|
|
815
|
+
var ZIP64_MAGIC_SHORT = 65535;
|
|
816
|
+
function isPredictableZipEntryName(name) {
|
|
817
|
+
if (name.length === 0) return false;
|
|
818
|
+
if (name.includes("\\")) return false;
|
|
819
|
+
if (name.includes("//")) return false;
|
|
820
|
+
if (name.startsWith("/")) return false;
|
|
821
|
+
if (name.startsWith("../")) return false;
|
|
822
|
+
if (name.endsWith("/")) return false;
|
|
823
|
+
return !/^\w+:/.test(name);
|
|
824
|
+
}
|
|
825
|
+
function predictStoredZipLength(entries) {
|
|
826
|
+
if (entries.length === 0) return null;
|
|
827
|
+
let localBytes = 0;
|
|
828
|
+
let centralBytes = 0;
|
|
829
|
+
for (const entry of entries) {
|
|
830
|
+
if (!Number.isSafeInteger(entry.size) || entry.size < 0) return null;
|
|
831
|
+
if (!isPredictableZipEntryName(entry.name)) return null;
|
|
832
|
+
const nameBytes = Buffer.byteLength(entry.name, "utf8");
|
|
833
|
+
const entryIsZip64 = entry.size > ZIP64_MAGIC;
|
|
834
|
+
const localHeaderOffset = localBytes;
|
|
835
|
+
localBytes += LOCAL_FILE_HEADER_BYTES + nameBytes + entry.size + (entryIsZip64 ? ZIP64_DATA_DESCRIPTOR_BYTES : DATA_DESCRIPTOR_BYTES);
|
|
836
|
+
centralBytes += CENTRAL_FILE_HEADER_BYTES + nameBytes + (entryIsZip64 || localHeaderOffset > ZIP64_MAGIC ? ZIP64_EXTRA_FIELD_BYTES : 0);
|
|
837
|
+
}
|
|
838
|
+
const archiveIsZip64 = entries.length > ZIP64_MAGIC_SHORT || localBytes > ZIP64_MAGIC || centralBytes > ZIP64_MAGIC;
|
|
839
|
+
const total = localBytes + centralBytes + (archiveIsZip64 ? ZIP64_END_OF_CENTRAL_DIRECTORY_BYTES : 0) + END_OF_CENTRAL_DIRECTORY_BYTES;
|
|
840
|
+
return Number.isSafeInteger(total) ? total : null;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
// src/zip-stream.ts
|
|
844
|
+
var DEFAULT_IDLE_TIMEOUT_MS = 3e4;
|
|
845
|
+
var OUTPUT_HIGH_WATER_MARK_BYTES = 256 * 1024;
|
|
846
|
+
var ARCHIVER_HIGH_WATER_MARK_BYTES = 64 * 1024;
|
|
847
|
+
var SIZE_PROBE_TIMEOUT_MS = 8e3;
|
|
848
|
+
var SIZE_PROBE_CONCURRENCY = 6;
|
|
849
|
+
var MAX_SIZE_PROBES = 32;
|
|
850
|
+
var activeStreams = /* @__PURE__ */ new Map();
|
|
795
851
|
function createDownloadZipStreamHandler(deps) {
|
|
796
852
|
const { queries, storage } = deps;
|
|
853
|
+
const idleTimeoutMs = deps.zipStreamIdleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
854
|
+
const contentLengthEnabled = deps.zipStreamContentLength !== false;
|
|
797
855
|
return async (req, ctx) => {
|
|
856
|
+
const requestStartedAt = Date.now();
|
|
798
857
|
const { token } = await ctx.params;
|
|
799
858
|
const url = new URL(req.url);
|
|
800
859
|
const resolution = await resolveZipBundle({
|
|
@@ -808,11 +867,46 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
808
867
|
if (!resolution.ok) {
|
|
809
868
|
return Response.json({ error: resolution.error }, { status: resolution.status });
|
|
810
869
|
}
|
|
811
|
-
const archive = archiver("zip", {
|
|
870
|
+
const archive = archiver("zip", {
|
|
871
|
+
store: true,
|
|
872
|
+
highWaterMark: ARCHIVER_HIGH_WATER_MARK_BYTES
|
|
873
|
+
});
|
|
812
874
|
archive.on("error", (err) => {
|
|
813
875
|
console.error("[zip-stream] archiver error:", err);
|
|
814
876
|
});
|
|
815
|
-
|
|
877
|
+
const fetches = new AbortController();
|
|
878
|
+
let abandonedFor = null;
|
|
879
|
+
let currentSource = null;
|
|
880
|
+
let finalized = false;
|
|
881
|
+
const abandon = (reason) => {
|
|
882
|
+
if (abandonedFor) return;
|
|
883
|
+
abandonedFor = reason;
|
|
884
|
+
const err = new Error(`zip stream abandoned: ${reason}`);
|
|
885
|
+
fetches.abort(err);
|
|
886
|
+
currentSource?.destroy(err);
|
|
887
|
+
if (!finalized) archive.abort();
|
|
888
|
+
archive.destroy(err);
|
|
889
|
+
};
|
|
890
|
+
const previous = activeStreams.get(token);
|
|
891
|
+
if (previous) {
|
|
892
|
+
console.warn(
|
|
893
|
+
`[zip-stream] superseding an in-flight stream for token ${token} (started ${Date.now() - previous.startedAt}ms ago)`
|
|
894
|
+
);
|
|
895
|
+
previous.supersede();
|
|
896
|
+
}
|
|
897
|
+
const registration = {
|
|
898
|
+
startedAt: requestStartedAt,
|
|
899
|
+
supersede: () => abandon("superseded")
|
|
900
|
+
};
|
|
901
|
+
activeStreams.set(token, registration);
|
|
902
|
+
const deregister = () => {
|
|
903
|
+
if (activeStreams.get(token) === registration) activeStreams.delete(token);
|
|
904
|
+
};
|
|
905
|
+
req.signal.addEventListener("abort", () => abandon("client-disconnect"));
|
|
906
|
+
const entrySizes = contentLengthEnabled ? await resolveEntrySizes(storage, resolution.files, fetches.signal) : null;
|
|
907
|
+
const contentLength = entrySizes ? predictStoredZipLength(
|
|
908
|
+
resolution.files.map((file, i) => ({ name: file.fileName, size: entrySizes[i] }))
|
|
909
|
+
) : null;
|
|
816
910
|
const failures = [];
|
|
817
911
|
const reportFailure = (failure) => {
|
|
818
912
|
failures.push(failure);
|
|
@@ -822,16 +916,61 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
822
916
|
console.warn("[zip-stream] onZipEntryFailure threw:", err);
|
|
823
917
|
}
|
|
824
918
|
};
|
|
919
|
+
let sentBytes = () => 0;
|
|
920
|
+
let summarised = false;
|
|
921
|
+
const summarise = (outcome) => {
|
|
922
|
+
if (summarised) return;
|
|
923
|
+
summarised = true;
|
|
924
|
+
deregister();
|
|
925
|
+
const sent = sentBytes();
|
|
926
|
+
const line = `[zip-stream] ${outcome} token=${token} entries=${resolution.files.length} failed=${failures.length} bytesSent=${sent} contentLength=${contentLength ?? "none"} durationMs=${Date.now() - requestStartedAt}`;
|
|
927
|
+
if (outcome !== "completed" || failures.length > 0 || contentLength !== null && sent !== contentLength) {
|
|
928
|
+
console.error(line);
|
|
929
|
+
} else {
|
|
930
|
+
console.log(line);
|
|
931
|
+
}
|
|
932
|
+
};
|
|
933
|
+
const output = createMeteredOutput(archive, {
|
|
934
|
+
idleTimeoutMs,
|
|
935
|
+
onIdle: () => {
|
|
936
|
+
console.error(
|
|
937
|
+
`[zip-stream] no progress for ${idleTimeoutMs}ms on token ${token}; treating the download as abandoned and releasing the storage stream`
|
|
938
|
+
);
|
|
939
|
+
abandon("consumer-idle");
|
|
940
|
+
summarise("abandoned:consumer-idle");
|
|
941
|
+
},
|
|
942
|
+
onCancel: () => {
|
|
943
|
+
abandon("response-cancelled");
|
|
944
|
+
summarise("abandoned:response-cancelled");
|
|
945
|
+
},
|
|
946
|
+
onClose: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "completed"),
|
|
947
|
+
onError: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "errored")
|
|
948
|
+
});
|
|
949
|
+
sentBytes = output.bytesSent;
|
|
825
950
|
(async () => {
|
|
826
|
-
for (const file of resolution.files) {
|
|
827
|
-
if (
|
|
951
|
+
for (const [index, file] of resolution.files.entries()) {
|
|
952
|
+
if (abandonedFor) return;
|
|
828
953
|
const baseName = file.fileName.split("/").pop() || file.fileName;
|
|
954
|
+
const expectedSize = contentLength === null ? void 0 : entrySizes?.[index];
|
|
955
|
+
const failFatally = (detail, bytesReceived) => {
|
|
956
|
+
console.error(
|
|
957
|
+
`[zip-stream] entry ${file.fileName} failed after Content-Length was committed (storageKey=${file.storageKey}, bytesReceived=${bytesReceived}, expected=${expectedSize}): ${detail} \u2014 aborting the response so the browser reports a failed download instead of a corrupt archive`
|
|
958
|
+
);
|
|
959
|
+
reportFailure({
|
|
960
|
+
token,
|
|
961
|
+
fileName: file.fileName,
|
|
962
|
+
storageKey: file.storageKey,
|
|
963
|
+
reason: detail,
|
|
964
|
+
bytesReceived
|
|
965
|
+
});
|
|
966
|
+
abandon("entry-failed-after-content-length");
|
|
967
|
+
};
|
|
829
968
|
try {
|
|
830
969
|
const url2 = await storage.getPresignedDownloadUrl(file.storageKey, {
|
|
831
970
|
filename: baseName,
|
|
832
971
|
contentType: file.contentType
|
|
833
972
|
});
|
|
834
|
-
const res = await fetch(url2, { signal:
|
|
973
|
+
const res = await fetch(url2, { signal: fetches.signal });
|
|
835
974
|
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
|
|
836
975
|
const body = Readable.fromWeb(
|
|
837
976
|
res.body
|
|
@@ -843,14 +982,19 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
843
982
|
cb(null, chunk);
|
|
844
983
|
}
|
|
845
984
|
});
|
|
985
|
+
currentSource = counter;
|
|
846
986
|
body.on("error", (err) => counter.destroy(err));
|
|
847
987
|
body.pipe(counter);
|
|
848
988
|
archive.append(counter, { name: file.fileName });
|
|
849
989
|
try {
|
|
850
990
|
await finished(counter);
|
|
851
991
|
} catch (err) {
|
|
852
|
-
if (
|
|
992
|
+
if (abandonedFor) return;
|
|
853
993
|
const detail = err instanceof Error ? err.message : String(err);
|
|
994
|
+
if (expectedSize !== void 0) {
|
|
995
|
+
failFatally(detail, bytesReceived);
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
854
998
|
console.error(
|
|
855
999
|
`[zip-stream] body stream error for ${file.fileName} (storageKey=${file.storageKey}, bytesReceived=${bytesReceived}): ${detail}`
|
|
856
1000
|
);
|
|
@@ -874,10 +1018,22 @@ Try downloading the track individually from your order page.
|
|
|
874
1018
|
appendErr
|
|
875
1019
|
);
|
|
876
1020
|
}
|
|
1021
|
+
continue;
|
|
1022
|
+
}
|
|
1023
|
+
if (expectedSize !== void 0 && bytesReceived !== expectedSize) {
|
|
1024
|
+
failFatally(
|
|
1025
|
+
`storage object is ${bytesReceived} bytes but the catalog says ${expectedSize}`,
|
|
1026
|
+
bytesReceived
|
|
1027
|
+
);
|
|
1028
|
+
return;
|
|
877
1029
|
}
|
|
878
1030
|
} catch (err) {
|
|
879
|
-
if (
|
|
1031
|
+
if (abandonedFor) return;
|
|
880
1032
|
const detail = err instanceof Error ? err.message : String(err);
|
|
1033
|
+
if (expectedSize !== void 0) {
|
|
1034
|
+
failFatally(detail, 0);
|
|
1035
|
+
return;
|
|
1036
|
+
}
|
|
881
1037
|
console.warn(`[zip-stream] fetch failed for ${file.fileName}:`, detail);
|
|
882
1038
|
reportFailure({
|
|
883
1039
|
token,
|
|
@@ -892,26 +1048,179 @@ Try downloading the track individually from your order page.
|
|
|
892
1048
|
`,
|
|
893
1049
|
{ name: `_FAILED_${baseName}.txt` }
|
|
894
1050
|
);
|
|
1051
|
+
} finally {
|
|
1052
|
+
currentSource = null;
|
|
895
1053
|
}
|
|
896
1054
|
}
|
|
1055
|
+
if (abandonedFor) return;
|
|
897
1056
|
if (failures.length > 0) {
|
|
898
1057
|
console.error(
|
|
899
1058
|
`[zip-stream] ${failures.length}/${resolution.files.length} entries failed for token ${token}: ${failures.map((f) => f.fileName).join(", ")}`
|
|
900
1059
|
);
|
|
901
1060
|
}
|
|
1061
|
+
finalized = true;
|
|
902
1062
|
await archive.finalize();
|
|
903
1063
|
})().catch((err) => {
|
|
904
1064
|
console.error("[zip-stream] pipeline error:", err);
|
|
905
|
-
|
|
1065
|
+
abandon("pipeline-error");
|
|
906
1066
|
});
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1067
|
+
const headers = {
|
|
1068
|
+
"Content-Type": "application/zip",
|
|
1069
|
+
"Content-Disposition": `attachment; filename="${resolution.zipName.replace(/"/g, "")}"`,
|
|
1070
|
+
"Cache-Control": "no-store"
|
|
1071
|
+
};
|
|
1072
|
+
if (contentLength !== null) headers["Content-Length"] = String(contentLength);
|
|
1073
|
+
return new Response(output.body, { headers });
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
function createMeteredOutput(source, hooks) {
|
|
1077
|
+
let bytesSent = 0;
|
|
1078
|
+
let lastHandoffAt = Date.now();
|
|
1079
|
+
let awaitingConsumer = false;
|
|
1080
|
+
let watchdog;
|
|
1081
|
+
const stopWatchdog = () => {
|
|
1082
|
+
if (watchdog !== void 0) {
|
|
1083
|
+
clearInterval(watchdog);
|
|
1084
|
+
watchdog = void 0;
|
|
1085
|
+
}
|
|
1086
|
+
};
|
|
1087
|
+
const waitForSource = () => new Promise((resolve, reject) => {
|
|
1088
|
+
const cleanup = () => {
|
|
1089
|
+
source.off("readable", onReadable);
|
|
1090
|
+
source.off("end", onEnd);
|
|
1091
|
+
source.off("close", onEnd);
|
|
1092
|
+
source.off("error", onError);
|
|
1093
|
+
};
|
|
1094
|
+
const onReadable = () => {
|
|
1095
|
+
cleanup();
|
|
1096
|
+
resolve();
|
|
1097
|
+
};
|
|
1098
|
+
const onEnd = () => {
|
|
1099
|
+
cleanup();
|
|
1100
|
+
resolve();
|
|
1101
|
+
};
|
|
1102
|
+
const onError = (err) => {
|
|
1103
|
+
cleanup();
|
|
1104
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
1105
|
+
};
|
|
1106
|
+
source.on("readable", onReadable);
|
|
1107
|
+
source.on("end", onEnd);
|
|
1108
|
+
source.on("close", onEnd);
|
|
1109
|
+
source.on("error", onError);
|
|
1110
|
+
});
|
|
1111
|
+
const body = new ReadableStream(
|
|
1112
|
+
{
|
|
1113
|
+
async pull(controller) {
|
|
1114
|
+
awaitingConsumer = false;
|
|
1115
|
+
try {
|
|
1116
|
+
for (; ; ) {
|
|
1117
|
+
const chunk = source.read();
|
|
1118
|
+
if (chunk !== null && chunk.length > 0) {
|
|
1119
|
+
bytesSent += chunk.length;
|
|
1120
|
+
lastHandoffAt = Date.now();
|
|
1121
|
+
awaitingConsumer = true;
|
|
1122
|
+
controller.enqueue(chunk);
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
if (source.errored) throw source.errored;
|
|
1126
|
+
if (source.readableEnded) {
|
|
1127
|
+
stopWatchdog();
|
|
1128
|
+
controller.close();
|
|
1129
|
+
hooks.onClose();
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
if (source.destroyed) {
|
|
1133
|
+
throw new Error("zip stream ended before the archive was finalized");
|
|
1134
|
+
}
|
|
1135
|
+
await waitForSource();
|
|
1136
|
+
}
|
|
1137
|
+
} catch (err) {
|
|
1138
|
+
stopWatchdog();
|
|
1139
|
+
hooks.onError();
|
|
1140
|
+
throw err;
|
|
1141
|
+
}
|
|
1142
|
+
},
|
|
1143
|
+
cancel() {
|
|
1144
|
+
stopWatchdog();
|
|
1145
|
+
hooks.onCancel();
|
|
1146
|
+
}
|
|
1147
|
+
},
|
|
1148
|
+
// Byte-counted so the cap is a real memory bound rather than a chunk
|
|
1149
|
+
// count: at most this much archive output is ever queued ahead of the
|
|
1150
|
+
// customer's connection.
|
|
1151
|
+
new ByteLengthQueuingStrategy({ highWaterMark: OUTPUT_HIGH_WATER_MARK_BYTES })
|
|
1152
|
+
);
|
|
1153
|
+
watchdog = setInterval(
|
|
1154
|
+
() => {
|
|
1155
|
+
if (!awaitingConsumer) return;
|
|
1156
|
+
if (Date.now() - lastHandoffAt < hooks.idleTimeoutMs) return;
|
|
1157
|
+
stopWatchdog();
|
|
1158
|
+
hooks.onIdle();
|
|
1159
|
+
},
|
|
1160
|
+
Math.max(100, Math.floor(hooks.idleTimeoutMs / 4))
|
|
1161
|
+
);
|
|
1162
|
+
watchdog.unref?.();
|
|
1163
|
+
return { body, bytesSent: () => bytesSent };
|
|
1164
|
+
}
|
|
1165
|
+
async function resolveEntrySizes(storage, files, signal) {
|
|
1166
|
+
const sizes = files.map((file) => file.byteSize);
|
|
1167
|
+
const pending = sizes.flatMap((size, i) => size === void 0 ? [i] : []);
|
|
1168
|
+
if (pending.length === 0) return sizes;
|
|
1169
|
+
if (pending.length > MAX_SIZE_PROBES) {
|
|
1170
|
+
console.warn(
|
|
1171
|
+
`[zip-stream] ${pending.length} entries have no recorded size; streaming without a Content-Length rather than probing them all`
|
|
1172
|
+
);
|
|
1173
|
+
return null;
|
|
1174
|
+
}
|
|
1175
|
+
let cursor = 0;
|
|
1176
|
+
let failed = false;
|
|
1177
|
+
const workers = Array.from(
|
|
1178
|
+
{ length: Math.min(SIZE_PROBE_CONCURRENCY, pending.length) },
|
|
1179
|
+
async () => {
|
|
1180
|
+
while (!failed) {
|
|
1181
|
+
const next = pending[cursor++];
|
|
1182
|
+
if (next === void 0) return;
|
|
1183
|
+
const size = await probeObjectSize(storage, files[next], signal);
|
|
1184
|
+
if (size === null) {
|
|
1185
|
+
failed = true;
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
sizes[next] = size;
|
|
912
1189
|
}
|
|
1190
|
+
}
|
|
1191
|
+
);
|
|
1192
|
+
await Promise.all(workers);
|
|
1193
|
+
return failed ? null : sizes;
|
|
1194
|
+
}
|
|
1195
|
+
async function probeObjectSize(storage, file, signal) {
|
|
1196
|
+
let res;
|
|
1197
|
+
try {
|
|
1198
|
+
const url = await storage.getPresignedDownloadUrl(file.storageKey, {
|
|
1199
|
+
filename: file.fileName.split("/").pop() || file.fileName,
|
|
1200
|
+
contentType: file.contentType
|
|
913
1201
|
});
|
|
914
|
-
|
|
1202
|
+
res = await fetch(url, {
|
|
1203
|
+
headers: { Range: "bytes=0-0" },
|
|
1204
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(SIZE_PROBE_TIMEOUT_MS)])
|
|
1205
|
+
});
|
|
1206
|
+
} catch (err) {
|
|
1207
|
+
console.warn(`[zip-stream] size probe failed for ${file.fileName}:`, err);
|
|
1208
|
+
return null;
|
|
1209
|
+
}
|
|
1210
|
+
try {
|
|
1211
|
+
const header = res.status === 206 ? /\/(\d+)$/.exec(res.headers.get("content-range")?.trim() ?? "")?.[1] : res.ok ? res.headers.get("content-length") : null;
|
|
1212
|
+
const size = header === null || header === void 0 ? NaN : Number(header);
|
|
1213
|
+
if (!Number.isSafeInteger(size) || size <= 0) {
|
|
1214
|
+
console.warn(
|
|
1215
|
+
`[zip-stream] size probe for ${file.fileName} returned HTTP ${res.status} without a usable length`
|
|
1216
|
+
);
|
|
1217
|
+
return null;
|
|
1218
|
+
}
|
|
1219
|
+
return size;
|
|
1220
|
+
} finally {
|
|
1221
|
+
await res.body?.cancel().catch(() => {
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
915
1224
|
}
|
|
916
1225
|
|
|
917
1226
|
// src/sw-zip-fallback.ts
|