@gigamusic/checkout 4.8.1 → 4.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/index.d.ts +67 -1
- package/dist/index.js +363 -26
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
- package/src/checkout.ts +10 -5
- package/src/fulfill.ts +6 -3
- package/src/metadata-chunks.ts +45 -0
- package/src/types.ts +27 -0
- package/src/zip-length.ts +128 -0
- package/src/zip-stream.ts +501 -18
- package/src/zip.ts +19 -0
package/README.md
CHANGED
|
@@ -66,6 +66,30 @@ export const GET = createDownloadZipStreamHandler({
|
|
|
66
66
|
Failures are also logged individually plus an `n/total entries failed` summary
|
|
67
67
|
line per request, so they're greppable without wiring anything up.
|
|
68
68
|
|
|
69
|
+
### Truncated archives
|
|
70
|
+
|
|
71
|
+
The server-side stream sends a real `Content-Length` whenever it can work out
|
|
72
|
+
every entry's exact size — from `track_files.fileSize`, falling back to a
|
|
73
|
+
ranged-GET probe for the nullable rows and for cover art. That matters because
|
|
74
|
+
a chunked zip response that dies mid-body (function timeout, OOM) is saved by
|
|
75
|
+
the browser as a *complete* file: valid audio bytes with no
|
|
76
|
+
End-of-Central-Directory record, which macOS Archive Utility reports as
|
|
77
|
+
"Error 79 – Inappropriate file type or format". With a length declared, the
|
|
78
|
+
browser flags the download as failed instead.
|
|
79
|
+
|
|
80
|
+
A declared length is a promise, so once it is out a failed entry can no longer
|
|
81
|
+
become a `_FAILED_*.txt` placeholder — the transfer is torn down and reported
|
|
82
|
+
instead. If your catalog's recorded sizes have drifted from what's in storage,
|
|
83
|
+
`zipStreamContentLength: false` restores the old chunked behaviour while you
|
|
84
|
+
repair the rows.
|
|
85
|
+
|
|
86
|
+
Downloads whose consumer stops accepting bytes are treated as abandoned after
|
|
87
|
+
`zipStreamIdleTimeoutMs` (default 30s) and released, and a second concurrent
|
|
88
|
+
stream for the same token supersedes the first — a customer retrying a stuck
|
|
89
|
+
download shouldn't leave the old one running until the function times out.
|
|
90
|
+
Every request ends with a `[zip-stream] completed …` line carrying the entry
|
|
91
|
+
count, bytes sent, bytes promised and duration.
|
|
92
|
+
|
|
69
93
|
## Bundles — discounted subsets of the catalog
|
|
70
94
|
|
|
71
95
|
`catalogPurchase` is all-or-nothing. For a curated pack ("the remix EPs", "2024
|
package/dist/index.d.ts
CHANGED
|
@@ -201,6 +201,33 @@ interface DownloadDeps {
|
|
|
201
201
|
* download that is otherwise still producing bytes.
|
|
202
202
|
*/
|
|
203
203
|
onZipEntryFailure?: (failure: ZipEntryFailure) => void;
|
|
204
|
+
/**
|
|
205
|
+
* How long `createDownloadZipStreamHandler` lets the customer's connection
|
|
206
|
+
* go without accepting a byte before it treats the download as abandoned,
|
|
207
|
+
* tears the archive down and stops paying for R2 egress. Defaults to 30s.
|
|
208
|
+
*
|
|
209
|
+
* This exists because `req.signal` cannot be relied on to fire when a
|
|
210
|
+
* customer closes a bulk download on Vercel — abandoned streams were
|
|
211
|
+
* observed running the function's full `maxDuration` and stacking up on one
|
|
212
|
+
* instance until it ran out of memory. Raise it only if you serve customers
|
|
213
|
+
* slow enough to spend that long on a single 256 KB window; lower it to
|
|
214
|
+
* reclaim capacity faster.
|
|
215
|
+
*/
|
|
216
|
+
zipStreamIdleTimeoutMs?: number;
|
|
217
|
+
/**
|
|
218
|
+
* Set `false` to make `createDownloadZipStreamHandler` stream without a
|
|
219
|
+
* `Content-Length`, as it did before the header was introduced.
|
|
220
|
+
*
|
|
221
|
+
* The header is what lets a browser notice that a zip arrived truncated
|
|
222
|
+
* instead of filing a corrupt archive as a finished download, so leaving it
|
|
223
|
+
* on is strongly preferred. The escape hatch is here for one scenario: the
|
|
224
|
+
* declared length is computed from `track_files.fileSize`, so a catalog
|
|
225
|
+
* whose recorded sizes have drifted from the objects in storage would fail
|
|
226
|
+
* every bulk download rather than silently shipping a mismatched body.
|
|
227
|
+
* Turning this off restores the old behaviour without a package downgrade
|
|
228
|
+
* while the rows are repaired. Defaults to `true`.
|
|
229
|
+
*/
|
|
230
|
+
zipStreamContentLength?: boolean;
|
|
204
231
|
/**
|
|
205
232
|
* Optional artist-name prefix prepended to zip filenames — both the SW
|
|
206
233
|
* manifest and the server-side stream output use it. Example: passing
|
|
@@ -412,7 +439,8 @@ interface RouteContext {
|
|
|
412
439
|
* whole archive. Because that turns a broken purchase into an
|
|
413
440
|
* apparently-successful download, every failure is reported through
|
|
414
441
|
* `deps.onZipEntryFailure` and summarised in a single log line at the end of
|
|
415
|
-
* the request.
|
|
442
|
+
* the request. The one exception is a failure that lands *after* a
|
|
443
|
+
* `Content-Length` has been promised — see below.
|
|
416
444
|
*
|
|
417
445
|
* Unlike the manifest path, this handler presigns each file *inside* the loop,
|
|
418
446
|
* immediately before fetching it, so signature expiry can't bite. Its own
|
|
@@ -421,6 +449,44 @@ interface RouteContext {
|
|
|
421
449
|
* (Vercel: 300s by default, up to 800s on Fluid compute). Multi-gigabyte
|
|
422
450
|
* bundles need that raised in the consuming route — `export const maxDuration`
|
|
423
451
|
* — or they'll be cut off mid-stream.
|
|
452
|
+
*
|
|
453
|
+
* ## Surviving a stream that dies mid-body
|
|
454
|
+
*
|
|
455
|
+
* A function that hits its timeout or its memory ceiling after the response
|
|
456
|
+
* headers are already out produces a *silent* truncation: the customer's
|
|
457
|
+
* browser sees a connection close on a chunked response and marks the download
|
|
458
|
+
* complete. The file is megabytes of perfectly good MP3 with no
|
|
459
|
+
* End-of-Central-Directory record, which macOS Archive Utility reports as the
|
|
460
|
+
* famously unhelpful "Error 79 – Inappropriate file type or format". Four
|
|
461
|
+
* things here exist to keep that from happening again:
|
|
462
|
+
*
|
|
463
|
+
* 1. **`Content-Length`.** Entries are stored uncompressed, so the archive's
|
|
464
|
+
* byte count is predictable from the entry sizes (`predictStoredZipLength`).
|
|
465
|
+
* Sizes come from `track_files.fileSize`, with a ranged-GET probe covering
|
|
466
|
+
* the rows where it's null and the cover art that has no row at all. If
|
|
467
|
+
* any size stays unknown, the header is omitted rather than guessed — a
|
|
468
|
+
* wrong `Content-Length` is worse than none. Once the header is out, a
|
|
469
|
+
* failed entry can no longer be papered over with a placeholder (the byte
|
|
470
|
+
* count would no longer match), so the transfer is torn down instead: the
|
|
471
|
+
* browser reports a failed download, which is the honest outcome.
|
|
472
|
+
* 2. **Demand-driven output, on small buffers.** The response body pulls from
|
|
473
|
+
* archiver only when the customer's connection has drained what came
|
|
474
|
+
* before, so a slow client throttles the R2 fetch rather than filling the
|
|
475
|
+
* heap — and archiver's 1 MiB default high-water mark, which it silently
|
|
476
|
+
* applies to four separate stream buffers, is cut to 64 KiB (see
|
|
477
|
+
* `ARCHIVER_HIGH_WATER_MARK_BYTES`).
|
|
478
|
+
* 3. **An idle watchdog.** `req.signal` is the documented disconnect signal
|
|
479
|
+
* but doesn't reliably fire on Vercel, so an abandoned download is instead
|
|
480
|
+
* detected by the consumer going quiet (`zipStreamIdleTimeoutMs`, 30s by
|
|
481
|
+
* default) and torn down within seconds rather than burning the full
|
|
482
|
+
* `maxDuration` of egress and memory.
|
|
483
|
+
* 4. **One stream per token.** A customer retrying a stuck download would
|
|
484
|
+
* otherwise stack zombie archives on a single instance; a second request
|
|
485
|
+
* supersedes the first (see `activeStreams`).
|
|
486
|
+
*
|
|
487
|
+
* Every request ends with a one-line summary — outcome, entries, bytes sent
|
|
488
|
+
* versus bytes promised, duration — so a future truncation shows up in the
|
|
489
|
+
* logs instead of only in a customer's inbox.
|
|
424
490
|
*/
|
|
425
491
|
declare function createDownloadZipStreamHandler(deps: DownloadDeps): (req: NextRequest, ctx: RouteContext) => Promise<Response>;
|
|
426
492
|
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,33 @@ import { Readable, Transform } from 'stream';
|
|
|
7
7
|
import { finished } from 'stream/promises';
|
|
8
8
|
import archiver from 'archiver';
|
|
9
9
|
|
|
10
|
+
// src/checkout.ts
|
|
11
|
+
|
|
12
|
+
// src/metadata-chunks.ts
|
|
13
|
+
var STRIPE_METADATA_VALUE_LIMIT = 500;
|
|
14
|
+
function chunkMetadataValue(key, value) {
|
|
15
|
+
const out = {};
|
|
16
|
+
for (let i = 0, offset = 0; offset < value.length || i === 0; i++) {
|
|
17
|
+
out[i === 0 ? key : `${key}_${i}`] = value.slice(
|
|
18
|
+
offset,
|
|
19
|
+
offset + STRIPE_METADATA_VALUE_LIMIT
|
|
20
|
+
);
|
|
21
|
+
offset += STRIPE_METADATA_VALUE_LIMIT;
|
|
22
|
+
}
|
|
23
|
+
return out;
|
|
24
|
+
}
|
|
25
|
+
function readChunkedMetadataValue(metadata, key) {
|
|
26
|
+
const head = metadata?.[key];
|
|
27
|
+
if (head === void 0 || head === null) return void 0;
|
|
28
|
+
let value = head;
|
|
29
|
+
for (let i = 1; ; i++) {
|
|
30
|
+
const part = metadata?.[`${key}_${i}`];
|
|
31
|
+
if (part === void 0 || part === null) break;
|
|
32
|
+
value += part;
|
|
33
|
+
}
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
|
|
10
37
|
// src/checkout.ts
|
|
11
38
|
var STRIPE_MIN_CHARGE_CENTS = 50;
|
|
12
39
|
var MANAGED_PAYMENTS_DISABLED = { enabled: false };
|
|
@@ -91,7 +118,7 @@ function createCheckoutHandler(deps) {
|
|
|
91
118
|
metadata: {
|
|
92
119
|
site,
|
|
93
120
|
catalog_purchase: "true",
|
|
94
|
-
release_ids
|
|
121
|
+
...chunkMetadataValue("release_ids", JSON.stringify(releases.map((r) => r.id))),
|
|
95
122
|
track_ids: "[]"
|
|
96
123
|
},
|
|
97
124
|
managed_payments: MANAGED_PAYMENTS_DISABLED,
|
|
@@ -226,14 +253,16 @@ function createCheckoutHandler(deps) {
|
|
|
226
253
|
const session = await stripe.checkout.sessions.create({
|
|
227
254
|
mode: "payment",
|
|
228
255
|
line_items: lineItems,
|
|
256
|
+
// Each value is spread across continuation keys past Stripe's 500-char
|
|
257
|
+
// cap; `fulfillCheckoutSession` reassembles them.
|
|
229
258
|
metadata: {
|
|
230
259
|
site,
|
|
231
|
-
release_ids
|
|
232
|
-
track_ids
|
|
233
|
-
amounts
|
|
260
|
+
...chunkMetadataValue("release_ids", JSON.stringify(resolvedReleaseIds)),
|
|
261
|
+
...chunkMetadataValue("track_ids", JSON.stringify(resolvedTrackIds)),
|
|
262
|
+
...chunkMetadataValue("amounts", JSON.stringify(chargedAmounts)),
|
|
234
263
|
// Ignored by fulfillment — bundles decompose into per-release order
|
|
235
264
|
// items — but invaluable when supporting "what did I actually buy?".
|
|
236
|
-
...bundleIds.length > 0 ?
|
|
265
|
+
...bundleIds.length > 0 ? chunkMetadataValue("bundle_ids", JSON.stringify(bundleIds)) : {}
|
|
237
266
|
},
|
|
238
267
|
managed_payments: MANAGED_PAYMENTS_DISABLED,
|
|
239
268
|
success_url: successUrl,
|
|
@@ -273,9 +302,11 @@ async function fulfillCheckoutSession(session, deps) {
|
|
|
273
302
|
if (existing) {
|
|
274
303
|
return { status: "already-recorded", order: existing };
|
|
275
304
|
}
|
|
276
|
-
const releaseIds = safeJsonIdList(
|
|
277
|
-
|
|
278
|
-
|
|
305
|
+
const releaseIds = safeJsonIdList(
|
|
306
|
+
readChunkedMetadataValue(session.metadata, "release_ids")
|
|
307
|
+
);
|
|
308
|
+
const trackIds = safeJsonIdList(readChunkedMetadataValue(session.metadata, "track_ids"));
|
|
309
|
+
const chargedAmounts = safeAmountMap(readChunkedMetadataValue(session.metadata, "amounts"));
|
|
279
310
|
if (releaseIds.length === 0 && trackIds.length === 0) {
|
|
280
311
|
return { status: "skipped", reason: "no-items" };
|
|
281
312
|
}
|
|
@@ -515,6 +546,9 @@ async function handleCoverArt(deps, token, releaseIdRaw) {
|
|
|
515
546
|
}
|
|
516
547
|
|
|
517
548
|
// src/zip.ts
|
|
549
|
+
function knownByteSize(fileSize) {
|
|
550
|
+
return typeof fileSize === "number" && Number.isSafeInteger(fileSize) && fileSize > 0 ? fileSize : void 0;
|
|
551
|
+
}
|
|
518
552
|
var ZIP_MANIFEST_EXPIRES_IN_SECONDS = 12 * 60 * 60;
|
|
519
553
|
function applyZipNamePrefix(prefix, base) {
|
|
520
554
|
const trimmed = prefix?.trim();
|
|
@@ -602,7 +636,8 @@ function resolveTrackList(order, trackIdsParam, format, audioContentType, fmt, z
|
|
|
602
636
|
fileName: zipEntryPath(ctx.release?.name ?? null, file.fileName),
|
|
603
637
|
storageKey: file.storageKey,
|
|
604
638
|
contentType: audioContentType,
|
|
605
|
-
source: { kind: "track", trackId: id, format }
|
|
639
|
+
source: { kind: "track", trackId: id, format },
|
|
640
|
+
byteSize: knownByteSize(file.fileSize)
|
|
606
641
|
});
|
|
607
642
|
}
|
|
608
643
|
if (files.length === 0) {
|
|
@@ -634,7 +669,8 @@ function resolveSingleRelease(order, releaseId, format, audioContentType, fmt, z
|
|
|
634
669
|
fileName: zipEntryPath(null, file.fileName),
|
|
635
670
|
storageKey: file.storageKey,
|
|
636
671
|
contentType: audioContentType,
|
|
637
|
-
source: { kind: "track", trackId: track.id, format }
|
|
672
|
+
source: { kind: "track", trackId: track.id, format },
|
|
673
|
+
byteSize: knownByteSize(file.fileSize)
|
|
638
674
|
};
|
|
639
675
|
}).filter((f) => f !== null);
|
|
640
676
|
if (files.length > 0 && release.coverImageUrl) {
|
|
@@ -667,7 +703,8 @@ function resolveWholeOrder(order, format, audioContentType, fmt, zipNamePrefix)
|
|
|
667
703
|
fileName: zipEntryPath(release.name, file.fileName),
|
|
668
704
|
storageKey: file.storageKey,
|
|
669
705
|
contentType: audioContentType,
|
|
670
|
-
source: { kind: "track", trackId: track.id, format }
|
|
706
|
+
source: { kind: "track", trackId: track.id, format },
|
|
707
|
+
byteSize: knownByteSize(file.fileSize)
|
|
671
708
|
});
|
|
672
709
|
}
|
|
673
710
|
if (entries.length > 0 && release.coverImageUrl) {
|
|
@@ -690,7 +727,8 @@ function resolveWholeOrder(order, format, audioContentType, fmt, zipNamePrefix)
|
|
|
690
727
|
fileName: zipEntryPath(release?.name ?? null, file.fileName),
|
|
691
728
|
storageKey: file.storageKey,
|
|
692
729
|
contentType: audioContentType,
|
|
693
|
-
source: { kind: "track", trackId: track.id, format }
|
|
730
|
+
source: { kind: "track", trackId: track.id, format },
|
|
731
|
+
byteSize: knownByteSize(file.fileSize)
|
|
694
732
|
});
|
|
695
733
|
}
|
|
696
734
|
}
|
|
@@ -795,9 +833,58 @@ function sourceUrl(downloadUrl, source) {
|
|
|
795
833
|
}
|
|
796
834
|
return url.toString();
|
|
797
835
|
}
|
|
836
|
+
|
|
837
|
+
// src/zip-length.ts
|
|
838
|
+
var LOCAL_FILE_HEADER_BYTES = 30;
|
|
839
|
+
var CENTRAL_FILE_HEADER_BYTES = 46;
|
|
840
|
+
var DATA_DESCRIPTOR_BYTES = 16;
|
|
841
|
+
var ZIP64_DATA_DESCRIPTOR_BYTES = 24;
|
|
842
|
+
var ZIP64_EXTRA_FIELD_BYTES = 28;
|
|
843
|
+
var END_OF_CENTRAL_DIRECTORY_BYTES = 22;
|
|
844
|
+
var ZIP64_END_OF_CENTRAL_DIRECTORY_BYTES = 56 + 20;
|
|
845
|
+
var ZIP64_MAGIC = 4294967295;
|
|
846
|
+
var ZIP64_MAGIC_SHORT = 65535;
|
|
847
|
+
function isPredictableZipEntryName(name) {
|
|
848
|
+
if (name.length === 0) return false;
|
|
849
|
+
if (name.includes("\\")) return false;
|
|
850
|
+
if (name.includes("//")) return false;
|
|
851
|
+
if (name.startsWith("/")) return false;
|
|
852
|
+
if (name.startsWith("../")) return false;
|
|
853
|
+
if (name.endsWith("/")) return false;
|
|
854
|
+
return !/^\w+:/.test(name);
|
|
855
|
+
}
|
|
856
|
+
function predictStoredZipLength(entries) {
|
|
857
|
+
if (entries.length === 0) return null;
|
|
858
|
+
let localBytes = 0;
|
|
859
|
+
let centralBytes = 0;
|
|
860
|
+
for (const entry of entries) {
|
|
861
|
+
if (!Number.isSafeInteger(entry.size) || entry.size < 0) return null;
|
|
862
|
+
if (!isPredictableZipEntryName(entry.name)) return null;
|
|
863
|
+
const nameBytes = Buffer.byteLength(entry.name, "utf8");
|
|
864
|
+
const entryIsZip64 = entry.size > ZIP64_MAGIC;
|
|
865
|
+
const localHeaderOffset = localBytes;
|
|
866
|
+
localBytes += LOCAL_FILE_HEADER_BYTES + nameBytes + entry.size + (entryIsZip64 ? ZIP64_DATA_DESCRIPTOR_BYTES : DATA_DESCRIPTOR_BYTES);
|
|
867
|
+
centralBytes += CENTRAL_FILE_HEADER_BYTES + nameBytes + (entryIsZip64 || localHeaderOffset > ZIP64_MAGIC ? ZIP64_EXTRA_FIELD_BYTES : 0);
|
|
868
|
+
}
|
|
869
|
+
const archiveIsZip64 = entries.length > ZIP64_MAGIC_SHORT || localBytes > ZIP64_MAGIC || centralBytes > ZIP64_MAGIC;
|
|
870
|
+
const total = localBytes + centralBytes + (archiveIsZip64 ? ZIP64_END_OF_CENTRAL_DIRECTORY_BYTES : 0) + END_OF_CENTRAL_DIRECTORY_BYTES;
|
|
871
|
+
return Number.isSafeInteger(total) ? total : null;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
// src/zip-stream.ts
|
|
875
|
+
var DEFAULT_IDLE_TIMEOUT_MS = 3e4;
|
|
876
|
+
var OUTPUT_HIGH_WATER_MARK_BYTES = 256 * 1024;
|
|
877
|
+
var ARCHIVER_HIGH_WATER_MARK_BYTES = 64 * 1024;
|
|
878
|
+
var SIZE_PROBE_TIMEOUT_MS = 8e3;
|
|
879
|
+
var SIZE_PROBE_CONCURRENCY = 6;
|
|
880
|
+
var MAX_SIZE_PROBES = 32;
|
|
881
|
+
var activeStreams = /* @__PURE__ */ new Map();
|
|
798
882
|
function createDownloadZipStreamHandler(deps) {
|
|
799
883
|
const { queries, storage } = deps;
|
|
884
|
+
const idleTimeoutMs = deps.zipStreamIdleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
885
|
+
const contentLengthEnabled = deps.zipStreamContentLength !== false;
|
|
800
886
|
return async (req, ctx) => {
|
|
887
|
+
const requestStartedAt = Date.now();
|
|
801
888
|
const { token } = await ctx.params;
|
|
802
889
|
const url = new URL(req.url);
|
|
803
890
|
const resolution = await resolveZipBundle({
|
|
@@ -811,11 +898,46 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
811
898
|
if (!resolution.ok) {
|
|
812
899
|
return Response.json({ error: resolution.error }, { status: resolution.status });
|
|
813
900
|
}
|
|
814
|
-
const archive = archiver("zip", {
|
|
901
|
+
const archive = archiver("zip", {
|
|
902
|
+
store: true,
|
|
903
|
+
highWaterMark: ARCHIVER_HIGH_WATER_MARK_BYTES
|
|
904
|
+
});
|
|
815
905
|
archive.on("error", (err) => {
|
|
816
906
|
console.error("[zip-stream] archiver error:", err);
|
|
817
907
|
});
|
|
818
|
-
|
|
908
|
+
const fetches = new AbortController();
|
|
909
|
+
let abandonedFor = null;
|
|
910
|
+
let currentSource = null;
|
|
911
|
+
let finalized = false;
|
|
912
|
+
const abandon = (reason) => {
|
|
913
|
+
if (abandonedFor) return;
|
|
914
|
+
abandonedFor = reason;
|
|
915
|
+
const err = new Error(`zip stream abandoned: ${reason}`);
|
|
916
|
+
fetches.abort(err);
|
|
917
|
+
currentSource?.destroy(err);
|
|
918
|
+
if (!finalized) archive.abort();
|
|
919
|
+
archive.destroy(err);
|
|
920
|
+
};
|
|
921
|
+
const previous = activeStreams.get(token);
|
|
922
|
+
if (previous) {
|
|
923
|
+
console.warn(
|
|
924
|
+
`[zip-stream] superseding an in-flight stream for token ${token} (started ${Date.now() - previous.startedAt}ms ago)`
|
|
925
|
+
);
|
|
926
|
+
previous.supersede();
|
|
927
|
+
}
|
|
928
|
+
const registration = {
|
|
929
|
+
startedAt: requestStartedAt,
|
|
930
|
+
supersede: () => abandon("superseded")
|
|
931
|
+
};
|
|
932
|
+
activeStreams.set(token, registration);
|
|
933
|
+
const deregister = () => {
|
|
934
|
+
if (activeStreams.get(token) === registration) activeStreams.delete(token);
|
|
935
|
+
};
|
|
936
|
+
req.signal.addEventListener("abort", () => abandon("client-disconnect"));
|
|
937
|
+
const entrySizes = contentLengthEnabled ? await resolveEntrySizes(storage, resolution.files, fetches.signal) : null;
|
|
938
|
+
const contentLength = entrySizes ? predictStoredZipLength(
|
|
939
|
+
resolution.files.map((file, i) => ({ name: file.fileName, size: entrySizes[i] }))
|
|
940
|
+
) : null;
|
|
819
941
|
const failures = [];
|
|
820
942
|
const reportFailure = (failure) => {
|
|
821
943
|
failures.push(failure);
|
|
@@ -825,16 +947,61 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
825
947
|
console.warn("[zip-stream] onZipEntryFailure threw:", err);
|
|
826
948
|
}
|
|
827
949
|
};
|
|
950
|
+
let sentBytes = () => 0;
|
|
951
|
+
let summarised = false;
|
|
952
|
+
const summarise = (outcome) => {
|
|
953
|
+
if (summarised) return;
|
|
954
|
+
summarised = true;
|
|
955
|
+
deregister();
|
|
956
|
+
const sent = sentBytes();
|
|
957
|
+
const line = `[zip-stream] ${outcome} token=${token} entries=${resolution.files.length} failed=${failures.length} bytesSent=${sent} contentLength=${contentLength ?? "none"} durationMs=${Date.now() - requestStartedAt}`;
|
|
958
|
+
if (outcome !== "completed" || failures.length > 0 || contentLength !== null && sent !== contentLength) {
|
|
959
|
+
console.error(line);
|
|
960
|
+
} else {
|
|
961
|
+
console.log(line);
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
const output = createMeteredOutput(archive, {
|
|
965
|
+
idleTimeoutMs,
|
|
966
|
+
onIdle: () => {
|
|
967
|
+
console.error(
|
|
968
|
+
`[zip-stream] no progress for ${idleTimeoutMs}ms on token ${token}; treating the download as abandoned and releasing the storage stream`
|
|
969
|
+
);
|
|
970
|
+
abandon("consumer-idle");
|
|
971
|
+
summarise("abandoned:consumer-idle");
|
|
972
|
+
},
|
|
973
|
+
onCancel: () => {
|
|
974
|
+
abandon("response-cancelled");
|
|
975
|
+
summarise("abandoned:response-cancelled");
|
|
976
|
+
},
|
|
977
|
+
onClose: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "completed"),
|
|
978
|
+
onError: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "errored")
|
|
979
|
+
});
|
|
980
|
+
sentBytes = output.bytesSent;
|
|
828
981
|
(async () => {
|
|
829
|
-
for (const file of resolution.files) {
|
|
830
|
-
if (
|
|
982
|
+
for (const [index, file] of resolution.files.entries()) {
|
|
983
|
+
if (abandonedFor) return;
|
|
831
984
|
const baseName = file.fileName.split("/").pop() || file.fileName;
|
|
985
|
+
const expectedSize = contentLength === null ? void 0 : entrySizes?.[index];
|
|
986
|
+
const failFatally = (detail, bytesReceived) => {
|
|
987
|
+
console.error(
|
|
988
|
+
`[zip-stream] entry ${file.fileName} failed after Content-Length was committed (storageKey=${file.storageKey}, bytesReceived=${bytesReceived}, expected=${expectedSize}): ${detail} \u2014 aborting the response so the browser reports a failed download instead of a corrupt archive`
|
|
989
|
+
);
|
|
990
|
+
reportFailure({
|
|
991
|
+
token,
|
|
992
|
+
fileName: file.fileName,
|
|
993
|
+
storageKey: file.storageKey,
|
|
994
|
+
reason: detail,
|
|
995
|
+
bytesReceived
|
|
996
|
+
});
|
|
997
|
+
abandon("entry-failed-after-content-length");
|
|
998
|
+
};
|
|
832
999
|
try {
|
|
833
1000
|
const url2 = await storage.getPresignedDownloadUrl(file.storageKey, {
|
|
834
1001
|
filename: baseName,
|
|
835
1002
|
contentType: file.contentType
|
|
836
1003
|
});
|
|
837
|
-
const res = await fetch(url2, { signal:
|
|
1004
|
+
const res = await fetch(url2, { signal: fetches.signal });
|
|
838
1005
|
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
|
|
839
1006
|
const body = Readable.fromWeb(
|
|
840
1007
|
res.body
|
|
@@ -846,14 +1013,19 @@ function createDownloadZipStreamHandler(deps) {
|
|
|
846
1013
|
cb(null, chunk);
|
|
847
1014
|
}
|
|
848
1015
|
});
|
|
1016
|
+
currentSource = counter;
|
|
849
1017
|
body.on("error", (err) => counter.destroy(err));
|
|
850
1018
|
body.pipe(counter);
|
|
851
1019
|
archive.append(counter, { name: file.fileName });
|
|
852
1020
|
try {
|
|
853
1021
|
await finished(counter);
|
|
854
1022
|
} catch (err) {
|
|
855
|
-
if (
|
|
1023
|
+
if (abandonedFor) return;
|
|
856
1024
|
const detail = err instanceof Error ? err.message : String(err);
|
|
1025
|
+
if (expectedSize !== void 0) {
|
|
1026
|
+
failFatally(detail, bytesReceived);
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
857
1029
|
console.error(
|
|
858
1030
|
`[zip-stream] body stream error for ${file.fileName} (storageKey=${file.storageKey}, bytesReceived=${bytesReceived}): ${detail}`
|
|
859
1031
|
);
|
|
@@ -877,10 +1049,22 @@ Try downloading the track individually from your order page.
|
|
|
877
1049
|
appendErr
|
|
878
1050
|
);
|
|
879
1051
|
}
|
|
1052
|
+
continue;
|
|
1053
|
+
}
|
|
1054
|
+
if (expectedSize !== void 0 && bytesReceived !== expectedSize) {
|
|
1055
|
+
failFatally(
|
|
1056
|
+
`storage object is ${bytesReceived} bytes but the catalog says ${expectedSize}`,
|
|
1057
|
+
bytesReceived
|
|
1058
|
+
);
|
|
1059
|
+
return;
|
|
880
1060
|
}
|
|
881
1061
|
} catch (err) {
|
|
882
|
-
if (
|
|
1062
|
+
if (abandonedFor) return;
|
|
883
1063
|
const detail = err instanceof Error ? err.message : String(err);
|
|
1064
|
+
if (expectedSize !== void 0) {
|
|
1065
|
+
failFatally(detail, 0);
|
|
1066
|
+
return;
|
|
1067
|
+
}
|
|
884
1068
|
console.warn(`[zip-stream] fetch failed for ${file.fileName}:`, detail);
|
|
885
1069
|
reportFailure({
|
|
886
1070
|
token,
|
|
@@ -895,26 +1079,179 @@ Try downloading the track individually from your order page.
|
|
|
895
1079
|
`,
|
|
896
1080
|
{ name: `_FAILED_${baseName}.txt` }
|
|
897
1081
|
);
|
|
1082
|
+
} finally {
|
|
1083
|
+
currentSource = null;
|
|
898
1084
|
}
|
|
899
1085
|
}
|
|
1086
|
+
if (abandonedFor) return;
|
|
900
1087
|
if (failures.length > 0) {
|
|
901
1088
|
console.error(
|
|
902
1089
|
`[zip-stream] ${failures.length}/${resolution.files.length} entries failed for token ${token}: ${failures.map((f) => f.fileName).join(", ")}`
|
|
903
1090
|
);
|
|
904
1091
|
}
|
|
1092
|
+
finalized = true;
|
|
905
1093
|
await archive.finalize();
|
|
906
1094
|
})().catch((err) => {
|
|
907
1095
|
console.error("[zip-stream] pipeline error:", err);
|
|
908
|
-
|
|
1096
|
+
abandon("pipeline-error");
|
|
909
1097
|
});
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
1098
|
+
const headers = {
|
|
1099
|
+
"Content-Type": "application/zip",
|
|
1100
|
+
"Content-Disposition": `attachment; filename="${resolution.zipName.replace(/"/g, "")}"`,
|
|
1101
|
+
"Cache-Control": "no-store"
|
|
1102
|
+
};
|
|
1103
|
+
if (contentLength !== null) headers["Content-Length"] = String(contentLength);
|
|
1104
|
+
return new Response(output.body, { headers });
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
function createMeteredOutput(source, hooks) {
|
|
1108
|
+
let bytesSent = 0;
|
|
1109
|
+
let lastHandoffAt = Date.now();
|
|
1110
|
+
let awaitingConsumer = false;
|
|
1111
|
+
let watchdog;
|
|
1112
|
+
const stopWatchdog = () => {
|
|
1113
|
+
if (watchdog !== void 0) {
|
|
1114
|
+
clearInterval(watchdog);
|
|
1115
|
+
watchdog = void 0;
|
|
1116
|
+
}
|
|
1117
|
+
};
|
|
1118
|
+
const waitForSource = () => new Promise((resolve, reject) => {
|
|
1119
|
+
const cleanup = () => {
|
|
1120
|
+
source.off("readable", onReadable);
|
|
1121
|
+
source.off("end", onEnd);
|
|
1122
|
+
source.off("close", onEnd);
|
|
1123
|
+
source.off("error", onError);
|
|
1124
|
+
};
|
|
1125
|
+
const onReadable = () => {
|
|
1126
|
+
cleanup();
|
|
1127
|
+
resolve();
|
|
1128
|
+
};
|
|
1129
|
+
const onEnd = () => {
|
|
1130
|
+
cleanup();
|
|
1131
|
+
resolve();
|
|
1132
|
+
};
|
|
1133
|
+
const onError = (err) => {
|
|
1134
|
+
cleanup();
|
|
1135
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
1136
|
+
};
|
|
1137
|
+
source.on("readable", onReadable);
|
|
1138
|
+
source.on("end", onEnd);
|
|
1139
|
+
source.on("close", onEnd);
|
|
1140
|
+
source.on("error", onError);
|
|
1141
|
+
});
|
|
1142
|
+
const body = new ReadableStream(
|
|
1143
|
+
{
|
|
1144
|
+
async pull(controller) {
|
|
1145
|
+
awaitingConsumer = false;
|
|
1146
|
+
try {
|
|
1147
|
+
for (; ; ) {
|
|
1148
|
+
const chunk = source.read();
|
|
1149
|
+
if (chunk !== null && chunk.length > 0) {
|
|
1150
|
+
bytesSent += chunk.length;
|
|
1151
|
+
lastHandoffAt = Date.now();
|
|
1152
|
+
awaitingConsumer = true;
|
|
1153
|
+
controller.enqueue(chunk);
|
|
1154
|
+
return;
|
|
1155
|
+
}
|
|
1156
|
+
if (source.errored) throw source.errored;
|
|
1157
|
+
if (source.readableEnded) {
|
|
1158
|
+
stopWatchdog();
|
|
1159
|
+
controller.close();
|
|
1160
|
+
hooks.onClose();
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
if (source.destroyed) {
|
|
1164
|
+
throw new Error("zip stream ended before the archive was finalized");
|
|
1165
|
+
}
|
|
1166
|
+
await waitForSource();
|
|
1167
|
+
}
|
|
1168
|
+
} catch (err) {
|
|
1169
|
+
stopWatchdog();
|
|
1170
|
+
hooks.onError();
|
|
1171
|
+
throw err;
|
|
1172
|
+
}
|
|
1173
|
+
},
|
|
1174
|
+
cancel() {
|
|
1175
|
+
stopWatchdog();
|
|
1176
|
+
hooks.onCancel();
|
|
1177
|
+
}
|
|
1178
|
+
},
|
|
1179
|
+
// Byte-counted so the cap is a real memory bound rather than a chunk
|
|
1180
|
+
// count: at most this much archive output is ever queued ahead of the
|
|
1181
|
+
// customer's connection.
|
|
1182
|
+
new ByteLengthQueuingStrategy({ highWaterMark: OUTPUT_HIGH_WATER_MARK_BYTES })
|
|
1183
|
+
);
|
|
1184
|
+
watchdog = setInterval(
|
|
1185
|
+
() => {
|
|
1186
|
+
if (!awaitingConsumer) return;
|
|
1187
|
+
if (Date.now() - lastHandoffAt < hooks.idleTimeoutMs) return;
|
|
1188
|
+
stopWatchdog();
|
|
1189
|
+
hooks.onIdle();
|
|
1190
|
+
},
|
|
1191
|
+
Math.max(100, Math.floor(hooks.idleTimeoutMs / 4))
|
|
1192
|
+
);
|
|
1193
|
+
watchdog.unref?.();
|
|
1194
|
+
return { body, bytesSent: () => bytesSent };
|
|
1195
|
+
}
|
|
1196
|
+
async function resolveEntrySizes(storage, files, signal) {
|
|
1197
|
+
const sizes = files.map((file) => file.byteSize);
|
|
1198
|
+
const pending = sizes.flatMap((size, i) => size === void 0 ? [i] : []);
|
|
1199
|
+
if (pending.length === 0) return sizes;
|
|
1200
|
+
if (pending.length > MAX_SIZE_PROBES) {
|
|
1201
|
+
console.warn(
|
|
1202
|
+
`[zip-stream] ${pending.length} entries have no recorded size; streaming without a Content-Length rather than probing them all`
|
|
1203
|
+
);
|
|
1204
|
+
return null;
|
|
1205
|
+
}
|
|
1206
|
+
let cursor = 0;
|
|
1207
|
+
let failed = false;
|
|
1208
|
+
const workers = Array.from(
|
|
1209
|
+
{ length: Math.min(SIZE_PROBE_CONCURRENCY, pending.length) },
|
|
1210
|
+
async () => {
|
|
1211
|
+
while (!failed) {
|
|
1212
|
+
const next = pending[cursor++];
|
|
1213
|
+
if (next === void 0) return;
|
|
1214
|
+
const size = await probeObjectSize(storage, files[next], signal);
|
|
1215
|
+
if (size === null) {
|
|
1216
|
+
failed = true;
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1219
|
+
sizes[next] = size;
|
|
915
1220
|
}
|
|
1221
|
+
}
|
|
1222
|
+
);
|
|
1223
|
+
await Promise.all(workers);
|
|
1224
|
+
return failed ? null : sizes;
|
|
1225
|
+
}
|
|
1226
|
+
async function probeObjectSize(storage, file, signal) {
|
|
1227
|
+
let res;
|
|
1228
|
+
try {
|
|
1229
|
+
const url = await storage.getPresignedDownloadUrl(file.storageKey, {
|
|
1230
|
+
filename: file.fileName.split("/").pop() || file.fileName,
|
|
1231
|
+
contentType: file.contentType
|
|
916
1232
|
});
|
|
917
|
-
|
|
1233
|
+
res = await fetch(url, {
|
|
1234
|
+
headers: { Range: "bytes=0-0" },
|
|
1235
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(SIZE_PROBE_TIMEOUT_MS)])
|
|
1236
|
+
});
|
|
1237
|
+
} catch (err) {
|
|
1238
|
+
console.warn(`[zip-stream] size probe failed for ${file.fileName}:`, err);
|
|
1239
|
+
return null;
|
|
1240
|
+
}
|
|
1241
|
+
try {
|
|
1242
|
+
const header = res.status === 206 ? /\/(\d+)$/.exec(res.headers.get("content-range")?.trim() ?? "")?.[1] : res.ok ? res.headers.get("content-length") : null;
|
|
1243
|
+
const size = header === null || header === void 0 ? NaN : Number(header);
|
|
1244
|
+
if (!Number.isSafeInteger(size) || size <= 0) {
|
|
1245
|
+
console.warn(
|
|
1246
|
+
`[zip-stream] size probe for ${file.fileName} returned HTTP ${res.status} without a usable length`
|
|
1247
|
+
);
|
|
1248
|
+
return null;
|
|
1249
|
+
}
|
|
1250
|
+
return size;
|
|
1251
|
+
} finally {
|
|
1252
|
+
await res.body?.cancel().catch(() => {
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
918
1255
|
}
|
|
919
1256
|
|
|
920
1257
|
// src/sw-zip-fallback.ts
|