@gigamusic/checkout 4.8.0 → 4.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/zip-stream.ts CHANGED
@@ -4,12 +4,69 @@ import { finished } from "node:stream/promises";
4
4
  import type { ReadableStream as NodeReadableStream } from "node:stream/web";
5
5
  import archiver from "archiver";
6
6
  import type { DownloadDeps, ZipEntryFailure } from "./types.js";
7
- import { resolveZipBundle } from "./zip.js";
7
+ import { resolveZipBundle, type ZipFile } from "./zip.js";
8
+ import { predictStoredZipLength } from "./zip-length.js";
8
9
 
9
10
  interface RouteContext {
10
11
  params: Promise<{ token: string }>;
11
12
  }
12
13
 
14
+ /** How long the customer's browser may go without accepting a byte before we call the download abandoned. */
15
+ const DEFAULT_IDLE_TIMEOUT_MS = 30_000;
16
+ /**
17
+ * Bytes the response may sit on ahead of the consumer. The output is
18
+ * demand-driven, so this is the whole of our buffering: nothing is pulled out
19
+ * of archiver — and therefore nothing is pulled out of R2 — until the
20
+ * customer's connection has taken what we already produced.
21
+ */
22
+ const OUTPUT_HIGH_WATER_MARK_BYTES = 256 * 1024;
23
+ /**
24
+ * High-water mark handed to archiver, overriding its 1 MiB default.
25
+ *
26
+ * That default is not one buffer: archiver passes its options straight down to
27
+ * `zip-stream`, so the `Archiver` transform and the `ZipArchiveOutputStream`
28
+ * beneath it each get 1 MiB on *both* their readable and writable sides — a
29
+ * measured ~4.6 MB resident per in-flight archive before the response has sent
30
+ * a byte. That is invisible on a healthy download and lethal on a stalled one:
31
+ * eight abandoned streams sharing a Fluid instance (which is exactly what the
32
+ * retry storm in production produced) hold ~37 MB of zip buffers on their own,
33
+ * on top of an open R2 socket and undici's own buffers each. 64 KiB per stage
34
+ * costs a few more event-loop turns and nothing measurable in throughput,
35
+ * since the network is the bottleneck either way.
36
+ */
37
+ const ARCHIVER_HIGH_WATER_MARK_BYTES = 64 * 1024;
38
+ /** Per-object timeout for the pre-flight size probe; a slow probe must not delay the archive. */
39
+ const SIZE_PROBE_TIMEOUT_MS = 8_000;
40
+ /** Probes run in parallel, but not unboundedly — this is TTFB the customer waits through. */
41
+ const SIZE_PROBE_CONCURRENCY = 6;
42
+ /**
43
+ * Above this many unsized entries, skip the probe entirely and stream without
44
+ * a `Content-Length`. A bundle that needs dozens of probes is one whose
45
+ * catalog rows are broken; the pre-flight cost isn't worth paying on every
46
+ * retry.
47
+ */
48
+ const MAX_SIZE_PROBES = 32;
49
+
50
+ interface ActiveStream {
51
+ startedAt: number;
52
+ supersede: () => void;
53
+ }
54
+
55
+ /**
56
+ * Zip streams currently in flight, keyed by download token.
57
+ *
58
+ * This is per-instance state and therefore *not* a global lock: two requests
59
+ * that land on different Vercel instances will not see each other. That's
60
+ * accepted deliberately. The failure this guards against — a customer whose
61
+ * download appears stuck clicking again, and again — reuses one instance,
62
+ * because Fluid compute packs concurrent requests for the same function onto
63
+ * the same instance. Catching the common case is what keeps eight zombie
64
+ * archives from stacking up and exhausting that instance's memory; a
65
+ * cross-instance guard would need shared state (Redis, or the DB) and is not
66
+ * worth the dependency for the remaining tail.
67
+ */
68
+ const activeStreams = new Map<string, ActiveStream>();
69
+
13
70
  /**
14
71
  * Server-side bulk-download fallback. Mirrors `createDownloadZipHandler`'s
15
72
  * auth + file-list logic via the shared `resolveZipBundle()`, but streams
@@ -31,7 +88,8 @@ interface RouteContext {
31
88
  * whole archive. Because that turns a broken purchase into an
32
89
  * apparently-successful download, every failure is reported through
33
90
  * `deps.onZipEntryFailure` and summarised in a single log line at the end of
34
- * the request.
91
+ * the request. The one exception is a failure that lands *after* a
92
+ * `Content-Length` has been promised — see below.
35
93
  *
36
94
  * Unlike the manifest path, this handler presigns each file *inside* the loop,
37
95
  * immediately before fetching it, so signature expiry can't bite. Its own
@@ -40,13 +98,54 @@ interface RouteContext {
40
98
  * (Vercel: 300s by default, up to 800s on Fluid compute). Multi-gigabyte
41
99
  * bundles need that raised in the consuming route — `export const maxDuration`
42
100
  * — or they'll be cut off mid-stream.
101
+ *
102
+ * ## Surviving a stream that dies mid-body
103
+ *
104
+ * A function that hits its timeout or its memory ceiling after the response
105
+ * headers are already out produces a *silent* truncation: the customer's
106
+ * browser sees a connection close on a chunked response and marks the download
107
+ * complete. The file is megabytes of perfectly good MP3 with no
108
+ * End-of-Central-Directory record, which macOS Archive Utility reports as the
109
+ * famously unhelpful "Error 79 – Inappropriate file type or format". Four
110
+ * things here exist to keep that from happening again:
111
+ *
112
+ * 1. **`Content-Length`.** Entries are stored uncompressed, so the archive's
113
+ * byte count is predictable from the entry sizes (`predictStoredZipLength`).
114
+ * Sizes come from `track_files.fileSize`, with a ranged-GET probe covering
115
+ * the rows where it's null and the cover art that has no row at all. If
116
+ * any size stays unknown, the header is omitted rather than guessed — a
117
+ * wrong `Content-Length` is worse than none. Once the header is out, a
118
+ * failed entry can no longer be papered over with a placeholder (the byte
119
+ * count would no longer match), so the transfer is torn down instead: the
120
+ * browser reports a failed download, which is the honest outcome.
121
+ * 2. **Demand-driven output, on small buffers.** The response body pulls from
122
+ * archiver only when the customer's connection has drained what came
123
+ * before, so a slow client throttles the R2 fetch rather than filling the
124
+ * heap — and archiver's 1 MiB default high-water mark, which it silently
125
+ * applies to four separate stream buffers, is cut to 64 KiB (see
126
+ * `ARCHIVER_HIGH_WATER_MARK_BYTES`).
127
+ * 3. **An idle watchdog.** `req.signal` is the documented disconnect signal
128
+ * but doesn't reliably fire on Vercel, so an abandoned download is instead
129
+ * detected by the consumer going quiet (`zipStreamIdleTimeoutMs`, 30s by
130
+ * default) and torn down within seconds rather than burning the full
131
+ * `maxDuration` of egress and memory.
132
+ * 4. **One stream per token.** A customer retrying a stuck download would
133
+ * otherwise stack zombie archives on a single instance; a second request
134
+ * supersedes the first (see `activeStreams`).
135
+ *
136
+ * Every request ends with a one-line summary — outcome, entries, bytes sent
137
+ * versus bytes promised, duration — so a future truncation shows up in the
138
+ * logs instead of only in a customer's inbox.
43
139
  */
44
140
  export function createDownloadZipStreamHandler(
45
141
  deps: DownloadDeps,
46
142
  ): (req: NextRequest, ctx: RouteContext) => Promise<Response> {
47
143
  const { queries, storage } = deps;
144
+ const idleTimeoutMs = deps.zipStreamIdleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
145
+ const contentLengthEnabled = deps.zipStreamContentLength !== false;
48
146
 
49
147
  return async (req: NextRequest, ctx: RouteContext): Promise<Response> => {
148
+ const requestStartedAt = Date.now();
50
149
  const { token } = await ctx.params;
51
150
  const url = new URL(req.url);
52
151
 
@@ -64,15 +163,68 @@ export function createDownloadZipStreamHandler(
64
163
 
65
164
  // `store: true` = no DEFLATE compression. MP3/WAV barely compress, and
66
165
  // the SW path serves uncompressed entries — staying consistent means
67
- // identical byte counts across the two routes.
68
- const archive = archiver("zip", { store: true });
166
+ // identical byte counts across the two routes. It's also what makes the
167
+ // archive's final size predictable; see `predictStoredZipLength`.
168
+ const archive = archiver("zip", {
169
+ store: true,
170
+ highWaterMark: ARCHIVER_HIGH_WATER_MARK_BYTES,
171
+ });
69
172
 
70
173
  archive.on("error", (err) => {
71
174
  console.error("[zip-stream] archiver error:", err);
72
175
  });
73
176
 
74
- // Tear the pipeline down if the client disconnects mid-download.
75
- req.signal.addEventListener("abort", () => archive.abort());
177
+ // Every storage fetch runs on our own signal rather than `req.signal`
178
+ // directly, so a disconnect, a supersede and the idle watchdog all have
179
+ // the same power to stop pulling bytes out of R2. `archive.abort()` alone
180
+ // doesn't: archiver kills its queue but never touches the source stream
181
+ // it's already draining.
182
+ const fetches = new AbortController();
183
+ let abandonedFor: string | null = null;
184
+ let currentSource: Transform | null = null;
185
+ let finalized = false;
186
+
187
+ /** Stop the whole pipeline — storage fetches, archiver, and the entry loop — recording why. */
188
+ const abandon = (reason: string) => {
189
+ if (abandonedFor) return;
190
+ abandonedFor = reason;
191
+ const err = new Error(`zip stream abandoned: ${reason}`);
192
+ fetches.abort(err);
193
+ currentSource?.destroy(err);
194
+ // `abort()` stops archiver accepting work; `destroy()` is what actually
195
+ // tears down the readable side, so a consumer still waiting on `pull`
196
+ // gets an error rather than a stream that never ends.
197
+ if (!finalized) archive.abort();
198
+ archive.destroy(err);
199
+ };
200
+
201
+ const previous = activeStreams.get(token);
202
+ if (previous) {
203
+ console.warn(
204
+ `[zip-stream] superseding an in-flight stream for token ${token} ` +
205
+ `(started ${Date.now() - previous.startedAt}ms ago)`,
206
+ );
207
+ previous.supersede();
208
+ }
209
+ const registration: ActiveStream = {
210
+ startedAt: requestStartedAt,
211
+ supersede: () => abandon("superseded"),
212
+ };
213
+ activeStreams.set(token, registration);
214
+ const deregister = () => {
215
+ if (activeStreams.get(token) === registration) activeStreams.delete(token);
216
+ };
217
+
218
+ req.signal.addEventListener("abort", () => abandon("client-disconnect"));
219
+
220
+ const entrySizes = contentLengthEnabled
221
+ ? await resolveEntrySizes(storage, resolution.files, fetches.signal)
222
+ : null;
223
+ const contentLength = entrySizes
224
+ ? predictStoredZipLength(
225
+ resolution.files.map((file, i) => ({ name: file.fileName, size: entrySizes[i]! })),
226
+ )
227
+ : null;
76
228
 
77
229
  const failures: ZipEntryFailure[] = [];
78
230
  /** Record one unusable entry and hand it to the consumer's reporter, never letting that reporter break the stream. */
@@ -85,6 +237,48 @@ export function createDownloadZipStreamHandler(
85
237
  }
86
238
  };
87
239
 
240
+ let sentBytes = () => 0;
241
+ let summarised = false;
242
+ /** One greppable line per request, so a truncated archive is visible in the logs rather than only in a support email. */
243
+ const summarise = (outcome: string) => {
244
+ if (summarised) return;
245
+ summarised = true;
246
+ deregister();
247
+ const sent = sentBytes();
248
+ const line =
249
+ `[zip-stream] ${outcome} token=${token} entries=${resolution.files.length} ` +
250
+ `failed=${failures.length} bytesSent=${sent} ` +
251
+ `contentLength=${contentLength ?? "none"} durationMs=${Date.now() - requestStartedAt}`;
252
+ if (
253
+ outcome !== "completed" ||
254
+ failures.length > 0 ||
255
+ (contentLength !== null && sent !== contentLength)
256
+ ) {
257
+ console.error(line);
258
+ } else {
259
+ console.log(line);
260
+ }
261
+ };
262
+
263
+ const output = createMeteredOutput(archive, {
264
+ idleTimeoutMs,
265
+ onIdle: () => {
266
+ console.error(
267
+ `[zip-stream] no progress for ${idleTimeoutMs}ms on token ${token}; ` +
268
+ `treating the download as abandoned and releasing the storage stream`,
269
+ );
270
+ abandon("consumer-idle");
271
+ summarise("abandoned:consumer-idle");
272
+ },
273
+ onCancel: () => {
274
+ abandon("response-cancelled");
275
+ summarise("abandoned:response-cancelled");
276
+ },
277
+ onClose: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "completed"),
278
+ onError: () => summarise(abandonedFor ? `abandoned:${abandonedFor}` : "errored"),
279
+ });
280
+ sentBytes = output.bytesSent;
281
+
88
282
  // Serialise storage fetches: only open file N's socket after archiver
89
283
  // has fully consumed file N-1. The naive parallel approach (open every
90
284
  // socket up front, let archiver drain them in order) leaves the later
@@ -94,15 +288,44 @@ export function createDownloadZipStreamHandler(
94
288
  // to the per-file streaming time — and archiver is the serial
95
289
  // bottleneck anyway, so end-to-end throughput is unchanged.
96
290
  (async () => {
97
- for (const file of resolution.files) {
98
- if (req.signal.aborted) return;
291
+ for (const [index, file] of resolution.files.entries()) {
292
+ if (abandonedFor) return;
99
293
  const baseName = file.fileName.split("/").pop() || file.fileName;
294
+ // Only a length that actually went out on the wire binds us: sizes
295
+ // resolved for an archive whose layout turned out unpredictable are
296
+ // just unused numbers, and that request keeps the placeholder policy.
297
+ const expectedSize = contentLength === null ? undefined : entrySizes?.[index];
298
+ /**
299
+ * Once a `Content-Length` is out on the wire the archive's byte count
300
+ * is a promise we can't renegotiate, so a placeholder (which has a
301
+ * different size) would itself produce the truncated download this
302
+ * handler exists to prevent. Fail the transfer instead: the browser
303
+ * flags it, the customer retries, and the operator gets a log line and
304
+ * an `onZipEntryFailure` call either way.
305
+ */
306
+ const failFatally = (detail: string, bytesReceived: number) => {
307
+ console.error(
308
+ `[zip-stream] entry ${file.fileName} failed after Content-Length was committed ` +
309
+ `(storageKey=${file.storageKey}, bytesReceived=${bytesReceived}, ` +
310
+ `expected=${expectedSize}): ${detail} — aborting the response so the ` +
311
+ `browser reports a failed download instead of a corrupt archive`,
312
+ );
313
+ reportFailure({
314
+ token,
315
+ fileName: file.fileName,
316
+ storageKey: file.storageKey,
317
+ reason: detail,
318
+ bytesReceived,
319
+ });
320
+ abandon("entry-failed-after-content-length");
321
+ };
322
+
100
323
  try {
101
324
  const url = await storage.getPresignedDownloadUrl(file.storageKey, {
102
325
  filename: baseName,
103
326
  contentType: file.contentType,
104
327
  });
105
- const res = await fetch(url, { signal: req.signal });
328
+ const res = await fetch(url, { signal: fetches.signal });
106
329
  if (!res.ok || !res.body) throw new Error(`HTTP ${res.status}`);
107
330
  // DOM `ReadableStream` and Node's web `ReadableStream` are
108
331
  // runtime-compatible but typed separately — cast at the boundary.
@@ -120,6 +343,7 @@ export function createDownloadZipStreamHandler(
120
343
  cb(null, chunk);
121
344
  },
122
345
  });
346
+ currentSource = counter;
123
347
  // Forward upstream errors onto the counter so the `finished`
124
348
  // await rejects and the loop can append a _FAILED_ placeholder.
125
349
  body.on("error", (err) => counter.destroy(err));
@@ -131,8 +355,12 @@ export function createDownloadZipStreamHandler(
131
355
  // readable side (entry fully written to the zip queue).
132
356
  await finished(counter);
133
357
  } catch (err) {
134
- if (req.signal.aborted) return;
358
+ if (abandonedFor) return;
135
359
  const detail = err instanceof Error ? err.message : String(err);
360
+ if (expectedSize !== undefined) {
361
+ failFatally(detail, bytesReceived);
362
+ return;
363
+ }
136
364
  console.error(
137
365
  `[zip-stream] body stream error for ${file.fileName} ` +
138
366
  `(storageKey=${file.storageKey}, bytesReceived=${bytesReceived}): ${detail}`,
@@ -156,10 +384,26 @@ export function createDownloadZipStreamHandler(
156
384
  appendErr,
157
385
  );
158
386
  }
387
+ continue;
388
+ }
389
+ // The declared length was computed from the catalog (or from a
390
+ // pre-flight probe); an object that turns out to be a different size
391
+ // means the promise on the wire is already wrong, so stop rather
392
+ // than ship a mismatched body.
393
+ if (expectedSize !== undefined && bytesReceived !== expectedSize) {
394
+ failFatally(
395
+ `storage object is ${bytesReceived} bytes but the catalog says ${expectedSize}`,
396
+ bytesReceived,
397
+ );
398
+ return;
159
399
  }
160
400
  } catch (err) {
161
- if (req.signal.aborted) return;
401
+ if (abandonedFor) return;
162
402
  const detail = err instanceof Error ? err.message : String(err);
403
+ if (expectedSize !== undefined) {
404
+ failFatally(detail, 0);
405
+ return;
406
+ }
163
407
  console.warn(`[zip-stream] fetch failed for ${file.fileName}:`, detail);
164
408
  reportFailure({
165
409
  token,
@@ -173,26 +417,265 @@ export function createDownloadZipStreamHandler(
173
417
  `Try downloading the track individually from your order page.\n`,
174
418
  { name: `_FAILED_${baseName}.txt` },
175
419
  );
420
+ } finally {
421
+ currentSource = null;
176
422
  }
177
423
  }
424
+ if (abandonedFor) return;
178
425
  if (failures.length > 0) {
179
426
  console.error(
180
427
  `[zip-stream] ${failures.length}/${resolution.files.length} entries failed ` +
181
428
  `for token ${token}: ${failures.map((f) => f.fileName).join(", ")}`,
182
429
  );
183
430
  }
431
+ finalized = true;
184
432
  await archive.finalize();
185
433
  })().catch((err) => {
186
434
  console.error("[zip-stream] pipeline error:", err);
187
- archive.abort();
435
+ abandon("pipeline-error");
436
+ });
437
+
438
+ const headers: Record<string, string> = {
439
+ "Content-Type": "application/zip",
440
+ "Content-Disposition": `attachment; filename="${resolution.zipName.replace(/"/g, "")}"`,
441
+ "Cache-Control": "no-store",
442
+ };
443
+ if (contentLength !== null) headers["Content-Length"] = String(contentLength);
444
+
445
+ return new Response(output.body, { headers });
446
+ };
447
+ }
448
+
449
+ interface MeteredOutput {
450
+ body: ReadableStream<Uint8Array>;
451
+ bytesSent: () => number;
452
+ }
453
+
454
+ interface MeteredOutputHooks {
455
+ idleTimeoutMs: number;
456
+ /** The consumer stopped taking bytes for `idleTimeoutMs` — the download has been abandoned. */
457
+ onIdle: () => void;
458
+ /** The platform cancelled the response body (the usual shape of a client disconnect). */
459
+ onCancel: () => void;
460
+ onClose: () => void;
461
+ onError: () => void;
462
+ }
463
+
464
+ /**
465
+ * Wrap archiver's output in a strictly demand-driven `ReadableStream`.
466
+ *
467
+ * `Readable.toWeb()` — what this replaces — runs the Node stream in flowing
468
+ * mode and pauses it once the web stream's queue is over its high-water mark.
469
+ * That works, but it hands the platform's response writer an already-flowing
470
+ * source and gives us no handle on whether the customer is actually taking
471
+ * bytes. Pulling instead of pushing gives both: nothing leaves archiver until
472
+ * the consumer asks, so a slow customer's connection is what paces the R2
473
+ * fetch, and "the consumer has not asked in a while" becomes an observable
474
+ * event.
475
+ *
476
+ * That second part is what makes an abandoned download detectable at all.
477
+ * `req.signal` is supposed to fire on disconnect and does so locally, but the
478
+ * production incident this was written for shows requests running the full
479
+ * 300-second timeout after the customer had plainly given up — seven retries
480
+ * from one order in 78 minutes, each of the earlier ones still burning R2
481
+ * egress and heap. A consumer that has gone quiet for `idleTimeoutMs` is
482
+ * treated as gone, whatever `req.signal` believes.
483
+ *
484
+ * The watchdog only counts time in which *we* are the ones waiting: while a
485
+ * `pull` is outstanding the archive is the slow side (a large track can take
486
+ * far longer than the idle timeout to fetch from R2), and that must never be
487
+ * mistaken for an absent customer.
488
+ */
489
+ function createMeteredOutput(source: Readable, hooks: MeteredOutputHooks): MeteredOutput {
490
+ let bytesSent = 0;
491
+ let lastHandoffAt = Date.now();
492
+ let awaitingConsumer = false;
493
+ let watchdog: ReturnType<typeof setInterval> | undefined;
494
+
495
+ const stopWatchdog = () => {
496
+ if (watchdog !== undefined) {
497
+ clearInterval(watchdog);
498
+ watchdog = undefined;
499
+ }
500
+ };
501
+
502
+ /** Resolve as soon as the archive has more bytes, has ended, or has failed. */
503
+ const waitForSource = () =>
504
+ new Promise<void>((resolve, reject) => {
505
+ const cleanup = () => {
506
+ source.off("readable", onReadable);
507
+ source.off("end", onEnd);
508
+ source.off("close", onEnd);
509
+ source.off("error", onError);
510
+ };
511
+ const onReadable = () => {
512
+ cleanup();
513
+ resolve();
514
+ };
515
+ const onEnd = () => {
516
+ cleanup();
517
+ resolve();
518
+ };
519
+ const onError = (err: unknown) => {
520
+ cleanup();
521
+ reject(err instanceof Error ? err : new Error(String(err)));
522
+ };
523
+ source.on("readable", onReadable);
524
+ source.on("end", onEnd);
525
+ // A destroy() with no error emits neither `end` nor `error`; without
526
+ // this the pull would wait forever on a stream that is already gone.
527
+ source.on("close", onEnd);
528
+ source.on("error", onError);
188
529
  });
189
530
 
190
- return new Response(Readable.toWeb(archive) as ReadableStream<Uint8Array>, {
191
- headers: {
192
- "Content-Type": "application/zip",
193
- "Content-Disposition": `attachment; filename="${resolution.zipName.replace(/"/g, "")}"`,
194
- "Cache-Control": "no-store",
531
+ const body = new ReadableStream<Uint8Array>(
532
+ {
533
+ async pull(controller) {
534
+ awaitingConsumer = false;
535
+ try {
536
+ for (;;) {
537
+ const chunk = source.read() as Buffer | null;
538
+ if (chunk !== null && chunk.length > 0) {
539
+ bytesSent += chunk.length;
540
+ lastHandoffAt = Date.now();
541
+ awaitingConsumer = true;
542
+ controller.enqueue(chunk);
543
+ return;
544
+ }
545
+ if (source.errored) throw source.errored;
546
+ if (source.readableEnded) {
547
+ stopWatchdog();
548
+ controller.close();
549
+ hooks.onClose();
550
+ return;
551
+ }
552
+ if (source.destroyed) {
553
+ throw new Error("zip stream ended before the archive was finalized");
554
+ }
555
+ await waitForSource();
556
+ }
557
+ } catch (err) {
558
+ stopWatchdog();
559
+ hooks.onError();
560
+ throw err;
561
+ }
562
+ },
563
+ cancel() {
564
+ stopWatchdog();
565
+ hooks.onCancel();
195
566
  },
567
+ },
568
+ // Byte-counted so the cap is a real memory bound rather than a chunk
569
+ // count: at most this much archive output is ever queued ahead of the
570
+ // customer's connection.
571
+ new ByteLengthQueuingStrategy({ highWaterMark: OUTPUT_HIGH_WATER_MARK_BYTES }),
572
+ );
573
+
574
+ watchdog = setInterval(
575
+ () => {
576
+ if (!awaitingConsumer) return;
577
+ if (Date.now() - lastHandoffAt < hooks.idleTimeoutMs) return;
578
+ stopWatchdog();
579
+ hooks.onIdle();
580
+ },
581
+ Math.max(100, Math.floor(hooks.idleTimeoutMs / 4)),
582
+ );
583
+ watchdog.unref?.();
584
+
585
+ return { body, bytesSent: () => bytesSent };
586
+ }
587
+
588
+ /**
589
+ * Exact byte length of every entry in the bundle, or `null` if even one is
590
+ * unknown — the caller must then skip `Content-Length` entirely rather than
591
+ * declare a number it can't keep.
592
+ *
593
+ * `track_files.fileSize` covers audio in the normal case, but the column is
594
+ * nullable (legacy rows store `0`) and cover art has no row at all, so the
595
+ * stragglers are measured directly against storage. The probe is a ranged GET
596
+ * for a single byte rather than a HEAD: presigned URLs are signed per HTTP
597
+ * method, so a `HEAD` against a GET signature is refused, while `Range` is an
598
+ * unsigned header and rides along fine. The `Content-Range` response carries
599
+ * the object's full length.
600
+ */
601
+ async function resolveEntrySizes(
602
+ storage: DownloadDeps["storage"],
603
+ files: ZipFile[],
604
+ signal: AbortSignal,
605
+ ): Promise<number[] | null> {
606
+ const sizes = files.map((file) => file.byteSize);
607
+ const pending = sizes.flatMap((size, i) => (size === undefined ? [i] : []));
608
+ if (pending.length === 0) return sizes as number[];
609
+ if (pending.length > MAX_SIZE_PROBES) {
610
+ console.warn(
611
+ `[zip-stream] ${pending.length} entries have no recorded size; ` +
612
+ `streaming without a Content-Length rather than probing them all`,
613
+ );
614
+ return null;
615
+ }
616
+
617
+ let cursor = 0;
618
+ let failed = false;
619
+ const workers = Array.from(
620
+ { length: Math.min(SIZE_PROBE_CONCURRENCY, pending.length) },
621
+ async () => {
622
+ while (!failed) {
623
+ const next = pending[cursor++];
624
+ if (next === undefined) return;
625
+ const size = await probeObjectSize(storage, files[next]!, signal);
626
+ if (size === null) {
627
+ failed = true;
628
+ return;
629
+ }
630
+ sizes[next] = size;
631
+ }
632
+ },
633
+ );
634
+ await Promise.all(workers);
635
+
636
+ return failed ? null : (sizes as number[]);
637
+ }
638
+
639
+ /** Measure one storage object with a single-byte ranged GET, or `null` if its size can't be established. */
640
+ async function probeObjectSize(
641
+ storage: DownloadDeps["storage"],
642
+ file: ZipFile,
643
+ signal: AbortSignal,
644
+ ): Promise<number | null> {
645
+ let res: Response;
646
+ try {
647
+ const url = await storage.getPresignedDownloadUrl(file.storageKey, {
648
+ filename: file.fileName.split("/").pop() || file.fileName,
649
+ contentType: file.contentType,
196
650
  });
197
- };
651
+ res = await fetch(url, {
652
+ headers: { Range: "bytes=0-0" },
653
+ signal: AbortSignal.any([signal, AbortSignal.timeout(SIZE_PROBE_TIMEOUT_MS)]),
654
+ });
655
+ } catch (err) {
656
+ console.warn(`[zip-stream] size probe failed for ${file.fileName}:`, err);
657
+ return null;
658
+ }
659
+
660
+ try {
661
+ // 206: the object's total length is the tail of `bytes 0-0/<total>`.
662
+ // 200: the range was ignored and `Content-Length` is the whole object.
663
+ const header =
664
+ res.status === 206
665
+ ? /\/(\d+)$/.exec(res.headers.get("content-range")?.trim() ?? "")?.[1]
666
+ : res.ok
667
+ ? res.headers.get("content-length")
668
+ : null;
669
+ const size = header === null || header === undefined ? NaN : Number(header);
670
+ if (!Number.isSafeInteger(size) || size <= 0) {
671
+ console.warn(
672
+ `[zip-stream] size probe for ${file.fileName} returned HTTP ${res.status} ` +
673
+ `without a usable length`,
674
+ );
675
+ return null;
676
+ }
677
+ return size;
678
+ } finally {
679
+ await res.body?.cancel().catch(() => {});
680
+ }
198
681
  }
package/src/zip.ts CHANGED
@@ -29,6 +29,21 @@ export interface ZipFile {
29
29
  storageKey: string;
30
30
  contentType: string;
31
31
  source?: ZipFileSource;
32
+ /**
33
+ * Exact byte length of the stored object, when the catalog knows it.
34
+ * `track_files.fileSize` is nullable (and legacy rows carry `0`), and cover
35
+ * art has no size row at all, so this is absent as often as not —
36
+ * `createDownloadZipStreamHandler` measures the stragglers itself before
37
+ * committing to a `Content-Length`.
38
+ */
39
+ byteSize?: number;
40
+ }
41
+
42
+ /** `track_files.fileSize` is nullable and legacy rows store `0`; only a positive integer is a real size. */
43
+ function knownByteSize(fileSize: number | null | undefined): number | undefined {
44
+ return typeof fileSize === "number" && Number.isSafeInteger(fileSize) && fileSize > 0
45
+ ? fileSize
46
+ : undefined;
32
47
  }
33
48
 
34
49
  interface ZipManifest {
@@ -234,6 +249,7 @@ function resolveTrackList(
234
249
  storageKey: file.storageKey,
235
250
  contentType: audioContentType,
236
251
  source: { kind: "track", trackId: id, format },
252
+ byteSize: knownByteSize(file.fileSize),
237
253
  });
238
254
  }
239
255
  if (files.length === 0) {
@@ -277,6 +293,7 @@ function resolveSingleRelease(
277
293
  storageKey: file.storageKey,
278
294
  contentType: audioContentType,
279
295
  source: { kind: "track", trackId: track.id, format },
296
+ byteSize: knownByteSize(file.fileSize),
280
297
  };
281
298
  })
282
299
  .filter((f): f is ZipFile => f !== null);
@@ -321,6 +338,7 @@ function resolveWholeOrder(
321
338
  storageKey: file.storageKey,
322
339
  contentType: audioContentType,
323
340
  source: { kind: "track", trackId: track.id, format },
341
+ byteSize: knownByteSize(file.fileSize),
324
342
  });
325
343
  }
326
344
  if (entries.length > 0 && release.coverImageUrl) {
@@ -344,6 +362,7 @@ function resolveWholeOrder(
344
362
  storageKey: file.storageKey,
345
363
  contentType: audioContentType,
346
364
  source: { kind: "track", trackId: track.id, format },
365
+ byteSize: knownByteSize(file.fileSize),
347
366
  });
348
367
  }
349
368
  }