cursedbelt-server 4.44.1 → 4.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,96 @@
1
+ /**
2
+ * A streaming ZIP writer — a download of many files as one, built as it is sent.
3
+ *
4
+ * Graduated from `apps/music/src/server/zip.ts` (2026-10-15, forge task desk-updates) when desk
5
+ * needed the same thing for "download this folder": two copies of a hand-rolled format writer
6
+ * is two places for the next bug. Folders (directory entries) and file dates arrived with it,
7
+ * because a folder download that loses its empty folders and stamps every file 1980 is not a
8
+ * backup.
9
+ *
10
+ * ── Why this is hand-written rather than a dependency ───────────────────────────────────
11
+ * Three properties are needed together, and the popular libraries give at most two:
12
+ *
13
+ * · **Store, never deflate.** Pictures, video, audio and `.xlsx` are already compressed.
14
+ * Deflating them spends CPU — on a Worker, the scarce thing — to produce a slightly LARGER
15
+ * file. Method 0 (store) is the correct encoding for them, and a few kilobytes of notes
16
+ * beside them are not worth a second code path.
17
+ * · **True streaming.** "Download this folder" can be thousands of files and many gigabytes.
18
+ * Nothing is buffered beyond one entry: bytes arrive from the store and leave for the
19
+ * browser in the same pass.
20
+ * · **ZIP64.** 4 GB is an entirely reachable selection, and the classic format's 32-bit size
21
+ * fields wrap silently at that point. A truncated archive that reports success is the worst
22
+ * available failure, so ZIP64 is written UNCONDITIONALLY rather than only when a threshold is
23
+ * crossed — a conditional format is a format that is only tested on the small path.
24
+ *
25
+ * ── Streaming means data descriptors ────────────────────────────────────────────────────
26
+ * A local file header must state the CRC-32 and the sizes, and none of those are known until
27
+ * the file's bytes have gone by. The format's answer is general-purpose flag bit 3: write zeros
28
+ * in the header, and follow the data with a DATA DESCRIPTOR carrying the real values. That is
29
+ * what makes a single forward pass possible, why this writer never seeks, and so why the
30
+ * response can be a `ReadableStream` and the server's memory stays flat whatever the size.
31
+ *
32
+ * ── The CPU it costs ────────────────────────────────────────────────────────────────────
33
+ * The CRC-32 is computed over every byte in the isolate: measured 2026-09-22 at ~280 MB of CPU
34
+ * per second (240 MB in 860 ms). A Worker's default budget is 30 s of CPU per request, so a
35
+ * caller on a Worker caps what one archive may hold — music at 4 GB / 2,000 files — and says so
36
+ * BEFORE the download starts, rather than letting the edge cut it off part-way.
37
+ */
38
+ /** Incremental CRC-32, so it can be fed one chunk at a time as bytes stream past. */
39
+ export declare class Crc32 {
40
+ private crc;
41
+ update(bytes: Uint8Array): void;
42
+ get value(): number;
43
+ }
44
+ /** One member of the archive: a file whose bytes are opened lazily, or a folder. */
45
+ export type ZipEntry = {
46
+ /** Path inside the archive, `/`-separated. Sanitized by {@link zipStream}. */
47
+ name: string;
48
+ /**
49
+ * Opens the bytes — a stream, or bytes already in hand. Called only when the entry's turn
50
+ * comes, so a 2,000-file archive does not open 2,000 connections at once. Returning `null`
51
+ * (or throwing) skips the entry: a file that vanished must not abort the rest.
52
+ */
53
+ open: () => Promise<ReadableStream<Uint8Array> | Uint8Array | null>;
54
+ /** When the file last changed. Omitted: the archive says 1980-01-01, the format's zero. */
55
+ modified?: Date | string | null;
56
+ directory?: false;
57
+ } | {
58
+ /** A folder — written so an EMPTY one survives the round trip. A trailing `/` is added. */
59
+ name: string;
60
+ directory: true;
61
+ modified?: Date | string | null;
62
+ } | {
63
+ /**
64
+ * A member the CALLER could not read before naming it (desk reads a picture's bytes to
65
+ * decide its extension). Nothing is written under this name; it is listed, with `missing`
66
+ * as the reason, in the archive's `MISSING FILES.txt`.
67
+ */
68
+ name: string;
69
+ missing: string;
70
+ };
71
+ /**
72
+ * Names inside a zip must not escape the extraction directory, and must not carry characters
73
+ * Windows refuses. `..` segments and absolute paths are the security half; the character set is
74
+ * the "it downloaded but won't extract" half. Each segment is capped, so a long name inside a
75
+ * deep folder keeps its folders.
76
+ */
77
+ export declare function safeEntryName(name: string, fallback?: string): string;
78
+ /** The MS-DOS date and time a zip header carries — local wall-clock, 2-second resolution, 1980 at the earliest. */
79
+ export declare function dosDateTime(when: Date | string | null | undefined): {
80
+ date: number;
81
+ time: number;
82
+ };
83
+ /**
84
+ * Build the archive as a stream.
85
+ *
86
+ * Duplicate file names are disambiguated rather than colliding — two different files that both
87
+ * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
88
+ * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
89
+ * once: its files already land in it. Names are compared the way the person's disk will compare
90
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
91
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
92
+ *
93
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
94
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
95
+ */
96
+ export declare function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array>;
@@ -0,0 +1,373 @@
1
+ /**
2
+ * A streaming ZIP writer — a download of many files as one, built as it is sent.
3
+ *
4
+ * Graduated from `apps/music/src/server/zip.ts` (2026-10-15, forge task desk-updates) when desk
5
+ * needed the same thing for "download this folder": two copies of a hand-rolled format writer
6
+ * is two places for the next bug. Folders (directory entries) and file dates arrived with it,
7
+ * because a folder download that loses its empty folders and stamps every file 1980 is not a
8
+ * backup.
9
+ *
10
+ * ── Why this is hand-written rather than a dependency ───────────────────────────────────
11
+ * Three properties are needed together, and the popular libraries give at most two:
12
+ *
13
+ * · **Store, never deflate.** Pictures, video, audio and `.xlsx` are already compressed.
14
+ * Deflating them spends CPU — on a Worker, the scarce thing — to produce a slightly LARGER
15
+ * file. Method 0 (store) is the correct encoding for them, and a few kilobytes of notes
16
+ * beside them are not worth a second code path.
17
+ * · **True streaming.** "Download this folder" can be thousands of files and many gigabytes.
18
+ * Nothing is buffered beyond one entry: bytes arrive from the store and leave for the
19
+ * browser in the same pass.
20
+ * · **ZIP64.** 4 GB is an entirely reachable selection, and the classic format's 32-bit size
21
+ * fields wrap silently at that point. A truncated archive that reports success is the worst
22
+ * available failure, so ZIP64 is written UNCONDITIONALLY rather than only when a threshold is
23
+ * crossed — a conditional format is a format that is only tested on the small path.
24
+ *
25
+ * ── Streaming means data descriptors ────────────────────────────────────────────────────
26
+ * A local file header must state the CRC-32 and the sizes, and none of those are known until
27
+ * the file's bytes have gone by. The format's answer is general-purpose flag bit 3: write zeros
28
+ * in the header, and follow the data with a DATA DESCRIPTOR carrying the real values. That is
29
+ * what makes a single forward pass possible, why this writer never seeks, and so why the
30
+ * response can be a `ReadableStream` and the server's memory stays flat whatever the size.
31
+ *
32
+ * ── The CPU it costs ────────────────────────────────────────────────────────────────────
33
+ * The CRC-32 is computed over every byte in the isolate: measured 2026-09-22 at ~280 MB of CPU
34
+ * per second (240 MB in 860 ms). A Worker's default budget is 30 s of CPU per request, so a
35
+ * caller on a Worker caps what one archive may hold — music at 4 GB / 2,000 files — and says so
36
+ * BEFORE the download starts, rather than letting the edge cut it off part-way.
37
+ */
38
+ // ── CRC-32 ──────────────────────────────────────────────────────────────────────────────
39
+ /** The standard IEEE 802.3 table, built once. */
40
+ const CRC_TABLE = (() => {
41
+ const table = new Uint32Array(256);
42
+ for (let i = 0; i < 256; i++) {
43
+ let c = i;
44
+ for (let k = 0; k < 8; k++)
45
+ c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
46
+ table[i] = c >>> 0;
47
+ }
48
+ return table;
49
+ })();
50
+ /** Incremental CRC-32, so it can be fed one chunk at a time as bytes stream past. */
51
+ export class Crc32 {
52
+ crc = 0xffffffff;
53
+ update(bytes) {
54
+ let crc = this.crc;
55
+ for (let i = 0; i < bytes.length; i++)
56
+ crc = CRC_TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
57
+ this.crc = crc;
58
+ }
59
+ get value() {
60
+ return (this.crc ^ 0xffffffff) >>> 0;
61
+ }
62
+ }
63
+ // ── Little-endian writing ───────────────────────────────────────────────────────────────
64
+ class ByteWriter {
65
+ parts = [];
66
+ u16(n) {
67
+ const b = new Uint8Array(2);
68
+ new DataView(b.buffer).setUint16(0, n, true);
69
+ this.parts.push(b);
70
+ return this;
71
+ }
72
+ u32(n) {
73
+ const b = new Uint8Array(4);
74
+ new DataView(b.buffer).setUint32(0, n >>> 0, true);
75
+ this.parts.push(b);
76
+ return this;
77
+ }
78
+ /** 64-bit, via BigInt — a whole-library download can pass 2^32, and Number would quietly lose precision past 2^53. */
79
+ u64(n) {
80
+ const b = new Uint8Array(8);
81
+ new DataView(b.buffer).setBigUint64(0, BigInt(n), true);
82
+ this.parts.push(b);
83
+ return this;
84
+ }
85
+ raw(bytes) {
86
+ this.parts.push(bytes);
87
+ return this;
88
+ }
89
+ done() {
90
+ const total = this.parts.reduce((n, p) => n + p.length, 0);
91
+ const out = new Uint8Array(total);
92
+ let at = 0;
93
+ for (const part of this.parts) {
94
+ out.set(part, at);
95
+ at += part.length;
96
+ }
97
+ return out;
98
+ }
99
+ }
100
+ /**
101
+ * The last entry of an archive that left anything out: one line per missing path and why.
102
+ * A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
103
+ * not is a backup that fails on the day it is restored — so the skip is never silent.
104
+ */
105
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
106
+ const encoder = new TextEncoder();
107
+ /** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
108
+ * names, which every accented and non-Latin file name needs. */
109
+ const VERSION_ZIP64 = 45;
110
+ const FLAG_DATA_DESCRIPTOR = 0x08;
111
+ const FLAG_UTF8 = 0x0800;
112
+ const METHOD_STORE = 0;
113
+ /** MS-DOS directory attribute, in the low byte of the external attributes. */
114
+ const ATTR_DIRECTORY = 0x10;
115
+ /**
116
+ * Names inside a zip must not escape the extraction directory, and must not carry characters
117
+ * Windows refuses. `..` segments and absolute paths are the security half; the character set is
118
+ * the "it downloaded but won't extract" half. Each segment is capped, so a long name inside a
119
+ * deep folder keeps its folders.
120
+ */
121
+ export function safeEntryName(name, fallback = "file") {
122
+ // Control characters (< 0x20) and the set Windows refuses. Filtered by code point rather than
123
+ // by a regex class, because a literal control character inside a regex is unreadable and the
124
+ // linter is right to object to it.
125
+ const FORBIDDEN = new Set(["<", ">", ":", '"', "|", "?", "*"]);
126
+ const segments = name
127
+ .split(/[/\\]/)
128
+ .filter((seg) => seg !== "" && seg !== "." && seg !== "..")
129
+ .map((seg) => {
130
+ let out = "";
131
+ for (const ch of seg)
132
+ out += ch.codePointAt(0) < 0x20 || FORBIDDEN.has(ch) ? "_" : ch;
133
+ // Windows also refuses a segment ending in a dot or a space; trimmed rather than replaced.
134
+ return out.slice(0, 200).replace(/[. ]+$/, "") || fallback;
135
+ });
136
+ return segments.join("/") || fallback;
137
+ }
138
+ /** The MS-DOS date and time a zip header carries — local wall-clock, 2-second resolution, 1980 at the earliest. */
139
+ export function dosDateTime(when) {
140
+ const at = when ? new Date(when) : null;
141
+ if (!at || Number.isNaN(at.getTime()) || at.getUTCFullYear() < 1980)
142
+ return { date: (0 << 9) | (1 << 5) | 1, time: 0 };
143
+ // UTC, not the server's zone: a Worker has none, and the same archive should not depend on where it was built.
144
+ const year = Math.min(at.getUTCFullYear(), 2107) - 1980;
145
+ return {
146
+ date: (year << 9) | ((at.getUTCMonth() + 1) << 5) | at.getUTCDate(),
147
+ time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | Math.floor(at.getUTCSeconds() / 2),
148
+ };
149
+ }
150
+ /**
151
+ * A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
152
+ * APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
153
+ * names equal under this fold are ONE file there — and the second would overwrite the first.
154
+ */
155
+ function foldName(name) {
156
+ return name.normalize("NFC").toLowerCase();
157
+ }
158
+ /** A reason for a person to read: one line, short, and never a signed URL's query string. */
159
+ function readableReason(text) {
160
+ return text
161
+ .replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
162
+ .replace(/\s+/g, " ")
163
+ .trim()
164
+ .slice(0, 200);
165
+ }
166
+ function asStream(body) {
167
+ if (!(body instanceof Uint8Array))
168
+ return body;
169
+ return new ReadableStream({
170
+ start(controller) {
171
+ if (body.byteLength > 0)
172
+ controller.enqueue(body);
173
+ controller.close();
174
+ },
175
+ });
176
+ }
177
+ /**
178
+ * Build the archive as a stream.
179
+ *
180
+ * Duplicate file names are disambiguated rather than colliding — two different files that both
181
+ * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
182
+ * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
183
+ * once: its files already land in it. Names are compared the way the person's disk will compare
184
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
185
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
186
+ *
187
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
188
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
189
+ */
190
+ export function zipStream(entries) {
191
+ const central = [];
192
+ let offset = 0n;
193
+ /** Every name written, as the extracting disk will compare it ({@link foldName}). */
194
+ const used = new Set();
195
+ const missing = [];
196
+ let reported = false;
197
+ const iterator = Symbol.asyncIterator in entries ? entries[Symbol.asyncIterator]() : entries[Symbol.iterator]();
198
+ /** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
199
+ const freeName = (name) => {
200
+ if (!used.has(foldName(name)))
201
+ return name;
202
+ const slash = name.lastIndexOf("/");
203
+ const dot = name.lastIndexOf(".");
204
+ const cut = dot > slash + 1 ? dot : name.length;
205
+ const stem = name.slice(0, cut);
206
+ const ext = name.slice(cut);
207
+ let n = 2;
208
+ while (used.has(foldName(`${stem} (${n})${ext}`)))
209
+ n++;
210
+ return `${stem} (${n})${ext}`;
211
+ };
212
+ /** One local header, the bytes, and the data descriptor; remembered for the central directory. */
213
+ const writeEntry = async (controller, name, body, modified) => {
214
+ const directory = body === null;
215
+ used.add(foldName(name));
216
+ const nameBytes = encoder.encode(name);
217
+ const { date, time } = dosDateTime(modified);
218
+ // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
219
+ const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
220
+ const header = new ByteWriter()
221
+ .u32(0x04034b50)
222
+ .u16(VERSION_ZIP64)
223
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
224
+ .u16(METHOD_STORE)
225
+ .u16(time)
226
+ .u16(date)
227
+ .u32(0) // crc, in the descriptor
228
+ .u32(0xffffffff) // compressed size → see ZIP64 extra
229
+ .u32(0xffffffff) // uncompressed size → see ZIP64 extra
230
+ .u16(nameBytes.length)
231
+ .u16(localExtra.length)
232
+ .raw(nameBytes)
233
+ .raw(localExtra)
234
+ .done();
235
+ controller.enqueue(header);
236
+ const entryOffset = offset;
237
+ offset += BigInt(header.length);
238
+ const crc = new Crc32();
239
+ let size = 0n;
240
+ if (body) {
241
+ const reader = body.getReader();
242
+ while (true) {
243
+ const { done, value } = await reader.read();
244
+ if (done)
245
+ break;
246
+ crc.update(value);
247
+ size += BigInt(value.length);
248
+ controller.enqueue(value);
249
+ }
250
+ }
251
+ offset += size;
252
+ // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
253
+ const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
254
+ controller.enqueue(descriptor);
255
+ offset += BigInt(descriptor.length);
256
+ central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
257
+ };
258
+ return new ReadableStream({
259
+ async pull(controller) {
260
+ // One entry per pull keeps backpressure honest: the next file is not opened until the
261
+ // consumer has taken the previous one.
262
+ //
263
+ // 🔴 The loop is not decoration. A `pull` that returns having enqueued NOTHING is not
264
+ // guaranteed to be called again, so an entry that turns out to be unreadable would stall
265
+ // the entire archive — the consumer waits forever for bytes that will never come. Skipping
266
+ // therefore has to continue to the next entry within the SAME pull. (Measured in music:
267
+ // two tests hung for the full 5 s timeout before this loop existed.)
268
+ while (true) {
269
+ const next = await iterator.next();
270
+ if (next.done)
271
+ break;
272
+ const entry = next.value;
273
+ if ("missing" in entry) {
274
+ missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
275
+ continue;
276
+ }
277
+ if (entry.directory === true) {
278
+ const name = `${safeEntryName(entry.name)}/`;
279
+ if (used.has(foldName(name)))
280
+ continue;
281
+ await writeEntry(controller, name, null, entry.modified);
282
+ return;
283
+ }
284
+ const name = freeName(safeEntryName(entry.name));
285
+ let failure = "its bytes are missing from the store";
286
+ const opened = await entry.open().catch((error) => {
287
+ failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
288
+ return null;
289
+ });
290
+ // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
291
+ // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
292
+ // note on the loop above.
293
+ if (!opened) {
294
+ missing.push({ name, reason: failure });
295
+ continue;
296
+ }
297
+ await writeEntry(controller, name, asStream(opened), entry.modified);
298
+ return;
299
+ }
300
+ if (missing.length > 0 && !reported) {
301
+ reported = true;
302
+ const lines = missing.map((m) => `${m.name} — ${m.reason}`);
303
+ const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
304
+ const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
305
+ await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
306
+ return;
307
+ }
308
+ // Central directory, then the ZIP64 end records, then the classic end record.
309
+ const start = offset;
310
+ for (const item of central) {
311
+ const extra = new ByteWriter().u16(0x0001).u16(24).u64(item.size).u64(item.size).u64(item.offset).done();
312
+ const record = new ByteWriter()
313
+ .u32(0x02014b50)
314
+ .u16(VERSION_ZIP64) // version made by (MS-DOS attributes)
315
+ .u16(VERSION_ZIP64) // version needed
316
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
317
+ .u16(METHOD_STORE)
318
+ .u16(item.time)
319
+ .u16(item.date)
320
+ .u32(item.crc)
321
+ .u32(0xffffffff)
322
+ .u32(0xffffffff)
323
+ .u16(item.name.length)
324
+ .u16(extra.length)
325
+ .u16(0) // comment length
326
+ .u16(0) // disk number
327
+ .u16(0) // internal attrs
328
+ .u32(item.directory ? ATTR_DIRECTORY : 0) // external attrs
329
+ .u32(0xffffffff) // local header offset → ZIP64 extra
330
+ .raw(item.name)
331
+ .raw(extra)
332
+ .done();
333
+ controller.enqueue(record);
334
+ offset += BigInt(record.length);
335
+ }
336
+ const centralSize = offset - start;
337
+ const zip64End = new ByteWriter()
338
+ .u32(0x06064b50)
339
+ .u64(44) // size of this record, minus its first 12 bytes
340
+ .u16(VERSION_ZIP64)
341
+ .u16(VERSION_ZIP64)
342
+ .u32(0)
343
+ .u32(0)
344
+ .u64(central.length)
345
+ .u64(central.length)
346
+ .u64(centralSize)
347
+ .u64(start)
348
+ .done();
349
+ controller.enqueue(zip64End);
350
+ const locator = new ByteWriter()
351
+ .u32(0x07064b50)
352
+ .u32(0)
353
+ .u64(offset) // where the ZIP64 end record starts
354
+ .u32(1)
355
+ .done();
356
+ controller.enqueue(locator);
357
+ // The classic record still has to be there for every reader to find the archive; its
358
+ // 0xffff/0xffffffff sentinels say "look at the ZIP64 record instead".
359
+ const end = new ByteWriter()
360
+ .u32(0x06054b50)
361
+ .u16(0)
362
+ .u16(0)
363
+ .u16(central.length > 0xffff ? 0xffff : central.length)
364
+ .u16(central.length > 0xffff ? 0xffff : central.length)
365
+ .u32(0xffffffff)
366
+ .u32(0xffffffff)
367
+ .u16(0)
368
+ .done();
369
+ controller.enqueue(end);
370
+ controller.close();
371
+ },
372
+ });
373
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "cursedbelt-server",
3
- "version": "4.44.1",
3
+ "version": "4.46.0",
4
4
  "license": "ISC",
5
5
  "type": "module",
6
6
  "description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
@@ -355,6 +355,12 @@
355
355
  "source": "./src/server/youtube/ytdlp.ts",
356
356
  "import": "./dist/server/youtube/ytdlp.js"
357
357
  },
358
+ "./zip-stream": {
359
+ "types": "./dist/server/zipStream.d.ts",
360
+ "bun": "./src/server/zipStream.ts",
361
+ "source": "./src/server/zipStream.ts",
362
+ "import": "./dist/server/zipStream.js"
363
+ },
358
364
  "./ws": {
359
365
  "types": "./dist/server/ws/index.d.ts",
360
366
  "bun": "./src/server/ws/index.ts",
@@ -243,6 +243,14 @@ const LEAVES = [
243
243
  */
244
244
  evaluates: 'createYtDlp',
245
245
  },
246
+ {
247
+ subpath: './zip-stream',
248
+ /**
249
+ * The streaming ZIP64 writer (4.45.0, forge task desk-updates) — out of Music so desk's folder
250
+ * downloads share it. Both run it on a Worker, so it imports nothing at all.
251
+ */
252
+ evaluates: 'zipStream',
253
+ },
246
254
  ] as const;
247
255
 
248
256
  /**
@@ -0,0 +1,320 @@
1
+ /**
2
+ * The zip writer is hand-rolled (see the file header for why), so it is verified against a REAL
3
+ * unzip rather than against itself: a self-consistent archive that no extractor accepts is exactly
4
+ * the failure a hand-rolled format writer produces. Moved here from `apps/music/src/server/zip.test.ts`
5
+ * with the writer (2026-10-15); the folder and date cases arrived with desk.
6
+ */
7
+
8
+ import { describe, expect, test } from 'bun:test';
9
+ import { existsSync, mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
10
+ import { tmpdir } from 'node:os';
11
+ import { join } from 'node:path';
12
+ import { Crc32, dosDateTime, safeEntryName, type ZipEntry, zipStream } from './zipStream';
13
+
14
+ const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text);
15
+
16
+ /** The report entry a skipped member lands in — spelt out, since a person reads it by this name. */
17
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
18
+
19
+ const entryFrom = (name: string, body: string): ZipEntry => ({
20
+ name,
21
+ open: async () =>
22
+ new ReadableStream<Uint8Array>({
23
+ start(controller) {
24
+ // Two chunks, so the streaming CRC and size accumulation are exercised rather than a
25
+ // single-chunk fast path that would hide an accumulator bug.
26
+ const half = Math.ceil(body.length / 2);
27
+ controller.enqueue(bytesOf(body.slice(0, half)));
28
+ controller.enqueue(bytesOf(body.slice(half)));
29
+ controller.close();
30
+ },
31
+ }),
32
+ });
33
+
34
+ const collect = async (stream: ReadableStream<Uint8Array>): Promise<Uint8Array> => new Uint8Array(await new Response(stream).arrayBuffer());
35
+
36
+ /** Run the system `unzip` over the archive in a throwaway directory. */
37
+ async function unzip(archive: Uint8Array, args: string[]): Promise<{ out: string; dir: string; cleanup: () => void }> {
38
+ const dir = mkdtempSync(join(tmpdir(), 'cbs-zip-'));
39
+ const path = join(dir, 'test.zip');
40
+ writeFileSync(path, archive);
41
+ const proc = Bun.spawn(['unzip', ...args, path], { cwd: dir, stdout: 'pipe', stderr: 'pipe' });
42
+ const out = await new Response(proc.stdout).text();
43
+ await proc.exited;
44
+ return { out, dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) };
45
+ }
46
+
47
+ /** Write the archive out and list it with the system `unzip` — the real test. */
48
+ async function unzipList(archive: Uint8Array): Promise<string> {
49
+ const run = await unzip(archive, ['-l']);
50
+ run.cleanup();
51
+ return run.out;
52
+ }
53
+
54
+ /** Extract and read one member back, proving the BYTES survive, not just the listing. */
55
+ async function unzipRead(archive: Uint8Array, member: string): Promise<string> {
56
+ const dir = mkdtempSync(join(tmpdir(), 'cbs-zip-'));
57
+ const path = join(dir, 'test.zip');
58
+ writeFileSync(path, archive);
59
+ const proc = Bun.spawn(['unzip', '-p', path, member], { stdout: 'pipe', stderr: 'pipe' });
60
+ const out = await new Response(proc.stdout).text();
61
+ await proc.exited;
62
+ rmSync(dir, { recursive: true, force: true });
63
+ return out;
64
+ }
65
+
66
+ describe("Crc32", () => {
67
+ test("matches the known IEEE check value for '123456789'", () => {
68
+ const crc = new Crc32();
69
+ crc.update(bytesOf("123456789"));
70
+ expect(crc.value).toBe(0xcbf43926);
71
+ });
72
+
73
+ test("is order-dependent across chunks, i.e. genuinely incremental", () => {
74
+ const whole = new Crc32();
75
+ whole.update(bytesOf("hello world"));
76
+ const split = new Crc32();
77
+ split.update(bytesOf("hello "));
78
+ split.update(bytesOf("world"));
79
+ expect(split.value).toBe(whole.value);
80
+ });
81
+ });
82
+
83
+ describe("safeEntryName", () => {
84
+ test("🔴 strips '..' and absolute paths — an archive must not escape its extract dir", () => {
85
+ expect(safeEntryName("../../etc/passwd")).toBe("etc/passwd");
86
+ expect(safeEntryName("/etc/passwd")).toBe("etc/passwd");
87
+ expect(safeEntryName("a/../../b")).toBe("a/b");
88
+ });
89
+
90
+ test("replaces characters Windows refuses, so the archive extracts everywhere", () => {
91
+ expect(safeEntryName('a:b"c|d?e*f.mp3')).toBe("a_b_c_d_e_f.mp3");
92
+ });
93
+
94
+ test("never yields an empty name", () => {
95
+ expect(safeEntryName("")).toBe("file");
96
+ expect(safeEntryName("../..")).toBe("file");
97
+ expect(safeEntryName("", "track")).toBe("track");
98
+ });
99
+
100
+ test("keeps ordinary nested paths intact", () => {
101
+ expect(safeEntryName("In Sides/01 - Belfast.mp3")).toBe("In Sides/01 - Belfast.mp3");
102
+ });
103
+ });
104
+
105
+ describe("zipStream", () => {
106
+ test("produces an archive a real unzip accepts and lists", async () => {
107
+ const archive = await collect(
108
+ zipStream([entryFrom("one.mp3", "first body"), entryFrom("two.mp3", "second body")]),
109
+ );
110
+ const listing = await unzipList(archive);
111
+ expect(listing).toContain("one.mp3");
112
+ expect(listing).toContain("two.mp3");
113
+ // unzip prints this banner for anything it could not parse.
114
+ expect(listing).not.toContain("cannot find zipfile directory");
115
+ });
116
+
117
+ test("the extracted BYTES round-trip exactly", async () => {
118
+ const body = "the quick brown fox jumps over the lazy dog";
119
+ const archive = await collect(zipStream([entryFrom("song.mp3", body)]));
120
+ expect(await unzipRead(archive, "song.mp3")).toBe(body);
121
+ });
122
+
123
+ test("🔴 duplicate names are disambiguated, not silently collapsed to one file", async () => {
124
+ // Two different rips of one song produce the same "Artist - Title.mp3". A zip with
125
+ // two identical names extracts as ONE file, losing the other without a word.
126
+ const archive = await collect(
127
+ zipStream([entryFrom("same.mp3", "copy one"), entryFrom("same.mp3", "copy two")]),
128
+ );
129
+ const listing = await unzipList(archive);
130
+ expect(listing).toContain("same.mp3");
131
+ expect(listing).toContain("same (2).mp3");
132
+ expect(await unzipRead(archive, "same.mp3")).toBe("copy one");
133
+ expect(await unzipRead(archive, "same (2).mp3")).toBe("copy two");
134
+ });
135
+
136
+ test("🔴 a member whose bytes have gone is SKIPPED, not fatal", async () => {
137
+ // One missing track must not abort a 2,000-file download that is otherwise fine.
138
+ const archive = await collect(
139
+ zipStream([
140
+ entryFrom("good.mp3", "here"),
141
+ { name: "gone.mp3", open: async () => null },
142
+ entryFrom("also-good.mp3", "also here"),
143
+ ]),
144
+ );
145
+ const listing = await unzipList(archive);
146
+ expect(listing).toContain("good.mp3");
147
+ expect(listing).toContain("also-good.mp3");
148
+ expect(listing).not.toContain("gone.mp3");
149
+ });
150
+
151
+ test("🔴 names differing only in case are TWO files — the later is numbered, never overwrites on a Mac", async () => {
152
+ // Desk lets a folder hold notes "Todo" and "todo". macOS (APFS) and Windows (NTFS) treat
153
+ // those as one file, so writing both names verbatim loses one on unzip — silently, or behind
154
+ // a "replace?" prompt the person did not expect.
155
+ const archive = await collect(zipStream([entryFrom("F/Todo.md", "upper"), entryFrom("f/todo.md", "lower"), entryFrom("F/TODO (2).md", "third")]));
156
+ const listing = await unzipList(archive);
157
+ expect(listing).toContain("F/Todo.md");
158
+ expect(listing).toContain("f/todo (2).md");
159
+ expect(listing).toContain("F/TODO (2) (2).md");
160
+ expect(await unzipRead(archive, "f/todo (2).md")).toBe("lower");
161
+ });
162
+
163
+ test("🔴 one name spelt with a composed and a decomposed accent is ONE name to APFS — numbered too", async () => {
164
+ const composed = "Caf\u00e9.txt";
165
+ const decomposed = "Cafe\u0301.txt";
166
+ expect(composed).not.toBe(decomposed);
167
+ const listing = await unzipList(await collect(zipStream([entryFrom(composed, "a"), entryFrom(decomposed, "b")])));
168
+ expect(listing).toContain("(2).txt");
169
+ });
170
+
171
+ test("a folder named twice in different case is written once", async () => {
172
+ const listing = await unzipList(await collect(zipStream([{ name: "Photos", directory: true }, { name: "photos", directory: true }])));
173
+ expect(listing.match(/ photos\//gi)?.length).toBe(1);
174
+ });
175
+
176
+ test(`🔴 what was skipped is LISTED in a final ${MISSING_FILES_NAME}, with the reason — a skip is never silent`, async () => {
177
+ const archive = await collect(
178
+ zipStream([
179
+ entryFrom("good.mp3", "here"),
180
+ { name: "gone.mp3", open: async () => null },
181
+ {
182
+ name: "Folder/broken.jpg",
183
+ open: async () => {
184
+ throw new Error("store answered 503 https://bytes.example/x?sig=SECRET");
185
+ },
186
+ },
187
+ { name: "Notes/unread.png", missing: "could not be read (timeout at https://store.example/b?token=SECRET2)" },
188
+ ]),
189
+ );
190
+ const report = await unzipRead(archive, MISSING_FILES_NAME);
191
+ expect(report).toContain("3 files were in the selection but could not be read");
192
+ expect(report).toContain("gone.mp3 — its bytes are missing from the store");
193
+ expect(report).toContain("Folder/broken.jpg — could not be read (store answered 503 https://bytes.example/x)");
194
+ expect(report).toContain("Notes/unread.png — could not be read (timeout at https://store.example/b)");
195
+ expect(report).not.toContain("SECRET2");
196
+ expect(report).not.toContain("SECRET");
197
+ expect(await unzipRead(archive, "good.mp3")).toBe("here");
198
+ });
199
+
200
+ test(`a complete archive carries no ${MISSING_FILES_NAME}`, async () => {
201
+ expect(await unzipList(await collect(zipStream([entryFrom("a.txt", "a")])))).not.toContain(MISSING_FILES_NAME);
202
+ });
203
+
204
+ test("an entry whose open() THROWS is skipped rather than killing the archive", async () => {
205
+ const archive = await collect(
206
+ zipStream([
207
+ {
208
+ name: "broken.mp3",
209
+ open: async () => {
210
+ throw new Error("upstream said no");
211
+ },
212
+ },
213
+ entryFrom("fine.mp3", "fine"),
214
+ ]),
215
+ );
216
+ expect(await unzipRead(archive, "fine.mp3")).toBe("fine");
217
+ });
218
+
219
+ test("an empty selection still produces a structurally valid (empty) archive", async () => {
220
+ const archive = await collect(zipStream([]));
221
+ const listing = await unzipList(archive);
222
+ expect(listing).not.toContain("cannot find zipfile directory");
223
+ });
224
+
225
+ test("nested paths survive, so a big archive extracts into album folders", async () => {
226
+ const archive = await collect(zipStream([entryFrom("In Sides/Belfast.mp3", "b")]));
227
+ expect(await unzipRead(archive, "In Sides/Belfast.mp3")).toBe("b");
228
+ });
229
+
230
+ test("UTF-8 names are stored as UTF-8 bytes with the UTF-8 flag set", async () => {
231
+ // Asserted on the ARCHIVE, not on `unzip -l`: macOS ships Info-ZIP 6.0, whose
232
+ // listing mangles non-ASCII for display regardless of what the file contains.
233
+ // This library has accented and non-Latin titles, so the encoding has to be right.
234
+ const name = "Björk - Jóga.mp3";
235
+ const archive = await collect(zipStream([entryFrom(name, "x")]));
236
+ const encoded = new TextEncoder().encode(name);
237
+ const found = archive.some((_, i) => encoded.every((b, k) => archive[i + k] === b));
238
+ expect(found).toBe(true);
239
+
240
+ // General-purpose flag bit 11 (0x0800) = "the name is UTF-8". It sits at offset 6
241
+ // of the local file header, which starts the archive.
242
+ const flags = archive[6]! | (archive[7]! << 8);
243
+ expect(flags & 0x0800).toBe(0x0800);
244
+ });
245
+
246
+ test("writes the ZIP64 end-of-central-directory record unconditionally", async () => {
247
+ // Written for every archive rather than only past 4 GB: a conditional format is a
248
+ // format that is only ever tested on the small path. 0x06064b50, little-endian.
249
+ const archive = await collect(zipStream([entryFrom("a.mp3", "a")]));
250
+ const signature = [0x50, 0x4b, 0x06, 0x06];
251
+ const found = archive.some((_, i) => signature.every((b, k) => archive[i + k] === b));
252
+ expect(found).toBe(true);
253
+ });
254
+
255
+ test("streams lazily — a later entry is not opened before an earlier one is consumed", async () => {
256
+ // This is what keeps a 2,000-file archive from opening 2,000 connections at once.
257
+ const opened: string[] = [];
258
+ const lazy = (name: string): ZipEntry => ({
259
+ name,
260
+ open: async () => {
261
+ opened.push(name);
262
+ return new ReadableStream<Uint8Array>({
263
+ start(controller) {
264
+ controller.enqueue(bytesOf("x"));
265
+ controller.close();
266
+ },
267
+ });
268
+ },
269
+ });
270
+ const stream = zipStream([lazy("1.mp3"), lazy("2.mp3"), lazy("3.mp3")]);
271
+ const reader = stream.getReader();
272
+ await reader.read();
273
+ expect(opened).toEqual(["1.mp3"]);
274
+ await reader.cancel();
275
+ });
276
+ });
277
+
278
+ describe('folders and dates (desk)', () => {
279
+ test('🔴 an EMPTY folder survives extraction — a folder download that drops it is not a backup', async () => {
280
+ const archive = await collect(zipStream([{ name: 'Project/Empty', directory: true }, entryFrom('Project/a.md', 'a')]));
281
+ const run = await unzip(archive, ['-o', '-q', '-d', 'out']);
282
+ try {
283
+ expect(statSync(join(run.dir, 'out/Project/Empty')).isDirectory()).toBe(true);
284
+ expect(existsSync(join(run.dir, 'out/Project/a.md'))).toBe(true);
285
+ } finally {
286
+ run.cleanup();
287
+ }
288
+ });
289
+
290
+ test('a folder named twice is written once', async () => {
291
+ const listing = await unzipList(await collect(zipStream([{ name: 'A', directory: true }, { name: 'A/', directory: true }])));
292
+ expect(listing.match(/ A\//g)?.length).toBe(1);
293
+ });
294
+
295
+ test("a duplicate inside a dotted folder numbers the FILE, not the folder's dot", async () => {
296
+ const listing = await unzipList(await collect(zipStream([entryFrom('v1.2/notes', 'a'), entryFrom('v1.2/notes', 'b')])));
297
+ expect(listing).toContain('v1.2/notes (2)');
298
+ });
299
+
300
+ test('a file keeps its date', async () => {
301
+ const listing = await unzipList(await collect(zipStream([{ ...entryFrom('dated.txt', 'x'), modified: '2026-10-15T12:34:56Z' }])));
302
+ expect(listing).toMatch(/10-15-2026\s+12:34/);
303
+ });
304
+
305
+ test('dosDateTime: before 1980 or unreadable is the format zero, never a wrapped year', () => {
306
+ expect(dosDateTime('1970-01-01T00:00:00Z')).toEqual({ date: 33, time: 0 });
307
+ expect(dosDateTime('not a date')).toEqual({ date: 33, time: 0 });
308
+ expect(dosDateTime(null)).toEqual({ date: 33, time: 0 });
309
+ });
310
+
311
+ test('bytes in hand and an async iterable of entries both work', async () => {
312
+ async function* entries(): AsyncGenerator<ZipEntry> {
313
+ yield { name: 'inline.txt', open: async () => bytesOf('inline bytes') };
314
+ yield { name: 'empty.txt', open: async () => new Uint8Array(0) };
315
+ }
316
+ const archive = await collect(zipStream(entries()));
317
+ expect(await unzipRead(archive, 'inline.txt')).toBe('inline bytes');
318
+ expect(await unzipList(archive)).toContain('empty.txt');
319
+ });
320
+ });
@@ -0,0 +1,446 @@
1
+ /**
2
+ * A streaming ZIP writer — a download of many files as one, built as it is sent.
3
+ *
4
+ * Graduated from `apps/music/src/server/zip.ts` (2026-10-15, forge task desk-updates) when desk
5
+ * needed the same thing for "download this folder": two copies of a hand-rolled format writer
6
+ * is two places for the next bug. Folders (directory entries) and file dates arrived with it,
7
+ * because a folder download that loses its empty folders and stamps every file 1980 is not a
8
+ * backup.
9
+ *
10
+ * ── Why this is hand-written rather than a dependency ───────────────────────────────────
11
+ * Three properties are needed together, and the popular libraries give at most two:
12
+ *
13
+ * · **Store, never deflate.** Pictures, video, audio and `.xlsx` are already compressed.
14
+ * Deflating them spends CPU — on a Worker, the scarce thing — to produce a slightly LARGER
15
+ * file. Method 0 (store) is the correct encoding for them, and a few kilobytes of notes
16
+ * beside them are not worth a second code path.
17
+ * · **True streaming.** "Download this folder" can be thousands of files and many gigabytes.
18
+ * Nothing is buffered beyond one entry: bytes arrive from the store and leave for the
19
+ * browser in the same pass.
20
+ * · **ZIP64.** 4 GB is an entirely reachable selection, and the classic format's 32-bit size
21
+ * fields wrap silently at that point. A truncated archive that reports success is the worst
22
+ * available failure, so ZIP64 is written UNCONDITIONALLY rather than only when a threshold is
23
+ * crossed — a conditional format is a format that is only tested on the small path.
24
+ *
25
+ * ── Streaming means data descriptors ────────────────────────────────────────────────────
26
+ * A local file header must state the CRC-32 and the sizes, and none of those are known until
27
+ * the file's bytes have gone by. The format's answer is general-purpose flag bit 3: write zeros
28
+ * in the header, and follow the data with a DATA DESCRIPTOR carrying the real values. That is
29
+ * what makes a single forward pass possible, why this writer never seeks, and so why the
30
+ * response can be a `ReadableStream` and the server's memory stays flat whatever the size.
31
+ *
32
+ * ── The CPU it costs ────────────────────────────────────────────────────────────────────
33
+ * The CRC-32 is computed over every byte in the isolate: measured 2026-09-22 at ~280 MB of CPU
34
+ * per second (240 MB in 860 ms). A Worker's default budget is 30 s of CPU per request, so a
35
+ * caller on a Worker caps what one archive may hold — music at 4 GB / 2,000 files — and says so
36
+ * BEFORE the download starts, rather than letting the edge cut it off part-way.
37
+ */
38
+
39
+ // ── CRC-32 ──────────────────────────────────────────────────────────────────────────────
40
+
41
+ /** The standard IEEE 802.3 table, built once. */
42
+ const CRC_TABLE: Uint32Array = (() => {
43
+ const table = new Uint32Array(256);
44
+ for (let i = 0; i < 256; i++) {
45
+ let c = i;
46
+ for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
47
+ table[i] = c >>> 0;
48
+ }
49
+ return table;
50
+ })();
51
+
52
+ /** Incremental CRC-32, so it can be fed one chunk at a time as bytes stream past. */
53
+ export class Crc32 {
54
+ private crc = 0xffffffff;
55
+
56
+ update(bytes: Uint8Array): void {
57
+ let crc = this.crc;
58
+ for (let i = 0; i < bytes.length; i++) crc = CRC_TABLE[(crc ^ (bytes[i] as number)) & 0xff]! ^ (crc >>> 8);
59
+ this.crc = crc;
60
+ }
61
+
62
+ get value(): number {
63
+ return (this.crc ^ 0xffffffff) >>> 0;
64
+ }
65
+ }
66
+
67
+ // ── Little-endian writing ───────────────────────────────────────────────────────────────
68
+
69
+ class ByteWriter {
70
+ private parts: Uint8Array[] = [];
71
+
72
+ u16(n: number): this {
73
+ const b = new Uint8Array(2);
74
+ new DataView(b.buffer).setUint16(0, n, true);
75
+ this.parts.push(b);
76
+ return this;
77
+ }
78
+
79
+ u32(n: number): this {
80
+ const b = new Uint8Array(4);
81
+ new DataView(b.buffer).setUint32(0, n >>> 0, true);
82
+ this.parts.push(b);
83
+ return this;
84
+ }
85
+
86
+ /** 64-bit, via BigInt — a whole-library download can pass 2^32, and Number would quietly lose precision past 2^53. */
87
+ u64(n: number | bigint): this {
88
+ const b = new Uint8Array(8);
89
+ new DataView(b.buffer).setBigUint64(0, BigInt(n), true);
90
+ this.parts.push(b);
91
+ return this;
92
+ }
93
+
94
+ raw(bytes: Uint8Array): this {
95
+ this.parts.push(bytes);
96
+ return this;
97
+ }
98
+
99
+ done(): Uint8Array {
100
+ const total = this.parts.reduce((n, p) => n + p.length, 0);
101
+ const out = new Uint8Array(total);
102
+ let at = 0;
103
+ for (const part of this.parts) {
104
+ out.set(part, at);
105
+ at += part.length;
106
+ }
107
+ return out;
108
+ }
109
+ }
110
+
111
+ /** One member of the archive: a file whose bytes are opened lazily, or a folder. */
112
+ export type ZipEntry =
113
+ | {
114
+ /** Path inside the archive, `/`-separated. Sanitized by {@link zipStream}. */
115
+ name: string;
116
+ /**
117
+ * Opens the bytes — a stream, or bytes already in hand. Called only when the entry's turn
118
+ * comes, so a 2,000-file archive does not open 2,000 connections at once. Returning `null`
119
+ * (or throwing) skips the entry: a file that vanished must not abort the rest.
120
+ */
121
+ open: () => Promise<ReadableStream<Uint8Array> | Uint8Array | null>;
122
+ /** When the file last changed. Omitted: the archive says 1980-01-01, the format's zero. */
123
+ modified?: Date | string | null;
124
+ directory?: false;
125
+ }
126
+ | {
127
+ /** A folder — written so an EMPTY one survives the round trip. A trailing `/` is added. */
128
+ name: string;
129
+ directory: true;
130
+ modified?: Date | string | null;
131
+ }
132
+ | {
133
+ /**
134
+ * A member the CALLER could not read before naming it (desk reads a picture's bytes to
135
+ * decide its extension). Nothing is written under this name; it is listed, with `missing`
136
+ * as the reason, in the archive's `MISSING FILES.txt`.
137
+ */
138
+ name: string;
139
+ missing: string;
140
+ };
141
+
142
+ /**
143
+ * The last entry of an archive that left anything out: one line per missing path and why.
144
+ * A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
145
+ * not is a backup that fails on the day it is restored — so the skip is never silent.
146
+ */
147
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
148
+
149
+ const encoder = new TextEncoder();
150
+
151
+ /** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
152
+ * names, which every accented and non-Latin file name needs. */
153
+ const VERSION_ZIP64 = 45;
154
+ const FLAG_DATA_DESCRIPTOR = 0x08;
155
+ const FLAG_UTF8 = 0x0800;
156
+ const METHOD_STORE = 0;
157
+ /** MS-DOS directory attribute, in the low byte of the external attributes. */
158
+ const ATTR_DIRECTORY = 0x10;
159
+
160
+ /**
161
+ * Names inside a zip must not escape the extraction directory, and must not carry characters
162
+ * Windows refuses. `..` segments and absolute paths are the security half; the character set is
163
+ * the "it downloaded but won't extract" half. Each segment is capped, so a long name inside a
164
+ * deep folder keeps its folders.
165
+ */
166
+ export function safeEntryName(name: string, fallback = "file"): string {
167
+ // Control characters (< 0x20) and the set Windows refuses. Filtered by code point rather than
168
+ // by a regex class, because a literal control character inside a regex is unreadable and the
169
+ // linter is right to object to it.
170
+ const FORBIDDEN = new Set(["<", ">", ":", '"', "|", "?", "*"]);
171
+ const segments = name
172
+ .split(/[/\\]/)
173
+ .filter((seg) => seg !== "" && seg !== "." && seg !== "..")
174
+ .map((seg) => {
175
+ let out = "";
176
+ for (const ch of seg) out += ch.codePointAt(0)! < 0x20 || FORBIDDEN.has(ch) ? "_" : ch;
177
+ // Windows also refuses a segment ending in a dot or a space; trimmed rather than replaced.
178
+ return out.slice(0, 200).replace(/[. ]+$/, "") || fallback;
179
+ });
180
+ return segments.join("/") || fallback;
181
+ }
182
+
183
+ /** The MS-DOS date and time a zip header carries — local wall-clock, 2-second resolution, 1980 at the earliest. */
184
+ export function dosDateTime(when: Date | string | null | undefined): { date: number; time: number } {
185
+ const at = when ? new Date(when) : null;
186
+ if (!at || Number.isNaN(at.getTime()) || at.getUTCFullYear() < 1980) return { date: (0 << 9) | (1 << 5) | 1, time: 0 };
187
+ // UTC, not the server's zone: a Worker has none, and the same archive should not depend on where it was built.
188
+ const year = Math.min(at.getUTCFullYear(), 2107) - 1980;
189
+ return {
190
+ date: (year << 9) | ((at.getUTCMonth() + 1) << 5) | at.getUTCDate(),
191
+ time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | Math.floor(at.getUTCSeconds() / 2),
192
+ };
193
+ }
194
+
195
+ /**
196
+ * A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
197
+ * APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
198
+ * names equal under this fold are ONE file there — and the second would overwrite the first.
199
+ */
200
+ function foldName(name: string): string {
201
+ return name.normalize("NFC").toLowerCase();
202
+ }
203
+
204
+ /** A reason for a person to read: one line, short, and never a signed URL's query string. */
205
+ function readableReason(text: string): string {
206
+ return text
207
+ .replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
208
+ .replace(/\s+/g, " ")
209
+ .trim()
210
+ .slice(0, 200);
211
+ }
212
+
213
+ function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream<Uint8Array> {
214
+ if (!(body instanceof Uint8Array)) return body;
215
+ return new ReadableStream<Uint8Array>({
216
+ start(controller) {
217
+ if (body.byteLength > 0) controller.enqueue(body);
218
+ controller.close();
219
+ },
220
+ });
221
+ }
222
+
223
+ /**
224
+ * Build the archive as a stream.
225
+ *
226
+ * Duplicate file names are disambiguated rather than colliding — two different files that both
227
+ * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
228
+ * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
229
+ * once: its files already land in it. Names are compared the way the person's disk will compare
230
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
231
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
232
+ *
233
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
234
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
235
+ */
236
+ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array> {
237
+ interface Central {
238
+ name: Uint8Array;
239
+ crc: number;
240
+ size: bigint;
241
+ offset: bigint;
242
+ date: number;
243
+ time: number;
244
+ directory: boolean;
245
+ }
246
+
247
+ const central: Central[] = [];
248
+ let offset = 0n;
249
+ /** Every name written, as the extracting disk will compare it ({@link foldName}). */
250
+ const used = new Set<string>();
251
+ const missing: { name: string; reason: string }[] = [];
252
+ let reported = false;
253
+ const iterator: AsyncIterator<ZipEntry> | Iterator<ZipEntry> =
254
+ Symbol.asyncIterator in entries ? (entries as AsyncIterable<ZipEntry>)[Symbol.asyncIterator]() : (entries as Iterable<ZipEntry>)[Symbol.iterator]();
255
+
256
+ /** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
257
+ const freeName = (name: string): string => {
258
+ if (!used.has(foldName(name))) return name;
259
+ const slash = name.lastIndexOf("/");
260
+ const dot = name.lastIndexOf(".");
261
+ const cut = dot > slash + 1 ? dot : name.length;
262
+ const stem = name.slice(0, cut);
263
+ const ext = name.slice(cut);
264
+ let n = 2;
265
+ while (used.has(foldName(`${stem} (${n})${ext}`))) n++;
266
+ return `${stem} (${n})${ext}`;
267
+ };
268
+
269
+ /** One local header, the bytes, and the data descriptor; remembered for the central directory. */
270
+ const writeEntry = async (
271
+ controller: ReadableStreamDefaultController<Uint8Array>,
272
+ name: string,
273
+ body: ReadableStream<Uint8Array> | null,
274
+ modified: Date | string | null | undefined,
275
+ ): Promise<void> => {
276
+ const directory = body === null;
277
+ used.add(foldName(name));
278
+ const nameBytes = encoder.encode(name);
279
+ const { date, time } = dosDateTime(modified);
280
+
281
+ // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
282
+ const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
283
+ const header = new ByteWriter()
284
+ .u32(0x04034b50)
285
+ .u16(VERSION_ZIP64)
286
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
287
+ .u16(METHOD_STORE)
288
+ .u16(time)
289
+ .u16(date)
290
+ .u32(0) // crc, in the descriptor
291
+ .u32(0xffffffff) // compressed size → see ZIP64 extra
292
+ .u32(0xffffffff) // uncompressed size → see ZIP64 extra
293
+ .u16(nameBytes.length)
294
+ .u16(localExtra.length)
295
+ .raw(nameBytes)
296
+ .raw(localExtra)
297
+ .done();
298
+
299
+ controller.enqueue(header);
300
+ const entryOffset = offset;
301
+ offset += BigInt(header.length);
302
+
303
+ const crc = new Crc32();
304
+ let size = 0n;
305
+ if (body) {
306
+ const reader = body.getReader();
307
+ while (true) {
308
+ const { done, value } = await reader.read();
309
+ if (done) break;
310
+ crc.update(value);
311
+ size += BigInt(value.length);
312
+ controller.enqueue(value);
313
+ }
314
+ }
315
+ offset += size;
316
+
317
+ // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
318
+ const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
319
+ controller.enqueue(descriptor);
320
+ offset += BigInt(descriptor.length);
321
+
322
+ central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
323
+ };
324
+
325
+ return new ReadableStream<Uint8Array>({
326
+ async pull(controller) {
327
+ // One entry per pull keeps backpressure honest: the next file is not opened until the
328
+ // consumer has taken the previous one.
329
+ //
330
+ // 🔴 The loop is not decoration. A `pull` that returns having enqueued NOTHING is not
331
+ // guaranteed to be called again, so an entry that turns out to be unreadable would stall
332
+ // the entire archive — the consumer waits forever for bytes that will never come. Skipping
333
+ // therefore has to continue to the next entry within the SAME pull. (Measured in music:
334
+ // two tests hung for the full 5 s timeout before this loop existed.)
335
+ while (true) {
336
+ const next = await iterator.next();
337
+ if (next.done) break;
338
+ const entry = next.value;
339
+
340
+ if ("missing" in entry) {
341
+ missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
342
+ continue;
343
+ }
344
+ if (entry.directory === true) {
345
+ const name = `${safeEntryName(entry.name)}/`;
346
+ if (used.has(foldName(name))) continue;
347
+ await writeEntry(controller, name, null, entry.modified);
348
+ return;
349
+ }
350
+
351
+ const name = freeName(safeEntryName(entry.name));
352
+ let failure = "its bytes are missing from the store";
353
+ const opened = await entry.open().catch((error: unknown) => {
354
+ failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
355
+ return null;
356
+ });
357
+ // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
358
+ // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
359
+ // note on the loop above.
360
+ if (!opened) {
361
+ missing.push({ name, reason: failure });
362
+ continue;
363
+ }
364
+ await writeEntry(controller, name, asStream(opened), entry.modified);
365
+ return;
366
+ }
367
+
368
+ if (missing.length > 0 && !reported) {
369
+ reported = true;
370
+ const lines = missing.map((m) => `${m.name} — ${m.reason}`);
371
+ const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
372
+ const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
373
+ await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
374
+ return;
375
+ }
376
+
377
+ // Central directory, then the ZIP64 end records, then the classic end record.
378
+ const start = offset;
379
+ for (const item of central) {
380
+ const extra = new ByteWriter().u16(0x0001).u16(24).u64(item.size).u64(item.size).u64(item.offset).done();
381
+ const record = new ByteWriter()
382
+ .u32(0x02014b50)
383
+ .u16(VERSION_ZIP64) // version made by (MS-DOS attributes)
384
+ .u16(VERSION_ZIP64) // version needed
385
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
386
+ .u16(METHOD_STORE)
387
+ .u16(item.time)
388
+ .u16(item.date)
389
+ .u32(item.crc)
390
+ .u32(0xffffffff)
391
+ .u32(0xffffffff)
392
+ .u16(item.name.length)
393
+ .u16(extra.length)
394
+ .u16(0) // comment length
395
+ .u16(0) // disk number
396
+ .u16(0) // internal attrs
397
+ .u32(item.directory ? ATTR_DIRECTORY : 0) // external attrs
398
+ .u32(0xffffffff) // local header offset → ZIP64 extra
399
+ .raw(item.name)
400
+ .raw(extra)
401
+ .done();
402
+ controller.enqueue(record);
403
+ offset += BigInt(record.length);
404
+ }
405
+ const centralSize = offset - start;
406
+
407
+ const zip64End = new ByteWriter()
408
+ .u32(0x06064b50)
409
+ .u64(44) // size of this record, minus its first 12 bytes
410
+ .u16(VERSION_ZIP64)
411
+ .u16(VERSION_ZIP64)
412
+ .u32(0)
413
+ .u32(0)
414
+ .u64(central.length)
415
+ .u64(central.length)
416
+ .u64(centralSize)
417
+ .u64(start)
418
+ .done();
419
+ controller.enqueue(zip64End);
420
+
421
+ const locator = new ByteWriter()
422
+ .u32(0x07064b50)
423
+ .u32(0)
424
+ .u64(offset) // where the ZIP64 end record starts
425
+ .u32(1)
426
+ .done();
427
+ controller.enqueue(locator);
428
+
429
+ // The classic record still has to be there for every reader to find the archive; its
430
+ // 0xffff/0xffffffff sentinels say "look at the ZIP64 record instead".
431
+ const end = new ByteWriter()
432
+ .u32(0x06054b50)
433
+ .u16(0)
434
+ .u16(0)
435
+ .u16(central.length > 0xffff ? 0xffff : central.length)
436
+ .u16(central.length > 0xffff ? 0xffff : central.length)
437
+ .u32(0xffffffff)
438
+ .u32(0xffffffff)
439
+ .u16(0)
440
+ .done();
441
+ controller.enqueue(end);
442
+
443
+ controller.close();
444
+ },
445
+ });
446
+ }