cursedbelt-server 4.45.0 → 4.46.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -417,13 +417,16 @@ export function createBinaryStore(cfg) {
417
417
  * caller asked for `paced` and the budget allows, otherwise thrown as a typed error that
418
418
  * keeps the `[binary-store] <op> <key>: 429` message batch callers already match.
419
419
  */
420
- const sendRead = async (op, key, url, init, opts) => {
420
+ const sendRead = async (op, key,
421
+ /** Signs the URL. Called for EVERY attempt: a read token lives 60 s and a paced read may wait
422
+ * 120 s, so a URL signed once went out dead after the first long retry-after (forge task 3302). */
423
+ mint, init, opts) => {
421
424
  const budget = opts?.paced
422
425
  ? (typeof opts.paced === 'object' ? opts.paced.maxWaitMs : undefined) ?? BINARY_PACED_MAX_WAIT_MS
423
426
  : 0;
424
427
  let waited = 0;
425
428
  for (let attempt = 0;; attempt++) {
426
- const res = await send(url, init);
429
+ const res = await send(await mint(), init);
427
430
  if (res.status !== 429)
428
431
  return res;
429
432
  const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
@@ -548,7 +551,7 @@ export function createBinaryStore(cfg) {
548
551
  if (!opts?.fresh && knownAbsent(key))
549
552
  return null;
550
553
  countWireRead(key);
551
- const res = await sendRead('get', key, await internalMediaUrl(key), undefined, opts);
554
+ const res = await sendRead('get', key, () => internalMediaUrl(key), undefined, opts);
552
555
  if (res.status === 404) {
553
556
  rememberAbsent(key);
554
557
  return null;
@@ -624,7 +627,7 @@ export function createBinaryStore(cfg) {
624
627
  return null;
625
628
  }
626
629
  countWireRead(key);
627
- const res = await sendRead('meta', key, await metaUrl(key), undefined, opts);
630
+ const res = await sendRead('meta', key, () => metaUrl(key), undefined, opts);
628
631
  if (res.status === 404) {
629
632
  rememberAbsent(key);
630
633
  return null;
@@ -671,7 +674,7 @@ export function createBinaryStore(cfg) {
671
674
  throw why;
672
675
  }
673
676
  countWireRead(key);
674
- const res = await sendRead('stat', key, await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
677
+ const res = await sendRead('stat', key, () => internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
675
678
  if (res.status === 404) {
676
679
  rememberAbsent(key);
677
680
  return null;
@@ -59,6 +59,14 @@ export type ZipEntry = {
59
59
  name: string;
60
60
  directory: true;
61
61
  modified?: Date | string | null;
62
+ } | {
63
+ /**
64
+ * A member the CALLER could not read before naming it (desk reads a picture's bytes to
65
+ * decide its extension). Nothing is written under this name; it is listed, with `missing`
66
+ * as the reason, in the archive's `MISSING FILES.txt`.
67
+ */
68
+ name: string;
69
+ missing: string;
62
70
  };
63
71
  /**
64
72
  * Names inside a zip must not escape the extraction directory, and must not carry characters
@@ -78,6 +86,11 @@ export declare function dosDateTime(when: Date | string | null | undefined): {
78
86
  * Duplicate file names are disambiguated rather than colliding — two different files that both
79
87
  * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
80
88
  * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
81
- * once: its files already land in it.
89
+ * once: its files already land in it. Names are compared the way the person's disk will compare
90
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
91
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
92
+ *
93
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
94
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
82
95
  */
83
96
  export declare function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array>;
@@ -97,6 +97,12 @@ class ByteWriter {
97
97
  return out;
98
98
  }
99
99
  }
100
+ /**
101
+ * The last entry of an archive that left anything out: one line per missing path and why.
102
+ * A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
103
+ * not is a backup that fails on the day it is restored — so the skip is never silent.
104
+ */
105
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
100
106
  const encoder = new TextEncoder();
101
107
  /** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
102
108
  * names, which every accented and non-Latin file name needs. */
@@ -141,6 +147,22 @@ export function dosDateTime(when) {
141
147
  time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | Math.floor(at.getUTCSeconds() / 2),
142
148
  };
143
149
  }
150
+ /**
151
+ * A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
152
+ * APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
153
+ * names equal under this fold are ONE file there — and the second would overwrite the first.
154
+ */
155
+ function foldName(name) {
156
+ return name.normalize("NFC").toLowerCase();
157
+ }
158
+ /** A reason for a person to read: one line, short, and never a signed URL's query string. */
159
+ function readableReason(text) {
160
+ return text
161
+ .replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
162
+ .replace(/\s+/g, " ")
163
+ .trim()
164
+ .slice(0, 200);
165
+ }
144
166
  function asStream(body) {
145
167
  if (!(body instanceof Uint8Array))
146
168
  return body;
@@ -158,13 +180,81 @@ function asStream(body) {
158
180
  * Duplicate file names are disambiguated rather than colliding — two different files that both
159
181
  * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
160
182
  * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
161
- * once: its files already land in it.
183
+ * once: its files already land in it. Names are compared the way the person's disk will compare
184
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
185
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
186
+ *
187
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
188
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
162
189
  */
163
190
  export function zipStream(entries) {
164
191
  const central = [];
165
192
  let offset = 0n;
193
+ /** Every name written, as the extracting disk will compare it ({@link foldName}). */
166
194
  const used = new Set();
195
+ const missing = [];
196
+ let reported = false;
167
197
  const iterator = Symbol.asyncIterator in entries ? entries[Symbol.asyncIterator]() : entries[Symbol.iterator]();
198
+ /** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
199
+ const freeName = (name) => {
200
+ if (!used.has(foldName(name)))
201
+ return name;
202
+ const slash = name.lastIndexOf("/");
203
+ const dot = name.lastIndexOf(".");
204
+ const cut = dot > slash + 1 ? dot : name.length;
205
+ const stem = name.slice(0, cut);
206
+ const ext = name.slice(cut);
207
+ let n = 2;
208
+ while (used.has(foldName(`${stem} (${n})${ext}`)))
209
+ n++;
210
+ return `${stem} (${n})${ext}`;
211
+ };
212
+ /** One local header, the bytes, and the data descriptor; remembered for the central directory. */
213
+ const writeEntry = async (controller, name, body, modified) => {
214
+ const directory = body === null;
215
+ used.add(foldName(name));
216
+ const nameBytes = encoder.encode(name);
217
+ const { date, time } = dosDateTime(modified);
218
+ // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
219
+ const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
220
+ const header = new ByteWriter()
221
+ .u32(0x04034b50)
222
+ .u16(VERSION_ZIP64)
223
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
224
+ .u16(METHOD_STORE)
225
+ .u16(time)
226
+ .u16(date)
227
+ .u32(0) // crc, in the descriptor
228
+ .u32(0xffffffff) // compressed size → see ZIP64 extra
229
+ .u32(0xffffffff) // uncompressed size → see ZIP64 extra
230
+ .u16(nameBytes.length)
231
+ .u16(localExtra.length)
232
+ .raw(nameBytes)
233
+ .raw(localExtra)
234
+ .done();
235
+ controller.enqueue(header);
236
+ const entryOffset = offset;
237
+ offset += BigInt(header.length);
238
+ const crc = new Crc32();
239
+ let size = 0n;
240
+ if (body) {
241
+ const reader = body.getReader();
242
+ while (true) {
243
+ const { done, value } = await reader.read();
244
+ if (done)
245
+ break;
246
+ crc.update(value);
247
+ size += BigInt(value.length);
248
+ controller.enqueue(value);
249
+ }
250
+ }
251
+ offset += size;
252
+ // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
253
+ const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
254
+ controller.enqueue(descriptor);
255
+ offset += BigInt(descriptor.length);
256
+ central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
257
+ };
168
258
  return new ReadableStream({
169
259
  async pull(controller) {
170
260
  // One entry per pull keeps backpressure honest: the next file is not opened until the
@@ -180,76 +270,39 @@ export function zipStream(entries) {
180
270
  if (next.done)
181
271
  break;
182
272
  const entry = next.value;
183
- const directory = entry.directory === true;
184
- let name = safeEntryName(entry.name);
185
- if (directory) {
186
- name = `${name}/`;
187
- if (used.has(name))
188
- continue;
189
- }
190
- else if (used.has(name)) {
191
- const slash = name.lastIndexOf("/");
192
- const dot = name.lastIndexOf(".");
193
- const cut = dot > slash + 1 ? dot : name.length;
194
- const stem = name.slice(0, cut);
195
- const ext = name.slice(cut);
196
- let n = 2;
197
- while (used.has(`${stem} (${n})${ext}`))
198
- n++;
199
- name = `${stem} (${n})${ext}`;
273
+ if ("missing" in entry) {
274
+ missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
275
+ continue;
200
276
  }
201
- let body = null;
202
- if (!directory) {
203
- const opened = await entry.open().catch(() => null);
204
- // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
205
- // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
206
- // note on the loop above.
207
- if (!opened)
277
+ if (entry.directory === true) {
278
+ const name = `${safeEntryName(entry.name)}/`;
279
+ if (used.has(foldName(name)))
208
280
  continue;
209
- body = asStream(opened);
281
+ await writeEntry(controller, name, null, entry.modified);
282
+ return;
210
283
  }
211
- used.add(name);
212
- const nameBytes = encoder.encode(name);
213
- const { date, time } = dosDateTime(entry.modified);
214
- // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
215
- const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
216
- const header = new ByteWriter()
217
- .u32(0x04034b50)
218
- .u16(VERSION_ZIP64)
219
- .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
220
- .u16(METHOD_STORE)
221
- .u16(time)
222
- .u16(date)
223
- .u32(0) // crc, in the descriptor
224
- .u32(0xffffffff) // compressed size → see ZIP64 extra
225
- .u32(0xffffffff) // uncompressed size → see ZIP64 extra
226
- .u16(nameBytes.length)
227
- .u16(localExtra.length)
228
- .raw(nameBytes)
229
- .raw(localExtra)
230
- .done();
231
- controller.enqueue(header);
232
- const entryOffset = offset;
233
- offset += BigInt(header.length);
234
- const crc = new Crc32();
235
- let size = 0n;
236
- if (body) {
237
- const reader = body.getReader();
238
- while (true) {
239
- const { done, value } = await reader.read();
240
- if (done)
241
- break;
242
- crc.update(value);
243
- size += BigInt(value.length);
244
- controller.enqueue(value);
245
- }
284
+ const name = freeName(safeEntryName(entry.name));
285
+ let failure = "its bytes are missing from the store";
286
+ const opened = await entry.open().catch((error) => {
287
+ failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
288
+ return null;
289
+ });
290
+ // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
291
+ // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
292
+ // note on the loop above.
293
+ if (!opened) {
294
+ missing.push({ name, reason: failure });
295
+ continue;
246
296
  }
247
- offset += size;
248
- // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
249
- const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
250
- controller.enqueue(descriptor);
251
- offset += BigInt(descriptor.length);
252
- central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
297
+ await writeEntry(controller, name, asStream(opened), entry.modified);
298
+ return;
299
+ }
300
+ if (missing.length > 0 && !reported) {
301
+ reported = true;
302
+ const lines = missing.map((m) => `${m.name} — ${m.reason}`);
303
+ const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
304
+ const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
305
+ await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
253
306
  return;
254
307
  }
255
308
  // Central directory, then the ZIP64 end records, then the classic end record.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "cursedbelt-server",
3
- "version": "4.45.0",
3
+ "version": "4.46.1",
4
4
  "license": "ISC",
5
5
  "type": "module",
6
6
  "description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
@@ -26,7 +26,7 @@
26
26
  * server; the RSA keypair is generated in-process and the signature is verified with
27
27
  * `node:crypto`, which is what makes the token assertions real rather than shape-deep.
28
28
  */
29
- import { afterAll, afterEach, beforeEach, describe, expect, it } from 'bun:test';
29
+ import { afterAll, afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test';
30
30
  import { generateKeyPairSync, verify as cryptoVerify } from 'node:crypto';
31
31
  import {
32
32
  AUTO_CHUNK_THRESHOLD_BYTES,
@@ -1056,6 +1056,29 @@ describe('🔴 a read the store PACES is waited out when asked, typed when not (
1056
1056
  expect(calls).toHaveLength(4);
1057
1057
  });
1058
1058
 
1059
+ it('🔴 every attempt goes with a token signed for IT — a 60 s token outlives no 120 s wait (forge task 3302)', async () => {
1060
+ // The URL was signed once, before the first attempt, so a paced read that waited out 30 s
1061
+ // refusals sent a dead token on its third try, and binary-server answered 403 — not 429.
1062
+ try {
1063
+ let n = 0;
1064
+ reply = () => (n++ < 3 ? limited('30') : ok(new Uint8Array([7, 8])));
1065
+ const store = makeStore({
1066
+ sleep: (ms) => {
1067
+ setSystemTime(new Date(Date.now() + ms));
1068
+ return Promise.resolve();
1069
+ },
1070
+ });
1071
+ await store.get('parts/0', { paced: true });
1072
+ expect(calls).toHaveLength(4);
1073
+ // The last attempt went 90 s after the first: its token must still have time left.
1074
+ const token = new URL(String(calls.at(-1)?.url)).searchParams.get('token') ?? '';
1075
+ const { exp } = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString('utf8')) as { exp: number };
1076
+ expect(exp * 1000).toBeGreaterThan(Date.now());
1077
+ } finally {
1078
+ setSystemTime();
1079
+ }
1080
+ });
1081
+
1059
1082
  it('without paced, a 429 is the typed error carrying retryAfterSec — and the old message', async () => {
1060
1083
  const { store, slept } = pacedStore(1, '7');
1061
1084
  const error = await store.get('parts/0').catch((e: unknown) => e);
@@ -728,7 +728,9 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
728
728
  const sendRead = async (
729
729
  op: string,
730
730
  key: string,
731
- url: string,
731
+ /** Signs the URL. Called for EVERY attempt: a read token lives 60 s and a paced read may wait
732
+ * 120 s, so a URL signed once went out dead after the first long retry-after (forge task 3302). */
733
+ mint: () => Promise<string>,
732
734
  init: RequestInit | undefined,
733
735
  opts: ReadOptions | undefined,
734
736
  ): Promise<Response> => {
@@ -737,7 +739,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
737
739
  : 0;
738
740
  let waited = 0;
739
741
  for (let attempt = 0; ; attempt++) {
740
- const res = await send(url, init);
742
+ const res = await send(await mint(), init);
741
743
  if (res.status !== 429) return res;
742
744
  const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
743
745
  const ms = retryAfterSec * 1000;
@@ -875,7 +877,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
875
877
  // short-circuit here — we want the bytes, and only the store has them.
876
878
  if (!opts?.fresh && knownAbsent(key)) return null;
877
879
  countWireRead(key);
878
- const res = await sendRead('get', key, await internalMediaUrl(key), undefined, opts);
880
+ const res = await sendRead('get', key, () => internalMediaUrl(key), undefined, opts);
879
881
  if (res.status === 404) {
880
882
  rememberAbsent(key);
881
883
  return null;
@@ -952,7 +954,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
952
954
  if (knownAbsent(key)) return null;
953
955
  }
954
956
  countWireRead(key);
955
- const res = await sendRead('meta', key, await metaUrl(key), undefined, opts);
957
+ const res = await sendRead('meta', key, () => metaUrl(key), undefined, opts);
956
958
  if (res.status === 404) {
957
959
  rememberAbsent(key);
958
960
  return null;
@@ -995,7 +997,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
995
997
  if (why instanceof BinaryStoreRateLimitedError) throw why;
996
998
  }
997
999
  countWireRead(key);
998
- const res = await sendRead('stat', key, await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
1000
+ const res = await sendRead('stat', key, () => internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
999
1001
  if (res.status === 404) {
1000
1002
  rememberAbsent(key);
1001
1003
  return null;
@@ -13,6 +13,9 @@ import { Crc32, dosDateTime, safeEntryName, type ZipEntry, zipStream } from './z
13
13
 
14
14
  const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text);
15
15
 
16
+ /** The report entry a skipped member lands in — spelt out, since a person reads it by this name. */
17
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
18
+
16
19
  const entryFrom = (name: string, body: string): ZipEntry => ({
17
20
  name,
18
21
  open: async () =>
@@ -145,6 +148,59 @@ describe("zipStream", () => {
145
148
  expect(listing).not.toContain("gone.mp3");
146
149
  });
147
150
 
151
+ test("🔴 names differing only in case are TWO files — the later is numbered, never overwrites on a Mac", async () => {
152
+ // Desk lets a folder hold notes "Todo" and "todo". macOS (APFS) and Windows (NTFS) treat
153
+ // those as one file, so writing both names verbatim loses one on unzip — silently, or behind
154
+ // a "replace?" prompt the person did not expect.
155
+ const archive = await collect(zipStream([entryFrom("F/Todo.md", "upper"), entryFrom("f/todo.md", "lower"), entryFrom("F/TODO (2).md", "third")]));
156
+ const listing = await unzipList(archive);
157
+ expect(listing).toContain("F/Todo.md");
158
+ expect(listing).toContain("f/todo (2).md");
159
+ expect(listing).toContain("F/TODO (2) (2).md");
160
+ expect(await unzipRead(archive, "f/todo (2).md")).toBe("lower");
161
+ });
162
+
163
+ test("🔴 one name spelt with a composed and a decomposed accent is ONE name to APFS — numbered too", async () => {
164
+ const composed = "Caf\u00e9.txt";
165
+ const decomposed = "Cafe\u0301.txt";
166
+ expect(composed).not.toBe(decomposed);
167
+ const listing = await unzipList(await collect(zipStream([entryFrom(composed, "a"), entryFrom(decomposed, "b")])));
168
+ expect(listing).toContain("(2).txt");
169
+ });
170
+
171
+ test("a folder named twice in different case is written once", async () => {
172
+ const listing = await unzipList(await collect(zipStream([{ name: "Photos", directory: true }, { name: "photos", directory: true }])));
173
+ expect(listing.match(/ photos\//gi)?.length).toBe(1);
174
+ });
175
+
176
+ test(`🔴 what was skipped is LISTED in a final ${MISSING_FILES_NAME}, with the reason — a skip is never silent`, async () => {
177
+ const archive = await collect(
178
+ zipStream([
179
+ entryFrom("good.mp3", "here"),
180
+ { name: "gone.mp3", open: async () => null },
181
+ {
182
+ name: "Folder/broken.jpg",
183
+ open: async () => {
184
+ throw new Error("store answered 503 https://bytes.example/x?sig=SECRET");
185
+ },
186
+ },
187
+ { name: "Notes/unread.png", missing: "could not be read (timeout at https://store.example/b?token=SECRET2)" },
188
+ ]),
189
+ );
190
+ const report = await unzipRead(archive, MISSING_FILES_NAME);
191
+ expect(report).toContain("3 files were in the selection but could not be read");
192
+ expect(report).toContain("gone.mp3 — its bytes are missing from the store");
193
+ expect(report).toContain("Folder/broken.jpg — could not be read (store answered 503 https://bytes.example/x)");
194
+ expect(report).toContain("Notes/unread.png — could not be read (timeout at https://store.example/b)");
195
+ expect(report).not.toContain("SECRET2");
196
+ expect(report).not.toContain("SECRET");
197
+ expect(await unzipRead(archive, "good.mp3")).toBe("here");
198
+ });
199
+
200
+ test(`a complete archive carries no ${MISSING_FILES_NAME}`, async () => {
201
+ expect(await unzipList(await collect(zipStream([entryFrom("a.txt", "a")])))).not.toContain(MISSING_FILES_NAME);
202
+ });
203
+
148
204
  test("an entry whose open() THROWS is skipped rather than killing the archive", async () => {
149
205
  const archive = await collect(
150
206
  zipStream([
@@ -128,8 +128,24 @@ export type ZipEntry =
128
128
  name: string;
129
129
  directory: true;
130
130
  modified?: Date | string | null;
131
+ }
132
+ | {
133
+ /**
134
+ * A member the CALLER could not read before naming it (desk reads a picture's bytes to
135
+ * decide its extension). Nothing is written under this name; it is listed, with `missing`
136
+ * as the reason, in the archive's `MISSING FILES.txt`.
137
+ */
138
+ name: string;
139
+ missing: string;
131
140
  };
132
141
 
142
+ /**
143
+ * The last entry of an archive that left anything out: one line per missing path and why.
144
+ * A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
145
+ * not is a backup that fails on the day it is restored — so the skip is never silent.
146
+ */
147
+ const MISSING_FILES_NAME = "MISSING FILES.txt";
148
+
133
149
  const encoder = new TextEncoder();
134
150
 
135
151
  /** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
@@ -176,6 +192,24 @@ export function dosDateTime(when: Date | string | null | undefined): { date: num
176
192
  };
177
193
  }
178
194
 
195
+ /**
196
+ * A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
197
+ * APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
198
+ * names equal under this fold are ONE file there — and the second would overwrite the first.
199
+ */
200
+ function foldName(name: string): string {
201
+ return name.normalize("NFC").toLowerCase();
202
+ }
203
+
204
+ /** A reason for a person to read: one line, short, and never a signed URL's query string. */
205
+ function readableReason(text: string): string {
206
+ return text
207
+ .replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
208
+ .replace(/\s+/g, " ")
209
+ .trim()
210
+ .slice(0, 200);
211
+ }
212
+
179
213
  function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream<Uint8Array> {
180
214
  if (!(body instanceof Uint8Array)) return body;
181
215
  return new ReadableStream<Uint8Array>({
@@ -192,7 +226,12 @@ function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream
192
226
  * Duplicate file names are disambiguated rather than colliding — two different files that both
193
227
  * want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
194
228
  * identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
195
- * once: its files already land in it.
229
+ * once: its files already land in it. Names are compared the way the person's disk will compare
230
+ * them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
231
+ * `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
232
+ *
233
+ * A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
234
+ * as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
196
235
  */
197
236
  export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array> {
198
237
  interface Central {
@@ -207,10 +246,82 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
207
246
 
208
247
  const central: Central[] = [];
209
248
  let offset = 0n;
249
+ /** Every name written, as the extracting disk will compare it ({@link foldName}). */
210
250
  const used = new Set<string>();
251
+ const missing: { name: string; reason: string }[] = [];
252
+ let reported = false;
211
253
  const iterator: AsyncIterator<ZipEntry> | Iterator<ZipEntry> =
212
254
  Symbol.asyncIterator in entries ? (entries as AsyncIterable<ZipEntry>)[Symbol.asyncIterator]() : (entries as Iterable<ZipEntry>)[Symbol.iterator]();
213
255
 
256
+ /** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
257
+ const freeName = (name: string): string => {
258
+ if (!used.has(foldName(name))) return name;
259
+ const slash = name.lastIndexOf("/");
260
+ const dot = name.lastIndexOf(".");
261
+ const cut = dot > slash + 1 ? dot : name.length;
262
+ const stem = name.slice(0, cut);
263
+ const ext = name.slice(cut);
264
+ let n = 2;
265
+ while (used.has(foldName(`${stem} (${n})${ext}`))) n++;
266
+ return `${stem} (${n})${ext}`;
267
+ };
268
+
269
+ /** One local header, the bytes, and the data descriptor; remembered for the central directory. */
270
+ const writeEntry = async (
271
+ controller: ReadableStreamDefaultController<Uint8Array>,
272
+ name: string,
273
+ body: ReadableStream<Uint8Array> | null,
274
+ modified: Date | string | null | undefined,
275
+ ): Promise<void> => {
276
+ const directory = body === null;
277
+ used.add(foldName(name));
278
+ const nameBytes = encoder.encode(name);
279
+ const { date, time } = dosDateTime(modified);
280
+
281
+ // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
282
+ const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
283
+ const header = new ByteWriter()
284
+ .u32(0x04034b50)
285
+ .u16(VERSION_ZIP64)
286
+ .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
287
+ .u16(METHOD_STORE)
288
+ .u16(time)
289
+ .u16(date)
290
+ .u32(0) // crc, in the descriptor
291
+ .u32(0xffffffff) // compressed size → see ZIP64 extra
292
+ .u32(0xffffffff) // uncompressed size → see ZIP64 extra
293
+ .u16(nameBytes.length)
294
+ .u16(localExtra.length)
295
+ .raw(nameBytes)
296
+ .raw(localExtra)
297
+ .done();
298
+
299
+ controller.enqueue(header);
300
+ const entryOffset = offset;
301
+ offset += BigInt(header.length);
302
+
303
+ const crc = new Crc32();
304
+ let size = 0n;
305
+ if (body) {
306
+ const reader = body.getReader();
307
+ while (true) {
308
+ const { done, value } = await reader.read();
309
+ if (done) break;
310
+ crc.update(value);
311
+ size += BigInt(value.length);
312
+ controller.enqueue(value);
313
+ }
314
+ }
315
+ offset += size;
316
+
317
+ // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
318
+ const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
319
+ controller.enqueue(descriptor);
320
+ offset += BigInt(descriptor.length);
321
+
322
+ central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
323
+ };
324
+
214
325
  return new ReadableStream<Uint8Array>({
215
326
  async pull(controller) {
216
327
  // One entry per pull keeps backpressure honest: the next file is not opened until the
@@ -226,77 +337,40 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
226
337
  if (next.done) break;
227
338
  const entry = next.value;
228
339
 
229
- const directory = entry.directory === true;
230
- let name = safeEntryName(entry.name);
231
- if (directory) {
232
- name = `${name}/`;
233
- if (used.has(name)) continue;
234
- } else if (used.has(name)) {
235
- const slash = name.lastIndexOf("/");
236
- const dot = name.lastIndexOf(".");
237
- const cut = dot > slash + 1 ? dot : name.length;
238
- const stem = name.slice(0, cut);
239
- const ext = name.slice(cut);
240
- let n = 2;
241
- while (used.has(`${stem} (${n})${ext}`)) n++;
242
- name = `${stem} (${n})${ext}`;
340
+ if ("missing" in entry) {
341
+ missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
342
+ continue;
243
343
  }
244
-
245
- let body: ReadableStream<Uint8Array> | null = null;
246
- if (!directory) {
247
- const opened = await entry.open().catch(() => null);
248
- // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
249
- // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
250
- // note on the loop above.
251
- if (!opened) continue;
252
- body = asStream(opened);
344
+ if (entry.directory === true) {
345
+ const name = `${safeEntryName(entry.name)}/`;
346
+ if (used.has(foldName(name))) continue;
347
+ await writeEntry(controller, name, null, entry.modified);
348
+ return;
253
349
  }
254
- used.add(name);
255
- const nameBytes = encoder.encode(name);
256
- const { date, time } = dosDateTime(entry.modified);
257
-
258
- // ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
259
- const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
260
- const header = new ByteWriter()
261
- .u32(0x04034b50)
262
- .u16(VERSION_ZIP64)
263
- .u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
264
- .u16(METHOD_STORE)
265
- .u16(time)
266
- .u16(date)
267
- .u32(0) // crc, in the descriptor
268
- .u32(0xffffffff) // compressed size → see ZIP64 extra
269
- .u32(0xffffffff) // uncompressed size → see ZIP64 extra
270
- .u16(nameBytes.length)
271
- .u16(localExtra.length)
272
- .raw(nameBytes)
273
- .raw(localExtra)
274
- .done();
275
350
 
276
- controller.enqueue(header);
277
- const entryOffset = offset;
278
- offset += BigInt(header.length);
279
-
280
- const crc = new Crc32();
281
- let size = 0n;
282
- if (body) {
283
- const reader = body.getReader();
284
- while (true) {
285
- const { done, value } = await reader.read();
286
- if (done) break;
287
- crc.update(value);
288
- size += BigInt(value.length);
289
- controller.enqueue(value);
290
- }
351
+ const name = freeName(safeEntryName(entry.name));
352
+ let failure = "its bytes are missing from the store";
353
+ const opened = await entry.open().catch((error: unknown) => {
354
+ failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
355
+ return null;
356
+ });
357
+ // A file whose bytes have gone is skipped, not fatal: one missing file must not abort
358
+ // a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
359
+ // note on the loop above.
360
+ if (!opened) {
361
+ missing.push({ name, reason: failure });
362
+ continue;
291
363
  }
292
- offset += size;
293
-
294
- // ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
295
- const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
296
- controller.enqueue(descriptor);
297
- offset += BigInt(descriptor.length);
364
+ await writeEntry(controller, name, asStream(opened), entry.modified);
365
+ return;
366
+ }
298
367
 
299
- central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
368
+ if (missing.length > 0 && !reported) {
369
+ reported = true;
370
+ const lines = missing.map((m) => `${m.name} — ${m.reason}`);
371
+ const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
372
+ const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
373
+ await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
300
374
  return;
301
375
  }
302
376