cursedbelt-server 4.45.0 → 4.46.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/storage/binaryStore.js +8 -5
- package/dist/server/zipStream.d.ts +14 -1
- package/dist/server/zipStream.js +120 -67
- package/package.json +1 -1
- package/src/server/storage/binaryStore.spec.ts +24 -1
- package/src/server/storage/binaryStore.ts +7 -5
- package/src/server/zipStream.spec.ts +56 -0
- package/src/server/zipStream.ts +141 -67
|
@@ -417,13 +417,16 @@ export function createBinaryStore(cfg) {
|
|
|
417
417
|
* caller asked for `paced` and the budget allows, otherwise thrown as a typed error that
|
|
418
418
|
* keeps the `[binary-store] <op> <key>: 429` message batch callers already match.
|
|
419
419
|
*/
|
|
420
|
-
const sendRead = async (op, key,
|
|
420
|
+
const sendRead = async (op, key,
|
|
421
|
+
/** Signs the URL. Called for EVERY attempt: a read token lives 60 s and a paced read may wait
|
|
422
|
+
* 120 s, so a URL signed once went out dead after the first long retry-after (forge task 3302). */
|
|
423
|
+
mint, init, opts) => {
|
|
421
424
|
const budget = opts?.paced
|
|
422
425
|
? (typeof opts.paced === 'object' ? opts.paced.maxWaitMs : undefined) ?? BINARY_PACED_MAX_WAIT_MS
|
|
423
426
|
: 0;
|
|
424
427
|
let waited = 0;
|
|
425
428
|
for (let attempt = 0;; attempt++) {
|
|
426
|
-
const res = await send(
|
|
429
|
+
const res = await send(await mint(), init);
|
|
427
430
|
if (res.status !== 429)
|
|
428
431
|
return res;
|
|
429
432
|
const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
|
|
@@ -548,7 +551,7 @@ export function createBinaryStore(cfg) {
|
|
|
548
551
|
if (!opts?.fresh && knownAbsent(key))
|
|
549
552
|
return null;
|
|
550
553
|
countWireRead(key);
|
|
551
|
-
const res = await sendRead('get', key,
|
|
554
|
+
const res = await sendRead('get', key, () => internalMediaUrl(key), undefined, opts);
|
|
552
555
|
if (res.status === 404) {
|
|
553
556
|
rememberAbsent(key);
|
|
554
557
|
return null;
|
|
@@ -624,7 +627,7 @@ export function createBinaryStore(cfg) {
|
|
|
624
627
|
return null;
|
|
625
628
|
}
|
|
626
629
|
countWireRead(key);
|
|
627
|
-
const res = await sendRead('meta', key,
|
|
630
|
+
const res = await sendRead('meta', key, () => metaUrl(key), undefined, opts);
|
|
628
631
|
if (res.status === 404) {
|
|
629
632
|
rememberAbsent(key);
|
|
630
633
|
return null;
|
|
@@ -671,7 +674,7 @@ export function createBinaryStore(cfg) {
|
|
|
671
674
|
throw why;
|
|
672
675
|
}
|
|
673
676
|
countWireRead(key);
|
|
674
|
-
const res = await sendRead('stat', key,
|
|
677
|
+
const res = await sendRead('stat', key, () => internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
|
|
675
678
|
if (res.status === 404) {
|
|
676
679
|
rememberAbsent(key);
|
|
677
680
|
return null;
|
|
@@ -59,6 +59,14 @@ export type ZipEntry = {
|
|
|
59
59
|
name: string;
|
|
60
60
|
directory: true;
|
|
61
61
|
modified?: Date | string | null;
|
|
62
|
+
} | {
|
|
63
|
+
/**
|
|
64
|
+
* A member the CALLER could not read before naming it (desk reads a picture's bytes to
|
|
65
|
+
* decide its extension). Nothing is written under this name; it is listed, with `missing`
|
|
66
|
+
* as the reason, in the archive's `MISSING FILES.txt`.
|
|
67
|
+
*/
|
|
68
|
+
name: string;
|
|
69
|
+
missing: string;
|
|
62
70
|
};
|
|
63
71
|
/**
|
|
64
72
|
* Names inside a zip must not escape the extraction directory, and must not carry characters
|
|
@@ -78,6 +86,11 @@ export declare function dosDateTime(when: Date | string | null | undefined): {
|
|
|
78
86
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
79
87
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
80
88
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
81
|
-
* once: its files already land in it.
|
|
89
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
90
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
91
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
92
|
+
*
|
|
93
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
94
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
82
95
|
*/
|
|
83
96
|
export declare function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array>;
|
package/dist/server/zipStream.js
CHANGED
|
@@ -97,6 +97,12 @@ class ByteWriter {
|
|
|
97
97
|
return out;
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* The last entry of an archive that left anything out: one line per missing path and why.
|
|
102
|
+
* A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
|
|
103
|
+
* not is a backup that fails on the day it is restored — so the skip is never silent.
|
|
104
|
+
*/
|
|
105
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
100
106
|
const encoder = new TextEncoder();
|
|
101
107
|
/** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
|
|
102
108
|
* names, which every accented and non-Latin file name needs. */
|
|
@@ -141,6 +147,22 @@ export function dosDateTime(when) {
|
|
|
141
147
|
time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | Math.floor(at.getUTCSeconds() / 2),
|
|
142
148
|
};
|
|
143
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
|
|
152
|
+
* APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
|
|
153
|
+
* names equal under this fold are ONE file there — and the second would overwrite the first.
|
|
154
|
+
*/
|
|
155
|
+
function foldName(name) {
|
|
156
|
+
return name.normalize("NFC").toLowerCase();
|
|
157
|
+
}
|
|
158
|
+
/** A reason for a person to read: one line, short, and never a signed URL's query string. */
|
|
159
|
+
function readableReason(text) {
|
|
160
|
+
return text
|
|
161
|
+
.replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
|
|
162
|
+
.replace(/\s+/g, " ")
|
|
163
|
+
.trim()
|
|
164
|
+
.slice(0, 200);
|
|
165
|
+
}
|
|
144
166
|
function asStream(body) {
|
|
145
167
|
if (!(body instanceof Uint8Array))
|
|
146
168
|
return body;
|
|
@@ -158,13 +180,81 @@ function asStream(body) {
|
|
|
158
180
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
159
181
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
160
182
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
161
|
-
* once: its files already land in it.
|
|
183
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
184
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
185
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
186
|
+
*
|
|
187
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
188
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
162
189
|
*/
|
|
163
190
|
export function zipStream(entries) {
|
|
164
191
|
const central = [];
|
|
165
192
|
let offset = 0n;
|
|
193
|
+
/** Every name written, as the extracting disk will compare it ({@link foldName}). */
|
|
166
194
|
const used = new Set();
|
|
195
|
+
const missing = [];
|
|
196
|
+
let reported = false;
|
|
167
197
|
const iterator = Symbol.asyncIterator in entries ? entries[Symbol.asyncIterator]() : entries[Symbol.iterator]();
|
|
198
|
+
/** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
|
|
199
|
+
const freeName = (name) => {
|
|
200
|
+
if (!used.has(foldName(name)))
|
|
201
|
+
return name;
|
|
202
|
+
const slash = name.lastIndexOf("/");
|
|
203
|
+
const dot = name.lastIndexOf(".");
|
|
204
|
+
const cut = dot > slash + 1 ? dot : name.length;
|
|
205
|
+
const stem = name.slice(0, cut);
|
|
206
|
+
const ext = name.slice(cut);
|
|
207
|
+
let n = 2;
|
|
208
|
+
while (used.has(foldName(`${stem} (${n})${ext}`)))
|
|
209
|
+
n++;
|
|
210
|
+
return `${stem} (${n})${ext}`;
|
|
211
|
+
};
|
|
212
|
+
/** One local header, the bytes, and the data descriptor; remembered for the central directory. */
|
|
213
|
+
const writeEntry = async (controller, name, body, modified) => {
|
|
214
|
+
const directory = body === null;
|
|
215
|
+
used.add(foldName(name));
|
|
216
|
+
const nameBytes = encoder.encode(name);
|
|
217
|
+
const { date, time } = dosDateTime(modified);
|
|
218
|
+
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
219
|
+
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
220
|
+
const header = new ByteWriter()
|
|
221
|
+
.u32(0x04034b50)
|
|
222
|
+
.u16(VERSION_ZIP64)
|
|
223
|
+
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
224
|
+
.u16(METHOD_STORE)
|
|
225
|
+
.u16(time)
|
|
226
|
+
.u16(date)
|
|
227
|
+
.u32(0) // crc, in the descriptor
|
|
228
|
+
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
229
|
+
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
230
|
+
.u16(nameBytes.length)
|
|
231
|
+
.u16(localExtra.length)
|
|
232
|
+
.raw(nameBytes)
|
|
233
|
+
.raw(localExtra)
|
|
234
|
+
.done();
|
|
235
|
+
controller.enqueue(header);
|
|
236
|
+
const entryOffset = offset;
|
|
237
|
+
offset += BigInt(header.length);
|
|
238
|
+
const crc = new Crc32();
|
|
239
|
+
let size = 0n;
|
|
240
|
+
if (body) {
|
|
241
|
+
const reader = body.getReader();
|
|
242
|
+
while (true) {
|
|
243
|
+
const { done, value } = await reader.read();
|
|
244
|
+
if (done)
|
|
245
|
+
break;
|
|
246
|
+
crc.update(value);
|
|
247
|
+
size += BigInt(value.length);
|
|
248
|
+
controller.enqueue(value);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
offset += size;
|
|
252
|
+
// ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
|
|
253
|
+
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
254
|
+
controller.enqueue(descriptor);
|
|
255
|
+
offset += BigInt(descriptor.length);
|
|
256
|
+
central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
|
|
257
|
+
};
|
|
168
258
|
return new ReadableStream({
|
|
169
259
|
async pull(controller) {
|
|
170
260
|
// One entry per pull keeps backpressure honest: the next file is not opened until the
|
|
@@ -180,76 +270,39 @@ export function zipStream(entries) {
|
|
|
180
270
|
if (next.done)
|
|
181
271
|
break;
|
|
182
272
|
const entry = next.value;
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
name = `${name}/`;
|
|
187
|
-
if (used.has(name))
|
|
188
|
-
continue;
|
|
189
|
-
}
|
|
190
|
-
else if (used.has(name)) {
|
|
191
|
-
const slash = name.lastIndexOf("/");
|
|
192
|
-
const dot = name.lastIndexOf(".");
|
|
193
|
-
const cut = dot > slash + 1 ? dot : name.length;
|
|
194
|
-
const stem = name.slice(0, cut);
|
|
195
|
-
const ext = name.slice(cut);
|
|
196
|
-
let n = 2;
|
|
197
|
-
while (used.has(`${stem} (${n})${ext}`))
|
|
198
|
-
n++;
|
|
199
|
-
name = `${stem} (${n})${ext}`;
|
|
273
|
+
if ("missing" in entry) {
|
|
274
|
+
missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
|
|
275
|
+
continue;
|
|
200
276
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
205
|
-
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
206
|
-
// note on the loop above.
|
|
207
|
-
if (!opened)
|
|
277
|
+
if (entry.directory === true) {
|
|
278
|
+
const name = `${safeEntryName(entry.name)}/`;
|
|
279
|
+
if (used.has(foldName(name)))
|
|
208
280
|
continue;
|
|
209
|
-
|
|
281
|
+
await writeEntry(controller, name, null, entry.modified);
|
|
282
|
+
return;
|
|
210
283
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
.
|
|
222
|
-
|
|
223
|
-
.u32(0) // crc, in the descriptor
|
|
224
|
-
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
225
|
-
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
226
|
-
.u16(nameBytes.length)
|
|
227
|
-
.u16(localExtra.length)
|
|
228
|
-
.raw(nameBytes)
|
|
229
|
-
.raw(localExtra)
|
|
230
|
-
.done();
|
|
231
|
-
controller.enqueue(header);
|
|
232
|
-
const entryOffset = offset;
|
|
233
|
-
offset += BigInt(header.length);
|
|
234
|
-
const crc = new Crc32();
|
|
235
|
-
let size = 0n;
|
|
236
|
-
if (body) {
|
|
237
|
-
const reader = body.getReader();
|
|
238
|
-
while (true) {
|
|
239
|
-
const { done, value } = await reader.read();
|
|
240
|
-
if (done)
|
|
241
|
-
break;
|
|
242
|
-
crc.update(value);
|
|
243
|
-
size += BigInt(value.length);
|
|
244
|
-
controller.enqueue(value);
|
|
245
|
-
}
|
|
284
|
+
const name = freeName(safeEntryName(entry.name));
|
|
285
|
+
let failure = "its bytes are missing from the store";
|
|
286
|
+
const opened = await entry.open().catch((error) => {
|
|
287
|
+
failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
|
|
288
|
+
return null;
|
|
289
|
+
});
|
|
290
|
+
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
291
|
+
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
292
|
+
// note on the loop above.
|
|
293
|
+
if (!opened) {
|
|
294
|
+
missing.push({ name, reason: failure });
|
|
295
|
+
continue;
|
|
246
296
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
297
|
+
await writeEntry(controller, name, asStream(opened), entry.modified);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
if (missing.length > 0 && !reported) {
|
|
301
|
+
reported = true;
|
|
302
|
+
const lines = missing.map((m) => `${m.name} — ${m.reason}`);
|
|
303
|
+
const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
|
|
304
|
+
const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
|
|
305
|
+
await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
|
|
253
306
|
return;
|
|
254
307
|
}
|
|
255
308
|
// Central directory, then the ZIP64 end records, then the classic end record.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.46.1",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* server; the RSA keypair is generated in-process and the signature is verified with
|
|
27
27
|
* `node:crypto`, which is what makes the token assertions real rather than shape-deep.
|
|
28
28
|
*/
|
|
29
|
-
import { afterAll, afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
29
|
+
import { afterAll, afterEach, beforeEach, describe, expect, it, setSystemTime } from 'bun:test';
|
|
30
30
|
import { generateKeyPairSync, verify as cryptoVerify } from 'node:crypto';
|
|
31
31
|
import {
|
|
32
32
|
AUTO_CHUNK_THRESHOLD_BYTES,
|
|
@@ -1056,6 +1056,29 @@ describe('🔴 a read the store PACES is waited out when asked, typed when not (
|
|
|
1056
1056
|
expect(calls).toHaveLength(4);
|
|
1057
1057
|
});
|
|
1058
1058
|
|
|
1059
|
+
it('🔴 every attempt goes with a token signed for IT — a 60 s token outlives no 120 s wait (forge task 3302)', async () => {
|
|
1060
|
+
// The URL was signed once, before the first attempt, so a paced read that waited out 30 s
|
|
1061
|
+
// refusals sent a dead token on its third try, and binary-server answered 403 — not 429.
|
|
1062
|
+
try {
|
|
1063
|
+
let n = 0;
|
|
1064
|
+
reply = () => (n++ < 3 ? limited('30') : ok(new Uint8Array([7, 8])));
|
|
1065
|
+
const store = makeStore({
|
|
1066
|
+
sleep: (ms) => {
|
|
1067
|
+
setSystemTime(new Date(Date.now() + ms));
|
|
1068
|
+
return Promise.resolve();
|
|
1069
|
+
},
|
|
1070
|
+
});
|
|
1071
|
+
await store.get('parts/0', { paced: true });
|
|
1072
|
+
expect(calls).toHaveLength(4);
|
|
1073
|
+
// The last attempt went 90 s after the first: its token must still have time left.
|
|
1074
|
+
const token = new URL(String(calls.at(-1)?.url)).searchParams.get('token') ?? '';
|
|
1075
|
+
const { exp } = JSON.parse(Buffer.from(token.split('.')[1] ?? '', 'base64url').toString('utf8')) as { exp: number };
|
|
1076
|
+
expect(exp * 1000).toBeGreaterThan(Date.now());
|
|
1077
|
+
} finally {
|
|
1078
|
+
setSystemTime();
|
|
1079
|
+
}
|
|
1080
|
+
});
|
|
1081
|
+
|
|
1059
1082
|
it('without paced, a 429 is the typed error carrying retryAfterSec — and the old message', async () => {
|
|
1060
1083
|
const { store, slept } = pacedStore(1, '7');
|
|
1061
1084
|
const error = await store.get('parts/0').catch((e: unknown) => e);
|
|
@@ -728,7 +728,9 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
728
728
|
const sendRead = async (
|
|
729
729
|
op: string,
|
|
730
730
|
key: string,
|
|
731
|
-
|
|
731
|
+
/** Signs the URL. Called for EVERY attempt: a read token lives 60 s and a paced read may wait
|
|
732
|
+
* 120 s, so a URL signed once went out dead after the first long retry-after (forge task 3302). */
|
|
733
|
+
mint: () => Promise<string>,
|
|
732
734
|
init: RequestInit | undefined,
|
|
733
735
|
opts: ReadOptions | undefined,
|
|
734
736
|
): Promise<Response> => {
|
|
@@ -737,7 +739,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
737
739
|
: 0;
|
|
738
740
|
let waited = 0;
|
|
739
741
|
for (let attempt = 0; ; attempt++) {
|
|
740
|
-
const res = await send(
|
|
742
|
+
const res = await send(await mint(), init);
|
|
741
743
|
if (res.status !== 429) return res;
|
|
742
744
|
const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
|
|
743
745
|
const ms = retryAfterSec * 1000;
|
|
@@ -875,7 +877,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
875
877
|
// short-circuit here — we want the bytes, and only the store has them.
|
|
876
878
|
if (!opts?.fresh && knownAbsent(key)) return null;
|
|
877
879
|
countWireRead(key);
|
|
878
|
-
const res = await sendRead('get', key,
|
|
880
|
+
const res = await sendRead('get', key, () => internalMediaUrl(key), undefined, opts);
|
|
879
881
|
if (res.status === 404) {
|
|
880
882
|
rememberAbsent(key);
|
|
881
883
|
return null;
|
|
@@ -952,7 +954,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
952
954
|
if (knownAbsent(key)) return null;
|
|
953
955
|
}
|
|
954
956
|
countWireRead(key);
|
|
955
|
-
const res = await sendRead('meta', key,
|
|
957
|
+
const res = await sendRead('meta', key, () => metaUrl(key), undefined, opts);
|
|
956
958
|
if (res.status === 404) {
|
|
957
959
|
rememberAbsent(key);
|
|
958
960
|
return null;
|
|
@@ -995,7 +997,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
995
997
|
if (why instanceof BinaryStoreRateLimitedError) throw why;
|
|
996
998
|
}
|
|
997
999
|
countWireRead(key);
|
|
998
|
-
const res = await sendRead('stat', key,
|
|
1000
|
+
const res = await sendRead('stat', key, () => internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
|
|
999
1001
|
if (res.status === 404) {
|
|
1000
1002
|
rememberAbsent(key);
|
|
1001
1003
|
return null;
|
|
@@ -13,6 +13,9 @@ import { Crc32, dosDateTime, safeEntryName, type ZipEntry, zipStream } from './z
|
|
|
13
13
|
|
|
14
14
|
const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text);
|
|
15
15
|
|
|
16
|
+
/** The report entry a skipped member lands in — spelt out, since a person reads it by this name. */
|
|
17
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
18
|
+
|
|
16
19
|
const entryFrom = (name: string, body: string): ZipEntry => ({
|
|
17
20
|
name,
|
|
18
21
|
open: async () =>
|
|
@@ -145,6 +148,59 @@ describe("zipStream", () => {
|
|
|
145
148
|
expect(listing).not.toContain("gone.mp3");
|
|
146
149
|
});
|
|
147
150
|
|
|
151
|
+
test("🔴 names differing only in case are TWO files — the later is numbered, never overwrites on a Mac", async () => {
|
|
152
|
+
// Desk lets a folder hold notes "Todo" and "todo". macOS (APFS) and Windows (NTFS) treat
|
|
153
|
+
// those as one file, so writing both names verbatim loses one on unzip — silently, or behind
|
|
154
|
+
// a "replace?" prompt the person did not expect.
|
|
155
|
+
const archive = await collect(zipStream([entryFrom("F/Todo.md", "upper"), entryFrom("f/todo.md", "lower"), entryFrom("F/TODO (2).md", "third")]));
|
|
156
|
+
const listing = await unzipList(archive);
|
|
157
|
+
expect(listing).toContain("F/Todo.md");
|
|
158
|
+
expect(listing).toContain("f/todo (2).md");
|
|
159
|
+
expect(listing).toContain("F/TODO (2) (2).md");
|
|
160
|
+
expect(await unzipRead(archive, "f/todo (2).md")).toBe("lower");
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("🔴 one name spelt with a composed and a decomposed accent is ONE name to APFS — numbered too", async () => {
|
|
164
|
+
const composed = "Caf\u00e9.txt";
|
|
165
|
+
const decomposed = "Cafe\u0301.txt";
|
|
166
|
+
expect(composed).not.toBe(decomposed);
|
|
167
|
+
const listing = await unzipList(await collect(zipStream([entryFrom(composed, "a"), entryFrom(decomposed, "b")])));
|
|
168
|
+
expect(listing).toContain("(2).txt");
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("a folder named twice in different case is written once", async () => {
|
|
172
|
+
const listing = await unzipList(await collect(zipStream([{ name: "Photos", directory: true }, { name: "photos", directory: true }])));
|
|
173
|
+
expect(listing.match(/ photos\//gi)?.length).toBe(1);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test(`🔴 what was skipped is LISTED in a final ${MISSING_FILES_NAME}, with the reason — a skip is never silent`, async () => {
|
|
177
|
+
const archive = await collect(
|
|
178
|
+
zipStream([
|
|
179
|
+
entryFrom("good.mp3", "here"),
|
|
180
|
+
{ name: "gone.mp3", open: async () => null },
|
|
181
|
+
{
|
|
182
|
+
name: "Folder/broken.jpg",
|
|
183
|
+
open: async () => {
|
|
184
|
+
throw new Error("store answered 503 https://bytes.example/x?sig=SECRET");
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
{ name: "Notes/unread.png", missing: "could not be read (timeout at https://store.example/b?token=SECRET2)" },
|
|
188
|
+
]),
|
|
189
|
+
);
|
|
190
|
+
const report = await unzipRead(archive, MISSING_FILES_NAME);
|
|
191
|
+
expect(report).toContain("3 files were in the selection but could not be read");
|
|
192
|
+
expect(report).toContain("gone.mp3 — its bytes are missing from the store");
|
|
193
|
+
expect(report).toContain("Folder/broken.jpg — could not be read (store answered 503 https://bytes.example/x)");
|
|
194
|
+
expect(report).toContain("Notes/unread.png — could not be read (timeout at https://store.example/b)");
|
|
195
|
+
expect(report).not.toContain("SECRET2");
|
|
196
|
+
expect(report).not.toContain("SECRET");
|
|
197
|
+
expect(await unzipRead(archive, "good.mp3")).toBe("here");
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test(`a complete archive carries no ${MISSING_FILES_NAME}`, async () => {
|
|
201
|
+
expect(await unzipList(await collect(zipStream([entryFrom("a.txt", "a")])))).not.toContain(MISSING_FILES_NAME);
|
|
202
|
+
});
|
|
203
|
+
|
|
148
204
|
test("an entry whose open() THROWS is skipped rather than killing the archive", async () => {
|
|
149
205
|
const archive = await collect(
|
|
150
206
|
zipStream([
|
package/src/server/zipStream.ts
CHANGED
|
@@ -128,8 +128,24 @@ export type ZipEntry =
|
|
|
128
128
|
name: string;
|
|
129
129
|
directory: true;
|
|
130
130
|
modified?: Date | string | null;
|
|
131
|
+
}
|
|
132
|
+
| {
|
|
133
|
+
/**
|
|
134
|
+
* A member the CALLER could not read before naming it (desk reads a picture's bytes to
|
|
135
|
+
* decide its extension). Nothing is written under this name; it is listed, with `missing`
|
|
136
|
+
* as the reason, in the archive's `MISSING FILES.txt`.
|
|
137
|
+
*/
|
|
138
|
+
name: string;
|
|
139
|
+
missing: string;
|
|
131
140
|
};
|
|
132
141
|
|
|
142
|
+
/**
|
|
143
|
+
* The last entry of an archive that left anything out: one line per missing path and why.
|
|
144
|
+
* A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
|
|
145
|
+
* not is a backup that fails on the day it is restored — so the skip is never silent.
|
|
146
|
+
*/
|
|
147
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
148
|
+
|
|
133
149
|
const encoder = new TextEncoder();
|
|
134
150
|
|
|
135
151
|
/** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
|
|
@@ -176,6 +192,24 @@ export function dosDateTime(when: Date | string | null | undefined): { date: num
|
|
|
176
192
|
};
|
|
177
193
|
}
|
|
178
194
|
|
|
195
|
+
/**
|
|
196
|
+
* A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
|
|
197
|
+
* APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
|
|
198
|
+
* names equal under this fold are ONE file there — and the second would overwrite the first.
|
|
199
|
+
*/
|
|
200
|
+
function foldName(name: string): string {
|
|
201
|
+
return name.normalize("NFC").toLowerCase();
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** A reason for a person to read: one line, short, and never a signed URL's query string. */
|
|
205
|
+
function readableReason(text: string): string {
|
|
206
|
+
return text
|
|
207
|
+
.replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
|
|
208
|
+
.replace(/\s+/g, " ")
|
|
209
|
+
.trim()
|
|
210
|
+
.slice(0, 200);
|
|
211
|
+
}
|
|
212
|
+
|
|
179
213
|
function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream<Uint8Array> {
|
|
180
214
|
if (!(body instanceof Uint8Array)) return body;
|
|
181
215
|
return new ReadableStream<Uint8Array>({
|
|
@@ -192,7 +226,12 @@ function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream
|
|
|
192
226
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
193
227
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
194
228
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
195
|
-
* once: its files already land in it.
|
|
229
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
230
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
231
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
232
|
+
*
|
|
233
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
234
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
196
235
|
*/
|
|
197
236
|
export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array> {
|
|
198
237
|
interface Central {
|
|
@@ -207,10 +246,82 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
|
|
|
207
246
|
|
|
208
247
|
const central: Central[] = [];
|
|
209
248
|
let offset = 0n;
|
|
249
|
+
/** Every name written, as the extracting disk will compare it ({@link foldName}). */
|
|
210
250
|
const used = new Set<string>();
|
|
251
|
+
const missing: { name: string; reason: string }[] = [];
|
|
252
|
+
let reported = false;
|
|
211
253
|
const iterator: AsyncIterator<ZipEntry> | Iterator<ZipEntry> =
|
|
212
254
|
Symbol.asyncIterator in entries ? (entries as AsyncIterable<ZipEntry>)[Symbol.asyncIterator]() : (entries as Iterable<ZipEntry>)[Symbol.iterator]();
|
|
213
255
|
|
|
256
|
+
/** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
|
|
257
|
+
const freeName = (name: string): string => {
|
|
258
|
+
if (!used.has(foldName(name))) return name;
|
|
259
|
+
const slash = name.lastIndexOf("/");
|
|
260
|
+
const dot = name.lastIndexOf(".");
|
|
261
|
+
const cut = dot > slash + 1 ? dot : name.length;
|
|
262
|
+
const stem = name.slice(0, cut);
|
|
263
|
+
const ext = name.slice(cut);
|
|
264
|
+
let n = 2;
|
|
265
|
+
while (used.has(foldName(`${stem} (${n})${ext}`))) n++;
|
|
266
|
+
return `${stem} (${n})${ext}`;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/** One local header, the bytes, and the data descriptor; remembered for the central directory. */
|
|
270
|
+
const writeEntry = async (
|
|
271
|
+
controller: ReadableStreamDefaultController<Uint8Array>,
|
|
272
|
+
name: string,
|
|
273
|
+
body: ReadableStream<Uint8Array> | null,
|
|
274
|
+
modified: Date | string | null | undefined,
|
|
275
|
+
): Promise<void> => {
|
|
276
|
+
const directory = body === null;
|
|
277
|
+
used.add(foldName(name));
|
|
278
|
+
const nameBytes = encoder.encode(name);
|
|
279
|
+
const { date, time } = dosDateTime(modified);
|
|
280
|
+
|
|
281
|
+
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
282
|
+
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
283
|
+
const header = new ByteWriter()
|
|
284
|
+
.u32(0x04034b50)
|
|
285
|
+
.u16(VERSION_ZIP64)
|
|
286
|
+
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
287
|
+
.u16(METHOD_STORE)
|
|
288
|
+
.u16(time)
|
|
289
|
+
.u16(date)
|
|
290
|
+
.u32(0) // crc, in the descriptor
|
|
291
|
+
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
292
|
+
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
293
|
+
.u16(nameBytes.length)
|
|
294
|
+
.u16(localExtra.length)
|
|
295
|
+
.raw(nameBytes)
|
|
296
|
+
.raw(localExtra)
|
|
297
|
+
.done();
|
|
298
|
+
|
|
299
|
+
controller.enqueue(header);
|
|
300
|
+
const entryOffset = offset;
|
|
301
|
+
offset += BigInt(header.length);
|
|
302
|
+
|
|
303
|
+
const crc = new Crc32();
|
|
304
|
+
let size = 0n;
|
|
305
|
+
if (body) {
|
|
306
|
+
const reader = body.getReader();
|
|
307
|
+
while (true) {
|
|
308
|
+
const { done, value } = await reader.read();
|
|
309
|
+
if (done) break;
|
|
310
|
+
crc.update(value);
|
|
311
|
+
size += BigInt(value.length);
|
|
312
|
+
controller.enqueue(value);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
offset += size;
|
|
316
|
+
|
|
317
|
+
// ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
|
|
318
|
+
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
319
|
+
controller.enqueue(descriptor);
|
|
320
|
+
offset += BigInt(descriptor.length);
|
|
321
|
+
|
|
322
|
+
central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
|
|
323
|
+
};
|
|
324
|
+
|
|
214
325
|
return new ReadableStream<Uint8Array>({
|
|
215
326
|
async pull(controller) {
|
|
216
327
|
// One entry per pull keeps backpressure honest: the next file is not opened until the
|
|
@@ -226,77 +337,40 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
|
|
|
226
337
|
if (next.done) break;
|
|
227
338
|
const entry = next.value;
|
|
228
339
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
name = `${name}/`;
|
|
233
|
-
if (used.has(name)) continue;
|
|
234
|
-
} else if (used.has(name)) {
|
|
235
|
-
const slash = name.lastIndexOf("/");
|
|
236
|
-
const dot = name.lastIndexOf(".");
|
|
237
|
-
const cut = dot > slash + 1 ? dot : name.length;
|
|
238
|
-
const stem = name.slice(0, cut);
|
|
239
|
-
const ext = name.slice(cut);
|
|
240
|
-
let n = 2;
|
|
241
|
-
while (used.has(`${stem} (${n})${ext}`)) n++;
|
|
242
|
-
name = `${stem} (${n})${ext}`;
|
|
340
|
+
if ("missing" in entry) {
|
|
341
|
+
missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
|
|
342
|
+
continue;
|
|
243
343
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
250
|
-
// note on the loop above.
|
|
251
|
-
if (!opened) continue;
|
|
252
|
-
body = asStream(opened);
|
|
344
|
+
if (entry.directory === true) {
|
|
345
|
+
const name = `${safeEntryName(entry.name)}/`;
|
|
346
|
+
if (used.has(foldName(name))) continue;
|
|
347
|
+
await writeEntry(controller, name, null, entry.modified);
|
|
348
|
+
return;
|
|
253
349
|
}
|
|
254
|
-
used.add(name);
|
|
255
|
-
const nameBytes = encoder.encode(name);
|
|
256
|
-
const { date, time } = dosDateTime(entry.modified);
|
|
257
|
-
|
|
258
|
-
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
259
|
-
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
260
|
-
const header = new ByteWriter()
|
|
261
|
-
.u32(0x04034b50)
|
|
262
|
-
.u16(VERSION_ZIP64)
|
|
263
|
-
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
264
|
-
.u16(METHOD_STORE)
|
|
265
|
-
.u16(time)
|
|
266
|
-
.u16(date)
|
|
267
|
-
.u32(0) // crc, in the descriptor
|
|
268
|
-
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
269
|
-
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
270
|
-
.u16(nameBytes.length)
|
|
271
|
-
.u16(localExtra.length)
|
|
272
|
-
.raw(nameBytes)
|
|
273
|
-
.raw(localExtra)
|
|
274
|
-
.done();
|
|
275
350
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
size += BigInt(value.length);
|
|
289
|
-
controller.enqueue(value);
|
|
290
|
-
}
|
|
351
|
+
const name = freeName(safeEntryName(entry.name));
|
|
352
|
+
let failure = "its bytes are missing from the store";
|
|
353
|
+
const opened = await entry.open().catch((error: unknown) => {
|
|
354
|
+
failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
|
|
355
|
+
return null;
|
|
356
|
+
});
|
|
357
|
+
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
358
|
+
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
359
|
+
// note on the loop above.
|
|
360
|
+
if (!opened) {
|
|
361
|
+
missing.push({ name, reason: failure });
|
|
362
|
+
continue;
|
|
291
363
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
296
|
-
controller.enqueue(descriptor);
|
|
297
|
-
offset += BigInt(descriptor.length);
|
|
364
|
+
await writeEntry(controller, name, asStream(opened), entry.modified);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
298
367
|
|
|
299
|
-
|
|
368
|
+
if (missing.length > 0 && !reported) {
|
|
369
|
+
reported = true;
|
|
370
|
+
const lines = missing.map((m) => `${m.name} — ${m.reason}`);
|
|
371
|
+
const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
|
|
372
|
+
const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
|
|
373
|
+
await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
|
|
300
374
|
return;
|
|
301
375
|
}
|
|
302
376
|
|