cursedbelt-server 4.45.0 → 4.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/zipStream.d.ts +14 -1
- package/dist/server/zipStream.js +120 -67
- package/package.json +1 -1
- package/src/server/zipStream.spec.ts +56 -0
- package/src/server/zipStream.ts +141 -67
|
@@ -59,6 +59,14 @@ export type ZipEntry = {
|
|
|
59
59
|
name: string;
|
|
60
60
|
directory: true;
|
|
61
61
|
modified?: Date | string | null;
|
|
62
|
+
} | {
|
|
63
|
+
/**
|
|
64
|
+
* A member the CALLER could not read before naming it (desk reads a picture's bytes to
|
|
65
|
+
* decide its extension). Nothing is written under this name; it is listed, with `missing`
|
|
66
|
+
* as the reason, in the archive's `MISSING FILES.txt`.
|
|
67
|
+
*/
|
|
68
|
+
name: string;
|
|
69
|
+
missing: string;
|
|
62
70
|
};
|
|
63
71
|
/**
|
|
64
72
|
* Names inside a zip must not escape the extraction directory, and must not carry characters
|
|
@@ -78,6 +86,11 @@ export declare function dosDateTime(when: Date | string | null | undefined): {
|
|
|
78
86
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
79
87
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
80
88
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
81
|
-
* once: its files already land in it.
|
|
89
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
90
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
91
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
92
|
+
*
|
|
93
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
94
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
82
95
|
*/
|
|
83
96
|
export declare function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array>;
|
package/dist/server/zipStream.js
CHANGED
|
@@ -97,6 +97,12 @@ class ByteWriter {
|
|
|
97
97
|
return out;
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* The last entry of an archive that left anything out: one line per missing path and why.
|
|
102
|
+
* A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
|
|
103
|
+
* not is a backup that fails on the day it is restored — so the skip is never silent.
|
|
104
|
+
*/
|
|
105
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
100
106
|
const encoder = new TextEncoder();
|
|
101
107
|
/** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
|
|
102
108
|
* names, which every accented and non-Latin file name needs. */
|
|
@@ -141,6 +147,22 @@ export function dosDateTime(when) {
|
|
|
141
147
|
time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | Math.floor(at.getUTCSeconds() / 2),
|
|
142
148
|
};
|
|
143
149
|
}
|
|
150
|
+
/**
|
|
151
|
+
* A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
|
|
152
|
+
* APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
|
|
153
|
+
* names equal under this fold are ONE file there — and the second would overwrite the first.
|
|
154
|
+
*/
|
|
155
|
+
function foldName(name) {
|
|
156
|
+
return name.normalize("NFC").toLowerCase();
|
|
157
|
+
}
|
|
158
|
+
/** A reason for a person to read: one line, short, and never a signed URL's query string. */
|
|
159
|
+
function readableReason(text) {
|
|
160
|
+
return text
|
|
161
|
+
.replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
|
|
162
|
+
.replace(/\s+/g, " ")
|
|
163
|
+
.trim()
|
|
164
|
+
.slice(0, 200);
|
|
165
|
+
}
|
|
144
166
|
function asStream(body) {
|
|
145
167
|
if (!(body instanceof Uint8Array))
|
|
146
168
|
return body;
|
|
@@ -158,13 +180,81 @@ function asStream(body) {
|
|
|
158
180
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
159
181
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
160
182
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
161
|
-
* once: its files already land in it.
|
|
183
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
184
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
185
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
186
|
+
*
|
|
187
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
188
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
162
189
|
*/
|
|
163
190
|
export function zipStream(entries) {
|
|
164
191
|
const central = [];
|
|
165
192
|
let offset = 0n;
|
|
193
|
+
/** Every name written, as the extracting disk will compare it ({@link foldName}). */
|
|
166
194
|
const used = new Set();
|
|
195
|
+
const missing = [];
|
|
196
|
+
let reported = false;
|
|
167
197
|
const iterator = Symbol.asyncIterator in entries ? entries[Symbol.asyncIterator]() : entries[Symbol.iterator]();
|
|
198
|
+
/** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
|
|
199
|
+
const freeName = (name) => {
|
|
200
|
+
if (!used.has(foldName(name)))
|
|
201
|
+
return name;
|
|
202
|
+
const slash = name.lastIndexOf("/");
|
|
203
|
+
const dot = name.lastIndexOf(".");
|
|
204
|
+
const cut = dot > slash + 1 ? dot : name.length;
|
|
205
|
+
const stem = name.slice(0, cut);
|
|
206
|
+
const ext = name.slice(cut);
|
|
207
|
+
let n = 2;
|
|
208
|
+
while (used.has(foldName(`${stem} (${n})${ext}`)))
|
|
209
|
+
n++;
|
|
210
|
+
return `${stem} (${n})${ext}`;
|
|
211
|
+
};
|
|
212
|
+
/** One local header, the bytes, and the data descriptor; remembered for the central directory. */
|
|
213
|
+
const writeEntry = async (controller, name, body, modified) => {
|
|
214
|
+
const directory = body === null;
|
|
215
|
+
used.add(foldName(name));
|
|
216
|
+
const nameBytes = encoder.encode(name);
|
|
217
|
+
const { date, time } = dosDateTime(modified);
|
|
218
|
+
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
219
|
+
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
220
|
+
const header = new ByteWriter()
|
|
221
|
+
.u32(0x04034b50)
|
|
222
|
+
.u16(VERSION_ZIP64)
|
|
223
|
+
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
224
|
+
.u16(METHOD_STORE)
|
|
225
|
+
.u16(time)
|
|
226
|
+
.u16(date)
|
|
227
|
+
.u32(0) // crc, in the descriptor
|
|
228
|
+
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
229
|
+
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
230
|
+
.u16(nameBytes.length)
|
|
231
|
+
.u16(localExtra.length)
|
|
232
|
+
.raw(nameBytes)
|
|
233
|
+
.raw(localExtra)
|
|
234
|
+
.done();
|
|
235
|
+
controller.enqueue(header);
|
|
236
|
+
const entryOffset = offset;
|
|
237
|
+
offset += BigInt(header.length);
|
|
238
|
+
const crc = new Crc32();
|
|
239
|
+
let size = 0n;
|
|
240
|
+
if (body) {
|
|
241
|
+
const reader = body.getReader();
|
|
242
|
+
while (true) {
|
|
243
|
+
const { done, value } = await reader.read();
|
|
244
|
+
if (done)
|
|
245
|
+
break;
|
|
246
|
+
crc.update(value);
|
|
247
|
+
size += BigInt(value.length);
|
|
248
|
+
controller.enqueue(value);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
offset += size;
|
|
252
|
+
// ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
|
|
253
|
+
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
254
|
+
controller.enqueue(descriptor);
|
|
255
|
+
offset += BigInt(descriptor.length);
|
|
256
|
+
central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
|
|
257
|
+
};
|
|
168
258
|
return new ReadableStream({
|
|
169
259
|
async pull(controller) {
|
|
170
260
|
// One entry per pull keeps backpressure honest: the next file is not opened until the
|
|
@@ -180,76 +270,39 @@ export function zipStream(entries) {
|
|
|
180
270
|
if (next.done)
|
|
181
271
|
break;
|
|
182
272
|
const entry = next.value;
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
name = `${name}/`;
|
|
187
|
-
if (used.has(name))
|
|
188
|
-
continue;
|
|
189
|
-
}
|
|
190
|
-
else if (used.has(name)) {
|
|
191
|
-
const slash = name.lastIndexOf("/");
|
|
192
|
-
const dot = name.lastIndexOf(".");
|
|
193
|
-
const cut = dot > slash + 1 ? dot : name.length;
|
|
194
|
-
const stem = name.slice(0, cut);
|
|
195
|
-
const ext = name.slice(cut);
|
|
196
|
-
let n = 2;
|
|
197
|
-
while (used.has(`${stem} (${n})${ext}`))
|
|
198
|
-
n++;
|
|
199
|
-
name = `${stem} (${n})${ext}`;
|
|
273
|
+
if ("missing" in entry) {
|
|
274
|
+
missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
|
|
275
|
+
continue;
|
|
200
276
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
205
|
-
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
206
|
-
// note on the loop above.
|
|
207
|
-
if (!opened)
|
|
277
|
+
if (entry.directory === true) {
|
|
278
|
+
const name = `${safeEntryName(entry.name)}/`;
|
|
279
|
+
if (used.has(foldName(name)))
|
|
208
280
|
continue;
|
|
209
|
-
|
|
281
|
+
await writeEntry(controller, name, null, entry.modified);
|
|
282
|
+
return;
|
|
210
283
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
.
|
|
222
|
-
|
|
223
|
-
.u32(0) // crc, in the descriptor
|
|
224
|
-
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
225
|
-
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
226
|
-
.u16(nameBytes.length)
|
|
227
|
-
.u16(localExtra.length)
|
|
228
|
-
.raw(nameBytes)
|
|
229
|
-
.raw(localExtra)
|
|
230
|
-
.done();
|
|
231
|
-
controller.enqueue(header);
|
|
232
|
-
const entryOffset = offset;
|
|
233
|
-
offset += BigInt(header.length);
|
|
234
|
-
const crc = new Crc32();
|
|
235
|
-
let size = 0n;
|
|
236
|
-
if (body) {
|
|
237
|
-
const reader = body.getReader();
|
|
238
|
-
while (true) {
|
|
239
|
-
const { done, value } = await reader.read();
|
|
240
|
-
if (done)
|
|
241
|
-
break;
|
|
242
|
-
crc.update(value);
|
|
243
|
-
size += BigInt(value.length);
|
|
244
|
-
controller.enqueue(value);
|
|
245
|
-
}
|
|
284
|
+
const name = freeName(safeEntryName(entry.name));
|
|
285
|
+
let failure = "its bytes are missing from the store";
|
|
286
|
+
const opened = await entry.open().catch((error) => {
|
|
287
|
+
failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
|
|
288
|
+
return null;
|
|
289
|
+
});
|
|
290
|
+
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
291
|
+
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
292
|
+
// note on the loop above.
|
|
293
|
+
if (!opened) {
|
|
294
|
+
missing.push({ name, reason: failure });
|
|
295
|
+
continue;
|
|
246
296
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
297
|
+
await writeEntry(controller, name, asStream(opened), entry.modified);
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
if (missing.length > 0 && !reported) {
|
|
301
|
+
reported = true;
|
|
302
|
+
const lines = missing.map((m) => `${m.name} — ${m.reason}`);
|
|
303
|
+
const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
|
|
304
|
+
const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
|
|
305
|
+
await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
|
|
253
306
|
return;
|
|
254
307
|
}
|
|
255
308
|
// Central directory, then the ZIP64 end records, then the classic end record.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.46.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
|
@@ -13,6 +13,9 @@ import { Crc32, dosDateTime, safeEntryName, type ZipEntry, zipStream } from './z
|
|
|
13
13
|
|
|
14
14
|
const bytesOf = (text: string): Uint8Array => new TextEncoder().encode(text);
|
|
15
15
|
|
|
16
|
+
/** The report entry a skipped member lands in — spelt out, since a person reads it by this name. */
|
|
17
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
18
|
+
|
|
16
19
|
const entryFrom = (name: string, body: string): ZipEntry => ({
|
|
17
20
|
name,
|
|
18
21
|
open: async () =>
|
|
@@ -145,6 +148,59 @@ describe("zipStream", () => {
|
|
|
145
148
|
expect(listing).not.toContain("gone.mp3");
|
|
146
149
|
});
|
|
147
150
|
|
|
151
|
+
test("🔴 names differing only in case are TWO files — the later is numbered, never overwrites on a Mac", async () => {
|
|
152
|
+
// Desk lets a folder hold notes "Todo" and "todo". macOS (APFS) and Windows (NTFS) treat
|
|
153
|
+
// those as one file, so writing both names verbatim loses one on unzip — silently, or behind
|
|
154
|
+
// a "replace?" prompt the person did not expect.
|
|
155
|
+
const archive = await collect(zipStream([entryFrom("F/Todo.md", "upper"), entryFrom("f/todo.md", "lower"), entryFrom("F/TODO (2).md", "third")]));
|
|
156
|
+
const listing = await unzipList(archive);
|
|
157
|
+
expect(listing).toContain("F/Todo.md");
|
|
158
|
+
expect(listing).toContain("f/todo (2).md");
|
|
159
|
+
expect(listing).toContain("F/TODO (2) (2).md");
|
|
160
|
+
expect(await unzipRead(archive, "f/todo (2).md")).toBe("lower");
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("🔴 one name spelt with a composed and a decomposed accent is ONE name to APFS — numbered too", async () => {
|
|
164
|
+
const composed = "Caf\u00e9.txt";
|
|
165
|
+
const decomposed = "Cafe\u0301.txt";
|
|
166
|
+
expect(composed).not.toBe(decomposed);
|
|
167
|
+
const listing = await unzipList(await collect(zipStream([entryFrom(composed, "a"), entryFrom(decomposed, "b")])));
|
|
168
|
+
expect(listing).toContain("(2).txt");
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("a folder named twice in different case is written once", async () => {
|
|
172
|
+
const listing = await unzipList(await collect(zipStream([{ name: "Photos", directory: true }, { name: "photos", directory: true }])));
|
|
173
|
+
expect(listing.match(/ photos\//gi)?.length).toBe(1);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test(`🔴 what was skipped is LISTED in a final ${MISSING_FILES_NAME}, with the reason — a skip is never silent`, async () => {
|
|
177
|
+
const archive = await collect(
|
|
178
|
+
zipStream([
|
|
179
|
+
entryFrom("good.mp3", "here"),
|
|
180
|
+
{ name: "gone.mp3", open: async () => null },
|
|
181
|
+
{
|
|
182
|
+
name: "Folder/broken.jpg",
|
|
183
|
+
open: async () => {
|
|
184
|
+
throw new Error("store answered 503 https://bytes.example/x?sig=SECRET");
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
{ name: "Notes/unread.png", missing: "could not be read (timeout at https://store.example/b?token=SECRET2)" },
|
|
188
|
+
]),
|
|
189
|
+
);
|
|
190
|
+
const report = await unzipRead(archive, MISSING_FILES_NAME);
|
|
191
|
+
expect(report).toContain("3 files were in the selection but could not be read");
|
|
192
|
+
expect(report).toContain("gone.mp3 — its bytes are missing from the store");
|
|
193
|
+
expect(report).toContain("Folder/broken.jpg — could not be read (store answered 503 https://bytes.example/x)");
|
|
194
|
+
expect(report).toContain("Notes/unread.png — could not be read (timeout at https://store.example/b)");
|
|
195
|
+
expect(report).not.toContain("SECRET2");
|
|
196
|
+
expect(report).not.toContain("SECRET");
|
|
197
|
+
expect(await unzipRead(archive, "good.mp3")).toBe("here");
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test(`a complete archive carries no ${MISSING_FILES_NAME}`, async () => {
|
|
201
|
+
expect(await unzipList(await collect(zipStream([entryFrom("a.txt", "a")])))).not.toContain(MISSING_FILES_NAME);
|
|
202
|
+
});
|
|
203
|
+
|
|
148
204
|
test("an entry whose open() THROWS is skipped rather than killing the archive", async () => {
|
|
149
205
|
const archive = await collect(
|
|
150
206
|
zipStream([
|
package/src/server/zipStream.ts
CHANGED
|
@@ -128,8 +128,24 @@ export type ZipEntry =
|
|
|
128
128
|
name: string;
|
|
129
129
|
directory: true;
|
|
130
130
|
modified?: Date | string | null;
|
|
131
|
+
}
|
|
132
|
+
| {
|
|
133
|
+
/**
|
|
134
|
+
* A member the CALLER could not read before naming it (desk reads a picture's bytes to
|
|
135
|
+
* decide its extension). Nothing is written under this name; it is listed, with `missing`
|
|
136
|
+
* as the reason, in the archive's `MISSING FILES.txt`.
|
|
137
|
+
*/
|
|
138
|
+
name: string;
|
|
139
|
+
missing: string;
|
|
131
140
|
};
|
|
132
141
|
|
|
142
|
+
/**
|
|
143
|
+
* The last entry of an archive that left anything out: one line per missing path and why.
|
|
144
|
+
* A skipped file must not end a 2,000-file download, but a zip that reads as complete when it is
|
|
145
|
+
* not is a backup that fails on the day it is restored — so the skip is never silent.
|
|
146
|
+
*/
|
|
147
|
+
const MISSING_FILES_NAME = "MISSING FILES.txt";
|
|
148
|
+
|
|
133
149
|
const encoder = new TextEncoder();
|
|
134
150
|
|
|
135
151
|
/** ZIP64 needs version 4.5; bit 3 = sizes/CRC follow in a data descriptor; bit 11 = UTF-8
|
|
@@ -176,6 +192,24 @@ export function dosDateTime(when: Date | string | null | undefined): { date: num
|
|
|
176
192
|
};
|
|
177
193
|
}
|
|
178
194
|
|
|
195
|
+
/**
|
|
196
|
+
* A name as the extracting disk compares it. APFS and NTFS are case-insensitive by default, and
|
|
197
|
+
* APFS also ignores Unicode composition (`é` typed as one code point or as `e` + accent), so two
|
|
198
|
+
* names equal under this fold are ONE file there — and the second would overwrite the first.
|
|
199
|
+
*/
|
|
200
|
+
function foldName(name: string): string {
|
|
201
|
+
return name.normalize("NFC").toLowerCase();
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** A reason for a person to read: one line, short, and never a signed URL's query string. */
|
|
205
|
+
function readableReason(text: string): string {
|
|
206
|
+
return text
|
|
207
|
+
.replace(/(\bhttps?:\/\/[^\s?]*)\?[^\s)]*/g, "$1")
|
|
208
|
+
.replace(/\s+/g, " ")
|
|
209
|
+
.trim()
|
|
210
|
+
.slice(0, 200);
|
|
211
|
+
}
|
|
212
|
+
|
|
179
213
|
function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream<Uint8Array> {
|
|
180
214
|
if (!(body instanceof Uint8Array)) return body;
|
|
181
215
|
return new ReadableStream<Uint8Array>({
|
|
@@ -192,7 +226,12 @@ function asStream(body: ReadableStream<Uint8Array> | Uint8Array): ReadableStream
|
|
|
192
226
|
* Duplicate file names are disambiguated rather than colliding — two different files that both
|
|
193
227
|
* want `Notes/Summary.md` produce `Summary.md` and `Summary (2).md`, because a zip with two
|
|
194
228
|
* identical names extracts as one file, silently losing the other. A duplicate FOLDER is written
|
|
195
|
-
* once: its files already land in it.
|
|
229
|
+
* once: its files already land in it. Names are compared the way the person's disk will compare
|
|
230
|
+
* them — case-folded and Unicode-normalised, since macOS and Windows both treat `Todo.md` and
|
|
231
|
+
* `todo.md` as ONE file — so the later one is numbered rather than overwriting the first on unzip.
|
|
232
|
+
*
|
|
233
|
+
* A member that cannot be read (its `open()` returns `null` or throws, or the caller yielded it
|
|
234
|
+
* as `missing`) is left out, and listed in a final `MISSING FILES.txt`.
|
|
196
235
|
*/
|
|
197
236
|
export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>): ReadableStream<Uint8Array> {
|
|
198
237
|
interface Central {
|
|
@@ -207,10 +246,82 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
|
|
|
207
246
|
|
|
208
247
|
const central: Central[] = [];
|
|
209
248
|
let offset = 0n;
|
|
249
|
+
/** Every name written, as the extracting disk will compare it ({@link foldName}). */
|
|
210
250
|
const used = new Set<string>();
|
|
251
|
+
const missing: { name: string; reason: string }[] = [];
|
|
252
|
+
let reported = false;
|
|
211
253
|
const iterator: AsyncIterator<ZipEntry> | Iterator<ZipEntry> =
|
|
212
254
|
Symbol.asyncIterator in entries ? (entries as AsyncIterable<ZipEntry>)[Symbol.asyncIterator]() : (entries as Iterable<ZipEntry>)[Symbol.iterator]();
|
|
213
255
|
|
|
256
|
+
/** A name not yet taken: `Summary.md` → `Summary (2).md`, numbering the file, never its folder's dot. */
|
|
257
|
+
const freeName = (name: string): string => {
|
|
258
|
+
if (!used.has(foldName(name))) return name;
|
|
259
|
+
const slash = name.lastIndexOf("/");
|
|
260
|
+
const dot = name.lastIndexOf(".");
|
|
261
|
+
const cut = dot > slash + 1 ? dot : name.length;
|
|
262
|
+
const stem = name.slice(0, cut);
|
|
263
|
+
const ext = name.slice(cut);
|
|
264
|
+
let n = 2;
|
|
265
|
+
while (used.has(foldName(`${stem} (${n})${ext}`))) n++;
|
|
266
|
+
return `${stem} (${n})${ext}`;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/** One local header, the bytes, and the data descriptor; remembered for the central directory. */
|
|
270
|
+
const writeEntry = async (
|
|
271
|
+
controller: ReadableStreamDefaultController<Uint8Array>,
|
|
272
|
+
name: string,
|
|
273
|
+
body: ReadableStream<Uint8Array> | null,
|
|
274
|
+
modified: Date | string | null | undefined,
|
|
275
|
+
): Promise<void> => {
|
|
276
|
+
const directory = body === null;
|
|
277
|
+
used.add(foldName(name));
|
|
278
|
+
const nameBytes = encoder.encode(name);
|
|
279
|
+
const { date, time } = dosDateTime(modified);
|
|
280
|
+
|
|
281
|
+
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
282
|
+
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
283
|
+
const header = new ByteWriter()
|
|
284
|
+
.u32(0x04034b50)
|
|
285
|
+
.u16(VERSION_ZIP64)
|
|
286
|
+
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
287
|
+
.u16(METHOD_STORE)
|
|
288
|
+
.u16(time)
|
|
289
|
+
.u16(date)
|
|
290
|
+
.u32(0) // crc, in the descriptor
|
|
291
|
+
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
292
|
+
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
293
|
+
.u16(nameBytes.length)
|
|
294
|
+
.u16(localExtra.length)
|
|
295
|
+
.raw(nameBytes)
|
|
296
|
+
.raw(localExtra)
|
|
297
|
+
.done();
|
|
298
|
+
|
|
299
|
+
controller.enqueue(header);
|
|
300
|
+
const entryOffset = offset;
|
|
301
|
+
offset += BigInt(header.length);
|
|
302
|
+
|
|
303
|
+
const crc = new Crc32();
|
|
304
|
+
let size = 0n;
|
|
305
|
+
if (body) {
|
|
306
|
+
const reader = body.getReader();
|
|
307
|
+
while (true) {
|
|
308
|
+
const { done, value } = await reader.read();
|
|
309
|
+
if (done) break;
|
|
310
|
+
crc.update(value);
|
|
311
|
+
size += BigInt(value.length);
|
|
312
|
+
controller.enqueue(value);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
offset += size;
|
|
316
|
+
|
|
317
|
+
// ZIP64 data descriptor: 8-byte sizes, unlike the 4-byte classic form.
|
|
318
|
+
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
319
|
+
controller.enqueue(descriptor);
|
|
320
|
+
offset += BigInt(descriptor.length);
|
|
321
|
+
|
|
322
|
+
central.push({ name: nameBytes, crc: crc.value, size, offset: entryOffset, date, time, directory });
|
|
323
|
+
};
|
|
324
|
+
|
|
214
325
|
return new ReadableStream<Uint8Array>({
|
|
215
326
|
async pull(controller) {
|
|
216
327
|
// One entry per pull keeps backpressure honest: the next file is not opened until the
|
|
@@ -226,77 +337,40 @@ export function zipStream(entries: Iterable<ZipEntry> | AsyncIterable<ZipEntry>)
|
|
|
226
337
|
if (next.done) break;
|
|
227
338
|
const entry = next.value;
|
|
228
339
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
name = `${name}/`;
|
|
233
|
-
if (used.has(name)) continue;
|
|
234
|
-
} else if (used.has(name)) {
|
|
235
|
-
const slash = name.lastIndexOf("/");
|
|
236
|
-
const dot = name.lastIndexOf(".");
|
|
237
|
-
const cut = dot > slash + 1 ? dot : name.length;
|
|
238
|
-
const stem = name.slice(0, cut);
|
|
239
|
-
const ext = name.slice(cut);
|
|
240
|
-
let n = 2;
|
|
241
|
-
while (used.has(`${stem} (${n})${ext}`)) n++;
|
|
242
|
-
name = `${stem} (${n})${ext}`;
|
|
340
|
+
if ("missing" in entry) {
|
|
341
|
+
missing.push({ name: safeEntryName(entry.name), reason: readableReason(entry.missing) || "could not be read" });
|
|
342
|
+
continue;
|
|
243
343
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
250
|
-
// note on the loop above.
|
|
251
|
-
if (!opened) continue;
|
|
252
|
-
body = asStream(opened);
|
|
344
|
+
if (entry.directory === true) {
|
|
345
|
+
const name = `${safeEntryName(entry.name)}/`;
|
|
346
|
+
if (used.has(foldName(name))) continue;
|
|
347
|
+
await writeEntry(controller, name, null, entry.modified);
|
|
348
|
+
return;
|
|
253
349
|
}
|
|
254
|
-
used.add(name);
|
|
255
|
-
const nameBytes = encoder.encode(name);
|
|
256
|
-
const { date, time } = dosDateTime(entry.modified);
|
|
257
|
-
|
|
258
|
-
// ZIP64 extra field, sizes zeroed — the data descriptor carries the truth.
|
|
259
|
-
const localExtra = new ByteWriter().u16(0x0001).u16(16).u64(0).u64(0).done();
|
|
260
|
-
const header = new ByteWriter()
|
|
261
|
-
.u32(0x04034b50)
|
|
262
|
-
.u16(VERSION_ZIP64)
|
|
263
|
-
.u16(FLAG_DATA_DESCRIPTOR | FLAG_UTF8)
|
|
264
|
-
.u16(METHOD_STORE)
|
|
265
|
-
.u16(time)
|
|
266
|
-
.u16(date)
|
|
267
|
-
.u32(0) // crc, in the descriptor
|
|
268
|
-
.u32(0xffffffff) // compressed size → see ZIP64 extra
|
|
269
|
-
.u32(0xffffffff) // uncompressed size → see ZIP64 extra
|
|
270
|
-
.u16(nameBytes.length)
|
|
271
|
-
.u16(localExtra.length)
|
|
272
|
-
.raw(nameBytes)
|
|
273
|
-
.raw(localExtra)
|
|
274
|
-
.done();
|
|
275
350
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
size += BigInt(value.length);
|
|
289
|
-
controller.enqueue(value);
|
|
290
|
-
}
|
|
351
|
+
const name = freeName(safeEntryName(entry.name));
|
|
352
|
+
let failure = "its bytes are missing from the store";
|
|
353
|
+
const opened = await entry.open().catch((error: unknown) => {
|
|
354
|
+
failure = `could not be read (${readableReason(error instanceof Error ? error.message : String(error)) || "unknown error"})`;
|
|
355
|
+
return null;
|
|
356
|
+
});
|
|
357
|
+
// A file whose bytes have gone is skipped, not fatal: one missing file must not abort
|
|
358
|
+
// a 2,000-file download that is otherwise fine. `continue`, never `return` — see the
|
|
359
|
+
// note on the loop above.
|
|
360
|
+
if (!opened) {
|
|
361
|
+
missing.push({ name, reason: failure });
|
|
362
|
+
continue;
|
|
291
363
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const descriptor = new ByteWriter().u32(0x08074b50).u32(crc.value).u64(size).u64(size).done();
|
|
296
|
-
controller.enqueue(descriptor);
|
|
297
|
-
offset += BigInt(descriptor.length);
|
|
364
|
+
await writeEntry(controller, name, asStream(opened), entry.modified);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
298
367
|
|
|
299
|
-
|
|
368
|
+
if (missing.length > 0 && !reported) {
|
|
369
|
+
reported = true;
|
|
370
|
+
const lines = missing.map((m) => `${m.name} — ${m.reason}`);
|
|
371
|
+
const count = missing.length === 1 ? "1 file was" : `${missing.length} files were`;
|
|
372
|
+
const text = `${count} in the selection but could not be read, so this archive does NOT contain ${missing.length === 1 ? "it" : "them"}:\n\n${lines.join("\n")}\n`;
|
|
373
|
+
await writeEntry(controller, freeName(MISSING_FILES_NAME), asStream(encoder.encode(text)), new Date());
|
|
300
374
|
return;
|
|
301
375
|
}
|
|
302
376
|
|