@bendyline/molen-pack 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,955 @@
1
+ import { PACK_MANIFEST_ENTRY, PACK_RESERVED_PREFIX, validate } from "@bendyline/molen-schema";
2
+ import { deflateSync, inflateSync } from "fflate";
3
+ //#region src/zip.ts
4
+ /**
5
+ * The small subset of the zip format packs use: stored (0) and deflated (8) members, UTF-8
6
+ * names, no zip64, no encryption, no extra fields on write. Output is deterministic: members are
7
+ * written in the order given with a fixed timestamp, so the same input always gives the same
8
+ * bytes. Deflate is fflate's, pinned to an exact version for the same reason.
9
+ */
10
+ const LOCAL_HEADER = 67324752;
11
+ const CENTRAL_HEADER = 33639248;
12
+ const END_OF_CENTRAL_DIRECTORY = 101010256;
13
+ const CENTRAL_HEADER_LENGTH = 46;
14
+ const UTF8_NAMES = 2048;
15
+ /** 1980-01-01 00:00, the earliest DOS date; every member carries it. */
16
+ const DOS_DATE = 33;
17
+ const MAX_U16 = 65535;
18
+ const MAX_U32 = 4294967295;
19
+ let crcTable;
20
+ function crc32(bytes) {
21
+ if (crcTable === void 0) {
22
+ crcTable = new Uint32Array(256);
23
+ for (let n = 0; n < 256; n++) {
24
+ let c = n;
25
+ for (let k = 0; k < 8; k++) c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
26
+ crcTable[n] = c >>> 0;
27
+ }
28
+ }
29
+ let crc = 4294967295;
30
+ for (let i = 0; i < bytes.length; i++) crc = crcTable[(crc ^ bytes[i]) & 255] ^ crc >>> 8;
31
+ return (crc ^ 4294967295) >>> 0;
32
+ }
33
+ /** Write members, in order, into one zip archive. */
34
+ function writeZip(members) {
35
+ if (members.length >= MAX_U16) throw new Error(`too many zip members (${members.length})`);
36
+ const encoder = new TextEncoder();
37
+ const encoded = [];
38
+ let offset = 0;
39
+ for (const member of members) {
40
+ const name = encoder.encode(member.name);
41
+ if (name.length > MAX_U16) throw new Error(`zip member name too long: ${member.name}`);
42
+ let method = 0;
43
+ let body = member.data;
44
+ if (member.compression !== "store" && member.data.length > 0) {
45
+ const deflated = deflateSync(member.data, { level: 9 });
46
+ if (member.compression === "deflate" || deflated.length < member.data.length * .9) {
47
+ method = 8;
48
+ body = deflated;
49
+ }
50
+ }
51
+ encoded.push({
52
+ name,
53
+ method,
54
+ crc: crc32(member.data),
55
+ size: member.data.length,
56
+ body,
57
+ offset
58
+ });
59
+ offset += 30 + name.length + body.length;
60
+ if (offset > MAX_U32) throw new Error("pack is larger than 4 GiB; zip64 is not supported");
61
+ }
62
+ const centralOffset = offset;
63
+ const centralSize = encoded.reduce((total, member) => total + CENTRAL_HEADER_LENGTH + member.name.length, 0);
64
+ const out = new Uint8Array(centralOffset + centralSize + 22);
65
+ const view = new DataView(out.buffer);
66
+ for (const member of encoded) {
67
+ let at = member.offset;
68
+ view.setUint32(at, LOCAL_HEADER, true);
69
+ view.setUint16(at + 4, 20, true);
70
+ view.setUint16(at + 6, UTF8_NAMES, true);
71
+ view.setUint16(at + 8, member.method, true);
72
+ view.setUint16(at + 10, 0, true);
73
+ view.setUint16(at + 12, DOS_DATE, true);
74
+ view.setUint32(at + 14, member.crc, true);
75
+ view.setUint32(at + 18, member.body.length, true);
76
+ view.setUint32(at + 22, member.size, true);
77
+ view.setUint16(at + 26, member.name.length, true);
78
+ view.setUint16(at + 28, 0, true);
79
+ at += 30;
80
+ out.set(member.name, at);
81
+ out.set(member.body, at + member.name.length);
82
+ }
83
+ let at = centralOffset;
84
+ for (const member of encoded) {
85
+ view.setUint32(at, CENTRAL_HEADER, true);
86
+ view.setUint16(at + 4, 20, true);
87
+ view.setUint16(at + 6, 20, true);
88
+ view.setUint16(at + 8, UTF8_NAMES, true);
89
+ view.setUint16(at + 10, member.method, true);
90
+ view.setUint16(at + 12, 0, true);
91
+ view.setUint16(at + 14, DOS_DATE, true);
92
+ view.setUint32(at + 16, member.crc, true);
93
+ view.setUint32(at + 20, member.body.length, true);
94
+ view.setUint32(at + 24, member.size, true);
95
+ view.setUint16(at + 28, member.name.length, true);
96
+ view.setUint32(at + 42, member.offset, true);
97
+ out.set(member.name, at + CENTRAL_HEADER_LENGTH);
98
+ at += CENTRAL_HEADER_LENGTH + member.name.length;
99
+ }
100
+ view.setUint32(at, END_OF_CENTRAL_DIRECTORY, true);
101
+ view.setUint16(at + 8, encoded.length, true);
102
+ view.setUint16(at + 10, encoded.length, true);
103
+ view.setUint32(at + 12, centralSize, true);
104
+ view.setUint32(at + 16, centralOffset, true);
105
+ return out;
106
+ }
107
+ /**
108
+ * Find the end record in the last bytes of an archive. `tailStart` is the absolute offset of
109
+ * `tail[0]`; when it is unknown (a cross-origin range response hides Content-Range) it is
110
+ * recovered from the record, since packs have nothing between the central directory and it.
111
+ */
112
+ function findZipEnd(tail, tailStart) {
113
+ const view = new DataView(tail.buffer, tail.byteOffset, tail.byteLength);
114
+ for (let at = tail.length - 22; at >= 0; at--) {
115
+ if (view.getUint32(at, true) !== END_OF_CENTRAL_DIRECTORY) continue;
116
+ const commentLength = view.getUint16(at + 20, true);
117
+ if (at + 22 + commentLength > tail.length) continue;
118
+ const entries = view.getUint16(at + 10, true);
119
+ const centralSize = view.getUint32(at + 12, true);
120
+ const centralOffset = view.getUint32(at + 16, true);
121
+ if (entries === MAX_U16 || centralSize === MAX_U32 || centralOffset === MAX_U32) throw new Error("zip64 archives are not supported");
122
+ const start = tailStart ?? centralOffset + centralSize - at;
123
+ return {
124
+ offset: start + at,
125
+ archiveSize: start + at + 22 + commentLength,
126
+ centralOffset,
127
+ centralSize,
128
+ entries
129
+ };
130
+ }
131
+ throw new Error("not a zip archive: no end-of-central-directory record");
132
+ }
133
+ function parseCentralDirectory(bytes, entries) {
134
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
135
+ const decoder = new TextDecoder();
136
+ const members = /* @__PURE__ */ new Map();
137
+ let at = 0;
138
+ for (let i = 0; i < entries; i++) {
139
+ if (at + CENTRAL_HEADER_LENGTH > bytes.length || view.getUint32(at, true) !== CENTRAL_HEADER) throw new Error("corrupt zip central directory");
140
+ const nameLength = view.getUint16(at + 28, true);
141
+ const extraLength = view.getUint16(at + 30, true);
142
+ const commentLength = view.getUint16(at + 32, true);
143
+ const name = decoder.decode(bytes.subarray(at + CENTRAL_HEADER_LENGTH, at + CENTRAL_HEADER_LENGTH + nameLength));
144
+ const member = {
145
+ name,
146
+ method: view.getUint16(at + 10, true),
147
+ crc: view.getUint32(at + 16, true),
148
+ compressedSize: view.getUint32(at + 20, true),
149
+ size: view.getUint32(at + 24, true),
150
+ localOffset: view.getUint32(at + 42, true)
151
+ };
152
+ if (member.compressedSize === MAX_U32 || member.localOffset === MAX_U32) throw new Error("zip64 archives are not supported");
153
+ members.set(name, member);
154
+ at += CENTRAL_HEADER_LENGTH + nameLength + extraLength + commentLength;
155
+ }
156
+ return members;
157
+ }
158
+ /** Offset of a member's data from its local header, read from the header itself. */
159
+ function localDataOffset(header) {
160
+ const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
161
+ if (view.getUint32(0, true) !== LOCAL_HEADER) throw new Error("corrupt zip local header");
162
+ return 30 + view.getUint16(26, true) + view.getUint16(28, true);
163
+ }
164
+ /** Decompress a member's data and check it against the central directory's CRC. */
165
+ function decodeMember(member, body) {
166
+ let data;
167
+ if (member.method === 0) data = body;
168
+ else if (member.method === 8) data = inflateSync(body, { out: new Uint8Array(member.size) });
169
+ else throw new Error(`zip member "${member.name}" uses unsupported method ${member.method}`);
170
+ if (data.length !== member.size || crc32(data) !== member.crc) throw new Error(`zip member "${member.name}" is corrupt (size or CRC mismatch)`);
171
+ return data;
172
+ }
173
+ //#endregion
174
+ //#region src/build.ts
175
+ /**
176
+ * Turn a set of files into a pack. Pure over bytes, so it runs in Node (molen pack build), a
177
+ * browser editor, or a test. The manifest is validated before any bytes are produced.
178
+ */
179
+ /** Largest uncompressed solid block; a group that exceeds it is split. */
180
+ const BLOCK_LIMIT = 1 << 20;
181
+ /** Text files at least this large are stored as their own member instead of in a block. */
182
+ const OWN_MEMBER_LIMIT = BLOCK_LIMIT / 2;
183
+ const TEXT_EXTENSIONS = new Set([
184
+ "json",
185
+ "txt",
186
+ "md",
187
+ "ts",
188
+ "mts",
189
+ "js",
190
+ "mjs",
191
+ "csv",
192
+ "svg",
193
+ "glsl",
194
+ "wgsl",
195
+ "html",
196
+ "css",
197
+ "xml",
198
+ "yaml",
199
+ "yml"
200
+ ]);
201
+ const MEDIA_TYPES = {
202
+ glb: "model/gltf-binary",
203
+ gltf: "model/gltf+json",
204
+ json: "application/json",
205
+ bin: "application/octet-stream",
206
+ png: "image/png",
207
+ jpg: "image/jpeg",
208
+ jpeg: "image/jpeg",
209
+ webp: "image/webp",
210
+ ktx2: "image/ktx2",
211
+ svg: "image/svg+xml",
212
+ md: "text/markdown",
213
+ txt: "text/plain",
214
+ csv: "text/csv",
215
+ html: "text/html",
216
+ css: "text/css",
217
+ xml: "application/xml",
218
+ yaml: "application/yaml",
219
+ yml: "application/yaml",
220
+ js: "text/javascript",
221
+ mjs: "text/javascript",
222
+ ts: "text/x-typescript",
223
+ mts: "text/x-typescript",
224
+ glsl: "text/plain",
225
+ wgsl: "text/plain",
226
+ wasm: "application/wasm",
227
+ pmtiles: "application/vnd.pmtiles",
228
+ mp3: "audio/mpeg",
229
+ ogg: "audio/ogg",
230
+ wav: "audio/wav"
231
+ };
232
+ function extension(path) {
233
+ const name = path.slice(path.lastIndexOf("/") + 1);
234
+ const dot = name.lastIndexOf(".");
235
+ return dot < 0 ? "" : name.slice(dot + 1).toLowerCase();
236
+ }
237
+ function mediaTypeOf(path) {
238
+ return MEDIA_TYPES[extension(path)] ?? "application/octet-stream";
239
+ }
240
+ function isTextPath(path) {
241
+ return TEXT_EXTENSIONS.has(extension(path));
242
+ }
243
+ async function sha256(bytes) {
244
+ const digest = await crypto.subtle.digest("SHA-256", bytes);
245
+ let hex = "";
246
+ for (const byte of new Uint8Array(digest)) hex += byte.toString(16).padStart(2, "0");
247
+ return `sha256:${hex}`;
248
+ }
249
+ /** Join a sidecar-relative path onto the sidecar's directory. */
250
+ function besideFile(file, relative) {
251
+ const slash = file.lastIndexOf("/");
252
+ return slash < 0 ? relative : `${file.slice(0, slash + 1)}${relative}`;
253
+ }
254
+ function sortedFiles(files) {
255
+ const seen = /* @__PURE__ */ new Set();
256
+ for (const file of files) {
257
+ if (seen.has(file.path)) throw new Error(`pack file "${file.path}" is listed twice`);
258
+ seen.add(file.path);
259
+ }
260
+ return [...files].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
261
+ }
262
+ /** Asset ids and variants declared by molen/asset@1 sidecars among the files. */
263
+ function sidecarIds(files, present) {
264
+ const ids = {};
265
+ const variants = /* @__PURE__ */ new Map();
266
+ const decoder = new TextDecoder();
267
+ for (const file of files) {
268
+ if (extension(file.path) !== "json" || file.bytes.length > BLOCK_LIMIT) continue;
269
+ let doc;
270
+ try {
271
+ doc = JSON.parse(decoder.decode(file.bytes));
272
+ } catch {
273
+ continue;
274
+ }
275
+ if (doc?.format !== "molen/asset@1" || typeof doc.id !== "string") continue;
276
+ const main = typeof doc.files?.main === "string" ? besideFile(file.path, doc.files.main) : "";
277
+ if (!present.has(main)) continue;
278
+ ids[doc.id] = main;
279
+ const found = {};
280
+ for (const [name, relative] of Object.entries(doc.files?.variants ?? {})) {
281
+ const path = besideFile(file.path, relative);
282
+ if (present.has(path)) found[name] = path;
283
+ }
284
+ if (Object.keys(found).length > 0) variants.set(main, found);
285
+ }
286
+ return {
287
+ ids,
288
+ variants
289
+ };
290
+ }
291
+ /**
292
+ * Describe files as a pack manifest with no blocks: entries, ids, roles and the content hash.
293
+ * The content hash covers paths and contents only, so a built pack and its unbuilt source
294
+ * directory report the same one.
295
+ */
296
+ async function describePack(files, options) {
297
+ const sorted = sortedFiles(files);
298
+ const derived = sidecarIds(sorted, new Set(sorted.map((file) => file.path)));
299
+ const entries = {};
300
+ let hashInput = "";
301
+ for (const file of sorted) {
302
+ const hash = await sha256(file.bytes);
303
+ const entry = {
304
+ size: file.bytes.length,
305
+ sha256: hash,
306
+ mediaType: mediaTypeOf(file.path)
307
+ };
308
+ const variants = derived.variants.get(file.path);
309
+ if (variants !== void 0) entry.variants = variants;
310
+ entries[file.path] = entry;
311
+ hashInput += `${file.path}\t${hash}\n`;
312
+ }
313
+ const ids = Object.fromEntries(Object.entries({
314
+ ...derived.ids,
315
+ ...options.ids
316
+ }).sort(([a], [b]) => a < b ? -1 : 1));
317
+ const provides = Object.fromEntries(Object.entries(options.provides ?? {}).sort(([a], [b]) => a < b ? -1 : 1).map(([role, paths]) => [role, typeof paths === "string" ? [paths] : [...paths]]));
318
+ const manifest = {
319
+ format: "molen/pack@1",
320
+ id: options.id,
321
+ version: options.version,
322
+ ...options.title !== void 0 ? { title: options.title } : {},
323
+ ...options.license !== void 0 ? { license: options.license } : {},
324
+ ...options.notice !== void 0 ? { notice: options.notice } : {},
325
+ contentHash: await sha256(new TextEncoder().encode(hashInput)),
326
+ provides,
327
+ ids,
328
+ blocks: {},
329
+ entries
330
+ };
331
+ assertValidManifest(manifest);
332
+ return manifest;
333
+ }
334
+ function assertValidManifest(manifest) {
335
+ const result = validate("pack", manifest);
336
+ if (!result.ok) throw new Error(`invalid pack "${manifest.id}":\n${result.formatted}`);
337
+ }
338
+ /** Top-level directory of a path; files at the root share one group. */
339
+ function groupOf(path) {
340
+ const slash = path.indexOf("/");
341
+ return slash < 0 ? "root" : path.slice(0, slash);
342
+ }
343
+ /** Build a pack: a deterministic zip whose last member is the manifest. */
344
+ async function createPack(files, options) {
345
+ const sorted = sortedFiles(files);
346
+ for (const file of sorted) if (file.path === PACK_MANIFEST_ENTRY || file.path.startsWith(PACK_RESERVED_PREFIX)) throw new Error(`"${file.path}" is reserved for the pack's own structure`);
347
+ const described = await describePack(sorted, options);
348
+ const entries = structuredClone(described.entries);
349
+ const members = [];
350
+ const blocks = {};
351
+ const blockMembers = [];
352
+ const pending = /* @__PURE__ */ new Map();
353
+ for (const file of sorted) if (options.solid !== false && isTextPath(file.path) && file.bytes.length < OWN_MEMBER_LIMIT) {
354
+ const group = groupOf(file.path);
355
+ pending.set(group, [...pending.get(group) ?? [], file]);
356
+ } else members.push({
357
+ name: file.path,
358
+ data: file.bytes,
359
+ compression: "auto"
360
+ });
361
+ for (const [group, groupFiles] of [...pending].sort(([a], [b]) => a < b ? -1 : 1)) {
362
+ let part = 1;
363
+ let chunk = [];
364
+ let size = 0;
365
+ const flush = () => {
366
+ if (chunk.length === 0) return;
367
+ const name = part === 1 ? group : `${group}.${part}`;
368
+ const data = new Uint8Array(size);
369
+ let offset = 0;
370
+ for (const file of chunk) {
371
+ data.set(file.bytes, offset);
372
+ entries[file.path] = {
373
+ ...entries[file.path],
374
+ block: name,
375
+ offset
376
+ };
377
+ offset += file.bytes.length;
378
+ }
379
+ const entry = `${PACK_RESERVED_PREFIX}blocks/${name}.blk`;
380
+ blocks[name] = {
381
+ entry,
382
+ size
383
+ };
384
+ blockMembers.push({
385
+ name: entry,
386
+ data,
387
+ compression: "auto"
388
+ });
389
+ part++;
390
+ chunk = [];
391
+ size = 0;
392
+ };
393
+ for (const file of groupFiles) {
394
+ if (size + file.bytes.length > BLOCK_LIMIT) flush();
395
+ chunk.push(file);
396
+ size += file.bytes.length;
397
+ }
398
+ flush();
399
+ }
400
+ const manifest = {
401
+ ...described,
402
+ blocks,
403
+ entries
404
+ };
405
+ assertValidManifest(manifest);
406
+ return {
407
+ bytes: writeZip([
408
+ ...members,
409
+ ...blockMembers,
410
+ {
411
+ name: PACK_MANIFEST_ENTRY,
412
+ data: new TextEncoder().encode(JSON.stringify(manifest)),
413
+ compression: "auto"
414
+ }
415
+ ]),
416
+ manifest
417
+ };
418
+ }
419
+ //#endregion
420
+ //#region src/source.ts
421
+ /** The server replaced the pack while it was being read (its ETag changed). */
422
+ var PackChangedError = class extends Error {
423
+ constructor(url) {
424
+ super(`pack ${url} changed on the server while it was being read; open it again`);
425
+ this.name = "PackChangedError";
426
+ }
427
+ };
428
+ var HttpStatusError = class extends Error {
429
+ url;
430
+ status;
431
+ constructor(url, status) {
432
+ super(`HTTP ${status} for ${url}`);
433
+ this.url = url;
434
+ this.status = status;
435
+ this.name = "HttpStatusError";
436
+ }
437
+ };
438
+ function blobReader(blob) {
439
+ return {
440
+ size: blob.size,
441
+ read: async (offset, length) => new Uint8Array(await blob.slice(offset, offset + length).arrayBuffer())
442
+ };
443
+ }
444
+ function isRetryable(error) {
445
+ if (error instanceof HttpStatusError) return error.status === 408 || error.status === 429 || error.status >= 500;
446
+ if (error instanceof PackChangedError) return false;
447
+ const name = error?.name;
448
+ return error instanceof TypeError || name === "TimeoutError";
449
+ }
450
+ function sleep(ms, signal) {
451
+ return new Promise((resolve, reject) => {
452
+ if (signal.aborted) return reject(signal.reason);
453
+ const timer = setTimeout(() => {
454
+ signal.removeEventListener("abort", abort);
455
+ resolve();
456
+ }, ms);
457
+ const abort = () => {
458
+ clearTimeout(timer);
459
+ reject(signal.reason);
460
+ };
461
+ signal.addEventListener("abort", abort, { once: true });
462
+ });
463
+ }
464
+ /** Run `task` with a per-attempt deadline, retrying transient failures with backoff. */
465
+ async function withRetry(task, retry, signal) {
466
+ const attempts = Math.max(1, retry.attempts ?? 3);
467
+ const base = retry.baseDelayMs ?? 250;
468
+ for (let attempt = 1;; attempt++) {
469
+ const deadline = AbortSignal.timeout(retry.timeoutMs ?? 3e4);
470
+ try {
471
+ return await task(AbortSignal.any([signal, deadline]));
472
+ } catch (error) {
473
+ if (signal.aborted) throw signal.reason;
474
+ if (attempt >= attempts || !isRetryable(error)) throw error;
475
+ await sleep(Math.min(base * 2 ** (attempt - 1), retry.maxDelayMs ?? 4e3), signal);
476
+ }
477
+ }
478
+ }
479
+ /** Bytes fetched when a remote pack is opened: the manifest and central directory live here. */
480
+ const TAIL_BYTES = 64 * 1024;
481
+ async function fetchBytes(response) {
482
+ return new Uint8Array(await response.arrayBuffer());
483
+ }
484
+ /**
485
+ * Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
486
+ * range request) has already sent the whole file, which is used as is.
487
+ */
488
+ async function openUrl(url, options) {
489
+ const { fetch: fetcher, retry, signal } = options;
490
+ if (options.mode === "whole" || options.mode === "auto" && options.sizeHint !== void 0 && options.sizeHint <= options.wholeThreshold) {
491
+ const bytes = await withRetry(async (attemptSignal) => {
492
+ const response = await fetcher(url, { signal: attemptSignal });
493
+ if (!response.ok) throw new HttpStatusError(url, response.status);
494
+ return fetchBytes(response);
495
+ }, retry, signal);
496
+ options.onResponse(bytes.length);
497
+ return {
498
+ kind: "whole",
499
+ bytes
500
+ };
501
+ }
502
+ const first = await withRetry(async (attemptSignal) => {
503
+ const response = await fetcher(url, {
504
+ headers: { Range: `bytes=-${TAIL_BYTES}` },
505
+ signal: attemptSignal
506
+ });
507
+ if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
508
+ return {
509
+ status: response.status,
510
+ bytes: await fetchBytes(response),
511
+ etag: response.headers.get("etag") ?? void 0,
512
+ range: response.headers.get("content-range") ?? void 0
513
+ };
514
+ }, retry, signal);
515
+ options.onResponse(first.bytes.length);
516
+ if (first.status === 200) return {
517
+ kind: "whole",
518
+ bytes: first.bytes
519
+ };
520
+ const total = first.range === void 0 ? void 0 : Number(first.range.split("/")[1]);
521
+ const etag = first.etag !== void 0 && !first.etag.startsWith("W/") ? first.etag : void 0;
522
+ return {
523
+ kind: "range",
524
+ tail: first.bytes,
525
+ ...total !== void 0 && Number.isFinite(total) ? { size: total } : {},
526
+ reader: (size) => ({
527
+ size,
528
+ read: (offset, length, readSignal) => withRetry(async (attemptSignal) => {
529
+ const headers = { Range: `bytes=${offset}-${offset + length - 1}` };
530
+ if (etag !== void 0) headers["If-Range"] = etag;
531
+ const response = await fetcher(url, {
532
+ headers,
533
+ signal: readSignal === void 0 ? attemptSignal : AbortSignal.any([attemptSignal, readSignal])
534
+ });
535
+ if (response.status === 200) {
536
+ if (etag !== void 0) throw new PackChangedError(url);
537
+ const all = await fetchBytes(response);
538
+ options.onResponse(all.length);
539
+ return all.subarray(offset, offset + length);
540
+ }
541
+ if (response.status !== 206) throw new HttpStatusError(url, response.status);
542
+ const bytes = await fetchBytes(response);
543
+ options.onResponse(bytes.length);
544
+ if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
545
+ return bytes;
546
+ }, retry, signal)
547
+ })
548
+ };
549
+ }
550
+ /**
551
+ * Queue reads for one source. Reads issued in the same tick are sorted, merged when close
552
+ * together, and run with bounded concurrency, high priority first.
553
+ */
554
+ var ReadScheduler = class {
555
+ reader;
556
+ signal;
557
+ queue = [];
558
+ flushing = false;
559
+ running = 0;
560
+ batches = [];
561
+ concurrency;
562
+ mergeGap;
563
+ maxSpan;
564
+ constructor(reader, signal, options = {}) {
565
+ this.reader = reader;
566
+ this.signal = signal;
567
+ this.concurrency = Math.max(1, options.concurrency ?? 6);
568
+ this.mergeGap = options.mergeGap ?? 32 * 1024;
569
+ this.maxSpan = options.maxSpan ?? 4 * 1024 * 1024;
570
+ }
571
+ read(offset, length, high = true, signal) {
572
+ if (length === 0) return Promise.resolve(new Uint8Array(0));
573
+ return new Promise((resolve, reject) => {
574
+ if (signal?.aborted) return reject(signal.reason);
575
+ const abort = () => reject(signal?.reason);
576
+ signal?.addEventListener("abort", abort, { once: true });
577
+ this.queue.push({
578
+ offset,
579
+ length,
580
+ high,
581
+ resolve: (bytes) => {
582
+ signal?.removeEventListener("abort", abort);
583
+ resolve(bytes);
584
+ },
585
+ reject: (error) => {
586
+ signal?.removeEventListener("abort", abort);
587
+ reject(error);
588
+ }
589
+ });
590
+ if (!this.flushing) {
591
+ this.flushing = true;
592
+ queueMicrotask(() => this.flush());
593
+ }
594
+ });
595
+ }
596
+ flush() {
597
+ this.flushing = false;
598
+ const waiters = this.queue.sort((a, b) => a.offset - b.offset);
599
+ this.queue = [];
600
+ for (const waiter of waiters) {
601
+ const last = this.batches.at(-1);
602
+ const end = waiter.offset + waiter.length;
603
+ if (last !== void 0 && waiter.offset - last.end <= this.mergeGap && Math.max(last.end, end) - last.start <= this.maxSpan && last.start <= waiter.offset) {
604
+ last.end = Math.max(last.end, end);
605
+ last.high ||= waiter.high;
606
+ last.waiters.push(waiter);
607
+ } else this.batches.push({
608
+ start: waiter.offset,
609
+ end,
610
+ high: waiter.high,
611
+ waiters: [waiter]
612
+ });
613
+ }
614
+ this.batches.sort((a, b) => Number(b.high) - Number(a.high));
615
+ this.pump();
616
+ }
617
+ pump() {
618
+ if (this.signal.aborted) {
619
+ for (const batch of this.batches.splice(0)) for (const waiter of batch.waiters) waiter.reject(this.signal.reason);
620
+ return;
621
+ }
622
+ while (this.running < this.concurrency && this.batches.length > 0) {
623
+ const batch = this.batches.shift();
624
+ this.running++;
625
+ this.reader.read(batch.start, batch.end - batch.start, this.signal).then((bytes) => {
626
+ for (const waiter of batch.waiters) {
627
+ const from = waiter.offset - batch.start;
628
+ waiter.resolve(bytes.subarray(from, from + waiter.length));
629
+ }
630
+ }, (error) => {
631
+ for (const waiter of batch.waiters) waiter.reject(error);
632
+ }).finally(() => {
633
+ this.running--;
634
+ this.pump();
635
+ });
636
+ }
637
+ }
638
+ };
639
+ //#endregion
640
+ //#region src/pack.ts
641
+ /**
642
+ * Open a pack from any source and read its files. Only the tail of a remote pack is fetched up
643
+ * front (manifest and central directory); files are read on demand, and small documents that
644
+ * share a solid block arrive together.
645
+ */
646
+ var PackEntryNotFoundError = class extends Error {
647
+ pack;
648
+ path;
649
+ constructor(pack, path) {
650
+ super(`pack "${pack}" has no file "${path}"`);
651
+ this.pack = pack;
652
+ this.path = path;
653
+ this.name = "PackEntryNotFoundError";
654
+ }
655
+ };
656
+ var PackIntegrityError = class extends Error {
657
+ constructor(pack, path, detail) {
658
+ super(`pack "${pack}" file "${path}" failed its integrity check: ${detail}`);
659
+ this.name = "PackIntegrityError";
660
+ }
661
+ };
662
+ var BasePack = class {
663
+ has(path) {
664
+ return this.manifest.entries[path] !== void 0;
665
+ }
666
+ paths() {
667
+ return Object.keys(this.manifest.entries).sort();
668
+ }
669
+ entry(path) {
670
+ const entry = this.manifest.entries[path];
671
+ if (entry === void 0) throw new PackEntryNotFoundError(this.manifest.id, path);
672
+ return entry;
673
+ }
674
+ async readText(path, options) {
675
+ return new TextDecoder().decode(await this.readBytes(path, options));
676
+ }
677
+ async readJson(path, options) {
678
+ const text = await this.readText(path, options);
679
+ try {
680
+ return JSON.parse(text);
681
+ } catch (error) {
682
+ throw new Error(`pack "${this.manifest.id}" file "${path}" is not valid JSON`, { cause: error });
683
+ }
684
+ }
685
+ };
686
+ /** Decoded zip members, least recently used first, bounded by total bytes. */
687
+ var ByteCache = class {
688
+ limit;
689
+ entries = /* @__PURE__ */ new Map();
690
+ sizes = /* @__PURE__ */ new Map();
691
+ total = 0;
692
+ constructor(limit) {
693
+ this.limit = limit;
694
+ }
695
+ get(key, load) {
696
+ const hit = this.entries.get(key);
697
+ if (hit !== void 0) {
698
+ this.entries.delete(key);
699
+ this.entries.set(key, hit);
700
+ return hit;
701
+ }
702
+ const pending = load();
703
+ this.entries.set(key, pending);
704
+ pending.then((bytes) => {
705
+ if (this.entries.get(key) !== pending) return;
706
+ if (bytes.length > this.limit) {
707
+ this.entries.delete(key);
708
+ return;
709
+ }
710
+ this.sizes.set(key, bytes.length);
711
+ this.total += bytes.length;
712
+ for (const [oldest] of this.entries) {
713
+ if (this.total <= this.limit) break;
714
+ const size = this.sizes.get(oldest);
715
+ if (size === void 0) continue;
716
+ this.entries.delete(oldest);
717
+ this.sizes.delete(oldest);
718
+ this.total -= size;
719
+ }
720
+ }, () => {
721
+ if (this.entries.get(key) === pending) this.entries.delete(key);
722
+ });
723
+ return pending;
724
+ }
725
+ clear() {
726
+ this.entries.clear();
727
+ this.sizes.clear();
728
+ this.total = 0;
729
+ }
730
+ };
731
+ function copy(bytes) {
732
+ return bytes.slice().buffer;
733
+ }
734
+ /** Settle with `promise`, or reject early when `signal` aborts; the work itself continues. */
735
+ function untilAborted(promise, signal) {
736
+ if (signal === void 0) return promise;
737
+ if (signal.aborted) return Promise.reject(signal.reason);
738
+ return new Promise((resolve, reject) => {
739
+ const abort = () => reject(signal.reason);
740
+ signal.addEventListener("abort", abort, { once: true });
741
+ promise.then((value) => {
742
+ signal.removeEventListener("abort", abort);
743
+ resolve(value);
744
+ }, (error) => {
745
+ signal.removeEventListener("abort", abort);
746
+ reject(error);
747
+ });
748
+ });
749
+ }
750
+ var ZipPack = class extends BasePack {
751
+ manifest;
752
+ label;
753
+ members;
754
+ size;
755
+ window;
756
+ scheduler;
757
+ controller;
758
+ options;
759
+ cache;
760
+ constructor(manifest, label, members, size, window, scheduler, controller, options) {
761
+ super();
762
+ this.manifest = manifest;
763
+ this.label = label;
764
+ this.members = members;
765
+ this.size = size;
766
+ this.window = window;
767
+ this.scheduler = scheduler;
768
+ this.controller = controller;
769
+ this.options = options;
770
+ this.cache = new ByteCache(options.maxCacheBytes ?? 16 * 1024 * 1024);
771
+ }
772
+ /** Loads are shared between callers, so they run under the pack's signal, not a caller's. */
773
+ range(offset, length, high) {
774
+ const { start, bytes } = this.window;
775
+ if (offset >= start && offset + length <= start + bytes.length) return Promise.resolve(bytes.subarray(offset - start, offset - start + length));
776
+ if (this.scheduler === void 0) return Promise.reject(/* @__PURE__ */ new Error(`pack ${this.label} is truncated`));
777
+ return this.scheduler.read(offset, length, high);
778
+ }
779
+ member(name, options) {
780
+ if (this.controller.signal.aborted) return Promise.reject(this.controller.signal.reason);
781
+ const member = this.members.get(name);
782
+ if (member === void 0) return Promise.reject(/* @__PURE__ */ new Error(`pack "${this.manifest.id}" is missing zip member "${name}"`));
783
+ const high = options?.priority !== "low";
784
+ return untilAborted(this.cache.get(name, async () => {
785
+ const guess = 30 + new TextEncoder().encode(name).length + 64;
786
+ const want = Math.min(guess + member.compressedSize, this.size - member.localOffset);
787
+ let chunk = await this.range(member.localOffset, want, high);
788
+ const dataStart = localDataOffset(chunk);
789
+ if (dataStart + member.compressedSize > chunk.length) chunk = await this.range(member.localOffset, dataStart + member.compressedSize, high);
790
+ const data = decodeMember(member, chunk.subarray(dataStart, dataStart + member.compressedSize));
791
+ return member.method === 0 ? data.slice() : data;
792
+ }), options?.signal);
793
+ }
794
+ async bytes(path, options) {
795
+ const entry = this.entry(path);
796
+ if (entry.block === void 0) return this.member(path, options);
797
+ const block = this.manifest.blocks[entry.block];
798
+ if (block === void 0) throw new Error(`pack "${this.manifest.id}" has no block "${entry.block}"`);
799
+ const data = await this.member(block.entry, options);
800
+ const offset = entry.offset ?? 0;
801
+ return data.subarray(offset, offset + entry.size);
802
+ }
803
+ async readBytes(path, options) {
804
+ const entry = this.entry(path);
805
+ const bytes = await this.bytes(path, options);
806
+ if (this.options.integrity === "sha256") {
807
+ const actual = await sha256(bytes);
808
+ if (actual !== entry.sha256) throw new PackIntegrityError(this.manifest.id, path, `sha256 is ${actual}`);
809
+ }
810
+ return copy(bytes);
811
+ }
812
+ async prefetch(paths, options) {
813
+ await Promise.all(paths.map((path) => this.bytes(path, options)));
814
+ }
815
+ close() {
816
+ this.controller.abort(/* @__PURE__ */ new Error(`pack "${this.manifest.id}" was closed`));
817
+ this.cache.clear();
818
+ }
819
+ };
820
+ var MemoryPack = class extends BasePack {
821
+ manifest;
822
+ label;
823
+ files;
824
+ constructor(manifest, label, files) {
825
+ super();
826
+ this.manifest = manifest;
827
+ this.label = label;
828
+ this.files = files;
829
+ }
830
+ async readBytes(path) {
831
+ this.entry(path);
832
+ return copy(this.files.get(path));
833
+ }
834
+ async prefetch() {}
835
+ close() {
836
+ this.files = /* @__PURE__ */ new Map();
837
+ }
838
+ };
839
+ /**
840
+ * A pack over loose files, with the same manifest a built pack of those files would have. For
841
+ * tests, development, and content generated at runtime.
842
+ */
843
+ async function packFromFiles(files, options, label = options.id) {
844
+ return new MemoryPack(await describePack(files, options), label, new Map(files.map((file) => [file.path, file.bytes])));
845
+ }
846
+ function isRangeReader(input) {
847
+ return typeof input === "object" && input !== null && typeof input.read === "function" && typeof input.size === "number";
848
+ }
849
+ function defaultLabel(input) {
850
+ if (typeof input === "string" || input instanceof URL) return String(input);
851
+ if (typeof Blob !== "undefined" && input instanceof Blob) return input.name ?? "blob";
852
+ if (isRangeReader(input)) return "reader";
853
+ return "bytes";
854
+ }
855
+ /** Open a pack. Reads only the manifest and zip directory; files are fetched when read. */
856
+ async function openPack(input, options = {}) {
857
+ const label = options.label ?? defaultLabel(input);
858
+ const controller = new AbortController();
859
+ const signal = options.signal === void 0 ? controller.signal : AbortSignal.any([options.signal, controller.signal]);
860
+ let bytesRead = 0;
861
+ let requests = 0;
862
+ const counted = (bytes) => {
863
+ bytesRead += bytes;
864
+ requests++;
865
+ options.onProgress?.({
866
+ label,
867
+ bytes: bytesRead,
868
+ requests
869
+ });
870
+ };
871
+ const counting = (reader) => ({
872
+ size: reader.size,
873
+ read: async (offset, length, readSignal) => {
874
+ const bytes = await reader.read(offset, length, readSignal);
875
+ counted(bytes.length);
876
+ return bytes;
877
+ }
878
+ });
879
+ let reader;
880
+ let tail;
881
+ let tailStart;
882
+ let size;
883
+ if (typeof input === "string" || input instanceof URL) {
884
+ const fetcher = options.fetch ?? globalThis.fetch;
885
+ if (fetcher === void 0) throw new Error("openPack needs fetch to read a URL");
886
+ const opened = await openUrl(String(input), {
887
+ fetch: fetcher,
888
+ retry: options.retry ?? {},
889
+ signal,
890
+ mode: options.mode ?? "auto",
891
+ ...options.sizeHint !== void 0 ? { sizeHint: options.sizeHint } : {},
892
+ wholeThreshold: options.wholeThreshold ?? 4 * 1024 * 1024,
893
+ onResponse: counted
894
+ });
895
+ if (opened.kind === "whole") {
896
+ tail = opened.bytes;
897
+ tailStart = 0;
898
+ size = opened.bytes.length;
899
+ } else {
900
+ tail = opened.tail;
901
+ if (opened.size !== void 0) {
902
+ size = opened.size;
903
+ tailStart = size - tail.length;
904
+ }
905
+ const end = findZipEnd(tail, tailStart);
906
+ size ??= end.archiveSize;
907
+ tailStart ??= size - tail.length;
908
+ reader = opened.reader(size);
909
+ }
910
+ } else if (input instanceof ArrayBuffer || input instanceof Uint8Array) {
911
+ tail = input instanceof Uint8Array ? input : new Uint8Array(input);
912
+ tailStart = 0;
913
+ size = tail.length;
914
+ } else {
915
+ if (typeof Blob !== "undefined" && input instanceof Blob) reader = counting(blobReader(input));
916
+ else if (isRangeReader(input)) reader = counting(input);
917
+ else throw new Error("openPack: unsupported input");
918
+ size = reader.size;
919
+ const length = Math.min(size, TAIL_BYTES);
920
+ tailStart = size - length;
921
+ tail = await reader.read(tailStart, length, signal);
922
+ }
923
+ const archiveSize = size;
924
+ const windowStart = tailStart;
925
+ const scheduler = reader === void 0 ? void 0 : new ReadScheduler(reader, signal, { ...options.concurrency !== void 0 ? { concurrency: options.concurrency } : {} });
926
+ const read = (offset, length) => {
927
+ if (offset >= windowStart && offset + length <= windowStart + tail.length) return Promise.resolve(tail.subarray(offset - windowStart, offset - windowStart + length));
928
+ if (scheduler === void 0) return Promise.reject(/* @__PURE__ */ new Error(`pack ${label} is truncated`));
929
+ return scheduler.read(offset, length);
930
+ };
931
+ const end = findZipEnd(tail, windowStart);
932
+ const members = parseCentralDirectory(await read(end.centralOffset, end.centralSize), end.entries);
933
+ const manifestMember = members.get(PACK_MANIFEST_ENTRY);
934
+ if (manifestMember === void 0) throw new Error(`${label} is not a Molen pack: it has no ${PACK_MANIFEST_ENTRY}`);
935
+ const header = await read(manifestMember.localOffset, Math.min(542, archiveSize - manifestMember.localOffset));
936
+ const manifestBytes = decodeMember(manifestMember, await read(manifestMember.localOffset + localDataOffset(header), manifestMember.compressedSize));
937
+ let manifestDoc;
938
+ try {
939
+ manifestDoc = JSON.parse(new TextDecoder().decode(manifestBytes));
940
+ } catch (error) {
941
+ throw new Error(`${label}: ${PACK_MANIFEST_ENTRY} is not valid JSON`, { cause: error });
942
+ }
943
+ const parsed = validate("pack", manifestDoc);
944
+ if (!parsed.ok) throw new Error(`${label}: invalid pack manifest\n${parsed.formatted}`);
945
+ const manifest = parsed.value;
946
+ if (options.expect !== void 0 && manifest.contentHash !== options.expect.contentHash) throw new Error(`${label}: pack "${manifest.id}" has contentHash ${manifest.contentHash}, expected ${options.expect.contentHash}`);
947
+ for (const [path, entry] of Object.entries(manifest.entries)) if (entry.block === void 0 && !members.has(path)) throw new Error(`${label}: pack "${manifest.id}" lists "${path}" but the zip has no such member`);
948
+ for (const [name, block] of Object.entries(manifest.blocks)) if (!members.has(block.entry)) throw new Error(`${label}: pack "${manifest.id}" block "${name}" is missing from the zip`);
949
+ return new ZipPack(manifest, label, members, archiveSize, {
950
+ start: windowStart,
951
+ bytes: tail
952
+ }, scheduler, controller, options);
953
+ }
954
+ //#endregion
955
+ export { PackChangedError as a, isTextPath as c, packFromFiles as i, mediaTypeOf as l, PackIntegrityError as n, createPack as o, openPack as r, describePack as s, PackEntryNotFoundError as t, sha256 as u };