@bendyline/molen-pack 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,4 +1,5 @@
1
- import { _ as describePack, a as PackIntegrityError, b as sha256, c as openPack, d as PackRetryOptions, f as RangeReader, g as createPack, h as PackOptions, i as PackInput, l as packFromFiles, m as PackFile, n as Pack, o as PackProgress, p as BuiltPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as PackChangedError, v as isTextPath, y as mediaTypeOf } from "./pack-DdapyeeP.mjs";
1
+ import { n as PackRetryOptions, r as RangeReader, t as PackChangedError } from "./source-7Pr2tntF.mjs";
2
+ import { _ as sha256, a as PackIntegrityError, c as openPack, d as PackFile, f as PackOptions, g as mediaTypeOf, h as isTextPath, i as PackInput, l as packFromFiles, m as describePack, n as Pack, o as PackProgress, p as createPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as BuiltPack } from "./pack-CF6pPp9f.mjs";
2
3
  import { PackIndex, PackIndexEntry, PackManifest } from "@bendyline/molen-schema";
3
4
 
4
5
  //#region src/set.d.ts
package/dist/index.mjs CHANGED
@@ -1,4 +1,5 @@
1
- import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-B-8f90rh.mjs";
1
+ import { a as createPack, c as mediaTypeOf, i as packFromFiles, l as sha256, n as PackIntegrityError, o as describePack, r as openPack, s as isTextPath, t as PackEntryNotFoundError } from "./pack-BIXihu9D.mjs";
2
+ import { t as PackChangedError } from "./source-MDAzioCi.mjs";
2
3
  //#region src/set.ts
3
4
  var Packs = class {
4
5
  list = [];
package/dist/node.d.mts CHANGED
@@ -1,4 +1,4 @@
1
- import { m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-DdapyeeP.mjs";
1
+ import { d as PackFile, f as PackOptions, n as Pack, t as OpenPackOptions } from "./pack-CF6pPp9f.mjs";
2
2
  import { PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
3
3
 
4
4
  //#region src/node.d.ts
@@ -54,6 +54,7 @@ declare function recordInPackIndex(outDir: string, id: string, entry: PackIndexE
54
54
  declare function buildPack(dir: string, options: {
55
55
  outDir: string;
56
56
  solid?: boolean;
57
+ compression?: PackOptions["compression"];
57
58
  }): Promise<BuiltPackFile>;
58
59
  /**
59
60
  * Write every file of a pack into `outDir`, with a `molen-pack.source.json` that builds it back
package/dist/node.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-B-8f90rh.mjs";
1
+ import { a as createPack, i as packFromFiles, l as sha256, r as openPack } from "./pack-BIXihu9D.mjs";
2
2
  import { validate } from "@bendyline/molen-schema";
3
3
  import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
4
  import { dirname, join, relative, resolve, sep } from "node:path";
@@ -162,7 +162,8 @@ async function buildPack(dir, options) {
162
162
  const source = await readPackSource(dir);
163
163
  const built = await createPack(source.files, {
164
164
  ...optionsOf(source.config),
165
- ...options.solid !== void 0 ? { solid: options.solid } : {}
165
+ ...options.solid !== void 0 ? { solid: options.solid } : {},
166
+ ...options.compression !== void 0 ? { compression: options.compression } : {}
166
167
  });
167
168
  const fileSha256 = await sha256(built.bytes);
168
169
  const id = built.manifest.id;
@@ -1,3 +1,4 @@
1
+ import { a as openUrl, i as blobReader, n as ReadScheduler, r as TAIL_BYTES } from "./source-MDAzioCi.mjs";
1
2
  import { PACK_MANIFEST_ENTRY, PACK_RESERVED_PREFIX, validate } from "@bendyline/molen-schema";
2
3
  import { deflateSync, inflateSync } from "fflate";
3
4
  //#region src/zip.ts
@@ -345,6 +346,7 @@ async function createPack(files, options) {
345
346
  const sorted = sortedFiles(files);
346
347
  for (const file of sorted) if (file.path === PACK_MANIFEST_ENTRY || file.path.startsWith(PACK_RESERVED_PREFIX)) throw new Error(`"${file.path}" is reserved for the pack's own structure`);
347
348
  const described = await describePack(sorted, options);
349
+ const compression = options.compression ?? "auto";
348
350
  const entries = structuredClone(described.entries);
349
351
  const members = [];
350
352
  const blocks = {};
@@ -356,7 +358,7 @@ async function createPack(files, options) {
356
358
  } else members.push({
357
359
  name: file.path,
358
360
  data: file.bytes,
359
- compression: "auto"
361
+ compression
360
362
  });
361
363
  for (const [group, groupFiles] of [...pending].sort(([a], [b]) => a < b ? -1 : 1)) {
362
364
  let part = 1;
@@ -384,7 +386,7 @@ async function createPack(files, options) {
384
386
  blockMembers.push({
385
387
  name: entry,
386
388
  data,
387
- compression: "auto"
389
+ compression
388
390
  });
389
391
  part++;
390
392
  chunk = [];
@@ -410,280 +412,13 @@ async function createPack(files, options) {
410
412
  {
411
413
  name: PACK_MANIFEST_ENTRY,
412
414
  data: new TextEncoder().encode(JSON.stringify(manifest)),
413
- compression: "auto"
415
+ compression
414
416
  }
415
417
  ]),
416
418
  manifest
417
419
  };
418
420
  }
419
421
  //#endregion
420
- //#region src/source.ts
421
- /** The server replaced the pack while it was being read (its ETag changed). */
422
- var PackChangedError = class extends Error {
423
- constructor(url) {
424
- super(`pack ${url} changed on the server while it was being read; open it again`);
425
- this.name = "PackChangedError";
426
- }
427
- };
428
- var HttpStatusError = class extends Error {
429
- url;
430
- status;
431
- constructor(url, status) {
432
- super(`HTTP ${status} for ${url}`);
433
- this.url = url;
434
- this.status = status;
435
- this.name = "HttpStatusError";
436
- }
437
- };
438
- function blobReader(blob) {
439
- return {
440
- size: blob.size,
441
- read: async (offset, length) => new Uint8Array(await blob.slice(offset, offset + length).arrayBuffer())
442
- };
443
- }
444
- function isRetryable(error) {
445
- if (error instanceof HttpStatusError) return error.status === 408 || error.status === 429 || error.status >= 500;
446
- if (error instanceof PackChangedError) return false;
447
- const name = error?.name;
448
- return error instanceof TypeError || name === "TimeoutError";
449
- }
450
- function sleep(ms, signal) {
451
- return new Promise((resolve, reject) => {
452
- if (signal.aborted) return reject(signal.reason);
453
- const timer = setTimeout(() => {
454
- signal.removeEventListener("abort", abort);
455
- resolve();
456
- }, ms);
457
- const abort = () => {
458
- clearTimeout(timer);
459
- reject(signal.reason);
460
- };
461
- signal.addEventListener("abort", abort, { once: true });
462
- });
463
- }
464
- /** Run `task` with a per-attempt deadline, retrying transient failures with backoff. */
465
- async function withRetry(task, retry, signal) {
466
- const attempts = Math.max(1, retry.attempts ?? 3);
467
- const base = retry.baseDelayMs ?? 250;
468
- for (let attempt = 1;; attempt++) {
469
- const deadline = AbortSignal.timeout(retry.timeoutMs ?? 3e4);
470
- try {
471
- return await task(AbortSignal.any([signal, deadline]));
472
- } catch (error) {
473
- if (signal.aborted) throw signal.reason;
474
- if (attempt >= attempts || !isRetryable(error)) throw error;
475
- await sleep(Math.min(base * 2 ** (attempt - 1), retry.maxDelayMs ?? 4e3), signal);
476
- }
477
- }
478
- }
479
- /** Bytes fetched when a remote pack is opened: the manifest and central directory live here. */
480
- const TAIL_BYTES = 64 * 1024;
481
- async function fetchBytes(response) {
482
- return new Uint8Array(await response.arrayBuffer());
483
- }
484
- /** Missing Content-Range is allowed for CORS hosts which do not expose this header. */
485
- function contentRange(value, url) {
486
- if (value === void 0) return void 0;
487
- const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
488
- const start = Number(match?.[1]);
489
- const end = Number(match?.[2]);
490
- const size = match?.[3] === "*" ? void 0 : Number(match?.[3]);
491
- if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || size !== void 0 && (!Number.isSafeInteger(size) || size <= end)) throw new Error(`invalid Content-Range for ${url}: ${value}`);
492
- return {
493
- start,
494
- end,
495
- size
496
- };
497
- }
498
- function strongEtag(value) {
499
- return value !== void 0 && !value.startsWith("W/") ? value : void 0;
500
- }
501
- /**
502
- * Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
503
- * range request) has already sent the whole file, which is used as is.
504
- */
505
- async function openUrl(url, options) {
506
- const { fetch: fetcher, retry, signal } = options;
507
- if (options.mode === "whole" || options.mode === "auto" && options.sizeHint !== void 0 && options.sizeHint <= options.wholeThreshold) {
508
- const bytes = await withRetry(async (attemptSignal) => {
509
- const response = await fetcher(url, { signal: attemptSignal });
510
- if (!response.ok) throw new HttpStatusError(url, response.status);
511
- return fetchBytes(response);
512
- }, retry, signal);
513
- options.onResponse(bytes.length);
514
- return {
515
- kind: "whole",
516
- bytes
517
- };
518
- }
519
- let first = await withRetry(async (attemptSignal) => {
520
- const response = await fetcher(url, {
521
- headers: { Range: `bytes=-${TAIL_BYTES}` },
522
- signal: attemptSignal
523
- });
524
- if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
525
- return {
526
- status: response.status,
527
- bytes: await fetchBytes(response),
528
- etag: response.headers.get("etag") ?? void 0,
529
- range: response.headers.get("content-range") ?? void 0
530
- };
531
- }, retry, signal);
532
- options.onResponse(first.bytes.length);
533
- if (first.status === 200) return {
534
- kind: "whole",
535
- bytes: first.bytes
536
- };
537
- let range = contentRange(first.range, url);
538
- const total = range?.size;
539
- if (total !== void 0 && (range?.start !== Math.max(0, total - 65536) || range?.end !== total - 1 || first.bytes.length !== Math.min(total, 65536))) {
540
- const validator = strongEtag(first.etag);
541
- const start = Math.max(0, total - TAIL_BYTES);
542
- first = await withRetry(async (attemptSignal) => {
543
- const headers = { Range: `bytes=${start}-${total - 1}` };
544
- if (validator !== void 0) headers["If-Range"] = validator;
545
- const response = await fetcher(url, {
546
- headers,
547
- signal: attemptSignal
548
- });
549
- if (validator !== void 0 && (response.status === 200 || response.headers.has("etag") && response.headers.get("etag") !== validator)) throw new PackChangedError(url);
550
- if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
551
- return {
552
- status: response.status,
553
- bytes: await fetchBytes(response),
554
- etag: response.headers.get("etag") ?? validator,
555
- range: response.headers.get("content-range") ?? void 0
556
- };
557
- }, retry, signal);
558
- options.onResponse(first.bytes.length);
559
- if (first.status === 200) return {
560
- kind: "whole",
561
- bytes: first.bytes
562
- };
563
- range = contentRange(first.range, url);
564
- if (first.bytes.length !== total - start || range !== void 0 && (range.start !== start || range.end !== total - 1 || range.size !== void 0 && range.size !== total)) throw new Error(`tail range ${start}-${total - 1} of ${url} was not honoured`);
565
- } else if (range !== void 0 && first.bytes.length !== range.end - range.start + 1) throw new Error(`Content-Range length does not match response bytes for ${url}`);
566
- const etag = strongEtag(first.etag);
567
- return {
568
- kind: "range",
569
- tail: first.bytes,
570
- ...total !== void 0 && Number.isFinite(total) ? { size: total } : {},
571
- reader: (size) => ({
572
- size,
573
- read: (offset, length, readSignal) => withRetry(async (attemptSignal) => {
574
- const headers = { Range: `bytes=${offset}-${offset + length - 1}` };
575
- if (etag !== void 0) headers["If-Range"] = etag;
576
- const response = await fetcher(url, {
577
- headers,
578
- signal: readSignal === void 0 ? attemptSignal : AbortSignal.any([attemptSignal, readSignal])
579
- });
580
- if (response.status === 200) {
581
- if (etag !== void 0) throw new PackChangedError(url);
582
- const all = await fetchBytes(response);
583
- options.onResponse(all.length);
584
- return all.subarray(offset, offset + length);
585
- }
586
- if (response.status !== 206) throw new HttpStatusError(url, response.status);
587
- const returnedRange = contentRange(response.headers.get("content-range") ?? void 0, url);
588
- if (returnedRange !== void 0 && (returnedRange.start !== offset || returnedRange.end !== offset + length - 1 || returnedRange.size !== void 0 && returnedRange.size !== size)) throw new Error(`range ${offset}+${length} of ${url} was not honoured`);
589
- const bytes = await fetchBytes(response);
590
- options.onResponse(bytes.length);
591
- if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
592
- return bytes;
593
- }, retry, signal)
594
- })
595
- };
596
- }
597
- /**
598
- * Queue reads for one source. Reads issued in the same tick are sorted, merged when close
599
- * together, and run with bounded concurrency, high priority first.
600
- */
601
- var ReadScheduler = class {
602
- reader;
603
- signal;
604
- queue = [];
605
- flushing = false;
606
- running = 0;
607
- batches = [];
608
- concurrency;
609
- mergeGap;
610
- maxSpan;
611
- constructor(reader, signal, options = {}) {
612
- this.reader = reader;
613
- this.signal = signal;
614
- this.concurrency = Math.max(1, options.concurrency ?? 6);
615
- this.mergeGap = options.mergeGap ?? 32 * 1024;
616
- this.maxSpan = options.maxSpan ?? 4 * 1024 * 1024;
617
- }
618
- read(offset, length, high = true, signal) {
619
- if (length === 0) return Promise.resolve(new Uint8Array(0));
620
- return new Promise((resolve, reject) => {
621
- if (signal?.aborted) return reject(signal.reason);
622
- const abort = () => reject(signal?.reason);
623
- signal?.addEventListener("abort", abort, { once: true });
624
- this.queue.push({
625
- offset,
626
- length,
627
- high,
628
- resolve: (bytes) => {
629
- signal?.removeEventListener("abort", abort);
630
- resolve(bytes);
631
- },
632
- reject: (error) => {
633
- signal?.removeEventListener("abort", abort);
634
- reject(error);
635
- }
636
- });
637
- if (!this.flushing) {
638
- this.flushing = true;
639
- queueMicrotask(() => this.flush());
640
- }
641
- });
642
- }
643
- flush() {
644
- this.flushing = false;
645
- const waiters = this.queue.sort((a, b) => a.offset - b.offset);
646
- this.queue = [];
647
- for (const waiter of waiters) {
648
- const last = this.batches.at(-1);
649
- const end = waiter.offset + waiter.length;
650
- if (last !== void 0 && waiter.offset - last.end <= this.mergeGap && Math.max(last.end, end) - last.start <= this.maxSpan && last.start <= waiter.offset) {
651
- last.end = Math.max(last.end, end);
652
- last.high ||= waiter.high;
653
- last.waiters.push(waiter);
654
- } else this.batches.push({
655
- start: waiter.offset,
656
- end,
657
- high: waiter.high,
658
- waiters: [waiter]
659
- });
660
- }
661
- this.batches.sort((a, b) => Number(b.high) - Number(a.high));
662
- this.pump();
663
- }
664
- pump() {
665
- if (this.signal.aborted) {
666
- for (const batch of this.batches.splice(0)) for (const waiter of batch.waiters) waiter.reject(this.signal.reason);
667
- return;
668
- }
669
- while (this.running < this.concurrency && this.batches.length > 0) {
670
- const batch = this.batches.shift();
671
- this.running++;
672
- this.reader.read(batch.start, batch.end - batch.start, this.signal).then((bytes) => {
673
- for (const waiter of batch.waiters) {
674
- const from = waiter.offset - batch.start;
675
- waiter.resolve(bytes.subarray(from, from + waiter.length));
676
- }
677
- }, (error) => {
678
- for (const waiter of batch.waiters) waiter.reject(error);
679
- }).finally(() => {
680
- this.running--;
681
- this.pump();
682
- });
683
- }
684
- }
685
- };
686
- //#endregion
687
422
  //#region src/pack.ts
688
423
  /**
689
424
  * Open a pack from any source and read its files. Only the tail of a remote pack is fetched up
@@ -999,4 +734,4 @@ async function openPack(input, options = {}) {
999
734
  }, scheduler, controller, options);
1000
735
  }
1001
736
  //#endregion
1002
- export { PackChangedError as a, isTextPath as c, packFromFiles as i, mediaTypeOf as l, PackIntegrityError as n, createPack as o, openPack as r, describePack as s, PackEntryNotFoundError as t, sha256 as u };
737
+ export { createPack as a, mediaTypeOf as c, packFromFiles as i, sha256 as l, PackIntegrityError as n, describePack as o, openPack as r, isTextPath as s, PackEntryNotFoundError as t };
@@ -1,3 +1,4 @@
1
+ import { n as PackRetryOptions, r as RangeReader } from "./source-7Pr2tntF.mjs";
1
2
  import { PackManifest } from "@bendyline/molen-schema";
2
3
 
3
4
  //#region src/build.d.ts
@@ -19,6 +20,12 @@ interface PackOptions {
19
20
  provides?: Record<string, string | readonly string[]>;
20
21
  /** Group small text files into compressed solid blocks (default true). */
21
22
  solid?: boolean;
23
+ /**
24
+ * `auto` (default) deflates a member when that saves at least 10%. `store` writes every member
25
+ * uncompressed: the manifest and contents are identical, only the archive is larger, so a pack
26
+ * built for local reading (a test run) skips the deflate that dominates build time.
27
+ */
28
+ compression?: "auto" | "store";
22
29
  }
23
30
  interface BuiltPack {
24
31
  bytes: Uint8Array;
@@ -36,31 +43,6 @@ declare function describePack(files: readonly PackFile[], options: PackOptions):
36
43
  /** Build a pack: a deterministic zip whose last member is the manifest. */
37
44
  declare function createPack(files: readonly PackFile[], options: PackOptions): Promise<BuiltPack>;
38
45
  //#endregion
39
- //#region src/source.d.ts
40
- /**
41
- * Byte sources a pack can be read from, and the transport policy for remote ones: tail-first
42
- * opening, range reads merged when they are close together, bounded concurrency, and retries.
43
- */
44
- /** Random access to a pack's bytes. Supply one for storage Molen does not know about. */
45
- interface RangeReader {
46
- /** Total size in bytes. */
47
- readonly size: number;
48
- read(offset: number, length: number, signal?: AbortSignal): Promise<Uint8Array>;
49
- }
50
- interface PackRetryOptions {
51
- /** Attempts per request, including the first (default 3). */
52
- attempts?: number;
53
- /** Delay before the first retry; doubles each time (default 250 ms). */
54
- baseDelayMs?: number;
55
- maxDelayMs?: number;
56
- /** Per-request deadline (default 30 s). */
57
- timeoutMs?: number;
58
- }
59
- /** The server replaced the pack while it was being read (its ETag changed). */
60
- declare class PackChangedError extends Error {
61
- constructor(url: string);
62
- }
63
- //#endregion
64
46
  //#region src/pack.d.ts
65
47
  /** Where a pack's bytes come from: a URL, a file the user picked, bytes, or your own reader. */
66
48
  type PackInput = string | URL | Blob | ArrayBuffer | Uint8Array | RangeReader;
@@ -132,4 +114,4 @@ declare function packFromFiles(files: readonly PackFile[], options: PackOptions,
132
114
  /** Open a pack. Reads only the manifest and zip directory; files are fetched when read. */
133
115
  declare function openPack(input: PackInput, options?: OpenPackOptions): Promise<Pack>;
134
116
  //#endregion
135
- export { describePack as _, PackIntegrityError as a, sha256 as b, openPack as c, PackRetryOptions as d, RangeReader as f, createPack as g, PackOptions as h, PackInput as i, packFromFiles as l, PackFile as m, Pack as n, PackProgress as o, BuiltPack as p, PackEntryNotFoundError as r, ReadOptions as s, OpenPackOptions as t, PackChangedError as u, isTextPath as v, mediaTypeOf as y };
117
+ export { sha256 as _, PackIntegrityError as a, openPack as c, PackFile as d, PackOptions as f, mediaTypeOf as g, isTextPath as h, PackInput as i, packFromFiles as l, describePack as m, Pack as n, PackProgress as o, createPack as p, PackEntryNotFoundError as r, ReadOptions as s, OpenPackOptions as t, BuiltPack as u };
@@ -0,0 +1,41 @@
1
+ //#region src/source.d.ts
2
+ /**
3
+ * Byte sources a pack can be read from, and the transport policy for remote ones: tail-first
4
+ * opening, range reads merged when they are close together, bounded concurrency, and retries.
5
+ */
6
+ /** Random access to a pack's bytes. Supply one for storage Molen does not know about. */
7
+ interface RangeReader {
8
+ /** Total size in bytes. */
9
+ readonly size: number;
10
+ read(offset: number, length: number, signal?: AbortSignal): Promise<Uint8Array>;
11
+ }
12
+ interface PackRetryOptions {
13
+ /** Attempts per request, including the first (default 3). */
14
+ attempts?: number;
15
+ /** Delay before the first retry; doubles each time (default 250 ms). */
16
+ baseDelayMs?: number;
17
+ maxDelayMs?: number;
18
+ /** Per-request deadline (default 30 s). */
19
+ timeoutMs?: number;
20
+ }
21
+ /** The server replaced the pack while it was being read (its ETag changed). */
22
+ declare class PackChangedError extends Error {
23
+ constructor(url: string);
24
+ }
25
+ interface UrlRangeReaderOptions {
26
+ /** Total size in bytes (a pack index lists it). */
27
+ size: number;
28
+ fetch?: typeof fetch;
29
+ retry?: PackRetryOptions;
30
+ /** Aborts every read. */
31
+ signal?: AbortSignal;
32
+ }
33
+ /**
34
+ * A `RangeReader` over a URL whose size is already known, with no opening request: the first
35
+ * response's strong ETag becomes the If-Range validator for every later read, so a file replaced
36
+ * mid-read fails with `PackChangedError` instead of mixing versions. Pair it with
37
+ * `cachingRangeReader` to open a pack from cached bytes.
38
+ */
39
+ declare function urlRangeReader(url: string, options: UrlRangeReaderOptions): RangeReader;
40
+ //#endregion
41
+ export { urlRangeReader as a, UrlRangeReaderOptions as i, PackRetryOptions as n, RangeReader as r, PackChangedError as t };