@bendyline/molen-pack 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -82,7 +82,7 @@ provides, and a `contentHash` over paths and contents that does not depend on th
82
82
 
83
83
  ## Status
84
84
 
85
- 0.x, on one fixed version line with every other `@bendyline/molen-*` package. The format is
85
+ 0.x, versioned independently of the other `@bendyline/molen-*` packages. The format is
86
86
  beta. Packs are written without zip64, so a pack must stay under 4 GiB and 65,535 members.
87
87
  Building the same files always produces the same bytes; that depends on the exact `fflate`
88
88
  version this package pins.
package/dist/index.d.mts CHANGED
@@ -1,4 +1,4 @@
1
- import { _ as describePack, a as PackIntegrityError, b as sha256, c as openPack, d as PackRetryOptions, f as RangeReader, g as createPack, h as PackOptions, i as PackInput, l as packFromFiles, m as PackFile, n as Pack, o as PackProgress, p as BuiltPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as PackChangedError, v as isTextPath, y as mediaTypeOf } from "./pack-DdapyeeP.mjs";
1
+ import { _ as describePack, a as PackIntegrityError, b as sha256, c as openPack, d as PackRetryOptions, f as RangeReader, g as createPack, h as PackOptions, i as PackInput, l as packFromFiles, m as PackFile, n as Pack, o as PackProgress, p as BuiltPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as PackChangedError, v as isTextPath, y as mediaTypeOf } from "./pack-DjufOvAS.mjs";
2
2
  import { PackIndex, PackIndexEntry, PackManifest } from "@bendyline/molen-schema";
3
3
 
4
4
  //#region src/set.d.ts
package/dist/index.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-CDBKnu89.mjs";
1
+ import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-DCQvXj1p.mjs";
2
2
  //#region src/set.ts
3
3
  var Packs = class {
4
4
  list = [];
package/dist/node.d.mts CHANGED
@@ -1,5 +1,5 @@
1
- import { m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-DdapyeeP.mjs";
2
- import { PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
1
+ import { h as PackOptions, m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-DjufOvAS.mjs";
2
+ import { PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
3
3
 
4
4
  //#region src/node.d.ts
5
5
  /** Build settings file in a pack's source directory. */
@@ -13,6 +13,12 @@ interface PackSource {
13
13
  config: PackSourceConfig;
14
14
  files: PackFile[];
15
15
  }
16
+ /** List validated source settings and included paths without loading file contents. */
17
+ declare function listPackSource(dir: string): Promise<{
18
+ dir: string;
19
+ config: PackSourceConfig;
20
+ paths: string[];
21
+ }>;
16
22
  /** Read a pack's source directory: its settings file and every file it includes. */
17
23
  declare function readPackSource(dir: string): Promise<PackSource>;
18
24
  /**
@@ -35,6 +41,12 @@ interface BuiltPackFile {
35
41
  indexPath: string;
36
42
  }
37
43
  /**
44
+ * List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
45
+ * the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
46
+ * index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
47
+ */
48
+ declare function recordInPackIndex(outDir: string, id: string, entry: PackIndexEntry): Promise<string>;
49
+ /**
38
50
  * Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
39
51
  * host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
40
52
  * the previous build of the same pack is removed.
@@ -42,6 +54,7 @@ interface BuiltPackFile {
42
54
  declare function buildPack(dir: string, options: {
43
55
  outDir: string;
44
56
  solid?: boolean;
57
+ compression?: PackOptions["compression"];
45
58
  }): Promise<BuiltPackFile>;
46
59
  /**
47
60
  * Write every file of a pack into `outDir`, with a `molen-pack.source.json` that builds it back
@@ -49,4 +62,4 @@ declare function buildPack(dir: string, options: {
49
62
  */
50
63
  declare function extractPack(pack: Pack, outDir: string): Promise<string[]>;
51
64
  //#endregion
52
- export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
65
+ export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
package/dist/node.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-CDBKnu89.mjs";
1
+ import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-DCQvXj1p.mjs";
2
2
  import { validate } from "@bendyline/molen-schema";
3
3
  import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
4
  import { dirname, join, relative, resolve, sep } from "node:path";
@@ -57,8 +57,8 @@ async function walk(dir, root, out) {
57
57
  else if (item.isFile()) out.push(relative(root, full).split(sep).join("/"));
58
58
  }
59
59
  }
60
- /** Read a pack's source directory: its settings file and every file it includes. */
61
- async function readPackSource(dir) {
60
+ /** List validated source settings and included paths without loading file contents. */
61
+ async function listPackSource(dir) {
62
62
  const root = resolve(dir);
63
63
  let raw;
64
64
  try {
@@ -73,14 +73,23 @@ async function readPackSource(dir) {
73
73
  const exclude = config.exclude.map(globToRegExp);
74
74
  const paths = [];
75
75
  await walk(root, root, paths);
76
- const selected = paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort();
77
76
  return {
78
77
  dir: root,
79
78
  config,
80
- files: await Promise.all(selected.map(async (path) => ({
81
- path,
82
- bytes: new Uint8Array(await readFile(join(root, ...path.split("/"))))
83
- })))
79
+ paths: paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort()
80
+ };
81
+ }
82
+ /** Read a pack's source directory: its settings file and every file it includes. */
83
+ async function readPackSource(dir) {
84
+ const source = await listPackSource(dir);
85
+ const files = await Promise.all(source.paths.map(async (path) => ({
86
+ path,
87
+ bytes: new Uint8Array(await readFile(join(source.dir, ...path.split("/"))))
88
+ })));
89
+ return {
90
+ dir: source.dir,
91
+ config: source.config,
92
+ files
84
93
  };
85
94
  }
86
95
  function optionsOf(config) {
@@ -126,6 +135,25 @@ async function readIndex(path) {
126
135
  return parsed.value;
127
136
  }
128
137
  /**
138
+ * List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
139
+ * the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
140
+ * index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
141
+ */
142
+ async function recordInPackIndex(outDir, id, entry) {
143
+ const indexPath = join(outDir, PACK_INDEX_FILE);
144
+ const index = await readIndex(indexPath);
145
+ const previous = index.packs[id]?.file;
146
+ index.packs[id] = entry;
147
+ const sorted = {
148
+ format: "molen/pack-index@1",
149
+ packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
150
+ };
151
+ await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
152
+ const stillListed = Object.values(sorted.packs).some((e) => e.file === previous);
153
+ if (previous !== void 0 && previous !== entry.file && !stillListed) await rm(join(outDir, previous), { force: true });
154
+ return indexPath;
155
+ }
156
+ /**
129
157
  * Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
130
158
  * host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
131
159
  * the previous build of the same pack is removed.
@@ -134,7 +162,8 @@ async function buildPack(dir, options) {
134
162
  const source = await readPackSource(dir);
135
163
  const built = await createPack(source.files, {
136
164
  ...optionsOf(source.config),
137
- ...options.solid !== void 0 ? { solid: options.solid } : {}
165
+ ...options.solid !== void 0 ? { solid: options.solid } : {},
166
+ ...options.compression !== void 0 ? { compression: options.compression } : {}
138
167
  });
139
168
  const fileSha256 = await sha256(built.bytes);
140
169
  const id = built.manifest.id;
@@ -143,22 +172,12 @@ async function buildPack(dir, options) {
143
172
  await mkdir(outDir, { recursive: true });
144
173
  const path = join(outDir, file);
145
174
  await writeFile(path, built.bytes);
146
- const indexPath = join(outDir, PACK_INDEX_FILE);
147
- const index = await readIndex(indexPath);
148
- const previous = index.packs[id]?.file;
149
- index.packs[id] = {
175
+ const indexPath = await recordInPackIndex(outDir, id, {
150
176
  version: built.manifest.version,
151
177
  file,
152
178
  contentHash: built.manifest.contentHash,
153
179
  size: built.bytes.length
154
- };
155
- const sorted = {
156
- format: "molen/pack-index@1",
157
- packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
158
- };
159
- await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
160
- const stillListed = Object.values(sorted.packs).some((entry) => entry.file === previous);
161
- if (previous !== void 0 && previous !== file && !stillListed) await rm(join(outDir, previous), { force: true });
180
+ });
162
181
  return {
163
182
  manifest: built.manifest,
164
183
  file,
@@ -200,4 +219,4 @@ async function extractPack(pack, outDir) {
200
219
  return written;
201
220
  }
202
221
  //#endregion
203
- export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
222
+ export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
@@ -345,6 +345,7 @@ async function createPack(files, options) {
345
345
  const sorted = sortedFiles(files);
346
346
  for (const file of sorted) if (file.path === PACK_MANIFEST_ENTRY || file.path.startsWith(PACK_RESERVED_PREFIX)) throw new Error(`"${file.path}" is reserved for the pack's own structure`);
347
347
  const described = await describePack(sorted, options);
348
+ const compression = options.compression ?? "auto";
348
349
  const entries = structuredClone(described.entries);
349
350
  const members = [];
350
351
  const blocks = {};
@@ -356,7 +357,7 @@ async function createPack(files, options) {
356
357
  } else members.push({
357
358
  name: file.path,
358
359
  data: file.bytes,
359
- compression: "auto"
360
+ compression
360
361
  });
361
362
  for (const [group, groupFiles] of [...pending].sort(([a], [b]) => a < b ? -1 : 1)) {
362
363
  let part = 1;
@@ -384,7 +385,7 @@ async function createPack(files, options) {
384
385
  blockMembers.push({
385
386
  name: entry,
386
387
  data,
387
- compression: "auto"
388
+ compression
388
389
  });
389
390
  part++;
390
391
  chunk = [];
@@ -410,7 +411,7 @@ async function createPack(files, options) {
410
411
  {
411
412
  name: PACK_MANIFEST_ENTRY,
412
413
  data: new TextEncoder().encode(JSON.stringify(manifest)),
413
- compression: "auto"
414
+ compression
414
415
  }
415
416
  ]),
416
417
  manifest
@@ -481,6 +482,23 @@ const TAIL_BYTES = 64 * 1024;
481
482
  async function fetchBytes(response) {
482
483
  return new Uint8Array(await response.arrayBuffer());
483
484
  }
485
+ /** Missing Content-Range is allowed for CORS hosts which do not expose this header. */
486
+ function contentRange(value, url) {
487
+ if (value === void 0) return void 0;
488
+ const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
489
+ const start = Number(match?.[1]);
490
+ const end = Number(match?.[2]);
491
+ const size = match?.[3] === "*" ? void 0 : Number(match?.[3]);
492
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || size !== void 0 && (!Number.isSafeInteger(size) || size <= end)) throw new Error(`invalid Content-Range for ${url}: ${value}`);
493
+ return {
494
+ start,
495
+ end,
496
+ size
497
+ };
498
+ }
499
+ function strongEtag(value) {
500
+ return value !== void 0 && !value.startsWith("W/") ? value : void 0;
501
+ }
484
502
  /**
485
503
  * Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
486
504
  * range request) has already sent the whole file, which is used as is.
@@ -499,7 +517,7 @@ async function openUrl(url, options) {
499
517
  bytes
500
518
  };
501
519
  }
502
- const first = await withRetry(async (attemptSignal) => {
520
+ let first = await withRetry(async (attemptSignal) => {
503
521
  const response = await fetcher(url, {
504
522
  headers: { Range: `bytes=-${TAIL_BYTES}` },
505
523
  signal: attemptSignal
@@ -517,8 +535,36 @@ async function openUrl(url, options) {
517
535
  kind: "whole",
518
536
  bytes: first.bytes
519
537
  };
520
- const total = first.range === void 0 ? void 0 : Number(first.range.split("/")[1]);
521
- const etag = first.etag !== void 0 && !first.etag.startsWith("W/") ? first.etag : void 0;
538
+ let range = contentRange(first.range, url);
539
+ const total = range?.size;
540
+ if (total !== void 0 && (range?.start !== Math.max(0, total - 65536) || range?.end !== total - 1 || first.bytes.length !== Math.min(total, 65536))) {
541
+ const validator = strongEtag(first.etag);
542
+ const start = Math.max(0, total - TAIL_BYTES);
543
+ first = await withRetry(async (attemptSignal) => {
544
+ const headers = { Range: `bytes=${start}-${total - 1}` };
545
+ if (validator !== void 0) headers["If-Range"] = validator;
546
+ const response = await fetcher(url, {
547
+ headers,
548
+ signal: attemptSignal
549
+ });
550
+ if (validator !== void 0 && (response.status === 200 || response.headers.has("etag") && response.headers.get("etag") !== validator)) throw new PackChangedError(url);
551
+ if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
552
+ return {
553
+ status: response.status,
554
+ bytes: await fetchBytes(response),
555
+ etag: response.headers.get("etag") ?? validator,
556
+ range: response.headers.get("content-range") ?? void 0
557
+ };
558
+ }, retry, signal);
559
+ options.onResponse(first.bytes.length);
560
+ if (first.status === 200) return {
561
+ kind: "whole",
562
+ bytes: first.bytes
563
+ };
564
+ range = contentRange(first.range, url);
565
+ if (first.bytes.length !== total - start || range !== void 0 && (range.start !== start || range.end !== total - 1 || range.size !== void 0 && range.size !== total)) throw new Error(`tail range ${start}-${total - 1} of ${url} was not honoured`);
566
+ } else if (range !== void 0 && first.bytes.length !== range.end - range.start + 1) throw new Error(`Content-Range length does not match response bytes for ${url}`);
567
+ const etag = strongEtag(first.etag);
522
568
  return {
523
569
  kind: "range",
524
570
  tail: first.bytes,
@@ -539,6 +585,8 @@ async function openUrl(url, options) {
539
585
  return all.subarray(offset, offset + length);
540
586
  }
541
587
  if (response.status !== 206) throw new HttpStatusError(url, response.status);
588
+ const returnedRange = contentRange(response.headers.get("content-range") ?? void 0, url);
589
+ if (returnedRange !== void 0 && (returnedRange.start !== offset || returnedRange.end !== offset + length - 1 || returnedRange.size !== void 0 && returnedRange.size !== size)) throw new Error(`range ${offset}+${length} of ${url} was not honoured`);
542
590
  const bytes = await fetchBytes(response);
543
591
  options.onResponse(bytes.length);
544
592
  if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
@@ -19,6 +19,12 @@ interface PackOptions {
19
19
  provides?: Record<string, string | readonly string[]>;
20
20
  /** Group small text files into compressed solid blocks (default true). */
21
21
  solid?: boolean;
22
+ /**
23
+ * `auto` (default) deflates a member when that saves at least 10%. `store` writes every member
24
+ * uncompressed: the manifest and contents are identical, only the archive is larger, so a pack
25
+ * built for local reading (a test run) skips the deflate that dominates build time.
26
+ */
27
+ compression?: "auto" | "store";
22
28
  }
23
29
  interface BuiltPack {
24
30
  bytes: Uint8Array;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/molen-pack",
3
- "version": "0.0.2",
3
+ "version": "0.0.4",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"
@@ -42,7 +42,7 @@
42
42
  ],
43
43
  "dependencies": {
44
44
  "fflate": "0.8.3",
45
- "@bendyline/molen-schema": "0.0.2"
45
+ "@bendyline/molen-schema": "0.0.4"
46
46
  },
47
47
  "devDependencies": {
48
48
  "@biomejs/biome": "^2.5.0",