@bendyline/molen-pack 0.0.8 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -62,20 +62,26 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
62
62
  const pack = await openDirPack('content/my-props');
63
63
  ```
64
64
 
65
- ## Host packs unzipped
65
+ ## Host packs on a CDN
66
66
 
67
- A CDN serves a pack just as well with its files loose: every file is its own cacheable URL, and a
68
- new release uploads only the packs that changed. `hostPacks` publishes the packs a pack index lists
69
- that way. Each version goes to `<pack id>/<12 hex of its contentHash>/` once and never changes, so
70
- those files can be served as immutable, and the `index.json` it writes last is the only document
71
- that changes between releases:
67
+ `hostPacks` publishes the packs a pack index lists for a CDN. Each version goes to
68
+ `<pack id>/<12 hex of its contentHash>/` once and never changes, so its files can be served as
69
+ immutable, and the `index.json` it writes last is the only document that changes between releases;
70
+ a new release uploads only the packs that changed.
71
+
72
+ A pack whose zip is at most `zipBelow` bytes (default `HOSTED_ZIP_BELOW`, 4 MiB) is published as
73
+ that zip: readers fetch an archive that small whole, so a pack of a thousand small documents costs
74
+ one request. A larger pack is published unzipped, every file at its own cacheable URL, so readers
75
+ fetch only the files they use. `zipBelow: 0` unzips every pack.
72
76
 
73
77
  ```ts
74
78
  import { hostPacks } from '@bendyline/molen-pack/node';
75
79
 
76
80
  // dist/packs/index.json lists built zips; _a/ becomes the directory to upload.
77
81
  await hostPacks('dist/packs/index.json', { outDir: '_a' });
78
- // _a/index.json → { "molen.sky": { "file": "molen.sky/b42226982545/molen-pack.json", … } }
82
+ // _a/index.json → {
83
+ // "molen.sky": { "file": "molen.sky/b42226982545/molen.sky-3f2a9c1e5b7d.zip", … },
84
+ // "molen.sounds": { "file": "molen.sounds/3d503f19b517/molen-pack.json", … } }
79
85
  ```
80
86
 
81
87
  `openPack` (and `openPacksFromIndex` in `@bendyline/molen-earth`) opens a URL ending in
package/dist/index.d.mts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { n as PackRetryOptions, r as RangeReader, t as PackChangedError } from "./source-7Pr2tntF.mjs";
2
- import { _ as describePack, a as PackInput, b as sha256, c as ReadOptions, d as openPack, f as packFromFiles, g as createPack, h as PackOptions, i as PackEntryNotFoundError, l as isDirectoryPackUrl, m as PackFile, n as OpenPackOptions, o as PackIntegrityError, p as BuiltPack, r as Pack, s as PackProgress, t as OpenDirectoryPackOptions, u as openDirectoryPack, v as isTextPath, y as mediaTypeOf } from "./pack-BwwY7qpa.mjs";
2
+ import { _ as describePack, a as PackInput, b as sha256, c as ReadOptions, d as openPack, f as packFromFiles, g as createPack, h as PackOptions, i as PackEntryNotFoundError, l as isDirectoryPackUrl, m as PackFile, n as OpenPackOptions, o as PackIntegrityError, p as BuiltPack, r as Pack, s as PackProgress, t as OpenDirectoryPackOptions, u as openDirectoryPack, v as isTextPath, y as mediaTypeOf } from "./pack-BkaC_X0Y.mjs";
3
3
  import { PackIndex, PackIndexEntry, PackManifest } from "@bendyline/molen-schema";
4
4
 
5
5
  //#region src/set.d.ts
package/dist/index.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { a as openPack, c as describePack, d as sha256, i as openDirectoryPack, l as isTextPath, n as PackIntegrityError, o as packFromFiles, r as isDirectoryPackUrl, s as createPack, t as PackEntryNotFoundError, u as mediaTypeOf } from "./pack-CymjXXOV.mjs";
1
+ import { a as openPack, c as describePack, d as sha256, i as openDirectoryPack, l as isTextPath, n as PackIntegrityError, o as packFromFiles, r as isDirectoryPackUrl, s as createPack, t as PackEntryNotFoundError, u as mediaTypeOf } from "./pack-C3g5LDPD.mjs";
2
2
  import { n as PackChangedError } from "./source-BMJGKBgK.mjs";
3
3
  //#region src/set.ts
4
4
  var Packs = class {
package/dist/node.d.mts CHANGED
@@ -1,4 +1,4 @@
1
- import { h as PackOptions, m as PackFile, n as OpenPackOptions, r as Pack } from "./pack-BwwY7qpa.mjs";
1
+ import { h as PackOptions, m as PackFile, n as OpenPackOptions, r as Pack } from "./pack-BkaC_X0Y.mjs";
2
2
  import { PackIndex, PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
3
3
 
4
4
  //#region src/node.d.ts
@@ -71,11 +71,22 @@ declare function writeDirectoryPack(pack: Pack, outDir: string): Promise<{
71
71
  manifest: PackManifest;
72
72
  manifestBytes: number;
73
73
  }>;
74
+ /**
75
+ * The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
76
+ * (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
77
+ * however many files it holds.
78
+ */
79
+ declare const HOSTED_ZIP_BELOW: number;
74
80
  interface HostPacksOptions {
75
81
  /** Where to publish: `<pack id>/<version>/…` per pack, then `index.json`. */
76
82
  outDir: string;
77
83
  /** Which pack ids to publish (default: every pack in the index). */
78
84
  include?(id: string): boolean;
85
+ /**
86
+ * Publish a pack as its zip when the zip is at most this many bytes (default
87
+ * `HOSTED_ZIP_BELOW`); larger packs are published unzipped. 0 unzips every pack.
88
+ */
89
+ zipBelow?: number;
79
90
  log?(line: string): void;
80
91
  }
81
92
  interface HostedPacks {
@@ -86,17 +97,21 @@ interface HostedPacks {
86
97
  written: string[];
87
98
  /** Pack ids whose version directory was already complete. */
88
99
  reused: string[];
100
+ /** Pack ids published as their zip rather than unzipped. */
101
+ zipped: string[];
89
102
  }
90
103
  /** A pack version's directory: the first 12 hex digits of its contentHash. */
91
104
  declare function hostedVersionDir(contentHash: string): string;
92
105
  /**
93
- * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
94
- * unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
95
- * changes, so a host can serve those files as immutable; `index.json`, written last and the only
96
- * document that changes between releases, names each version's `molen-pack.json`. Complete
97
- * version directories from an earlier run are kept; older versions are left in place for clients
98
- * still holding an older index.
106
+ * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
107
+ * CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
108
+ * host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
109
+ * published as that zip, which readers fetch whole in one request; a larger one is published
110
+ * unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
111
+ * written last and the only document that changes between releases, names each version's zip or
112
+ * `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
113
+ * place for clients still holding an older index.
99
114
  */
100
115
  declare function hostPacks(sourceIndexPath: string, options: HostPacksOptions): Promise<HostedPacks>;
101
116
  //#endregion
102
- export { BuiltPackFile, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
117
+ export { BuiltPackFile, HOSTED_ZIP_BELOW, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
package/dist/node.mjs CHANGED
@@ -1,7 +1,8 @@
1
- import { a as openPack, d as sha256, o as packFromFiles, s as createPack } from "./pack-CymjXXOV.mjs";
1
+ import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, r as isDirectoryPackUrl, s as createPack } from "./pack-C3g5LDPD.mjs";
2
2
  import { PACK_MANIFEST_ENTRY, validate } from "@bendyline/molen-schema";
3
- import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
- import { dirname, join, relative, resolve, sep } from "node:path";
3
+ import { copyFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
+ import { basename, dirname, join, relative, resolve, sep } from "node:path";
5
+ import { fileURLToPath, pathToFileURL } from "node:url";
5
6
  //#region src/node.ts
6
7
  /** Build settings file in a pack's source directory. */
7
8
  const PACK_SOURCE_FILE = "molen-pack.source.json";
@@ -101,7 +102,8 @@ function optionsOf(config) {
101
102
  ...config.notice !== void 0 ? { notice: config.notice } : {},
102
103
  ids: config.ids,
103
104
  provides: config.provides,
104
- solid: config.solid
105
+ solid: config.solid,
106
+ independent: config.independent
105
107
  };
106
108
  }
107
109
  /**
@@ -118,6 +120,17 @@ async function openDirPack(dir, options = {}) {
118
120
  /** Open a pack from a URL, a built pack file, or a pack source directory. */
119
121
  async function openPackAt(source, options = {}) {
120
122
  if (/^https?:\/\//.test(source)) return openPack(source, options);
123
+ if (source.endsWith(PACK_MANIFEST_ENTRY)) {
124
+ const base = pathToFileURL(resolve(source));
125
+ return openDirectoryPack(base, {
126
+ ...options,
127
+ fetch: async (input) => {
128
+ const url = new URL(String(input));
129
+ if (url.protocol !== "file:" || !url.href.startsWith(new URL(".", base).href)) throw new Error("Pack file outside its directory");
130
+ return new Response(await readFile(fileURLToPath(url)));
131
+ }
132
+ });
133
+ }
121
134
  return (await stat(source)).isDirectory() ? openDirPack(source, options) : openFilePack(source, options);
122
135
  }
123
136
  async function readIndex(path) {
@@ -213,7 +226,8 @@ async function extractPack(pack, outDir) {
213
226
  exclude: [],
214
227
  ids: manifest.ids,
215
228
  provides: manifest.provides,
216
- solid: Object.keys(manifest.blocks).length > 0
229
+ solid: Object.keys(manifest.blocks).length > 0,
230
+ independent: Object.entries(manifest.entries).filter(([, entry]) => entry.block === void 0).map(([path]) => path)
217
231
  };
218
232
  await writeFile(join(root, PACK_SOURCE_FILE), `${JSON.stringify(config, null, 2)}\n`);
219
233
  return written;
@@ -248,17 +262,25 @@ async function writeDirectoryPack(pack, outDir) {
248
262
  manifestBytes: new TextEncoder().encode(text).length
249
263
  };
250
264
  }
265
+ /**
266
+ * The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
267
+ * (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
268
+ * however many files it holds.
269
+ */
270
+ const HOSTED_ZIP_BELOW = 4 * 1024 * 1024;
251
271
  /** A pack version's directory: the first 12 hex digits of its contentHash. */
252
272
  function hostedVersionDir(contentHash) {
253
273
  return contentHash.replace("sha256:", "").slice(0, 12);
254
274
  }
255
275
  /**
256
- * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
257
- * unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
258
- * changes, so a host can serve those files as immutable; `index.json`, written last and the only
259
- * document that changes between releases, names each version's `molen-pack.json`. Complete
260
- * version directories from an earlier run are kept; older versions are left in place for clients
261
- * still holding an older index.
276
+ * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
277
+ * CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
278
+ * host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
279
+ * published as that zip, which readers fetch whole in one request; a larger one is published
280
+ * unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
281
+ * written last and the only document that changes between releases, names each version's zip or
282
+ * `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
283
+ * place for clients still holding an older index.
262
284
  */
263
285
  async function hostPacks(sourceIndexPath, options) {
264
286
  const source = resolve(sourceIndexPath);
@@ -269,15 +291,43 @@ async function hostPacks(sourceIndexPath, options) {
269
291
  });
270
292
  const parsed = validate("pack-index", JSON.parse(raw));
271
293
  if (!parsed.ok) throw new Error(`${source} is not a pack index:\n${parsed.formatted}`);
294
+ const zipBelow = options.zipBelow ?? 4194304;
272
295
  const packs = {};
273
296
  const written = [];
274
297
  const reused = [];
298
+ const zipped = [];
275
299
  const ids = Object.keys(parsed.value.packs).sort();
276
300
  for (const id of ids) {
277
301
  if (options.include !== void 0 && !options.include(id)) continue;
278
302
  const entry = parsed.value.packs[id];
279
303
  const version = hostedVersionDir(entry.contentHash);
280
304
  const dir = join(outDir, id, version);
305
+ const sourceFile = join(dirname(source), ...entry.file.split("/"));
306
+ if (!isDirectoryPackUrl(entry.file) && (await stat(sourceFile)).size <= zipBelow) {
307
+ const name = basename(sourceFile);
308
+ const target = join(dir, name);
309
+ let complete = false;
310
+ try {
311
+ (await openFilePack(target, { expect: { contentHash: entry.contentHash } })).close();
312
+ complete = true;
313
+ } catch {}
314
+ if (complete) reused.push(id);
315
+ else {
316
+ (await openFilePack(sourceFile, { expect: { contentHash: entry.contentHash } })).close();
317
+ await mkdir(dir, { recursive: true });
318
+ await copyFile(sourceFile, target);
319
+ written.push(id);
320
+ log(`${id}@${entry.version} → ${id}/${version}/${name}`);
321
+ }
322
+ zipped.push(id);
323
+ packs[id] = {
324
+ version: entry.version,
325
+ file: `${id}/${version}/${name}`,
326
+ contentHash: entry.contentHash,
327
+ size: (await stat(target)).size
328
+ };
329
+ continue;
330
+ }
281
331
  const manifestPath = join(dir, PACK_MANIFEST_ENTRY);
282
332
  let manifestBytes;
283
333
  try {
@@ -291,7 +341,7 @@ async function hostPacks(sourceIndexPath, options) {
291
341
  recursive: true,
292
342
  force: true
293
343
  });
294
- const pack = await openFilePack(join(dirname(source), ...entry.file.split("/")), { expect: { contentHash: entry.contentHash } });
344
+ const pack = await openPackAt(sourceFile, { expect: { contentHash: entry.contentHash } });
295
345
  try {
296
346
  manifestBytes = (await writeDirectoryPack(pack, dir)).manifestBytes;
297
347
  } finally {
@@ -320,8 +370,9 @@ async function hostPacks(sourceIndexPath, options) {
320
370
  indexPath,
321
371
  index,
322
372
  written,
323
- reused
373
+ reused,
374
+ zipped
324
375
  };
325
376
  }
326
377
  //#endregion
327
- export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
378
+ export { HOSTED_ZIP_BELOW, PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
@@ -20,6 +20,8 @@ interface PackOptions {
20
20
  provides?: Record<string, string | readonly string[]>;
21
21
  /** Group small text files into compressed solid blocks (default true). */
22
22
  solid?: boolean;
23
+ /** Files or directory prefixes (ending /) kept as independent ZIP members. */
24
+ independent?: readonly string[];
23
25
  /**
24
26
  * `auto` (default) deflates a member when that saves at least 10%. `store` writes every member
25
27
  * uncompressed: the manifest and contents are identical, only the archive is larger, so a pack
@@ -352,7 +352,7 @@ async function createPack(files, options) {
352
352
  const blocks = {};
353
353
  const blockMembers = [];
354
354
  const pending = /* @__PURE__ */ new Map();
355
- for (const file of sorted) if (options.solid !== false && isTextPath(file.path) && file.bytes.length < OWN_MEMBER_LIMIT) {
355
+ for (const file of sorted) if (options.solid !== false && !options.independent?.some((path) => file.path === path || path.endsWith("/") && file.path.startsWith(path)) && isTextPath(file.path) && file.bytes.length < OWN_MEMBER_LIMIT) {
356
356
  const group = groupOf(file.path);
357
357
  pending.set(group, [...pending.get(group) ?? [], file]);
358
358
  } else members.push({
@@ -475,6 +475,11 @@ var ByteCache = class {
475
475
  constructor(limit) {
476
476
  this.limit = limit;
477
477
  }
478
+ delete(key) {
479
+ this.entries.delete(key);
480
+ this.total -= this.sizes.get(key) ?? 0;
481
+ this.sizes.delete(key);
482
+ }
478
483
  get(key, load) {
479
484
  const hit = this.entries.get(key);
480
485
  if (hit !== void 0) {
@@ -632,6 +637,7 @@ var DirectoryPack = class extends BasePack {
632
637
  reader;
633
638
  controller;
634
639
  options;
640
+ flights = /* @__PURE__ */ new Map();
635
641
  cache;
636
642
  active = 0;
637
643
  waiting = [];
@@ -671,19 +677,49 @@ var DirectoryPack = class extends BasePack {
671
677
  }
672
678
  bytes(path, options) {
673
679
  if (this.controller.signal.aborted) return Promise.reject(this.controller.signal.reason);
680
+ if (options?.signal?.aborted) return Promise.reject(options.signal.reason);
674
681
  const entry = this.entry(path);
675
- return untilAborted(this.cache.get(path, async () => {
682
+ const load = this.cache.get(path, async () => {
676
683
  if (entry.size === 0) return new Uint8Array(0);
677
- const bytes = await this.slot(options?.priority !== "low", () => this.reader(this.url(path), entry).read(0, entry.size, this.controller.signal));
678
- this.bytesRead += bytes.length;
679
- this.requests++;
680
- this.options.onProgress?.({
681
- label: this.label,
682
- bytes: this.bytesRead,
683
- requests: this.requests
684
- });
685
- return bytes;
686
- }), options?.signal);
684
+ const flight = {
685
+ controller: new AbortController(),
686
+ users: 0,
687
+ settled: false
688
+ };
689
+ this.flights.set(path, flight);
690
+ const signal = AbortSignal.any([this.controller.signal, flight.controller.signal]);
691
+ try {
692
+ const bytes = await this.slot(options?.priority !== "low", async () => {
693
+ signal.throwIfAborted();
694
+ return this.reader(this.url(path), entry).read(0, entry.size, signal);
695
+ });
696
+ signal.throwIfAborted();
697
+ this.bytesRead += bytes.length;
698
+ this.requests++;
699
+ this.options.onProgress?.({
700
+ label: this.label,
701
+ bytes: this.bytesRead,
702
+ requests: this.requests
703
+ });
704
+ return bytes;
705
+ } finally {
706
+ flight.settled = true;
707
+ if (this.flights.get(path) === flight) this.flights.delete(path);
708
+ }
709
+ });
710
+ const flight = this.flights.get(path);
711
+ if (!flight) return untilAborted(load, options?.signal);
712
+ flight.users++;
713
+ return untilAborted(load, options?.signal).finally(() => {
714
+ flight.users--;
715
+ if (!flight.users && !flight.settled) {
716
+ flight.controller.abort(new DOMException("No remaining pack readers", "AbortError"));
717
+ if (this.flights.get(path) === flight) {
718
+ this.flights.delete(path);
719
+ this.cache.delete(path);
720
+ }
721
+ }
722
+ });
687
723
  }
688
724
  async readBytes(path, options) {
689
725
  const entry = this.entry(path);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/molen-pack",
3
- "version": "0.0.8",
3
+ "version": "0.0.10",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"
@@ -45,8 +45,8 @@
45
45
  "dist"
46
46
  ],
47
47
  "dependencies": {
48
- "fflate": "0.8.3",
49
- "@bendyline/molen-schema": "0.0.7"
48
+ "@bendyline/molen-schema": "0.0.9",
49
+ "fflate": "0.8.3"
50
50
  },
51
51
  "devDependencies": {
52
52
  "@biomejs/biome": "^2.5.0",