@bendyline/molen-pack 0.0.9 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -62,20 +62,26 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
62
62
  const pack = await openDirPack('content/my-props');
63
63
  ```
64
64
 
65
- ## Host packs unzipped
65
+ ## Host packs on a CDN
66
66
 
67
- A CDN serves a pack just as well with its files loose: every file is its own cacheable URL, and a
68
- new release uploads only the packs that changed. `hostPacks` publishes the packs a pack index lists
69
- that way. Each version goes to `<pack id>/<12 hex of its contentHash>/` once and never changes, so
70
- those files can be served as immutable, and the `index.json` it writes last is the only document
71
- that changes between releases:
67
+ `hostPacks` publishes the packs a pack index lists for a CDN. Each version goes to
68
+ `<pack id>/<12 hex of its contentHash>/` once and never changes, so its files can be served as
69
+ immutable, and the `index.json` it writes last is the only document that changes between releases;
70
+ a new release uploads only the packs that changed.
71
+
72
+ A pack whose zip is at most `zipBelow` bytes (default `HOSTED_ZIP_BELOW`, 4 MiB) is published as
73
+ that zip: readers fetch an archive that small whole, so a pack of a thousand small documents costs
74
+ one request. A larger pack is published unzipped, every file at its own cacheable URL, so readers
75
+ fetch only the files they use. `zipBelow: 0` unzips every pack.
72
76
 
73
77
  ```ts
74
78
  import { hostPacks } from '@bendyline/molen-pack/node';
75
79
 
76
80
  // dist/packs/index.json lists built zips; _a/ becomes the directory to upload.
77
81
  await hostPacks('dist/packs/index.json', { outDir: '_a' });
78
- // _a/index.json → { "molen.sky": { "file": "molen.sky/b42226982545/molen-pack.json", … } }
82
+ // _a/index.json → {
83
+ // "molen.sky": { "file": "molen.sky/b42226982545/molen.sky-3f2a9c1e5b7d.zip", … },
84
+ // "molen.sounds": { "file": "molen.sounds/3d503f19b517/molen-pack.json", … } }
79
85
  ```
80
86
 
81
87
  `openPack` (and `openPacksFromIndex` in `@bendyline/molen-earth`) opens a URL ending in
package/dist/node.d.mts CHANGED
@@ -71,11 +71,22 @@ declare function writeDirectoryPack(pack: Pack, outDir: string): Promise<{
71
71
  manifest: PackManifest;
72
72
  manifestBytes: number;
73
73
  }>;
74
+ /**
75
+ * The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
76
+ * (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
77
+ * however many files it holds.
78
+ */
79
+ declare const HOSTED_ZIP_BELOW: number;
74
80
  interface HostPacksOptions {
75
81
  /** Where to publish: `<pack id>/<version>/…` per pack, then `index.json`. */
76
82
  outDir: string;
77
83
  /** Which pack ids to publish (default: every pack in the index). */
78
84
  include?(id: string): boolean;
85
+ /**
86
+ * Publish a pack as its zip when the zip is at most this many bytes (default
87
+ * `HOSTED_ZIP_BELOW`); larger packs are published unzipped. 0 unzips every pack.
88
+ */
89
+ zipBelow?: number;
79
90
  log?(line: string): void;
80
91
  }
81
92
  interface HostedPacks {
@@ -86,17 +97,21 @@ interface HostedPacks {
86
97
  written: string[];
87
98
  /** Pack ids whose version directory was already complete. */
88
99
  reused: string[];
100
+ /** Pack ids published as their zip rather than unzipped. */
101
+ zipped: string[];
89
102
  }
90
103
  /** A pack version's directory: the first 12 hex digits of its contentHash. */
91
104
  declare function hostedVersionDir(contentHash: string): string;
92
105
  /**
93
- * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
94
- * unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
95
- * changes, so a host can serve those files as immutable; `index.json`, written last and the only
96
- * document that changes between releases, names each version's `molen-pack.json`. Complete
97
- * version directories from an earlier run are kept; older versions are left in place for clients
98
- * still holding an older index.
106
+ * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
107
+ * CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
108
+ * host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
109
+ * published as that zip, which readers fetch whole in one request; a larger one is published
110
+ * unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
111
+ * written last and the only document that changes between releases, names each version's zip or
112
+ * `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
113
+ * place for clients still holding an older index.
99
114
  */
100
115
  declare function hostPacks(sourceIndexPath: string, options: HostPacksOptions): Promise<HostedPacks>;
101
116
  //#endregion
102
- export { BuiltPackFile, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
117
+ export { BuiltPackFile, HOSTED_ZIP_BELOW, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
package/dist/node.mjs CHANGED
@@ -1,7 +1,7 @@
1
- import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, s as createPack } from "./pack-C3g5LDPD.mjs";
1
+ import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, r as isDirectoryPackUrl, s as createPack } from "./pack-C3g5LDPD.mjs";
2
2
  import { PACK_MANIFEST_ENTRY, validate } from "@bendyline/molen-schema";
3
- import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
- import { dirname, join, relative, resolve, sep } from "node:path";
3
+ import { copyFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
4
+ import { basename, dirname, join, relative, resolve, sep } from "node:path";
5
5
  import { fileURLToPath, pathToFileURL } from "node:url";
6
6
  //#region src/node.ts
7
7
  /** Build settings file in a pack's source directory. */
@@ -262,17 +262,25 @@ async function writeDirectoryPack(pack, outDir) {
262
262
  manifestBytes: new TextEncoder().encode(text).length
263
263
  };
264
264
  }
265
+ /**
266
+ * The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
267
+ * (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
268
+ * however many files it holds.
269
+ */
270
+ const HOSTED_ZIP_BELOW = 4 * 1024 * 1024;
265
271
  /** A pack version's directory: the first 12 hex digits of its contentHash. */
266
272
  function hostedVersionDir(contentHash) {
267
273
  return contentHash.replace("sha256:", "").slice(0, 12);
268
274
  }
269
275
  /**
270
- * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
271
- * unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
272
- * changes, so a host can serve those files as immutable; `index.json`, written last and the only
273
- * document that changes between releases, names each version's `molen-pack.json`. Complete
274
- * version directories from an earlier run are kept; older versions are left in place for clients
275
- * still holding an older index.
276
+ * Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
277
+ * CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
278
+ * host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
279
+ * published as that zip, which readers fetch whole in one request; a larger one is published
280
+ * unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
281
+ * written last and the only document that changes between releases, names each version's zip or
282
+ * `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
283
+ * place for clients still holding an older index.
276
284
  */
277
285
  async function hostPacks(sourceIndexPath, options) {
278
286
  const source = resolve(sourceIndexPath);
@@ -283,15 +291,43 @@ async function hostPacks(sourceIndexPath, options) {
283
291
  });
284
292
  const parsed = validate("pack-index", JSON.parse(raw));
285
293
  if (!parsed.ok) throw new Error(`${source} is not a pack index:\n${parsed.formatted}`);
294
+ const zipBelow = options.zipBelow ?? 4194304;
286
295
  const packs = {};
287
296
  const written = [];
288
297
  const reused = [];
298
+ const zipped = [];
289
299
  const ids = Object.keys(parsed.value.packs).sort();
290
300
  for (const id of ids) {
291
301
  if (options.include !== void 0 && !options.include(id)) continue;
292
302
  const entry = parsed.value.packs[id];
293
303
  const version = hostedVersionDir(entry.contentHash);
294
304
  const dir = join(outDir, id, version);
305
+ const sourceFile = join(dirname(source), ...entry.file.split("/"));
306
+ if (!isDirectoryPackUrl(entry.file) && (await stat(sourceFile)).size <= zipBelow) {
307
+ const name = basename(sourceFile);
308
+ const target = join(dir, name);
309
+ let complete = false;
310
+ try {
311
+ (await openFilePack(target, { expect: { contentHash: entry.contentHash } })).close();
312
+ complete = true;
313
+ } catch {}
314
+ if (complete) reused.push(id);
315
+ else {
316
+ (await openFilePack(sourceFile, { expect: { contentHash: entry.contentHash } })).close();
317
+ await mkdir(dir, { recursive: true });
318
+ await copyFile(sourceFile, target);
319
+ written.push(id);
320
+ log(`${id}@${entry.version} → ${id}/${version}/${name}`);
321
+ }
322
+ zipped.push(id);
323
+ packs[id] = {
324
+ version: entry.version,
325
+ file: `${id}/${version}/${name}`,
326
+ contentHash: entry.contentHash,
327
+ size: (await stat(target)).size
328
+ };
329
+ continue;
330
+ }
295
331
  const manifestPath = join(dir, PACK_MANIFEST_ENTRY);
296
332
  let manifestBytes;
297
333
  try {
@@ -305,7 +341,7 @@ async function hostPacks(sourceIndexPath, options) {
305
341
  recursive: true,
306
342
  force: true
307
343
  });
308
- const pack = await openPackAt(join(dirname(source), ...entry.file.split("/")), { expect: { contentHash: entry.contentHash } });
344
+ const pack = await openPackAt(sourceFile, { expect: { contentHash: entry.contentHash } });
309
345
  try {
310
346
  manifestBytes = (await writeDirectoryPack(pack, dir)).manifestBytes;
311
347
  } finally {
@@ -334,8 +370,9 @@ async function hostPacks(sourceIndexPath, options) {
334
370
  indexPath,
335
371
  index,
336
372
  written,
337
- reused
373
+ reused,
374
+ zipped
338
375
  };
339
376
  }
340
377
  //#endregion
341
- export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
378
+ export { HOSTED_ZIP_BELOW, PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/molen-pack",
3
- "version": "0.0.9",
3
+ "version": "0.0.11",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"
@@ -45,7 +45,7 @@
45
45
  "dist"
46
46
  ],
47
47
  "dependencies": {
48
- "@bendyline/molen-schema": "0.0.8",
48
+ "@bendyline/molen-schema": "0.0.10",
49
49
  "fflate": "0.8.3"
50
50
  },
51
51
  "devDependencies": {