@bendyline/molen-pack 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/dist/node.d.mts +22 -7
- package/dist/node.mjs +49 -12
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -62,20 +62,26 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
|
|
|
62
62
|
const pack = await openDirPack('content/my-props');
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
-
## Host packs
|
|
65
|
+
## Host packs on a CDN
|
|
66
66
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
67
|
+
`hostPacks` publishes the packs a pack index lists for a CDN. Each version goes to
|
|
68
|
+
`<pack id>/<12 hex of its contentHash>/` once and never changes, so its files can be served as
|
|
69
|
+
immutable, and the `index.json` it writes last is the only document that changes between releases;
|
|
70
|
+
a new release uploads only the packs that changed.
|
|
71
|
+
|
|
72
|
+
A pack whose zip is at most `zipBelow` bytes (default `HOSTED_ZIP_BELOW`, 4 MiB) is published as
|
|
73
|
+
that zip: readers fetch an archive that small whole, so a pack of a thousand small documents costs
|
|
74
|
+
one request. A larger pack is published unzipped, every file at its own cacheable URL, so readers
|
|
75
|
+
fetch only the files they use. `zipBelow: 0` unzips every pack.
|
|
72
76
|
|
|
73
77
|
```ts
|
|
74
78
|
import { hostPacks } from '@bendyline/molen-pack/node';
|
|
75
79
|
|
|
76
80
|
// dist/packs/index.json lists built zips; _a/ becomes the directory to upload.
|
|
77
81
|
await hostPacks('dist/packs/index.json', { outDir: '_a' });
|
|
78
|
-
// _a/index.json → {
|
|
82
|
+
// _a/index.json → {
|
|
83
|
+
// "molen.sky": { "file": "molen.sky/b42226982545/molen.sky-3f2a9c1e5b7d.zip", … },
|
|
84
|
+
// "molen.sounds": { "file": "molen.sounds/3d503f19b517/molen-pack.json", … } }
|
|
79
85
|
```
|
|
80
86
|
|
|
81
87
|
`openPack` (and `openPacksFromIndex` in `@bendyline/molen-earth`) opens a URL ending in
|
package/dist/node.d.mts
CHANGED
|
@@ -71,11 +71,22 @@ declare function writeDirectoryPack(pack: Pack, outDir: string): Promise<{
|
|
|
71
71
|
manifest: PackManifest;
|
|
72
72
|
manifestBytes: number;
|
|
73
73
|
}>;
|
|
74
|
+
/**
|
|
75
|
+
* The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
|
|
76
|
+
* (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
|
|
77
|
+
* however many files it holds.
|
|
78
|
+
*/
|
|
79
|
+
declare const HOSTED_ZIP_BELOW: number;
|
|
74
80
|
interface HostPacksOptions {
|
|
75
81
|
/** Where to publish: `<pack id>/<version>/…` per pack, then `index.json`. */
|
|
76
82
|
outDir: string;
|
|
77
83
|
/** Which pack ids to publish (default: every pack in the index). */
|
|
78
84
|
include?(id: string): boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Publish a pack as its zip when the zip is at most this many bytes (default
|
|
87
|
+
* `HOSTED_ZIP_BELOW`); larger packs are published unzipped. 0 unzips every pack.
|
|
88
|
+
*/
|
|
89
|
+
zipBelow?: number;
|
|
79
90
|
log?(line: string): void;
|
|
80
91
|
}
|
|
81
92
|
interface HostedPacks {
|
|
@@ -86,17 +97,21 @@ interface HostedPacks {
|
|
|
86
97
|
written: string[];
|
|
87
98
|
/** Pack ids whose version directory was already complete. */
|
|
88
99
|
reused: string[];
|
|
100
|
+
/** Pack ids published as their zip rather than unzipped. */
|
|
101
|
+
zipped: string[];
|
|
89
102
|
}
|
|
90
103
|
/** A pack version's directory: the first 12 hex digits of its contentHash. */
|
|
91
104
|
declare function hostedVersionDir(contentHash: string): string;
|
|
92
105
|
/**
|
|
93
|
-
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
106
|
+
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
|
|
107
|
+
* CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
|
|
108
|
+
* host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
|
|
109
|
+
* published as that zip, which readers fetch whole in one request; a larger one is published
|
|
110
|
+
* unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
|
|
111
|
+
* written last and the only document that changes between releases, names each version's zip or
|
|
112
|
+
* `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
|
|
113
|
+
* place for clients still holding an older index.
|
|
99
114
|
*/
|
|
100
115
|
declare function hostPacks(sourceIndexPath: string, options: HostPacksOptions): Promise<HostedPacks>;
|
|
101
116
|
//#endregion
|
|
102
|
-
export { BuiltPackFile, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
|
117
|
+
export { BuiltPackFile, HOSTED_ZIP_BELOW, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
package/dist/node.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, s as createPack } from "./pack-C3g5LDPD.mjs";
|
|
1
|
+
import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, r as isDirectoryPackUrl, s as createPack } from "./pack-C3g5LDPD.mjs";
|
|
2
2
|
import { PACK_MANIFEST_ENTRY, validate } from "@bendyline/molen-schema";
|
|
3
|
-
import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
-
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { copyFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
6
6
|
//#region src/node.ts
|
|
7
7
|
/** Build settings file in a pack's source directory. */
|
|
@@ -262,17 +262,25 @@ async function writeDirectoryPack(pack, outDir) {
|
|
|
262
262
|
manifestBytes: new TextEncoder().encode(text).length
|
|
263
263
|
};
|
|
264
264
|
}
|
|
265
|
+
/**
|
|
266
|
+
* The largest zip `hostPacks` publishes as is (4 MiB): readers fetch an archive this small whole
|
|
267
|
+
* (`openPack`'s `wholeThreshold`, `cachingRangeReader`'s `wholeBelow`), so it costs one request
|
|
268
|
+
* however many files it holds.
|
|
269
|
+
*/
|
|
270
|
+
const HOSTED_ZIP_BELOW = 4 * 1024 * 1024;
|
|
265
271
|
/** A pack version's directory: the first 12 hex digits of its contentHash. */
|
|
266
272
|
function hostedVersionDir(contentHash) {
|
|
267
273
|
return contentHash.replace("sha256:", "").slice(0, 12);
|
|
268
274
|
}
|
|
269
275
|
/**
|
|
270
|
-
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
*
|
|
274
|
-
*
|
|
275
|
-
*
|
|
276
|
+
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips) for a
|
|
277
|
+
* CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never changes, so a
|
|
278
|
+
* host can serve those files as immutable. A pack whose zip is at most `zipBelow` bytes is
|
|
279
|
+
* published as that zip, which readers fetch whole in one request; a larger one is published
|
|
280
|
+
* unzipped, every file at its own path, so readers fetch only the files they use. `index.json`,
|
|
281
|
+
* written last and the only document that changes between releases, names each version's zip or
|
|
282
|
+
* `molen-pack.json`. Complete versions from an earlier run are kept; older versions are left in
|
|
283
|
+
* place for clients still holding an older index.
|
|
276
284
|
*/
|
|
277
285
|
async function hostPacks(sourceIndexPath, options) {
|
|
278
286
|
const source = resolve(sourceIndexPath);
|
|
@@ -283,15 +291,43 @@ async function hostPacks(sourceIndexPath, options) {
|
|
|
283
291
|
});
|
|
284
292
|
const parsed = validate("pack-index", JSON.parse(raw));
|
|
285
293
|
if (!parsed.ok) throw new Error(`${source} is not a pack index:\n${parsed.formatted}`);
|
|
294
|
+
const zipBelow = options.zipBelow ?? 4194304;
|
|
286
295
|
const packs = {};
|
|
287
296
|
const written = [];
|
|
288
297
|
const reused = [];
|
|
298
|
+
const zipped = [];
|
|
289
299
|
const ids = Object.keys(parsed.value.packs).sort();
|
|
290
300
|
for (const id of ids) {
|
|
291
301
|
if (options.include !== void 0 && !options.include(id)) continue;
|
|
292
302
|
const entry = parsed.value.packs[id];
|
|
293
303
|
const version = hostedVersionDir(entry.contentHash);
|
|
294
304
|
const dir = join(outDir, id, version);
|
|
305
|
+
const sourceFile = join(dirname(source), ...entry.file.split("/"));
|
|
306
|
+
if (!isDirectoryPackUrl(entry.file) && (await stat(sourceFile)).size <= zipBelow) {
|
|
307
|
+
const name = basename(sourceFile);
|
|
308
|
+
const target = join(dir, name);
|
|
309
|
+
let complete = false;
|
|
310
|
+
try {
|
|
311
|
+
(await openFilePack(target, { expect: { contentHash: entry.contentHash } })).close();
|
|
312
|
+
complete = true;
|
|
313
|
+
} catch {}
|
|
314
|
+
if (complete) reused.push(id);
|
|
315
|
+
else {
|
|
316
|
+
(await openFilePack(sourceFile, { expect: { contentHash: entry.contentHash } })).close();
|
|
317
|
+
await mkdir(dir, { recursive: true });
|
|
318
|
+
await copyFile(sourceFile, target);
|
|
319
|
+
written.push(id);
|
|
320
|
+
log(`${id}@${entry.version} → ${id}/${version}/${name}`);
|
|
321
|
+
}
|
|
322
|
+
zipped.push(id);
|
|
323
|
+
packs[id] = {
|
|
324
|
+
version: entry.version,
|
|
325
|
+
file: `${id}/${version}/${name}`,
|
|
326
|
+
contentHash: entry.contentHash,
|
|
327
|
+
size: (await stat(target)).size
|
|
328
|
+
};
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
295
331
|
const manifestPath = join(dir, PACK_MANIFEST_ENTRY);
|
|
296
332
|
let manifestBytes;
|
|
297
333
|
try {
|
|
@@ -305,7 +341,7 @@ async function hostPacks(sourceIndexPath, options) {
|
|
|
305
341
|
recursive: true,
|
|
306
342
|
force: true
|
|
307
343
|
});
|
|
308
|
-
const pack = await openPackAt(
|
|
344
|
+
const pack = await openPackAt(sourceFile, { expect: { contentHash: entry.contentHash } });
|
|
309
345
|
try {
|
|
310
346
|
manifestBytes = (await writeDirectoryPack(pack, dir)).manifestBytes;
|
|
311
347
|
} finally {
|
|
@@ -334,8 +370,9 @@ async function hostPacks(sourceIndexPath, options) {
|
|
|
334
370
|
indexPath,
|
|
335
371
|
index,
|
|
336
372
|
written,
|
|
337
|
-
reused
|
|
373
|
+
reused,
|
|
374
|
+
zipped
|
|
338
375
|
};
|
|
339
376
|
}
|
|
340
377
|
//#endregion
|
|
341
|
-
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
|
378
|
+
export { HOSTED_ZIP_BELOW, PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.10",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"dist"
|
|
46
46
|
],
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@bendyline/molen-schema": "0.0.
|
|
48
|
+
"@bendyline/molen-schema": "0.0.9",
|
|
49
49
|
"fflate": "0.8.3"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|