@bendyline/molen-pack 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -1
- package/dist/cache.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +3 -3
- package/dist/node.d.mts +40 -3
- package/dist/node.mjs +108 -3
- package/dist/{pack-CF6pPp9f.d.mts → pack-BwwY7qpa.d.mts} +24 -3
- package/dist/{pack-BIXihu9D.mjs → pack-CymjXXOV.mjs} +136 -4
- package/dist/{source-MDAzioCi.mjs → source-BMJGKBgK.mjs} +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -62,6 +62,26 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
|
|
|
62
62
|
const pack = await openDirPack('content/my-props');
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
+
## Host packs unzipped
|
|
66
|
+
|
|
67
|
+
A CDN serves a pack just as well with its files loose: every file is its own cacheable URL, and a
|
|
68
|
+
new release uploads only the packs that changed. `hostPacks` publishes the packs a pack index lists
|
|
69
|
+
that way. Each version goes to `<pack id>/<12 hex of its contentHash>/` once and never changes, so
|
|
70
|
+
those files can be served as immutable, and the `index.json` it writes last is the only document
|
|
71
|
+
that changes between releases:
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import { hostPacks } from '@bendyline/molen-pack/node';
|
|
75
|
+
|
|
76
|
+
// dist/packs/index.json lists built zips; _a/ becomes the directory to upload.
|
|
77
|
+
await hostPacks('dist/packs/index.json', { outDir: '_a' });
|
|
78
|
+
// _a/index.json → { "molen.sky": { "file": "molen.sky/b42226982545/molen-pack.json", … } }
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`openPack` (and `openPacksFromIndex` in `@bendyline/molen-earth`) opens a URL ending in
|
|
82
|
+
`molen-pack.json` as an unzipped pack, reading each file from its path beside the manifest.
|
|
83
|
+
`writeDirectoryPack(pack, dir)` writes a single pack in that layout.
|
|
84
|
+
|
|
65
85
|
## Cache what you read
|
|
66
86
|
|
|
67
87
|
`./cache` keeps the bytes of range-read archives (packs, PMTiles) across visits. Each archive is
|
|
@@ -98,12 +118,13 @@ deleted. Store failures (private browsing, quota) only cost network, never a fai
|
|
|
98
118
|
- `openPack(input, options)` — a `Pack` from a URL, `Blob`, bytes, or a `RangeReader`, with
|
|
99
119
|
`readBytes` / `readText` / `readJson` / `prefetch`. Options cover whole-or-range mode, retries,
|
|
100
120
|
concurrency, progress, an expected `contentHash`, and sha256 checks per file.
|
|
121
|
+
`openDirectoryPack(url)` opens an unzipped pack from its `molen-pack.json`.
|
|
101
122
|
- `createPackSet(packs)` — several packs read as one, with later packs overriding earlier ones,
|
|
102
123
|
`provided(role)` to find the files a pack declares for a role, and `assetProvider()`.
|
|
103
124
|
- `createPack(files, options)` / `describePack` / `packFromFiles` — build a pack, or a pack-shaped
|
|
104
125
|
view of loose files, from bytes in any environment.
|
|
105
126
|
- `./node`: `buildPack`, `openFilePack`, `openDirPack`, `openPackAt`, `readPackSource`,
|
|
106
|
-
`extractPack`.
|
|
127
|
+
`extractPack`, `writeDirectoryPack`, `hostPacks`.
|
|
107
128
|
- `./cache`: `createBlockCache`, `createIndexedDbByteStore`, `createMemoryByteStore`,
|
|
108
129
|
`cachingRangeSource`, `cachingRangeReader`, `urlRangeReader`, `cachingDocumentFetch`.
|
|
109
130
|
|
package/dist/cache.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { s as urlRangeReader } from "./source-BMJGKBgK.mjs";
|
|
2
2
|
/** Recency bias per priority: added to a block's last-used time to order eviction. */
|
|
3
3
|
const PRIORITY_BIAS_MS = {
|
|
4
4
|
high: 7 * (1440 * 60 * 1e3),
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as PackRetryOptions, r as RangeReader, t as PackChangedError } from "./source-7Pr2tntF.mjs";
|
|
2
|
-
import { _ as
|
|
2
|
+
import { _ as describePack, a as PackInput, b as sha256, c as ReadOptions, d as openPack, f as packFromFiles, g as createPack, h as PackOptions, i as PackEntryNotFoundError, l as isDirectoryPackUrl, m as PackFile, n as OpenPackOptions, o as PackIntegrityError, p as BuiltPack, r as Pack, s as PackProgress, t as OpenDirectoryPackOptions, u as openDirectoryPack, v as isTextPath, y as mediaTypeOf } from "./pack-BwwY7qpa.mjs";
|
|
3
3
|
import { PackIndex, PackIndexEntry, PackManifest } from "@bendyline/molen-schema";
|
|
4
4
|
|
|
5
5
|
//#region src/set.d.ts
|
|
@@ -39,4 +39,4 @@ interface PackSet {
|
|
|
39
39
|
}
|
|
40
40
|
declare function createPackSet(packs?: readonly Pack[]): PackSet;
|
|
41
41
|
//#endregion
|
|
42
|
-
export { type AssetProviderLike, type BuiltPack, type OpenPackOptions, type Pack, type PackAssetProviderOptions, PackChangedError, PackEntryNotFoundError, type PackFile, type PackIndex, type PackIndexEntry, type PackInput, PackIntegrityError, type PackManifest, type PackOptions, type PackProgress, type PackRetryOptions, type PackSet, type RangeReader, type ReadOptions, createPack, createPackSet, describePack, isTextPath, mediaTypeOf, openPack, packFromFiles, sha256 };
|
|
42
|
+
export { type AssetProviderLike, type BuiltPack, type OpenDirectoryPackOptions, type OpenPackOptions, type Pack, type PackAssetProviderOptions, PackChangedError, PackEntryNotFoundError, type PackFile, type PackIndex, type PackIndexEntry, type PackInput, PackIntegrityError, type PackManifest, type PackOptions, type PackProgress, type PackRetryOptions, type PackSet, type RangeReader, type ReadOptions, createPack, createPackSet, describePack, isDirectoryPackUrl, isTextPath, mediaTypeOf, openDirectoryPack, openPack, packFromFiles, sha256 };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as
|
|
2
|
-
import {
|
|
1
|
+
import { a as openPack, c as describePack, d as sha256, i as openDirectoryPack, l as isTextPath, n as PackIntegrityError, o as packFromFiles, r as isDirectoryPackUrl, s as createPack, t as PackEntryNotFoundError, u as mediaTypeOf } from "./pack-CymjXXOV.mjs";
|
|
2
|
+
import { n as PackChangedError } from "./source-BMJGKBgK.mjs";
|
|
3
3
|
//#region src/set.ts
|
|
4
4
|
var Packs = class {
|
|
5
5
|
list = [];
|
|
@@ -103,4 +103,4 @@ function createPackSet(packs = []) {
|
|
|
103
103
|
return new Packs(packs);
|
|
104
104
|
}
|
|
105
105
|
//#endregion
|
|
106
|
-
export { PackChangedError, PackEntryNotFoundError, PackIntegrityError, createPack, createPackSet, describePack, isTextPath, mediaTypeOf, openPack, packFromFiles, sha256 };
|
|
106
|
+
export { PackChangedError, PackEntryNotFoundError, PackIntegrityError, createPack, createPackSet, describePack, isDirectoryPackUrl, isTextPath, mediaTypeOf, openDirectoryPack, openPack, packFromFiles, sha256 };
|
package/dist/node.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
1
|
+
import { h as PackOptions, m as PackFile, n as OpenPackOptions, r as Pack } from "./pack-BwwY7qpa.mjs";
|
|
2
|
+
import { PackIndex, PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
3
3
|
|
|
4
4
|
//#region src/node.d.ts
|
|
5
5
|
/** Build settings file in a pack's source directory. */
|
|
@@ -61,5 +61,42 @@ declare function buildPack(dir: string, options: {
|
|
|
61
61
|
* to the same content.
|
|
62
62
|
*/
|
|
63
63
|
declare function extractPack(pack: Pack, outDir: string): Promise<string[]>;
|
|
64
|
+
/**
|
|
65
|
+
* Write `pack` unzipped, for static hosting: every file at its own path under `outDir`, then a
|
|
66
|
+
* `molen-pack.json` that lists them loose (no solid blocks) with the same contentHash. Serve the
|
|
67
|
+
* directory and `openPack('<url>/molen-pack.json')` reads it back. The manifest is written last,
|
|
68
|
+
* so a directory without one is an interrupted write.
|
|
69
|
+
*/
|
|
70
|
+
declare function writeDirectoryPack(pack: Pack, outDir: string): Promise<{
|
|
71
|
+
manifest: PackManifest;
|
|
72
|
+
manifestBytes: number;
|
|
73
|
+
}>;
|
|
74
|
+
interface HostPacksOptions {
|
|
75
|
+
/** Where to publish: `<pack id>/<version>/…` per pack, then `index.json`. */
|
|
76
|
+
outDir: string;
|
|
77
|
+
/** Which pack ids to publish (default: every pack in the index). */
|
|
78
|
+
include?(id: string): boolean;
|
|
79
|
+
log?(line: string): void;
|
|
80
|
+
}
|
|
81
|
+
interface HostedPacks {
|
|
82
|
+
/** Path of the written `index.json`. */
|
|
83
|
+
indexPath: string;
|
|
84
|
+
index: PackIndex;
|
|
85
|
+
/** Pack ids written by this call. */
|
|
86
|
+
written: string[];
|
|
87
|
+
/** Pack ids whose version directory was already complete. */
|
|
88
|
+
reused: string[];
|
|
89
|
+
}
|
|
90
|
+
/** A pack version's directory: the first 12 hex digits of its contentHash. */
|
|
91
|
+
declare function hostedVersionDir(contentHash: string): string;
|
|
92
|
+
/**
|
|
93
|
+
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
|
|
94
|
+
* unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
|
|
95
|
+
* changes, so a host can serve those files as immutable; `index.json`, written last and the only
|
|
96
|
+
* document that changes between releases, names each version's `molen-pack.json`. Complete
|
|
97
|
+
* version directories from an earlier run are kept; older versions are left in place for clients
|
|
98
|
+
* still holding an older index.
|
|
99
|
+
*/
|
|
100
|
+
declare function hostPacks(sourceIndexPath: string, options: HostPacksOptions): Promise<HostedPacks>;
|
|
64
101
|
//#endregion
|
|
65
|
-
export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
|
102
|
+
export { BuiltPackFile, HostPacksOptions, HostedPacks, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
package/dist/node.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as
|
|
2
|
-
import { validate } from "@bendyline/molen-schema";
|
|
1
|
+
import { a as openPack, d as sha256, o as packFromFiles, s as createPack } from "./pack-CymjXXOV.mjs";
|
|
2
|
+
import { PACK_MANIFEST_ENTRY, validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
//#region src/node.ts
|
|
@@ -218,5 +218,110 @@ async function extractPack(pack, outDir) {
|
|
|
218
218
|
await writeFile(join(root, PACK_SOURCE_FILE), `${JSON.stringify(config, null, 2)}\n`);
|
|
219
219
|
return written;
|
|
220
220
|
}
|
|
221
|
+
/**
|
|
222
|
+
* Write `pack` unzipped, for static hosting: every file at its own path under `outDir`, then a
|
|
223
|
+
* `molen-pack.json` that lists them loose (no solid blocks) with the same contentHash. Serve the
|
|
224
|
+
* directory and `openPack('<url>/molen-pack.json')` reads it back. The manifest is written last,
|
|
225
|
+
* so a directory without one is an interrupted write.
|
|
226
|
+
*/
|
|
227
|
+
async function writeDirectoryPack(pack, outDir) {
|
|
228
|
+
const root = resolve(outDir);
|
|
229
|
+
for (const path of pack.paths()) {
|
|
230
|
+
const target = join(root, ...path.split("/"));
|
|
231
|
+
if (!target.startsWith(root + sep)) throw new Error(`refusing to write outside ${root}: ${path}`);
|
|
232
|
+
await mkdir(dirname(target), { recursive: true });
|
|
233
|
+
await writeFile(target, new Uint8Array(await pack.readBytes(path)));
|
|
234
|
+
}
|
|
235
|
+
const entries = Object.fromEntries(Object.entries(pack.manifest.entries).map(([path, { block: _block, offset: _offset, ...entry }]) => [path, entry]));
|
|
236
|
+
const manifest = {
|
|
237
|
+
...pack.manifest,
|
|
238
|
+
blocks: {},
|
|
239
|
+
entries
|
|
240
|
+
};
|
|
241
|
+
const parsed = validate("pack", manifest);
|
|
242
|
+
if (!parsed.ok) throw new Error(`${pack.label}: unzipped manifest is invalid\n${parsed.formatted}`);
|
|
243
|
+
const text = `${JSON.stringify(manifest)}\n`;
|
|
244
|
+
await mkdir(root, { recursive: true });
|
|
245
|
+
await writeFile(join(root, PACK_MANIFEST_ENTRY), text);
|
|
246
|
+
return {
|
|
247
|
+
manifest,
|
|
248
|
+
manifestBytes: new TextEncoder().encode(text).length
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
/** A pack version's directory: the first 12 hex digits of its contentHash. */
|
|
252
|
+
function hostedVersionDir(contentHash) {
|
|
253
|
+
return contentHash.replace("sha256:", "").slice(0, 12);
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Publish the packs a pack index lists (a `molen/pack-index@1` file beside its built zips)
|
|
257
|
+
* unzipped, for a CDN. Each pack version goes to `<id>/<12 hex of its contentHash>/` once and never
|
|
258
|
+
* changes, so a host can serve those files as immutable; `index.json`, written last and the only
|
|
259
|
+
* document that changes between releases, names each version's `molen-pack.json`. Complete
|
|
260
|
+
* version directories from an earlier run are kept; older versions are left in place for clients
|
|
261
|
+
* still holding an older index.
|
|
262
|
+
*/
|
|
263
|
+
async function hostPacks(sourceIndexPath, options) {
|
|
264
|
+
const source = resolve(sourceIndexPath);
|
|
265
|
+
const outDir = resolve(options.outDir);
|
|
266
|
+
const log = options.log ?? (() => void 0);
|
|
267
|
+
const raw = await readFile(source, "utf8").catch((error) => {
|
|
268
|
+
throw new Error(`${source}: cannot read the pack index`, { cause: error });
|
|
269
|
+
});
|
|
270
|
+
const parsed = validate("pack-index", JSON.parse(raw));
|
|
271
|
+
if (!parsed.ok) throw new Error(`${source} is not a pack index:\n${parsed.formatted}`);
|
|
272
|
+
const packs = {};
|
|
273
|
+
const written = [];
|
|
274
|
+
const reused = [];
|
|
275
|
+
const ids = Object.keys(parsed.value.packs).sort();
|
|
276
|
+
for (const id of ids) {
|
|
277
|
+
if (options.include !== void 0 && !options.include(id)) continue;
|
|
278
|
+
const entry = parsed.value.packs[id];
|
|
279
|
+
const version = hostedVersionDir(entry.contentHash);
|
|
280
|
+
const dir = join(outDir, id, version);
|
|
281
|
+
const manifestPath = join(dir, PACK_MANIFEST_ENTRY);
|
|
282
|
+
let manifestBytes;
|
|
283
|
+
try {
|
|
284
|
+
if (JSON.parse(await readFile(manifestPath, "utf8")).contentHash === entry.contentHash) {
|
|
285
|
+
manifestBytes = (await stat(manifestPath)).size;
|
|
286
|
+
reused.push(id);
|
|
287
|
+
}
|
|
288
|
+
} catch {}
|
|
289
|
+
if (manifestBytes === void 0) {
|
|
290
|
+
await rm(dir, {
|
|
291
|
+
recursive: true,
|
|
292
|
+
force: true
|
|
293
|
+
});
|
|
294
|
+
const pack = await openFilePack(join(dirname(source), ...entry.file.split("/")), { expect: { contentHash: entry.contentHash } });
|
|
295
|
+
try {
|
|
296
|
+
manifestBytes = (await writeDirectoryPack(pack, dir)).manifestBytes;
|
|
297
|
+
} finally {
|
|
298
|
+
pack.close();
|
|
299
|
+
}
|
|
300
|
+
written.push(id);
|
|
301
|
+
log(`${id}@${entry.version} → ${id}/${version}/`);
|
|
302
|
+
}
|
|
303
|
+
packs[id] = {
|
|
304
|
+
version: entry.version,
|
|
305
|
+
file: `${id}/${version}/${PACK_MANIFEST_ENTRY}`,
|
|
306
|
+
contentHash: entry.contentHash,
|
|
307
|
+
size: manifestBytes
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
const index = {
|
|
311
|
+
format: "molen/pack-index@1",
|
|
312
|
+
packs
|
|
313
|
+
};
|
|
314
|
+
const checked = validate("pack-index", index);
|
|
315
|
+
if (!checked.ok) throw new Error(`hosted pack index is invalid:\n${checked.formatted}`);
|
|
316
|
+
await mkdir(outDir, { recursive: true });
|
|
317
|
+
const indexPath = join(outDir, PACK_INDEX_FILE);
|
|
318
|
+
await writeFile(indexPath, `${JSON.stringify(index, null, 2)}\n`);
|
|
319
|
+
return {
|
|
320
|
+
indexPath,
|
|
321
|
+
index,
|
|
322
|
+
written,
|
|
323
|
+
reused
|
|
324
|
+
};
|
|
325
|
+
}
|
|
221
326
|
//#endregion
|
|
222
|
-
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
|
327
|
+
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, hostPacks, hostedVersionDir, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex, writeDirectoryPack };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as PackRetryOptions, r as RangeReader } from "./source-7Pr2tntF.mjs";
|
|
2
|
-
import { PackManifest } from "@bendyline/molen-schema";
|
|
2
|
+
import { PackEntry, PackManifest } from "@bendyline/molen-schema";
|
|
3
3
|
|
|
4
4
|
//#region src/build.d.ts
|
|
5
5
|
interface PackFile {
|
|
@@ -106,12 +106,33 @@ declare class PackEntryNotFoundError extends Error {
|
|
|
106
106
|
declare class PackIntegrityError extends Error {
|
|
107
107
|
constructor(pack: string, path: string, detail: string);
|
|
108
108
|
}
|
|
109
|
+
interface OpenDirectoryPackOptions extends Pick<OpenPackOptions, "fetch" | "signal" | "retry" | "concurrency" | "integrity" | "expect" | "maxCacheBytes" | "onProgress" | "label"> {
|
|
110
|
+
/**
|
|
111
|
+
* How to read one file, given its URL and manifest entry (default: `urlRangeReader` over
|
|
112
|
+
* `fetch`). Wrap that in `cachingRangeReader` keyed by `entry.sha256` to keep files in a byte
|
|
113
|
+
* cache; identical files then share one copy across packs and versions.
|
|
114
|
+
*/
|
|
115
|
+
fileReader?(url: string, entry: PackEntry): RangeReader;
|
|
116
|
+
}
|
|
117
|
+
/** Whether `url` names an unzipped pack (its `molen-pack.json`) rather than a zip. */
|
|
118
|
+
declare function isDirectoryPackUrl(url: string | URL): boolean;
|
|
119
|
+
/**
|
|
120
|
+
* Open an unzipped pack from the URL of its `molen-pack.json`: the manifest lists every file
|
|
121
|
+
* loose (no solid blocks), and each file is read from its path beside the manifest when first
|
|
122
|
+
* needed. `writeDirectoryPack` in `@bendyline/molen-pack/node` writes this layout; it suits a CDN
|
|
123
|
+
* better than a zip, since every file is its own cacheable URL. Relative URLs resolve against the
|
|
124
|
+
* page.
|
|
125
|
+
*/
|
|
126
|
+
declare function openDirectoryPack(manifestUrl: string | URL, options?: OpenDirectoryPackOptions): Promise<Pack>;
|
|
109
127
|
/**
|
|
110
128
|
* A pack over loose files, with the same manifest a built pack of those files would have. For
|
|
111
129
|
* tests, development, and content generated at runtime.
|
|
112
130
|
*/
|
|
113
131
|
declare function packFromFiles(files: readonly PackFile[], options: PackOptions, label?: string): Promise<Pack>;
|
|
114
|
-
/**
|
|
132
|
+
/**
|
|
133
|
+
* Open a pack. Reads only the manifest and zip directory; files are fetched when read. A URL whose
|
|
134
|
+
* last segment is `molen-pack.json` opens an unzipped pack (`openDirectoryPack`).
|
|
135
|
+
*/
|
|
115
136
|
declare function openPack(input: PackInput, options?: OpenPackOptions): Promise<Pack>;
|
|
116
137
|
//#endregion
|
|
117
|
-
export {
|
|
138
|
+
export { describePack as _, PackInput as a, sha256 as b, ReadOptions as c, openPack as d, packFromFiles as f, createPack as g, PackOptions as h, PackEntryNotFoundError as i, isDirectoryPackUrl as l, PackFile as m, OpenPackOptions as n, PackIntegrityError as o, BuiltPack as p, Pack as r, PackProgress as s, OpenDirectoryPackOptions as t, openDirectoryPack as u, isTextPath as v, mediaTypeOf as y };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as
|
|
1
|
+
import { a as blobReader, c as withRetry, i as TAIL_BYTES, o as openUrl, r as ReadScheduler, s as urlRangeReader, t as HttpStatusError } from "./source-BMJGKBgK.mjs";
|
|
2
2
|
import { PACK_MANIFEST_ENTRY, PACK_RESERVED_PREFIX, validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { deflateSync, inflateSync } from "fflate";
|
|
4
4
|
//#region src/zip.ts
|
|
@@ -423,7 +423,8 @@ async function createPack(files, options) {
|
|
|
423
423
|
/**
|
|
424
424
|
* Open a pack from any source and read its files. Only the tail of a remote pack is fetched up
|
|
425
425
|
* front (manifest and central directory); files are read on demand, and small documents that
|
|
426
|
-
* share a solid block arrive together.
|
|
426
|
+
* share a solid block arrive together. An unzipped pack — its files served loose beside its
|
|
427
|
+
* `molen-pack.json` — opens from that manifest's URL and reads each file with its own request.
|
|
427
428
|
*/
|
|
428
429
|
var PackEntryNotFoundError = class extends Error {
|
|
429
430
|
pack;
|
|
@@ -618,6 +619,130 @@ var MemoryPack = class extends BasePack {
|
|
|
618
619
|
this.files = /* @__PURE__ */ new Map();
|
|
619
620
|
}
|
|
620
621
|
};
|
|
622
|
+
/** Whether `url` names an unzipped pack (its `molen-pack.json`) rather than a zip. */
|
|
623
|
+
function isDirectoryPackUrl(url) {
|
|
624
|
+
const { pathname } = new URL(String(url), "http://pack.invalid/");
|
|
625
|
+
return pathname.split("/").at(-1) === PACK_MANIFEST_ENTRY;
|
|
626
|
+
}
|
|
627
|
+
/** A pack whose files sit loose beside its manifest; each read fetches one file. */
|
|
628
|
+
var DirectoryPack = class extends BasePack {
|
|
629
|
+
manifest;
|
|
630
|
+
label;
|
|
631
|
+
base;
|
|
632
|
+
reader;
|
|
633
|
+
controller;
|
|
634
|
+
options;
|
|
635
|
+
cache;
|
|
636
|
+
active = 0;
|
|
637
|
+
waiting = [];
|
|
638
|
+
bytesRead = 0;
|
|
639
|
+
requests = 0;
|
|
640
|
+
constructor(manifest, label, base, reader, controller, options) {
|
|
641
|
+
super();
|
|
642
|
+
this.manifest = manifest;
|
|
643
|
+
this.label = label;
|
|
644
|
+
this.base = base;
|
|
645
|
+
this.reader = reader;
|
|
646
|
+
this.controller = controller;
|
|
647
|
+
this.options = options;
|
|
648
|
+
this.cache = new ByteCache(options.maxCacheBytes ?? 16 * 1024 * 1024);
|
|
649
|
+
}
|
|
650
|
+
/** A file's URL beside the manifest, each path segment escaped. */
|
|
651
|
+
url(path) {
|
|
652
|
+
return new URL(path.split("/").map(encodeURIComponent).join("/"), this.base).href;
|
|
653
|
+
}
|
|
654
|
+
/** Run `task` within the concurrency limit, high-priority reads first. */
|
|
655
|
+
slot(high, task) {
|
|
656
|
+
return new Promise((resolve, reject) => {
|
|
657
|
+
const start = () => {
|
|
658
|
+
this.active++;
|
|
659
|
+
task().then(resolve, reject).finally(() => {
|
|
660
|
+
this.active--;
|
|
661
|
+
const next = this.waiting.findIndex((waiter) => waiter.high);
|
|
662
|
+
this.waiting.splice(next === -1 ? 0 : next, 1)[0]?.start();
|
|
663
|
+
});
|
|
664
|
+
};
|
|
665
|
+
if (this.active < (this.options.concurrency ?? 6)) start();
|
|
666
|
+
else this.waiting.push({
|
|
667
|
+
high,
|
|
668
|
+
start
|
|
669
|
+
});
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
bytes(path, options) {
|
|
673
|
+
if (this.controller.signal.aborted) return Promise.reject(this.controller.signal.reason);
|
|
674
|
+
const entry = this.entry(path);
|
|
675
|
+
return untilAborted(this.cache.get(path, async () => {
|
|
676
|
+
if (entry.size === 0) return new Uint8Array(0);
|
|
677
|
+
const bytes = await this.slot(options?.priority !== "low", () => this.reader(this.url(path), entry).read(0, entry.size, this.controller.signal));
|
|
678
|
+
this.bytesRead += bytes.length;
|
|
679
|
+
this.requests++;
|
|
680
|
+
this.options.onProgress?.({
|
|
681
|
+
label: this.label,
|
|
682
|
+
bytes: this.bytesRead,
|
|
683
|
+
requests: this.requests
|
|
684
|
+
});
|
|
685
|
+
return bytes;
|
|
686
|
+
}), options?.signal);
|
|
687
|
+
}
|
|
688
|
+
async readBytes(path, options) {
|
|
689
|
+
const entry = this.entry(path);
|
|
690
|
+
const bytes = await this.bytes(path, options);
|
|
691
|
+
if (bytes.length !== entry.size) throw new PackIntegrityError(this.manifest.id, path, `${bytes.length} bytes, expected ${entry.size}`);
|
|
692
|
+
if (this.options.integrity === "sha256") {
|
|
693
|
+
const actual = await sha256(bytes);
|
|
694
|
+
if (actual !== entry.sha256) throw new PackIntegrityError(this.manifest.id, path, `sha256 is ${actual}`);
|
|
695
|
+
}
|
|
696
|
+
return copy(bytes);
|
|
697
|
+
}
|
|
698
|
+
async prefetch(paths, options) {
|
|
699
|
+
await Promise.all(paths.map((path) => this.bytes(path, options)));
|
|
700
|
+
}
|
|
701
|
+
close() {
|
|
702
|
+
this.controller.abort(/* @__PURE__ */ new Error(`pack "${this.manifest.id}" was closed`));
|
|
703
|
+
this.cache.clear();
|
|
704
|
+
}
|
|
705
|
+
};
|
|
706
|
+
/**
|
|
707
|
+
* Open an unzipped pack from the URL of its `molen-pack.json`: the manifest lists every file
|
|
708
|
+
* loose (no solid blocks), and each file is read from its path beside the manifest when first
|
|
709
|
+
* needed. `writeDirectoryPack` in `@bendyline/molen-pack/node` writes this layout; it suits a CDN
|
|
710
|
+
* better than a zip, since every file is its own cacheable URL. Relative URLs resolve against the
|
|
711
|
+
* page.
|
|
712
|
+
*/
|
|
713
|
+
async function openDirectoryPack(manifestUrl, options = {}) {
|
|
714
|
+
const page = globalThis.location?.href;
|
|
715
|
+
const href = new URL(String(manifestUrl), page).href;
|
|
716
|
+
const label = options.label ?? href;
|
|
717
|
+
const fetcher = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
718
|
+
const controller = new AbortController();
|
|
719
|
+
const signal = options.signal === void 0 ? controller.signal : AbortSignal.any([options.signal, controller.signal]);
|
|
720
|
+
const retry = options.retry ?? {};
|
|
721
|
+
const text = await withRetry(async (attemptSignal) => {
|
|
722
|
+
const response = await fetcher(href, { signal: attemptSignal });
|
|
723
|
+
if (!response.ok) throw new HttpStatusError(href, response.status);
|
|
724
|
+
return response.text();
|
|
725
|
+
}, retry, signal);
|
|
726
|
+
let manifestDoc;
|
|
727
|
+
try {
|
|
728
|
+
manifestDoc = JSON.parse(text);
|
|
729
|
+
} catch (error) {
|
|
730
|
+
throw new Error(`${label}: ${PACK_MANIFEST_ENTRY} is not valid JSON`, { cause: error });
|
|
731
|
+
}
|
|
732
|
+
const parsed = validate("pack", manifestDoc);
|
|
733
|
+
if (!parsed.ok) throw new Error(`${label}: invalid pack manifest\n${parsed.formatted}`);
|
|
734
|
+
const manifest = parsed.value;
|
|
735
|
+
if (options.expect !== void 0 && manifest.contentHash !== options.expect.contentHash) throw new Error(`${label}: pack "${manifest.id}" has contentHash ${manifest.contentHash}, expected ${options.expect.contentHash}`);
|
|
736
|
+
const blocked = Object.entries(manifest.entries).find(([, entry]) => entry.block !== void 0);
|
|
737
|
+
if (Object.keys(manifest.blocks).length > 0 || blocked !== void 0) throw new Error(`${label}: an unzipped pack lists every file loose, but "${manifest.id}" has solid blocks`);
|
|
738
|
+
const reader = options.fileReader ?? ((url, entry) => urlRangeReader(url, {
|
|
739
|
+
size: entry.size,
|
|
740
|
+
fetch: fetcher,
|
|
741
|
+
retry,
|
|
742
|
+
signal
|
|
743
|
+
}));
|
|
744
|
+
return new DirectoryPack(manifest, label, new URL(href), reader, controller, options);
|
|
745
|
+
}
|
|
621
746
|
/**
|
|
622
747
|
* A pack over loose files, with the same manifest a built pack of those files would have. For
|
|
623
748
|
* tests, development, and content generated at runtime.
|
|
@@ -634,8 +759,15 @@ function defaultLabel(input) {
|
|
|
634
759
|
if (isRangeReader(input)) return "reader";
|
|
635
760
|
return "bytes";
|
|
636
761
|
}
|
|
637
|
-
/**
|
|
762
|
+
/**
|
|
763
|
+
* Open a pack. Reads only the manifest and zip directory; files are fetched when read. A URL whose
|
|
764
|
+
* last segment is `molen-pack.json` opens an unzipped pack (`openDirectoryPack`).
|
|
765
|
+
*/
|
|
638
766
|
async function openPack(input, options = {}) {
|
|
767
|
+
if ((typeof input === "string" || input instanceof URL) && isDirectoryPackUrl(input)) {
|
|
768
|
+
const { mode: _mode, sizeHint: _sizeHint, wholeThreshold: _whole, ...directory } = options;
|
|
769
|
+
return openDirectoryPack(input, directory);
|
|
770
|
+
}
|
|
639
771
|
const label = options.label ?? defaultLabel(input);
|
|
640
772
|
const controller = new AbortController();
|
|
641
773
|
const signal = options.signal === void 0 ? controller.signal : AbortSignal.any([options.signal, controller.signal]);
|
|
@@ -734,4 +866,4 @@ async function openPack(input, options = {}) {
|
|
|
734
866
|
}, scheduler, controller, options);
|
|
735
867
|
}
|
|
736
868
|
//#endregion
|
|
737
|
-
export {
|
|
869
|
+
export { openPack as a, describePack as c, sha256 as d, openDirectoryPack as i, isTextPath as l, PackIntegrityError as n, packFromFiles as o, isDirectoryPackUrl as r, createPack as s, PackEntryNotFoundError as t, mediaTypeOf as u };
|
|
@@ -296,4 +296,4 @@ var ReadScheduler = class {
|
|
|
296
296
|
}
|
|
297
297
|
};
|
|
298
298
|
//#endregion
|
|
299
|
-
export {
|
|
299
|
+
export { blobReader as a, withRetry as c, TAIL_BYTES as i, PackChangedError as n, openUrl as o, ReadScheduler as r, urlRangeReader as s, HttpStatusError as t };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
],
|
|
47
47
|
"dependencies": {
|
|
48
48
|
"fflate": "0.8.3",
|
|
49
|
-
"@bendyline/molen-schema": "0.0.
|
|
49
|
+
"@bendyline/molen-schema": "0.0.7"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@biomejs/biome": "^2.5.0",
|