@bendyline/molen-pack 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/index.mjs +1 -1
- package/dist/node.d.mts +14 -2
- package/dist/node.mjs +39 -21
- package/dist/{pack-CDBKnu89.mjs → pack-B-8f90rh.mjs} +50 -3
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -43,17 +43,22 @@ the small documents. Other files are read on demand with HTTP range requests; re
|
|
|
43
43
|
together are merged when they sit close in the file, at most six run at once, and transient
|
|
44
44
|
failures are retried. A server that ignores `Range` just sends the whole file, which is used as is.
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
Molen's own packs (entities, the default style pack, the Earth catalogs, the star table) are
|
|
47
|
+
published at [molen.dev/packs](https://molen.dev/packs/index.json).
|
|
48
|
+
`npx molen pack fetch https://molen.dev/packs/index.json` downloads them and pins each one's
|
|
49
|
+
`contentHash` in `project.json`.
|
|
50
|
+
|
|
51
|
+
Build your own packs with the CLI (`molen pack build <dir> --out-dir <d>`) or in code:
|
|
47
52
|
|
|
48
53
|
```ts
|
|
49
54
|
import { buildPack, openDirPack } from '@bendyline/molen-pack/node';
|
|
50
55
|
|
|
51
56
|
// A directory with a molen-pack.source.json describing the pack.
|
|
52
|
-
const built = await buildPack('content/
|
|
53
|
-
built.file; // '
|
|
57
|
+
const built = await buildPack('content/my-props', { outDir: 'dist/packs' });
|
|
58
|
+
built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
|
|
54
59
|
|
|
55
60
|
// The same content, unzipped: handy in tests and during development.
|
|
56
|
-
const pack = await openDirPack('content/
|
|
61
|
+
const pack = await openDirPack('content/my-props');
|
|
57
62
|
```
|
|
58
63
|
|
|
59
64
|
## What's in it
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-
|
|
1
|
+
import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-B-8f90rh.mjs";
|
|
2
2
|
//#region src/set.ts
|
|
3
3
|
var Packs = class {
|
|
4
4
|
list = [];
|
package/dist/node.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-DdapyeeP.mjs";
|
|
2
|
-
import { PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
2
|
+
import { PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
3
3
|
|
|
4
4
|
//#region src/node.d.ts
|
|
5
5
|
/** Build settings file in a pack's source directory. */
|
|
@@ -13,6 +13,12 @@ interface PackSource {
|
|
|
13
13
|
config: PackSourceConfig;
|
|
14
14
|
files: PackFile[];
|
|
15
15
|
}
|
|
16
|
+
/** List validated source settings and included paths without loading file contents. */
|
|
17
|
+
declare function listPackSource(dir: string): Promise<{
|
|
18
|
+
dir: string;
|
|
19
|
+
config: PackSourceConfig;
|
|
20
|
+
paths: string[];
|
|
21
|
+
}>;
|
|
16
22
|
/** Read a pack's source directory: its settings file and every file it includes. */
|
|
17
23
|
declare function readPackSource(dir: string): Promise<PackSource>;
|
|
18
24
|
/**
|
|
@@ -35,6 +41,12 @@ interface BuiltPackFile {
|
|
|
35
41
|
indexPath: string;
|
|
36
42
|
}
|
|
37
43
|
/**
|
|
44
|
+
* List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
|
|
45
|
+
* the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
|
|
46
|
+
* index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
|
|
47
|
+
*/
|
|
48
|
+
declare function recordInPackIndex(outDir: string, id: string, entry: PackIndexEntry): Promise<string>;
|
|
49
|
+
/**
|
|
38
50
|
* Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
|
|
39
51
|
* host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
|
|
40
52
|
* the previous build of the same pack is removed.
|
|
@@ -49,4 +61,4 @@ declare function buildPack(dir: string, options: {
|
|
|
49
61
|
*/
|
|
50
62
|
declare function extractPack(pack: Pack, outDir: string): Promise<string[]>;
|
|
51
63
|
//#endregion
|
|
52
|
-
export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
|
|
64
|
+
export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
package/dist/node.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-
|
|
1
|
+
import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-B-8f90rh.mjs";
|
|
2
2
|
import { validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
@@ -57,8 +57,8 @@ async function walk(dir, root, out) {
|
|
|
57
57
|
else if (item.isFile()) out.push(relative(root, full).split(sep).join("/"));
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
/**
|
|
61
|
-
async function
|
|
60
|
+
/** List validated source settings and included paths without loading file contents. */
|
|
61
|
+
async function listPackSource(dir) {
|
|
62
62
|
const root = resolve(dir);
|
|
63
63
|
let raw;
|
|
64
64
|
try {
|
|
@@ -73,14 +73,23 @@ async function readPackSource(dir) {
|
|
|
73
73
|
const exclude = config.exclude.map(globToRegExp);
|
|
74
74
|
const paths = [];
|
|
75
75
|
await walk(root, root, paths);
|
|
76
|
-
const selected = paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort();
|
|
77
76
|
return {
|
|
78
77
|
dir: root,
|
|
79
78
|
config,
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
79
|
+
paths: paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort()
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/** Read a pack's source directory: its settings file and every file it includes. */
|
|
83
|
+
async function readPackSource(dir) {
|
|
84
|
+
const source = await listPackSource(dir);
|
|
85
|
+
const files = await Promise.all(source.paths.map(async (path) => ({
|
|
86
|
+
path,
|
|
87
|
+
bytes: new Uint8Array(await readFile(join(source.dir, ...path.split("/"))))
|
|
88
|
+
})));
|
|
89
|
+
return {
|
|
90
|
+
dir: source.dir,
|
|
91
|
+
config: source.config,
|
|
92
|
+
files
|
|
84
93
|
};
|
|
85
94
|
}
|
|
86
95
|
function optionsOf(config) {
|
|
@@ -126,6 +135,25 @@ async function readIndex(path) {
|
|
|
126
135
|
return parsed.value;
|
|
127
136
|
}
|
|
128
137
|
/**
|
|
138
|
+
* List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
|
|
139
|
+
* the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
|
|
140
|
+
* index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
|
|
141
|
+
*/
|
|
142
|
+
async function recordInPackIndex(outDir, id, entry) {
|
|
143
|
+
const indexPath = join(outDir, PACK_INDEX_FILE);
|
|
144
|
+
const index = await readIndex(indexPath);
|
|
145
|
+
const previous = index.packs[id]?.file;
|
|
146
|
+
index.packs[id] = entry;
|
|
147
|
+
const sorted = {
|
|
148
|
+
format: "molen/pack-index@1",
|
|
149
|
+
packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
|
|
150
|
+
};
|
|
151
|
+
await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
|
|
152
|
+
const stillListed = Object.values(sorted.packs).some((e) => e.file === previous);
|
|
153
|
+
if (previous !== void 0 && previous !== entry.file && !stillListed) await rm(join(outDir, previous), { force: true });
|
|
154
|
+
return indexPath;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
129
157
|
* Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
|
|
130
158
|
* host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
|
|
131
159
|
* the previous build of the same pack is removed.
|
|
@@ -143,22 +171,12 @@ async function buildPack(dir, options) {
|
|
|
143
171
|
await mkdir(outDir, { recursive: true });
|
|
144
172
|
const path = join(outDir, file);
|
|
145
173
|
await writeFile(path, built.bytes);
|
|
146
|
-
const indexPath =
|
|
147
|
-
const index = await readIndex(indexPath);
|
|
148
|
-
const previous = index.packs[id]?.file;
|
|
149
|
-
index.packs[id] = {
|
|
174
|
+
const indexPath = await recordInPackIndex(outDir, id, {
|
|
150
175
|
version: built.manifest.version,
|
|
151
176
|
file,
|
|
152
177
|
contentHash: built.manifest.contentHash,
|
|
153
178
|
size: built.bytes.length
|
|
154
|
-
};
|
|
155
|
-
const sorted = {
|
|
156
|
-
format: "molen/pack-index@1",
|
|
157
|
-
packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
|
|
158
|
-
};
|
|
159
|
-
await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
|
|
160
|
-
const stillListed = Object.values(sorted.packs).some((entry) => entry.file === previous);
|
|
161
|
-
if (previous !== void 0 && previous !== file && !stillListed) await rm(join(outDir, previous), { force: true });
|
|
179
|
+
});
|
|
162
180
|
return {
|
|
163
181
|
manifest: built.manifest,
|
|
164
182
|
file,
|
|
@@ -200,4 +218,4 @@ async function extractPack(pack, outDir) {
|
|
|
200
218
|
return written;
|
|
201
219
|
}
|
|
202
220
|
//#endregion
|
|
203
|
-
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
|
|
221
|
+
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
|
@@ -481,6 +481,23 @@ const TAIL_BYTES = 64 * 1024;
|
|
|
481
481
|
async function fetchBytes(response) {
|
|
482
482
|
return new Uint8Array(await response.arrayBuffer());
|
|
483
483
|
}
|
|
484
|
+
/** Missing Content-Range is allowed for CORS hosts which do not expose this header. */
|
|
485
|
+
function contentRange(value, url) {
|
|
486
|
+
if (value === void 0) return void 0;
|
|
487
|
+
const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
|
|
488
|
+
const start = Number(match?.[1]);
|
|
489
|
+
const end = Number(match?.[2]);
|
|
490
|
+
const size = match?.[3] === "*" ? void 0 : Number(match?.[3]);
|
|
491
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || size !== void 0 && (!Number.isSafeInteger(size) || size <= end)) throw new Error(`invalid Content-Range for ${url}: ${value}`);
|
|
492
|
+
return {
|
|
493
|
+
start,
|
|
494
|
+
end,
|
|
495
|
+
size
|
|
496
|
+
};
|
|
497
|
+
}
|
|
498
|
+
function strongEtag(value) {
|
|
499
|
+
return value !== void 0 && !value.startsWith("W/") ? value : void 0;
|
|
500
|
+
}
|
|
484
501
|
/**
|
|
485
502
|
* Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
|
|
486
503
|
* range request) has already sent the whole file, which is used as is.
|
|
@@ -499,7 +516,7 @@ async function openUrl(url, options) {
|
|
|
499
516
|
bytes
|
|
500
517
|
};
|
|
501
518
|
}
|
|
502
|
-
|
|
519
|
+
let first = await withRetry(async (attemptSignal) => {
|
|
503
520
|
const response = await fetcher(url, {
|
|
504
521
|
headers: { Range: `bytes=-${TAIL_BYTES}` },
|
|
505
522
|
signal: attemptSignal
|
|
@@ -517,8 +534,36 @@ async function openUrl(url, options) {
|
|
|
517
534
|
kind: "whole",
|
|
518
535
|
bytes: first.bytes
|
|
519
536
|
};
|
|
520
|
-
|
|
521
|
-
const
|
|
537
|
+
let range = contentRange(first.range, url);
|
|
538
|
+
const total = range?.size;
|
|
539
|
+
if (total !== void 0 && (range?.start !== Math.max(0, total - 65536) || range?.end !== total - 1 || first.bytes.length !== Math.min(total, 65536))) {
|
|
540
|
+
const validator = strongEtag(first.etag);
|
|
541
|
+
const start = Math.max(0, total - TAIL_BYTES);
|
|
542
|
+
first = await withRetry(async (attemptSignal) => {
|
|
543
|
+
const headers = { Range: `bytes=${start}-${total - 1}` };
|
|
544
|
+
if (validator !== void 0) headers["If-Range"] = validator;
|
|
545
|
+
const response = await fetcher(url, {
|
|
546
|
+
headers,
|
|
547
|
+
signal: attemptSignal
|
|
548
|
+
});
|
|
549
|
+
if (validator !== void 0 && (response.status === 200 || response.headers.has("etag") && response.headers.get("etag") !== validator)) throw new PackChangedError(url);
|
|
550
|
+
if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
551
|
+
return {
|
|
552
|
+
status: response.status,
|
|
553
|
+
bytes: await fetchBytes(response),
|
|
554
|
+
etag: response.headers.get("etag") ?? validator,
|
|
555
|
+
range: response.headers.get("content-range") ?? void 0
|
|
556
|
+
};
|
|
557
|
+
}, retry, signal);
|
|
558
|
+
options.onResponse(first.bytes.length);
|
|
559
|
+
if (first.status === 200) return {
|
|
560
|
+
kind: "whole",
|
|
561
|
+
bytes: first.bytes
|
|
562
|
+
};
|
|
563
|
+
range = contentRange(first.range, url);
|
|
564
|
+
if (first.bytes.length !== total - start || range !== void 0 && (range.start !== start || range.end !== total - 1 || range.size !== void 0 && range.size !== total)) throw new Error(`tail range ${start}-${total - 1} of ${url} was not honoured`);
|
|
565
|
+
} else if (range !== void 0 && first.bytes.length !== range.end - range.start + 1) throw new Error(`Content-Range length does not match response bytes for ${url}`);
|
|
566
|
+
const etag = strongEtag(first.etag);
|
|
522
567
|
return {
|
|
523
568
|
kind: "range",
|
|
524
569
|
tail: first.bytes,
|
|
@@ -539,6 +584,8 @@ async function openUrl(url, options) {
|
|
|
539
584
|
return all.subarray(offset, offset + length);
|
|
540
585
|
}
|
|
541
586
|
if (response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
587
|
+
const returnedRange = contentRange(response.headers.get("content-range") ?? void 0, url);
|
|
588
|
+
if (returnedRange !== void 0 && (returnedRange.start !== offset || returnedRange.end !== offset + length - 1 || returnedRange.size !== void 0 && returnedRange.size !== size)) throw new Error(`range ${offset}+${length} of ${url} was not honoured`);
|
|
542
589
|
const bytes = await fetchBytes(response);
|
|
543
590
|
options.onResponse(bytes.length);
|
|
544
591
|
if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.3",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -42,14 +42,14 @@
|
|
|
42
42
|
],
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"fflate": "0.8.3",
|
|
45
|
-
"@bendyline/molen-schema": "0.0.
|
|
45
|
+
"@bendyline/molen-schema": "0.0.3"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@biomejs/biome": "^2.5.0",
|
|
49
49
|
"@types/node": "^24",
|
|
50
50
|
"tsdown": "^0.22.2",
|
|
51
51
|
"typescript": "^6.0.3",
|
|
52
|
-
"vitest": "^
|
|
52
|
+
"vitest": "^5.0.1"
|
|
53
53
|
},
|
|
54
54
|
"scripts": {
|
|
55
55
|
"build": "tsdown",
|