@bendyline/molen-pack 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +1 -1
- package/dist/node.d.mts +16 -3
- package/dist/node.mjs +41 -22
- package/dist/{pack-CDBKnu89.mjs → pack-DCQvXj1p.mjs} +54 -6
- package/dist/{pack-DdapyeeP.d.mts → pack-DjufOvAS.d.mts} +6 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -82,7 +82,7 @@ provides, and a `contentHash` over paths and contents that does not depend on th
|
|
|
82
82
|
|
|
83
83
|
## Status
|
|
84
84
|
|
|
85
|
-
0.x,
|
|
85
|
+
0.x, versioned independently of the other `@bendyline/molen-*` packages. The format is
|
|
86
86
|
beta. Packs are written without zip64, so a pack must stay under 4 GiB and 65,535 members.
|
|
87
87
|
Building the same files always produces the same bytes; that depends on the exact `fflate`
|
|
88
88
|
version this package pins.
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { _ as describePack, a as PackIntegrityError, b as sha256, c as openPack, d as PackRetryOptions, f as RangeReader, g as createPack, h as PackOptions, i as PackInput, l as packFromFiles, m as PackFile, n as Pack, o as PackProgress, p as BuiltPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as PackChangedError, v as isTextPath, y as mediaTypeOf } from "./pack-
|
|
1
|
+
import { _ as describePack, a as PackIntegrityError, b as sha256, c as openPack, d as PackRetryOptions, f as RangeReader, g as createPack, h as PackOptions, i as PackInput, l as packFromFiles, m as PackFile, n as Pack, o as PackProgress, p as BuiltPack, r as PackEntryNotFoundError, s as ReadOptions, t as OpenPackOptions, u as PackChangedError, v as isTextPath, y as mediaTypeOf } from "./pack-DjufOvAS.mjs";
|
|
2
2
|
import { PackIndex, PackIndexEntry, PackManifest } from "@bendyline/molen-schema";
|
|
3
3
|
|
|
4
4
|
//#region src/set.d.ts
|
package/dist/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-
|
|
1
|
+
import { a as PackChangedError, c as isTextPath, i as packFromFiles, l as mediaTypeOf, n as PackIntegrityError, o as createPack, r as openPack, s as describePack, t as PackEntryNotFoundError, u as sha256 } from "./pack-DCQvXj1p.mjs";
|
|
2
2
|
//#region src/set.ts
|
|
3
3
|
var Packs = class {
|
|
4
4
|
list = [];
|
package/dist/node.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-
|
|
2
|
-
import { PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
1
|
+
import { h as PackOptions, m as PackFile, n as Pack, t as OpenPackOptions } from "./pack-DjufOvAS.mjs";
|
|
2
|
+
import { PackIndexEntry, PackManifest, PackSourceConfig } from "@bendyline/molen-schema";
|
|
3
3
|
|
|
4
4
|
//#region src/node.d.ts
|
|
5
5
|
/** Build settings file in a pack's source directory. */
|
|
@@ -13,6 +13,12 @@ interface PackSource {
|
|
|
13
13
|
config: PackSourceConfig;
|
|
14
14
|
files: PackFile[];
|
|
15
15
|
}
|
|
16
|
+
/** List validated source settings and included paths without loading file contents. */
|
|
17
|
+
declare function listPackSource(dir: string): Promise<{
|
|
18
|
+
dir: string;
|
|
19
|
+
config: PackSourceConfig;
|
|
20
|
+
paths: string[];
|
|
21
|
+
}>;
|
|
16
22
|
/** Read a pack's source directory: its settings file and every file it includes. */
|
|
17
23
|
declare function readPackSource(dir: string): Promise<PackSource>;
|
|
18
24
|
/**
|
|
@@ -35,6 +41,12 @@ interface BuiltPackFile {
|
|
|
35
41
|
indexPath: string;
|
|
36
42
|
}
|
|
37
43
|
/**
|
|
44
|
+
* List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
|
|
45
|
+
* the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
|
|
46
|
+
* index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
|
|
47
|
+
*/
|
|
48
|
+
declare function recordInPackIndex(outDir: string, id: string, entry: PackIndexEntry): Promise<string>;
|
|
49
|
+
/**
|
|
38
50
|
* Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
|
|
39
51
|
* host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
|
|
40
52
|
* the previous build of the same pack is removed.
|
|
@@ -42,6 +54,7 @@ interface BuiltPackFile {
|
|
|
42
54
|
declare function buildPack(dir: string, options: {
|
|
43
55
|
outDir: string;
|
|
44
56
|
solid?: boolean;
|
|
57
|
+
compression?: PackOptions["compression"];
|
|
45
58
|
}): Promise<BuiltPackFile>;
|
|
46
59
|
/**
|
|
47
60
|
* Write every file of a pack into `outDir`, with a `molen-pack.source.json` that builds it back
|
|
@@ -49,4 +62,4 @@ declare function buildPack(dir: string, options: {
|
|
|
49
62
|
*/
|
|
50
63
|
declare function extractPack(pack: Pack, outDir: string): Promise<string[]>;
|
|
51
64
|
//#endregion
|
|
52
|
-
export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
|
|
65
|
+
export { BuiltPackFile, PACK_INDEX_FILE, PACK_SOURCE_FILE, PackSource, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
package/dist/node.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-
|
|
1
|
+
import { i as packFromFiles, o as createPack, r as openPack, u as sha256 } from "./pack-DCQvXj1p.mjs";
|
|
2
2
|
import { validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
@@ -57,8 +57,8 @@ async function walk(dir, root, out) {
|
|
|
57
57
|
else if (item.isFile()) out.push(relative(root, full).split(sep).join("/"));
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
/**
|
|
61
|
-
async function
|
|
60
|
+
/** List validated source settings and included paths without loading file contents. */
|
|
61
|
+
async function listPackSource(dir) {
|
|
62
62
|
const root = resolve(dir);
|
|
63
63
|
let raw;
|
|
64
64
|
try {
|
|
@@ -73,14 +73,23 @@ async function readPackSource(dir) {
|
|
|
73
73
|
const exclude = config.exclude.map(globToRegExp);
|
|
74
74
|
const paths = [];
|
|
75
75
|
await walk(root, root, paths);
|
|
76
|
-
const selected = paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort();
|
|
77
76
|
return {
|
|
78
77
|
dir: root,
|
|
79
78
|
config,
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
79
|
+
paths: paths.filter((path) => path !== PACK_SOURCE_FILE).filter((path) => include.some((glob) => glob.test(path))).filter((path) => !exclude.some((glob) => glob.test(path))).sort()
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
/** Read a pack's source directory: its settings file and every file it includes. */
|
|
83
|
+
async function readPackSource(dir) {
|
|
84
|
+
const source = await listPackSource(dir);
|
|
85
|
+
const files = await Promise.all(source.paths.map(async (path) => ({
|
|
86
|
+
path,
|
|
87
|
+
bytes: new Uint8Array(await readFile(join(source.dir, ...path.split("/"))))
|
|
88
|
+
})));
|
|
89
|
+
return {
|
|
90
|
+
dir: source.dir,
|
|
91
|
+
config: source.config,
|
|
92
|
+
files
|
|
84
93
|
};
|
|
85
94
|
}
|
|
86
95
|
function optionsOf(config) {
|
|
@@ -126,6 +135,25 @@ async function readIndex(path) {
|
|
|
126
135
|
return parsed.value;
|
|
127
136
|
}
|
|
128
137
|
/**
|
|
138
|
+
* List a pack file in `outDir/index.json` (a molen/pack-index@1 document), replacing the entry for
|
|
139
|
+
* the same pack id and deleting that entry's previous file when nothing else lists it. Returns the
|
|
140
|
+
* index path. A browser opens packs from that index (`openPacksFromIndex` in molen-earth).
|
|
141
|
+
*/
|
|
142
|
+
async function recordInPackIndex(outDir, id, entry) {
|
|
143
|
+
const indexPath = join(outDir, PACK_INDEX_FILE);
|
|
144
|
+
const index = await readIndex(indexPath);
|
|
145
|
+
const previous = index.packs[id]?.file;
|
|
146
|
+
index.packs[id] = entry;
|
|
147
|
+
const sorted = {
|
|
148
|
+
format: "molen/pack-index@1",
|
|
149
|
+
packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
|
|
150
|
+
};
|
|
151
|
+
await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
|
|
152
|
+
const stillListed = Object.values(sorted.packs).some((e) => e.file === previous);
|
|
153
|
+
if (previous !== void 0 && previous !== entry.file && !stillListed) await rm(join(outDir, previous), { force: true });
|
|
154
|
+
return indexPath;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
129
157
|
* Build a pack source directory into `outDir`. The file name carries a hash of its bytes, so a
|
|
130
158
|
* host can cache it forever; `index.json` in `outDir` maps the pack id to its current file, and
|
|
131
159
|
* the previous build of the same pack is removed.
|
|
@@ -134,7 +162,8 @@ async function buildPack(dir, options) {
|
|
|
134
162
|
const source = await readPackSource(dir);
|
|
135
163
|
const built = await createPack(source.files, {
|
|
136
164
|
...optionsOf(source.config),
|
|
137
|
-
...options.solid !== void 0 ? { solid: options.solid } : {}
|
|
165
|
+
...options.solid !== void 0 ? { solid: options.solid } : {},
|
|
166
|
+
...options.compression !== void 0 ? { compression: options.compression } : {}
|
|
138
167
|
});
|
|
139
168
|
const fileSha256 = await sha256(built.bytes);
|
|
140
169
|
const id = built.manifest.id;
|
|
@@ -143,22 +172,12 @@ async function buildPack(dir, options) {
|
|
|
143
172
|
await mkdir(outDir, { recursive: true });
|
|
144
173
|
const path = join(outDir, file);
|
|
145
174
|
await writeFile(path, built.bytes);
|
|
146
|
-
const indexPath =
|
|
147
|
-
const index = await readIndex(indexPath);
|
|
148
|
-
const previous = index.packs[id]?.file;
|
|
149
|
-
index.packs[id] = {
|
|
175
|
+
const indexPath = await recordInPackIndex(outDir, id, {
|
|
150
176
|
version: built.manifest.version,
|
|
151
177
|
file,
|
|
152
178
|
contentHash: built.manifest.contentHash,
|
|
153
179
|
size: built.bytes.length
|
|
154
|
-
};
|
|
155
|
-
const sorted = {
|
|
156
|
-
format: "molen/pack-index@1",
|
|
157
|
-
packs: Object.fromEntries(Object.entries(index.packs).sort(([a], [b]) => a < b ? -1 : 1))
|
|
158
|
-
};
|
|
159
|
-
await writeFile(indexPath, `${JSON.stringify(sorted, null, 2)}\n`);
|
|
160
|
-
const stillListed = Object.values(sorted.packs).some((entry) => entry.file === previous);
|
|
161
|
-
if (previous !== void 0 && previous !== file && !stillListed) await rm(join(outDir, previous), { force: true });
|
|
180
|
+
});
|
|
162
181
|
return {
|
|
163
182
|
manifest: built.manifest,
|
|
164
183
|
file,
|
|
@@ -200,4 +219,4 @@ async function extractPack(pack, outDir) {
|
|
|
200
219
|
return written;
|
|
201
220
|
}
|
|
202
221
|
//#endregion
|
|
203
|
-
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, openDirPack, openFilePack, openPackAt, readPackSource };
|
|
222
|
+
export { PACK_INDEX_FILE, PACK_SOURCE_FILE, buildPack, extractPack, listPackSource, openDirPack, openFilePack, openPackAt, readPackSource, recordInPackIndex };
|
|
@@ -345,6 +345,7 @@ async function createPack(files, options) {
|
|
|
345
345
|
const sorted = sortedFiles(files);
|
|
346
346
|
for (const file of sorted) if (file.path === PACK_MANIFEST_ENTRY || file.path.startsWith(PACK_RESERVED_PREFIX)) throw new Error(`"${file.path}" is reserved for the pack's own structure`);
|
|
347
347
|
const described = await describePack(sorted, options);
|
|
348
|
+
const compression = options.compression ?? "auto";
|
|
348
349
|
const entries = structuredClone(described.entries);
|
|
349
350
|
const members = [];
|
|
350
351
|
const blocks = {};
|
|
@@ -356,7 +357,7 @@ async function createPack(files, options) {
|
|
|
356
357
|
} else members.push({
|
|
357
358
|
name: file.path,
|
|
358
359
|
data: file.bytes,
|
|
359
|
-
compression
|
|
360
|
+
compression
|
|
360
361
|
});
|
|
361
362
|
for (const [group, groupFiles] of [...pending].sort(([a], [b]) => a < b ? -1 : 1)) {
|
|
362
363
|
let part = 1;
|
|
@@ -384,7 +385,7 @@ async function createPack(files, options) {
|
|
|
384
385
|
blockMembers.push({
|
|
385
386
|
name: entry,
|
|
386
387
|
data,
|
|
387
|
-
compression
|
|
388
|
+
compression
|
|
388
389
|
});
|
|
389
390
|
part++;
|
|
390
391
|
chunk = [];
|
|
@@ -410,7 +411,7 @@ async function createPack(files, options) {
|
|
|
410
411
|
{
|
|
411
412
|
name: PACK_MANIFEST_ENTRY,
|
|
412
413
|
data: new TextEncoder().encode(JSON.stringify(manifest)),
|
|
413
|
-
compression
|
|
414
|
+
compression
|
|
414
415
|
}
|
|
415
416
|
]),
|
|
416
417
|
manifest
|
|
@@ -481,6 +482,23 @@ const TAIL_BYTES = 64 * 1024;
|
|
|
481
482
|
async function fetchBytes(response) {
|
|
482
483
|
return new Uint8Array(await response.arrayBuffer());
|
|
483
484
|
}
|
|
485
|
+
/** Missing Content-Range is allowed for CORS hosts which do not expose this header. */
|
|
486
|
+
function contentRange(value, url) {
|
|
487
|
+
if (value === void 0) return void 0;
|
|
488
|
+
const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
|
|
489
|
+
const start = Number(match?.[1]);
|
|
490
|
+
const end = Number(match?.[2]);
|
|
491
|
+
const size = match?.[3] === "*" ? void 0 : Number(match?.[3]);
|
|
492
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || size !== void 0 && (!Number.isSafeInteger(size) || size <= end)) throw new Error(`invalid Content-Range for ${url}: ${value}`);
|
|
493
|
+
return {
|
|
494
|
+
start,
|
|
495
|
+
end,
|
|
496
|
+
size
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
function strongEtag(value) {
|
|
500
|
+
return value !== void 0 && !value.startsWith("W/") ? value : void 0;
|
|
501
|
+
}
|
|
484
502
|
/**
|
|
485
503
|
* Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
|
|
486
504
|
* range request) has already sent the whole file, which is used as is.
|
|
@@ -499,7 +517,7 @@ async function openUrl(url, options) {
|
|
|
499
517
|
bytes
|
|
500
518
|
};
|
|
501
519
|
}
|
|
502
|
-
|
|
520
|
+
let first = await withRetry(async (attemptSignal) => {
|
|
503
521
|
const response = await fetcher(url, {
|
|
504
522
|
headers: { Range: `bytes=-${TAIL_BYTES}` },
|
|
505
523
|
signal: attemptSignal
|
|
@@ -517,8 +535,36 @@ async function openUrl(url, options) {
|
|
|
517
535
|
kind: "whole",
|
|
518
536
|
bytes: first.bytes
|
|
519
537
|
};
|
|
520
|
-
|
|
521
|
-
const
|
|
538
|
+
let range = contentRange(first.range, url);
|
|
539
|
+
const total = range?.size;
|
|
540
|
+
if (total !== void 0 && (range?.start !== Math.max(0, total - 65536) || range?.end !== total - 1 || first.bytes.length !== Math.min(total, 65536))) {
|
|
541
|
+
const validator = strongEtag(first.etag);
|
|
542
|
+
const start = Math.max(0, total - TAIL_BYTES);
|
|
543
|
+
first = await withRetry(async (attemptSignal) => {
|
|
544
|
+
const headers = { Range: `bytes=${start}-${total - 1}` };
|
|
545
|
+
if (validator !== void 0) headers["If-Range"] = validator;
|
|
546
|
+
const response = await fetcher(url, {
|
|
547
|
+
headers,
|
|
548
|
+
signal: attemptSignal
|
|
549
|
+
});
|
|
550
|
+
if (validator !== void 0 && (response.status === 200 || response.headers.has("etag") && response.headers.get("etag") !== validator)) throw new PackChangedError(url);
|
|
551
|
+
if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
552
|
+
return {
|
|
553
|
+
status: response.status,
|
|
554
|
+
bytes: await fetchBytes(response),
|
|
555
|
+
etag: response.headers.get("etag") ?? validator,
|
|
556
|
+
range: response.headers.get("content-range") ?? void 0
|
|
557
|
+
};
|
|
558
|
+
}, retry, signal);
|
|
559
|
+
options.onResponse(first.bytes.length);
|
|
560
|
+
if (first.status === 200) return {
|
|
561
|
+
kind: "whole",
|
|
562
|
+
bytes: first.bytes
|
|
563
|
+
};
|
|
564
|
+
range = contentRange(first.range, url);
|
|
565
|
+
if (first.bytes.length !== total - start || range !== void 0 && (range.start !== start || range.end !== total - 1 || range.size !== void 0 && range.size !== total)) throw new Error(`tail range ${start}-${total - 1} of ${url} was not honoured`);
|
|
566
|
+
} else if (range !== void 0 && first.bytes.length !== range.end - range.start + 1) throw new Error(`Content-Range length does not match response bytes for ${url}`);
|
|
567
|
+
const etag = strongEtag(first.etag);
|
|
522
568
|
return {
|
|
523
569
|
kind: "range",
|
|
524
570
|
tail: first.bytes,
|
|
@@ -539,6 +585,8 @@ async function openUrl(url, options) {
|
|
|
539
585
|
return all.subarray(offset, offset + length);
|
|
540
586
|
}
|
|
541
587
|
if (response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
588
|
+
const returnedRange = contentRange(response.headers.get("content-range") ?? void 0, url);
|
|
589
|
+
if (returnedRange !== void 0 && (returnedRange.start !== offset || returnedRange.end !== offset + length - 1 || returnedRange.size !== void 0 && returnedRange.size !== size)) throw new Error(`range ${offset}+${length} of ${url} was not honoured`);
|
|
542
590
|
const bytes = await fetchBytes(response);
|
|
543
591
|
options.onResponse(bytes.length);
|
|
544
592
|
if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
|
|
@@ -19,6 +19,12 @@ interface PackOptions {
|
|
|
19
19
|
provides?: Record<string, string | readonly string[]>;
|
|
20
20
|
/** Group small text files into compressed solid blocks (default true). */
|
|
21
21
|
solid?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* `auto` (default) deflates a member when that saves at least 10%. `store` writes every member
|
|
24
|
+
* uncompressed: the manifest and contents are identical, only the archive is larger, so a pack
|
|
25
|
+
* built for local reading (a test run) skips the deflate that dominates build time.
|
|
26
|
+
*/
|
|
27
|
+
compression?: "auto" | "store";
|
|
22
28
|
}
|
|
23
29
|
interface BuiltPack {
|
|
24
30
|
bytes: Uint8Array;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
],
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"fflate": "0.8.3",
|
|
45
|
-
"@bendyline/molen-schema": "0.0.
|
|
45
|
+
"@bendyline/molen-schema": "0.0.4"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@biomejs/biome": "^2.5.0",
|