@bendyline/molen-pack 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cache.d.mts
CHANGED
|
@@ -17,6 +17,12 @@ import { a as urlRangeReader, i as UrlRangeReaderOptions, r as RangeReader } fro
|
|
|
17
17
|
* the validator of a keyed archive mid-session makes the cache drop that archive's bytes and
|
|
18
18
|
* re-read, never mix versions.
|
|
19
19
|
*
|
|
20
|
+
* Only a strong ETag vouches for stored bytes. A URL can be answered by more than one server
|
|
21
|
+
* over time (a development server on a port, then a proxy to production), and blocks kept from a
|
|
22
|
+
* server that sent no strong ETag are of unknown origin: the first strong one drops them. A
|
|
23
|
+
* server sending none for an archive stored under a strong one is read past the cache for the
|
|
24
|
+
* session, leaving the stored version in place. Mixed, two versions' blocks decode as garbage.
|
|
25
|
+
*
|
|
20
26
|
* The store holds a byte budget. Blocks are evicted least recently used first, with a priority
|
|
21
27
|
* that biases recency: `high` blocks count as a week newer than they are, `low` ones as a day
|
|
22
28
|
* older, so a planet base layer outlives a street-level tile seen at the same time.
|
|
@@ -214,6 +220,12 @@ interface CachingRangeReaderOptions {
|
|
|
214
220
|
* 4 MiB), so a small pack costs one request on its first visit and none after.
|
|
215
221
|
*/
|
|
216
222
|
wholeBelow?: number;
|
|
223
|
+
/**
|
|
224
|
+
* The file is only ever read whole (an unzipped pack's files): the spans the cache is missing
|
|
225
|
+
* come from one request for the whole file, however large. Over HTTP that request carries no
|
|
226
|
+
* Range (see `urlRangeReader`), so a CDN can compress it: a pack's JSON is a twentieth the size.
|
|
227
|
+
*/
|
|
228
|
+
wholeFile?: boolean;
|
|
217
229
|
}
|
|
218
230
|
/**
|
|
219
231
|
* A pack `RangeReader` whose bytes come from `cache` when it has them. Key it by the pack's
|
package/dist/cache.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { s as urlRangeReader } from "./source-
|
|
1
|
+
import { s as urlRangeReader } from "./source-BARbHFRX.mjs";
|
|
2
2
|
/** Recency bias per priority: added to a block's last-used time to order eviction. */
|
|
3
3
|
const PRIORITY_BIAS_MS = {
|
|
4
4
|
high: 7 * (1440 * 60 * 1e3),
|
|
@@ -125,7 +125,8 @@ function createBlockCache(store = createMemoryByteStore(), options = {}) {
|
|
|
125
125
|
etag: fixedEtag,
|
|
126
126
|
fixedEtag: fixedEtag !== void 0,
|
|
127
127
|
meta: void 0,
|
|
128
|
-
unstable: false
|
|
128
|
+
unstable: false,
|
|
129
|
+
unvalidated: false
|
|
129
130
|
};
|
|
130
131
|
states.set(key, state);
|
|
131
132
|
} else if (archive.size !== void 0 && state.declaredSize === void 0) {
|
|
@@ -144,6 +145,7 @@ function createBlockCache(store = createMemoryByteStore(), options = {}) {
|
|
|
144
145
|
}
|
|
145
146
|
state.size ??= stored.size;
|
|
146
147
|
if (!state.fixedEtag) state.etag ??= stored.etag;
|
|
148
|
+
state.unvalidated = !state.fixedEtag && state.etag === void 0;
|
|
147
149
|
})();
|
|
148
150
|
return state.meta;
|
|
149
151
|
};
|
|
@@ -265,7 +267,22 @@ function createBlockCache(store = createMemoryByteStore(), options = {}) {
|
|
|
265
267
|
for (let n = 0; n < count; n++) settle(start + n, changed);
|
|
266
268
|
return;
|
|
267
269
|
}
|
|
270
|
+
if (learned !== void 0 && state.etag === void 0 && state.unvalidated) {
|
|
271
|
+
state.unvalidated = false;
|
|
272
|
+
await dropState(state);
|
|
273
|
+
state.etag = learned;
|
|
274
|
+
const changed = new GenerationChanged(state.key);
|
|
275
|
+
for (let n = 0; n < count; n++) settle(start + n, changed);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
if (learned === void 0 && state.etag !== void 0 && !state.fixedEtag) {
|
|
279
|
+
state.unstable = true;
|
|
280
|
+
const changed = new GenerationChanged(state.key);
|
|
281
|
+
for (let n = 0; n < count; n++) settle(start + n, changed);
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
268
284
|
if (learned !== void 0 && state.etag === void 0) state.etag = learned;
|
|
285
|
+
if (state.etag === void 0 && !state.fixedEtag) state.unvalidated = true;
|
|
269
286
|
if (bytes.length < want) {
|
|
270
287
|
if (state.size !== void 0) {
|
|
271
288
|
const short = /* @__PURE__ */ new Error(`${state.url}: ${bytes.length} bytes at ${offset}, expected ${want} (size ${state.size})`);
|
|
@@ -443,6 +460,7 @@ function createBlockCache(store = createMemoryByteStore(), options = {}) {
|
|
|
443
460
|
} catch (error) {
|
|
444
461
|
if (!(error instanceof GenerationChanged)) throw error;
|
|
445
462
|
}
|
|
463
|
+
if (state.unstable) return passthrough(source, offset, length, signal, etag);
|
|
446
464
|
try {
|
|
447
465
|
return await readBlocks(state, offset, length, signal, etag, source);
|
|
448
466
|
} catch (error) {
|
|
@@ -741,22 +759,34 @@ function cachingRangeSource(source, cache, archive) {
|
|
|
741
759
|
*/
|
|
742
760
|
function cachingRangeReader(reader, cache, archive, options = {}) {
|
|
743
761
|
const wholeBelow = options.wholeBelow ?? 4 * 1024 * 1024;
|
|
762
|
+
let fetching;
|
|
744
763
|
const handle = cache.open({
|
|
745
764
|
...archive,
|
|
746
765
|
size: archive.size ?? reader.size
|
|
747
|
-
}, async (offset, length, signal) =>
|
|
766
|
+
}, async (offset, length, signal) => {
|
|
767
|
+
if (options.wholeFile !== true) return { bytes: await reader.read(offset, length, signal) };
|
|
768
|
+
fetching ??= reader.read(0, reader.size, signal).catch((error) => {
|
|
769
|
+
fetching = void 0;
|
|
770
|
+
throw error;
|
|
771
|
+
});
|
|
772
|
+
return { bytes: (await fetching).subarray(offset, offset + length) };
|
|
773
|
+
});
|
|
748
774
|
let whole;
|
|
749
775
|
return {
|
|
750
776
|
size: reader.size,
|
|
751
777
|
read: async (offset, length, signal) => {
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
whole
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
778
|
+
try {
|
|
779
|
+
if (reader.size <= wholeBelow) {
|
|
780
|
+
whole ??= handle.read(0, reader.size).catch((error) => {
|
|
781
|
+
whole = void 0;
|
|
782
|
+
throw error;
|
|
783
|
+
});
|
|
784
|
+
await whole;
|
|
785
|
+
}
|
|
786
|
+
return (await handle.read(offset, length, signal)).bytes;
|
|
787
|
+
} finally {
|
|
788
|
+
fetching = void 0;
|
|
758
789
|
}
|
|
759
|
-
return (await handle.read(offset, length, signal)).bytes;
|
|
760
790
|
}
|
|
761
791
|
};
|
|
762
792
|
}
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as openPack, c as describePack, d as sha256, i as openDirectoryPack, l as isTextPath, n as PackIntegrityError, o as packFromFiles, r as isDirectoryPackUrl, s as createPack, t as PackEntryNotFoundError, u as mediaTypeOf } from "./pack-
|
|
2
|
-
import { n as PackChangedError } from "./source-
|
|
1
|
+
import { a as openPack, c as describePack, d as sha256, i as openDirectoryPack, l as isTextPath, n as PackIntegrityError, o as packFromFiles, r as isDirectoryPackUrl, s as createPack, t as PackEntryNotFoundError, u as mediaTypeOf } from "./pack-Bx-rovba.mjs";
|
|
2
|
+
import { n as PackChangedError } from "./source-BARbHFRX.mjs";
|
|
3
3
|
//#region src/set.ts
|
|
4
4
|
var Packs = class {
|
|
5
5
|
list = [];
|
package/dist/node.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, r as isDirectoryPackUrl, s as createPack } from "./pack-
|
|
1
|
+
import { a as openPack, d as sha256, i as openDirectoryPack, o as packFromFiles, r as isDirectoryPackUrl, s as createPack } from "./pack-Bx-rovba.mjs";
|
|
2
2
|
import { PACK_MANIFEST_ENTRY, validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { copyFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as blobReader, c as withRetry, i as TAIL_BYTES, o as openUrl, r as ReadScheduler, s as urlRangeReader, t as HttpStatusError } from "./source-
|
|
1
|
+
import { a as blobReader, c as withRetry, i as TAIL_BYTES, o as openUrl, r as ReadScheduler, s as urlRangeReader, t as HttpStatusError } from "./source-BARbHFRX.mjs";
|
|
2
2
|
import { PACK_MANIFEST_ENTRY, PACK_RESERVED_PREFIX, validate } from "@bendyline/molen-schema";
|
|
3
3
|
import { deflateSync, inflateSync } from "fflate";
|
|
4
4
|
//#region src/zip.ts
|
|
@@ -160,19 +160,37 @@ async function openUrl(url, options) {
|
|
|
160
160
|
})
|
|
161
161
|
};
|
|
162
162
|
}
|
|
163
|
-
/**
|
|
163
|
+
/** An ETag's opaque value: a CDN weakens the ETag of a response it compresses (`W/"…"`). */
|
|
164
|
+
function etagValue(value) {
|
|
165
|
+
return value?.replace(/^W\//, "");
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Range reads of one URL, validated with If-Range when a strong ETag is known. A read of the
|
|
169
|
+
* whole file asks for it without a Range: a CDN compresses a full response but never a range,
|
|
170
|
+
* and a pack's JSON then crosses the network at a twentieth of its size. Its length is checked,
|
|
171
|
+
* and its ETag against the known one (compressed responses carry a weakened copy of it).
|
|
172
|
+
*/
|
|
164
173
|
function httpRangeReader(url, options) {
|
|
165
174
|
const { fetch: fetcher, retry, signal, size } = options;
|
|
166
175
|
let etag = options.etag;
|
|
167
176
|
return {
|
|
168
177
|
size,
|
|
169
178
|
read: (offset, length, readSignal) => withRetry(async (attemptSignal) => {
|
|
170
|
-
const
|
|
171
|
-
|
|
179
|
+
const whole = offset === 0 && length === size && size > 0;
|
|
180
|
+
const headers = whole ? {} : { Range: `bytes=${offset}-${offset + length - 1}` };
|
|
181
|
+
if (etag !== void 0 && !whole) headers["If-Range"] = etag;
|
|
172
182
|
const response = await fetcher(url, {
|
|
173
183
|
headers,
|
|
174
184
|
signal: readSignal === void 0 ? attemptSignal : AbortSignal.any([attemptSignal, readSignal])
|
|
175
185
|
});
|
|
186
|
+
if (whole && response.status === 200) {
|
|
187
|
+
if (etag !== void 0 && response.headers.has("etag") && etagValue(response.headers.get("etag")) !== etagValue(etag)) throw new PackChangedError(url);
|
|
188
|
+
const all = await fetchBytes(response);
|
|
189
|
+
options.onResponse(all.length);
|
|
190
|
+
if (all.length !== size) throw new PackChangedError(url);
|
|
191
|
+
if (options.learnEtag && etag === void 0) etag = strongEtag(response.headers.get("etag") ?? void 0);
|
|
192
|
+
return all;
|
|
193
|
+
}
|
|
176
194
|
if (response.status === 200) {
|
|
177
195
|
if (etag !== void 0) throw new PackChangedError(url);
|
|
178
196
|
const all = await fetchBytes(response);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.13",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"dist"
|
|
46
46
|
],
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@bendyline/molen-schema": "0.0.
|
|
48
|
+
"@bendyline/molen-schema": "0.0.12",
|
|
49
49
|
"fflate": "0.8.3"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|