@bendyline/molen-pack 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/dist/cache.d.mts +263 -0
- package/dist/cache.mjs +1292 -0
- package/dist/index.d.mts +4 -3
- package/dist/index.mjs +10 -9
- package/dist/node.d.mts +1 -1
- package/dist/node.mjs +1 -1
- package/dist/{pack-DCQvXj1p.mjs → pack-BIXihu9D.mjs} +2 -268
- package/dist/{pack-DjufOvAS.d.mts → pack-CF6pPp9f.d.mts} +2 -26
- package/dist/source-7Pr2tntF.d.mts +41 -0
- package/dist/source-MDAzioCi.mjs +299 -0
- package/package.json +7 -2
package/dist/cache.mjs
ADDED
|
@@ -0,0 +1,1292 @@
|
|
|
1
|
+
import { o as urlRangeReader } from "./source-MDAzioCi.mjs";
|
|
2
|
+
/** Recency bias per priority: added to a block's last-used time to order eviction. */
|
|
3
|
+
const PRIORITY_BIAS_MS = {
|
|
4
|
+
high: 7 * (1440 * 60 * 1e3),
|
|
5
|
+
normal: 0,
|
|
6
|
+
low: -864e5
|
|
7
|
+
};
|
|
8
|
+
/** A block's place in the eviction order: lower ranks go first. */
|
|
9
|
+
function evictionRank(priority, touched) {
|
|
10
|
+
return touched + PRIORITY_BIAS_MS[priority];
|
|
11
|
+
}
|
|
12
|
+
/** Whether an error says the server's bytes changed while they were being read. */
|
|
13
|
+
function isChangedError(error) {
|
|
14
|
+
const named = error;
|
|
15
|
+
const name = named?.name;
|
|
16
|
+
return name === "EtagMismatch" || name === "PackChangedError" || named?.constructor?.name === "EtagMismatch";
|
|
17
|
+
}
|
|
18
|
+
function strongEtag(value) {
|
|
19
|
+
return value !== null && value !== void 0 && value !== "" && !value.startsWith("W/") ? value : void 0;
|
|
20
|
+
}
|
|
21
|
+
/** The server replaced a keyed archive; the readers retry from a clean slate. */
|
|
22
|
+
var GenerationChanged = class extends Error {
|
|
23
|
+
constructor(key) {
|
|
24
|
+
super(`archive ${key} changed on the server`);
|
|
25
|
+
this.name = "GenerationChanged";
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
function abortError() {
|
|
29
|
+
return typeof DOMException === "function" ? new DOMException("The operation was aborted.", "AbortError") : Object.assign(/* @__PURE__ */ new Error("The operation was aborted."), { name: "AbortError" });
|
|
30
|
+
}
|
|
31
|
+
/** Settle with `promise`, or reject early when `signal` aborts; the work itself continues. */
|
|
32
|
+
function untilAborted(promise, signal) {
|
|
33
|
+
if (signal === void 0) return promise;
|
|
34
|
+
if (signal.aborted) return Promise.reject(signal.reason ?? abortError());
|
|
35
|
+
return new Promise((resolve, reject) => {
|
|
36
|
+
const abort = () => reject(signal.reason ?? abortError());
|
|
37
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
38
|
+
promise.then((value) => {
|
|
39
|
+
signal.removeEventListener("abort", abort);
|
|
40
|
+
resolve(value);
|
|
41
|
+
}, (error) => {
|
|
42
|
+
signal.removeEventListener("abort", abort);
|
|
43
|
+
reject(error);
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
function defer() {
|
|
48
|
+
let resolve;
|
|
49
|
+
let reject;
|
|
50
|
+
const promise = new Promise((res, rej) => {
|
|
51
|
+
resolve = res;
|
|
52
|
+
reject = rej;
|
|
53
|
+
});
|
|
54
|
+
promise.catch(() => void 0);
|
|
55
|
+
const deferred = {
|
|
56
|
+
promise,
|
|
57
|
+
settled: false,
|
|
58
|
+
resolve: (value) => {
|
|
59
|
+
if (deferred.settled) return;
|
|
60
|
+
deferred.settled = true;
|
|
61
|
+
resolve(value);
|
|
62
|
+
},
|
|
63
|
+
reject: (error) => {
|
|
64
|
+
if (deferred.settled) return;
|
|
65
|
+
deferred.settled = true;
|
|
66
|
+
reject(error);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
return deferred;
|
|
70
|
+
}
|
|
71
|
+
const blockKey = (key, index) => `${key}\u0000${index}`;
|
|
72
|
+
/** Create a cache over `store` (default: an in-memory store with a 64 MiB budget). */
|
|
73
|
+
function createBlockCache(store = createMemoryByteStore(), options = {}) {
|
|
74
|
+
const B = options.blockBytes ?? 64 * 1024;
|
|
75
|
+
if (!Number.isSafeInteger(B) || B < 1024) throw new Error("blockBytes must be at least 1024");
|
|
76
|
+
const memoryLimit = options.memoryBytes ?? 32 * 1024 * 1024;
|
|
77
|
+
const maxSpanBlocks = Math.max(1, Math.floor((options.maxSpanBytes ?? 4 * 1024 * 1024) / B));
|
|
78
|
+
const flushDelay = options.flushDelayMs ?? 50;
|
|
79
|
+
const touchInterval = options.touchIntervalMs ?? 6e4;
|
|
80
|
+
const highWater = options.evictHighWater ?? 1.05;
|
|
81
|
+
const lowWater = options.evictLowWater ?? .9;
|
|
82
|
+
const now = options.now ?? Date.now;
|
|
83
|
+
const states = /* @__PURE__ */ new Map();
|
|
84
|
+
/** In-memory LRU, oldest first. */
|
|
85
|
+
const memory = /* @__PURE__ */ new Map();
|
|
86
|
+
let memoryTotal = 0;
|
|
87
|
+
/** Blocks waiting to be written to the store; readable until then. */
|
|
88
|
+
const pending = /* @__PURE__ */ new Map();
|
|
89
|
+
const inflight = /* @__PURE__ */ new Map();
|
|
90
|
+
const lastTouch = /* @__PURE__ */ new Map();
|
|
91
|
+
const dueTouches = /* @__PURE__ */ new Map();
|
|
92
|
+
const cleanedGenerations = /* @__PURE__ */ new Set();
|
|
93
|
+
let flushTimer;
|
|
94
|
+
let flushChain = Promise.resolve();
|
|
95
|
+
let disposed = false;
|
|
96
|
+
const counters = {
|
|
97
|
+
memoryHits: 0,
|
|
98
|
+
storeHits: 0,
|
|
99
|
+
misses: 0,
|
|
100
|
+
networkRequests: 0,
|
|
101
|
+
networkBytes: 0,
|
|
102
|
+
uncachedRequests: 0,
|
|
103
|
+
evictedBytes: 0
|
|
104
|
+
};
|
|
105
|
+
let storeStats = {
|
|
106
|
+
backend: store.kind,
|
|
107
|
+
bytes: 0,
|
|
108
|
+
blocks: 0,
|
|
109
|
+
archives: 0,
|
|
110
|
+
budgetBytes: store.budgetBytes,
|
|
111
|
+
degraded: false
|
|
112
|
+
};
|
|
113
|
+
store.stats().then((stats) => {
|
|
114
|
+
storeStats = stats;
|
|
115
|
+
});
|
|
116
|
+
const stateFor = (key, archive, fixedEtag) => {
|
|
117
|
+
let state = states.get(key);
|
|
118
|
+
if (state === void 0) {
|
|
119
|
+
state = {
|
|
120
|
+
key,
|
|
121
|
+
url: archive.url,
|
|
122
|
+
priority: archive.priority ?? "normal",
|
|
123
|
+
declaredSize: archive.size,
|
|
124
|
+
size: archive.size,
|
|
125
|
+
etag: fixedEtag,
|
|
126
|
+
fixedEtag: fixedEtag !== void 0,
|
|
127
|
+
meta: void 0,
|
|
128
|
+
unstable: false
|
|
129
|
+
};
|
|
130
|
+
states.set(key, state);
|
|
131
|
+
} else if (archive.size !== void 0 && state.declaredSize === void 0) {
|
|
132
|
+
state.declaredSize = archive.size;
|
|
133
|
+
state.size ??= archive.size;
|
|
134
|
+
}
|
|
135
|
+
return state;
|
|
136
|
+
};
|
|
137
|
+
const loadMeta = (state) => {
|
|
138
|
+
state.meta ??= (async () => {
|
|
139
|
+
const stored = await store.getArchive(state.key);
|
|
140
|
+
if (stored === void 0) return;
|
|
141
|
+
if (state.declaredSize !== void 0 && stored.size !== void 0 && stored.size !== state.declaredSize) {
|
|
142
|
+
await store.deleteArchive(state.key);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
state.size ??= stored.size;
|
|
146
|
+
if (!state.fixedEtag) state.etag ??= stored.etag;
|
|
147
|
+
})();
|
|
148
|
+
return state.meta;
|
|
149
|
+
};
|
|
150
|
+
const memoryGet = (state, index) => {
|
|
151
|
+
const k = blockKey(state.key, index);
|
|
152
|
+
const hit = memory.get(k);
|
|
153
|
+
if (hit !== void 0) {
|
|
154
|
+
memory.delete(k);
|
|
155
|
+
memory.set(k, hit);
|
|
156
|
+
return hit.bytes;
|
|
157
|
+
}
|
|
158
|
+
return pending.get(k)?.bytes;
|
|
159
|
+
};
|
|
160
|
+
const memorySet = (state, index, bytes) => {
|
|
161
|
+
if (bytes.length > memoryLimit) return;
|
|
162
|
+
const k = blockKey(state.key, index);
|
|
163
|
+
const existing = memory.get(k);
|
|
164
|
+
if (existing !== void 0) {
|
|
165
|
+
memoryTotal -= existing.bytes.length;
|
|
166
|
+
memory.delete(k);
|
|
167
|
+
}
|
|
168
|
+
memory.set(k, {
|
|
169
|
+
state,
|
|
170
|
+
index,
|
|
171
|
+
bytes
|
|
172
|
+
});
|
|
173
|
+
memoryTotal += bytes.length;
|
|
174
|
+
for (const [oldest, entry] of memory) {
|
|
175
|
+
if (memoryTotal <= memoryLimit) break;
|
|
176
|
+
memory.delete(oldest);
|
|
177
|
+
memoryTotal -= entry.bytes.length;
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
const scheduleFlush = (delay) => {
|
|
181
|
+
if (disposed) return;
|
|
182
|
+
if (flushTimer !== void 0) {
|
|
183
|
+
if (delay > 0) return;
|
|
184
|
+
clearTimeout(flushTimer);
|
|
185
|
+
}
|
|
186
|
+
flushTimer = setTimeout(() => {
|
|
187
|
+
flushTimer = void 0;
|
|
188
|
+
flush();
|
|
189
|
+
}, delay);
|
|
190
|
+
};
|
|
191
|
+
const touchSoon = (state, index) => {
|
|
192
|
+
const k = blockKey(state.key, index);
|
|
193
|
+
const t = now();
|
|
194
|
+
if (t - (lastTouch.get(k) ?? Number.NEGATIVE_INFINITY) < touchInterval) return;
|
|
195
|
+
lastTouch.set(k, t);
|
|
196
|
+
let due = dueTouches.get(state);
|
|
197
|
+
if (due === void 0) {
|
|
198
|
+
due = /* @__PURE__ */ new Set();
|
|
199
|
+
dueTouches.set(state, due);
|
|
200
|
+
}
|
|
201
|
+
due.add(index);
|
|
202
|
+
scheduleFlush(flushDelay);
|
|
203
|
+
};
|
|
204
|
+
const queueWrite = (state, index, bytes) => {
|
|
205
|
+
const k = blockKey(state.key, index);
|
|
206
|
+
pending.set(k, {
|
|
207
|
+
state,
|
|
208
|
+
index,
|
|
209
|
+
bytes
|
|
210
|
+
});
|
|
211
|
+
lastTouch.set(k, now());
|
|
212
|
+
scheduleFlush(pending.size >= 64 ? 0 : flushDelay);
|
|
213
|
+
};
|
|
214
|
+
const info = (state) => ({
|
|
215
|
+
key: state.key,
|
|
216
|
+
url: state.url,
|
|
217
|
+
priority: state.priority,
|
|
218
|
+
...state.size !== void 0 ? { size: state.size } : {},
|
|
219
|
+
...state.etag !== void 0 ? { etag: state.etag } : {}
|
|
220
|
+
});
|
|
221
|
+
const forget = (state) => {
|
|
222
|
+
for (const [k, entry] of memory) {
|
|
223
|
+
if (entry.state !== state) continue;
|
|
224
|
+
memory.delete(k);
|
|
225
|
+
memoryTotal -= entry.bytes.length;
|
|
226
|
+
}
|
|
227
|
+
for (const [k, entry] of pending) if (entry.state === state) pending.delete(k);
|
|
228
|
+
const prefix = `${state.key}\u0000`;
|
|
229
|
+
for (const k of lastTouch.keys()) if (k.startsWith(prefix)) lastTouch.delete(k);
|
|
230
|
+
dueTouches.delete(state);
|
|
231
|
+
state.size = state.declaredSize;
|
|
232
|
+
if (!state.fixedEtag) state.etag = void 0;
|
|
233
|
+
state.meta = Promise.resolve();
|
|
234
|
+
};
|
|
235
|
+
const dropState = async (state) => {
|
|
236
|
+
forget(state);
|
|
237
|
+
await store.deleteArchive(state.key);
|
|
238
|
+
};
|
|
239
|
+
const fetchSpan = async (state, start, count, batch, etag, source, settle) => {
|
|
240
|
+
const offset = start * B;
|
|
241
|
+
let want = count * B;
|
|
242
|
+
if (state.size !== void 0) want = Math.min(want, state.size - offset);
|
|
243
|
+
if (want <= 0) {
|
|
244
|
+
for (let n = 0; n < count; n++) settle(start + n, void 0);
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
counters.networkRequests++;
|
|
248
|
+
counters.misses += count;
|
|
249
|
+
let response;
|
|
250
|
+
try {
|
|
251
|
+
response = await source(offset, want, batch.controller.signal, etag);
|
|
252
|
+
} catch (error) {
|
|
253
|
+
if (isChangedError(error)) await dropState(state);
|
|
254
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
255
|
+
for (let n = 0; n < count; n++) settle(start + n, failure);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
const bytes = response.bytes;
|
|
259
|
+
counters.networkBytes += bytes.length;
|
|
260
|
+
const learned = strongEtag(response.etag);
|
|
261
|
+
if (learned !== void 0 && state.etag !== void 0 && learned !== state.etag) {
|
|
262
|
+
await dropState(state);
|
|
263
|
+
if (!state.fixedEtag) state.etag = learned;
|
|
264
|
+
const changed = new GenerationChanged(state.key);
|
|
265
|
+
for (let n = 0; n < count; n++) settle(start + n, changed);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
if (learned !== void 0 && state.etag === void 0) state.etag = learned;
|
|
269
|
+
if (bytes.length < want) {
|
|
270
|
+
if (state.size !== void 0) {
|
|
271
|
+
const short = /* @__PURE__ */ new Error(`${state.url}: ${bytes.length} bytes at ${offset}, expected ${want} (size ${state.size})`);
|
|
272
|
+
for (let n = 0; n < count; n++) settle(start + n, short);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
state.size = offset + bytes.length;
|
|
276
|
+
}
|
|
277
|
+
for (let n = 0; n < count; n++) {
|
|
278
|
+
const index = start + n;
|
|
279
|
+
const from = n * B;
|
|
280
|
+
if (from >= bytes.length) {
|
|
281
|
+
settle(index, void 0);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const block = bytes.slice(from, Math.min(from + B, bytes.length));
|
|
285
|
+
memorySet(state, index, block);
|
|
286
|
+
queueWrite(state, index, block);
|
|
287
|
+
settle(index, block);
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
const startBatch = (state, fresh, etag, source) => {
|
|
291
|
+
const deferreds = /* @__PURE__ */ new Map();
|
|
292
|
+
const batch = {
|
|
293
|
+
waiters: 0,
|
|
294
|
+
done: false,
|
|
295
|
+
controller: new AbortController(),
|
|
296
|
+
abort: () => {
|
|
297
|
+
if (batch.done) return;
|
|
298
|
+
batch.done = true;
|
|
299
|
+
const reason = abortError();
|
|
300
|
+
batch.controller.abort(reason);
|
|
301
|
+
for (const index of fresh) settle(index, reason);
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
const settle = (index, value) => {
|
|
305
|
+
const deferred = deferreds.get(index);
|
|
306
|
+
if (deferred === void 0 || deferred.settled) return;
|
|
307
|
+
const k = blockKey(state.key, index);
|
|
308
|
+
if (inflight.get(k)?.batch === batch) inflight.delete(k);
|
|
309
|
+
if (value instanceof Error) deferred.reject(value);
|
|
310
|
+
else deferred.resolve(value);
|
|
311
|
+
};
|
|
312
|
+
for (const index of fresh) {
|
|
313
|
+
const deferred = defer();
|
|
314
|
+
deferreds.set(index, deferred);
|
|
315
|
+
inflight.set(blockKey(state.key, index), {
|
|
316
|
+
batch,
|
|
317
|
+
promise: deferred.promise
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
(async () => {
|
|
321
|
+
const stored = await store.getBlocks(state.key, fresh);
|
|
322
|
+
const missing = [];
|
|
323
|
+
fresh.forEach((index, n) => {
|
|
324
|
+
const hit = stored[n];
|
|
325
|
+
if (hit !== void 0) {
|
|
326
|
+
counters.storeHits++;
|
|
327
|
+
memorySet(state, index, hit);
|
|
328
|
+
touchSoon(state, index);
|
|
329
|
+
settle(index, hit);
|
|
330
|
+
} else missing.push(index);
|
|
331
|
+
});
|
|
332
|
+
if (missing.length === 0 || batch.done) {
|
|
333
|
+
batch.done = true;
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const spans = [];
|
|
337
|
+
for (const index of missing) {
|
|
338
|
+
const last = spans[spans.length - 1];
|
|
339
|
+
if (last !== void 0 && last[0] + last[1] === index && last[1] < maxSpanBlocks) last[1]++;
|
|
340
|
+
else spans.push([index, 1]);
|
|
341
|
+
}
|
|
342
|
+
await Promise.all(spans.map(([start, count]) => fetchSpan(state, start, count, batch, etag, source, settle)));
|
|
343
|
+
batch.done = true;
|
|
344
|
+
})().catch((error) => {
|
|
345
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
346
|
+
for (const index of fresh) settle(index, failure);
|
|
347
|
+
batch.done = true;
|
|
348
|
+
});
|
|
349
|
+
return batch;
|
|
350
|
+
};
|
|
351
|
+
const passthrough = async (source, offset, length, signal, etag) => {
|
|
352
|
+
counters.uncachedRequests++;
|
|
353
|
+
const response = await source(offset, length, signal ?? new AbortController().signal, etag);
|
|
354
|
+
counters.networkBytes += response.bytes.length;
|
|
355
|
+
return {
|
|
356
|
+
bytes: response.bytes,
|
|
357
|
+
...response.etag !== void 0 ? { etag: response.etag } : {}
|
|
358
|
+
};
|
|
359
|
+
};
|
|
360
|
+
const readBlocks = async (state, offset, length, signal, etag, source) => {
|
|
361
|
+
let end = offset + length;
|
|
362
|
+
if (state.size !== void 0) end = Math.min(end, state.size);
|
|
363
|
+
const done = (bytes) => ({
|
|
364
|
+
bytes,
|
|
365
|
+
...state.etag !== void 0 ? { etag: state.etag } : {}
|
|
366
|
+
});
|
|
367
|
+
if (end <= offset) return done(new Uint8Array(0));
|
|
368
|
+
const out = new Uint8Array(end - offset);
|
|
369
|
+
const place = (index, block) => {
|
|
370
|
+
const blockStart = index * B;
|
|
371
|
+
const from = Math.max(offset, blockStart);
|
|
372
|
+
const to = Math.min(offset + out.length, blockStart + block.length);
|
|
373
|
+
if (to > from) out.set(block.subarray(from - blockStart, to - blockStart), from - offset);
|
|
374
|
+
};
|
|
375
|
+
const first = Math.floor(offset / B);
|
|
376
|
+
const last = Math.floor((end - 1) / B);
|
|
377
|
+
const waits = [];
|
|
378
|
+
const batches = /* @__PURE__ */ new Set();
|
|
379
|
+
const fresh = [];
|
|
380
|
+
for (let index = first; index <= last; index++) {
|
|
381
|
+
const hit = memoryGet(state, index);
|
|
382
|
+
if (hit !== void 0) {
|
|
383
|
+
counters.memoryHits++;
|
|
384
|
+
place(index, hit);
|
|
385
|
+
touchSoon(state, index);
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
const flight = inflight.get(blockKey(state.key, index));
|
|
389
|
+
if (flight !== void 0) {
|
|
390
|
+
waits.push({
|
|
391
|
+
index,
|
|
392
|
+
promise: flight.promise
|
|
393
|
+
});
|
|
394
|
+
batches.add(flight.batch);
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
fresh.push(index);
|
|
398
|
+
}
|
|
399
|
+
if (fresh.length > 0) {
|
|
400
|
+
const batch = startBatch(state, fresh, etag, source);
|
|
401
|
+
batches.add(batch);
|
|
402
|
+
for (const index of fresh) {
|
|
403
|
+
const flight = inflight.get(blockKey(state.key, index));
|
|
404
|
+
if (flight !== void 0) waits.push({
|
|
405
|
+
index,
|
|
406
|
+
promise: flight.promise
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
if (waits.length === 0) return done(out);
|
|
411
|
+
for (const batch of batches) batch.waiters++;
|
|
412
|
+
let released = false;
|
|
413
|
+
const release = (aborted) => {
|
|
414
|
+
if (released) return;
|
|
415
|
+
released = true;
|
|
416
|
+
for (const batch of batches) {
|
|
417
|
+
batch.waiters--;
|
|
418
|
+
if (aborted && batch.waiters <= 0) batch.abort();
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
let blocks;
|
|
422
|
+
try {
|
|
423
|
+
blocks = await untilAborted(Promise.all(waits.map((wait) => wait.promise)), signal);
|
|
424
|
+
} catch (error) {
|
|
425
|
+
release(signal?.aborted === true);
|
|
426
|
+
throw error;
|
|
427
|
+
}
|
|
428
|
+
release(false);
|
|
429
|
+
waits.forEach((wait, n) => {
|
|
430
|
+
const block = blocks[n];
|
|
431
|
+
if (block !== void 0) place(wait.index, block);
|
|
432
|
+
});
|
|
433
|
+
if (state.size !== void 0 && offset + out.length > state.size) return done(out.slice(0, Math.max(0, state.size - offset)));
|
|
434
|
+
return done(out);
|
|
435
|
+
};
|
|
436
|
+
const readState = async (state, offset, length, signal, etag, source) => {
|
|
437
|
+
if (signal?.aborted) throw signal.reason ?? abortError();
|
|
438
|
+
await untilAborted(loadMeta(state), signal);
|
|
439
|
+
if (disposed || state.unstable) return passthrough(source, offset, length, signal, etag);
|
|
440
|
+
if (etag !== void 0 && state.etag !== void 0 && etag !== state.etag) return passthrough(source, offset, length, signal, etag);
|
|
441
|
+
try {
|
|
442
|
+
return await readBlocks(state, offset, length, signal, etag, source);
|
|
443
|
+
} catch (error) {
|
|
444
|
+
if (!(error instanceof GenerationChanged)) throw error;
|
|
445
|
+
}
|
|
446
|
+
try {
|
|
447
|
+
return await readBlocks(state, offset, length, signal, etag, source);
|
|
448
|
+
} catch (error) {
|
|
449
|
+
if (!(error instanceof GenerationChanged)) throw error;
|
|
450
|
+
state.unstable = true;
|
|
451
|
+
await dropState(state);
|
|
452
|
+
return passthrough(source, offset, length, signal, etag);
|
|
453
|
+
}
|
|
454
|
+
};
|
|
455
|
+
const cleanGenerations = (url, keep) => {
|
|
456
|
+
if (cleanedGenerations.has(keep)) return;
|
|
457
|
+
cleanedGenerations.add(keep);
|
|
458
|
+
(async () => {
|
|
459
|
+
for (const archive of await store.archivesForUrl(url)) if (archive.key !== keep && archive.key.startsWith(`${url}@`)) {
|
|
460
|
+
const stale = states.get(archive.key);
|
|
461
|
+
if (stale !== void 0) forget(stale);
|
|
462
|
+
await store.deleteArchive(archive.key);
|
|
463
|
+
}
|
|
464
|
+
})();
|
|
465
|
+
};
|
|
466
|
+
const writeOut = async () => {
|
|
467
|
+
if (flushTimer !== void 0) {
|
|
468
|
+
clearTimeout(flushTimer);
|
|
469
|
+
flushTimer = void 0;
|
|
470
|
+
}
|
|
471
|
+
const t = now();
|
|
472
|
+
while (pending.size > 0) {
|
|
473
|
+
const byState = /* @__PURE__ */ new Map();
|
|
474
|
+
for (const entry of pending.values()) {
|
|
475
|
+
let list = byState.get(entry.state);
|
|
476
|
+
if (list === void 0) {
|
|
477
|
+
list = [];
|
|
478
|
+
byState.set(entry.state, list);
|
|
479
|
+
}
|
|
480
|
+
list.push({
|
|
481
|
+
index: entry.index,
|
|
482
|
+
bytes: entry.bytes
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
pending.clear();
|
|
486
|
+
await Promise.all([...byState].map(([state, blocks]) => store.putBlocks(info(state), blocks, t)));
|
|
487
|
+
}
|
|
488
|
+
if (dueTouches.size > 0) {
|
|
489
|
+
const touches = [...dueTouches];
|
|
490
|
+
dueTouches.clear();
|
|
491
|
+
await Promise.all(touches.map(([state, indices]) => store.touch(state.key, [...indices], t)));
|
|
492
|
+
}
|
|
493
|
+
let stats = await store.stats();
|
|
494
|
+
if (stats.bytes > stats.budgetBytes * highWater) {
|
|
495
|
+
counters.evictedBytes += await store.evict(Math.floor(stats.budgetBytes * lowWater));
|
|
496
|
+
stats = await store.stats();
|
|
497
|
+
}
|
|
498
|
+
storeStats = stats;
|
|
499
|
+
};
|
|
500
|
+
const flush = () => {
|
|
501
|
+
flushChain = flushChain.then(writeOut, writeOut);
|
|
502
|
+
return flushChain;
|
|
503
|
+
};
|
|
504
|
+
const open = (archive, source) => {
|
|
505
|
+
if (archive.keyBy === "etag") {
|
|
506
|
+
let current;
|
|
507
|
+
return {
|
|
508
|
+
read: async (offset, length, signal, etag) => {
|
|
509
|
+
if (etag === void 0) {
|
|
510
|
+
const read = await passthrough(source, offset, length, signal, void 0);
|
|
511
|
+
const learned = strongEtag(read.etag);
|
|
512
|
+
if (learned !== void 0) cleanGenerations(archive.url, `${archive.url}@${learned}`);
|
|
513
|
+
return read;
|
|
514
|
+
}
|
|
515
|
+
current = stateFor(`${archive.url}@${etag}`, archive, etag);
|
|
516
|
+
return readState(current, offset, length, signal, etag, source);
|
|
517
|
+
},
|
|
518
|
+
size: () => current?.size ?? archive.size,
|
|
519
|
+
etag: () => current?.etag,
|
|
520
|
+
drop: async () => {
|
|
521
|
+
for (const stored of await store.archivesForUrl(archive.url)) {
|
|
522
|
+
const state = states.get(stored.key);
|
|
523
|
+
if (state !== void 0) forget(state);
|
|
524
|
+
await store.deleteArchive(stored.key);
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
if (archive.key === void 0 || archive.key === "") throw new Error(`cache archive ${archive.url} needs a key (or keyBy: 'etag')`);
|
|
530
|
+
const state = stateFor(archive.key, archive);
|
|
531
|
+
return {
|
|
532
|
+
read: (offset, length, signal, etag) => readState(state, offset, length, signal, etag, source),
|
|
533
|
+
size: () => state.size,
|
|
534
|
+
etag: () => state.etag,
|
|
535
|
+
drop: () => dropState(state)
|
|
536
|
+
};
|
|
537
|
+
};
|
|
538
|
+
return {
|
|
539
|
+
store,
|
|
540
|
+
blockBytes: B,
|
|
541
|
+
open,
|
|
542
|
+
drop: async (key) => {
|
|
543
|
+
const state = states.get(key);
|
|
544
|
+
if (state !== void 0) forget(state);
|
|
545
|
+
await store.deleteArchive(key);
|
|
546
|
+
},
|
|
547
|
+
stats: () => ({
|
|
548
|
+
backend: storeStats.backend,
|
|
549
|
+
degraded: storeStats.degraded,
|
|
550
|
+
budgetBytes: storeStats.budgetBytes,
|
|
551
|
+
blockBytes: B,
|
|
552
|
+
memoryBytes: memoryTotal,
|
|
553
|
+
memoryBlocks: memory.size,
|
|
554
|
+
pendingBlocks: pending.size,
|
|
555
|
+
storeBytes: storeStats.bytes,
|
|
556
|
+
storeBlocks: storeStats.blocks,
|
|
557
|
+
storeArchives: storeStats.archives,
|
|
558
|
+
...counters
|
|
559
|
+
}),
|
|
560
|
+
flush,
|
|
561
|
+
setBudget: (bytes) => {
|
|
562
|
+
store.setBudget(bytes);
|
|
563
|
+
scheduleFlush(0);
|
|
564
|
+
},
|
|
565
|
+
clear: async () => {
|
|
566
|
+
memory.clear();
|
|
567
|
+
memoryTotal = 0;
|
|
568
|
+
pending.clear();
|
|
569
|
+
lastTouch.clear();
|
|
570
|
+
dueTouches.clear();
|
|
571
|
+
cleanedGenerations.clear();
|
|
572
|
+
for (const state of states.values()) {
|
|
573
|
+
state.size = state.declaredSize;
|
|
574
|
+
if (!state.fixedEtag) state.etag = void 0;
|
|
575
|
+
state.meta = Promise.resolve();
|
|
576
|
+
}
|
|
577
|
+
await store.clear();
|
|
578
|
+
storeStats = await store.stats();
|
|
579
|
+
},
|
|
580
|
+
dispose: () => {
|
|
581
|
+
if (disposed) return;
|
|
582
|
+
flush().finally(() => store.close?.());
|
|
583
|
+
disposed = true;
|
|
584
|
+
if (flushTimer !== void 0) clearTimeout(flushTimer);
|
|
585
|
+
flushTimer = void 0;
|
|
586
|
+
}
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
/** A `ByteStore` in memory: for tests, private browsing, and hosts without IndexedDB. */
|
|
590
|
+
function createMemoryByteStore(options = {}) {
|
|
591
|
+
let budget = options.budgetBytes ?? 64 * 1024 * 1024;
|
|
592
|
+
const archives = /* @__PURE__ */ new Map();
|
|
593
|
+
let total = 0;
|
|
594
|
+
let blockCount = 0;
|
|
595
|
+
const summary = (archive) => ({
|
|
596
|
+
...archive.info,
|
|
597
|
+
bytes: archive.bytes,
|
|
598
|
+
blocks: archive.blocks.size
|
|
599
|
+
});
|
|
600
|
+
const remove = (key) => {
|
|
601
|
+
const archive = archives.get(key);
|
|
602
|
+
if (archive === void 0) return;
|
|
603
|
+
total -= archive.bytes;
|
|
604
|
+
blockCount -= archive.blocks.size;
|
|
605
|
+
archives.delete(key);
|
|
606
|
+
};
|
|
607
|
+
return {
|
|
608
|
+
kind: "memory",
|
|
609
|
+
get budgetBytes() {
|
|
610
|
+
return budget;
|
|
611
|
+
},
|
|
612
|
+
getBlocks: async (key, indices) => {
|
|
613
|
+
const archive = archives.get(key);
|
|
614
|
+
return indices.map((index) => archive?.blocks.get(index)?.bytes);
|
|
615
|
+
},
|
|
616
|
+
putBlocks: async (info, blocks, now) => {
|
|
617
|
+
let archive = archives.get(info.key);
|
|
618
|
+
if (archive === void 0) {
|
|
619
|
+
archive = {
|
|
620
|
+
info,
|
|
621
|
+
blocks: /* @__PURE__ */ new Map(),
|
|
622
|
+
bytes: 0
|
|
623
|
+
};
|
|
624
|
+
archives.set(info.key, archive);
|
|
625
|
+
} else archive.info = info;
|
|
626
|
+
const rank = evictionRank(info.priority, now);
|
|
627
|
+
for (const block of blocks) {
|
|
628
|
+
const previous = archive.blocks.get(block.index);
|
|
629
|
+
if (previous !== void 0) {
|
|
630
|
+
archive.bytes -= previous.bytes.length;
|
|
631
|
+
total -= previous.bytes.length;
|
|
632
|
+
} else blockCount++;
|
|
633
|
+
archive.blocks.set(block.index, {
|
|
634
|
+
bytes: block.bytes,
|
|
635
|
+
rank
|
|
636
|
+
});
|
|
637
|
+
archive.bytes += block.bytes.length;
|
|
638
|
+
total += block.bytes.length;
|
|
639
|
+
}
|
|
640
|
+
},
|
|
641
|
+
touch: async (key, indices, now) => {
|
|
642
|
+
const archive = archives.get(key);
|
|
643
|
+
if (archive === void 0) return;
|
|
644
|
+
const rank = evictionRank(archive.info.priority, now);
|
|
645
|
+
for (const index of indices) {
|
|
646
|
+
const block = archive.blocks.get(index);
|
|
647
|
+
if (block !== void 0) block.rank = rank;
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
getArchive: async (key) => {
|
|
651
|
+
const archive = archives.get(key);
|
|
652
|
+
return archive === void 0 ? void 0 : summary(archive);
|
|
653
|
+
},
|
|
654
|
+
updateArchive: async (info) => {
|
|
655
|
+
const archive = archives.get(info.key);
|
|
656
|
+
if (archive === void 0) archives.set(info.key, {
|
|
657
|
+
info,
|
|
658
|
+
blocks: /* @__PURE__ */ new Map(),
|
|
659
|
+
bytes: 0
|
|
660
|
+
});
|
|
661
|
+
else archive.info = info;
|
|
662
|
+
},
|
|
663
|
+
archivesForUrl: async (url) => [...archives.values()].filter((archive) => archive.info.url === url).map(summary),
|
|
664
|
+
deleteArchive: async (key) => remove(key),
|
|
665
|
+
evict: async (target) => {
|
|
666
|
+
if (total <= target) return 0;
|
|
667
|
+
const order = [];
|
|
668
|
+
for (const archive of archives.values()) for (const [index, block] of archive.blocks) order.push({
|
|
669
|
+
archive,
|
|
670
|
+
index,
|
|
671
|
+
rank: block.rank
|
|
672
|
+
});
|
|
673
|
+
order.sort((a, b) => a.rank - b.rank);
|
|
674
|
+
let freed = 0;
|
|
675
|
+
for (const { archive, index } of order) {
|
|
676
|
+
if (total <= target) break;
|
|
677
|
+
const block = archive.blocks.get(index);
|
|
678
|
+
if (block === void 0) continue;
|
|
679
|
+
archive.blocks.delete(index);
|
|
680
|
+
archive.bytes -= block.bytes.length;
|
|
681
|
+
total -= block.bytes.length;
|
|
682
|
+
blockCount--;
|
|
683
|
+
freed += block.bytes.length;
|
|
684
|
+
if (archive.blocks.size === 0) archives.delete(archive.info.key);
|
|
685
|
+
}
|
|
686
|
+
return freed;
|
|
687
|
+
},
|
|
688
|
+
setBudget: (bytes) => {
|
|
689
|
+
budget = Math.max(0, bytes);
|
|
690
|
+
},
|
|
691
|
+
stats: async () => ({
|
|
692
|
+
backend: "memory",
|
|
693
|
+
bytes: total,
|
|
694
|
+
blocks: blockCount,
|
|
695
|
+
archives: archives.size,
|
|
696
|
+
budgetBytes: budget,
|
|
697
|
+
degraded: false
|
|
698
|
+
}),
|
|
699
|
+
clear: async () => {
|
|
700
|
+
archives.clear();
|
|
701
|
+
total = 0;
|
|
702
|
+
blockCount = 0;
|
|
703
|
+
}
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
//#endregion
|
|
707
|
+
//#region src/cache-adapters.ts
|
|
708
|
+
function exactBuffer(bytes) {
|
|
709
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes.buffer : bytes.slice().buffer;
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* A PMTiles source whose bytes come from `cache` when it has them. The inner source must honour
|
|
713
|
+
* Range requests. Its validator handling is kept: a request carrying the archive's ETag still
|
|
714
|
+
* reaches the inner source on a miss, so a replaced file fails with the inner source's own
|
|
715
|
+
* mismatch error (which also drops the cached bytes). Use `keyBy: 'etag'` for a stable URL whose
|
|
716
|
+
* contents are replaced in place; then the first read of a session (the header) always goes to
|
|
717
|
+
* the network and names the version the rest are cached under.
|
|
718
|
+
*/
|
|
719
|
+
function cachingRangeSource(source, cache, archive) {
|
|
720
|
+
const handle = cache.open(archive, async (offset, length, signal, etag) => {
|
|
721
|
+
const response = await source.getBytes(offset, length, signal, etag);
|
|
722
|
+
return {
|
|
723
|
+
bytes: new Uint8Array(response.data),
|
|
724
|
+
...response.etag !== void 0 ? { etag: response.etag } : {}
|
|
725
|
+
};
|
|
726
|
+
});
|
|
727
|
+
return {
|
|
728
|
+
getKey: () => source.getKey(),
|
|
729
|
+
getBytes: async (offset, length, signal, etag) => {
|
|
730
|
+
const read = await handle.read(offset, length, signal, etag);
|
|
731
|
+
return {
|
|
732
|
+
data: exactBuffer(read.bytes),
|
|
733
|
+
...read.etag !== void 0 ? { etag: read.etag } : {}
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* A pack `RangeReader` whose bytes come from `cache` when it has them. Key it by the pack's
|
|
740
|
+
* content hash (`contentHash` in a pack index) so a cached copy can never be another version.
|
|
741
|
+
*/
|
|
742
|
+
function cachingRangeReader(reader, cache, archive, options = {}) {
|
|
743
|
+
const wholeBelow = options.wholeBelow ?? 4 * 1024 * 1024;
|
|
744
|
+
const handle = cache.open({
|
|
745
|
+
...archive,
|
|
746
|
+
size: archive.size ?? reader.size
|
|
747
|
+
}, async (offset, length, signal) => ({ bytes: await reader.read(offset, length, signal) }));
|
|
748
|
+
let whole;
|
|
749
|
+
return {
|
|
750
|
+
size: reader.size,
|
|
751
|
+
read: async (offset, length, signal) => {
|
|
752
|
+
if (reader.size <= wholeBelow) {
|
|
753
|
+
whole ??= handle.read(0, reader.size).catch((error) => {
|
|
754
|
+
whole = void 0;
|
|
755
|
+
throw error;
|
|
756
|
+
});
|
|
757
|
+
await whole;
|
|
758
|
+
}
|
|
759
|
+
return (await handle.read(offset, length, signal)).bytes;
|
|
760
|
+
}
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
//#endregion
|
|
764
|
+
//#region src/document-cache.ts
|
|
765
|
+
/** Response header naming where a cached document came from: `hit`, `revalidated` or `stale`. */
|
|
766
|
+
const DOCUMENT_CACHE_HEADER = "x-molen-cache";
|
|
767
|
+
const documentKey = (url) => `doc:${url}`;
|
|
768
|
+
function requestUrl(input) {
|
|
769
|
+
const raw = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
770
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(raw)) return raw;
|
|
771
|
+
const base = globalThis.location?.href;
|
|
772
|
+
return base === void 0 ? raw : new URL(raw, base).href;
|
|
773
|
+
}
|
|
774
|
+
function cacheable(input, init) {
|
|
775
|
+
const request = typeof input === "object" && "method" in input ? input : void 0;
|
|
776
|
+
if ((init?.method ?? request?.method ?? "GET").toUpperCase() !== "GET") return false;
|
|
777
|
+
if ((init?.cache ?? request?.cache) === "no-store") return false;
|
|
778
|
+
return !new Headers(init?.headers ?? request?.headers).has("range");
|
|
779
|
+
}
|
|
780
|
+
/** Wrap `fetchImpl` so GETs of small documents are kept in `store`. */
|
|
781
|
+
function cachingDocumentFetch(store, fetchImpl = (input, init) => globalThis.fetch(input, init), options = {}) {
|
|
782
|
+
const timeoutMs = options.timeoutMs ?? 4e3;
|
|
783
|
+
const maxBytes = options.maxBytes ?? 2 * 1024 * 1024;
|
|
784
|
+
const now = options.now ?? Date.now;
|
|
785
|
+
const maxAge = (url) => typeof options.maxAgeMs === "function" ? options.maxAgeMs(url) : options.maxAgeMs ?? 0;
|
|
786
|
+
const save = async (url, response) => {
|
|
787
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
788
|
+
if (bytes.length <= maxBytes) {
|
|
789
|
+
const etag = response.headers.get("etag") ?? void 0;
|
|
790
|
+
const type = response.headers.get("content-type") ?? void 0;
|
|
791
|
+
await store.putBlocks({
|
|
792
|
+
key: documentKey(url),
|
|
793
|
+
url,
|
|
794
|
+
priority: "high",
|
|
795
|
+
size: bytes.length,
|
|
796
|
+
validated: now(),
|
|
797
|
+
...etag !== void 0 ? { etag } : {},
|
|
798
|
+
...type !== void 0 ? { type } : {}
|
|
799
|
+
}, [{
|
|
800
|
+
index: 0,
|
|
801
|
+
bytes
|
|
802
|
+
}], now());
|
|
803
|
+
}
|
|
804
|
+
return bytes;
|
|
805
|
+
};
|
|
806
|
+
const respond = (bytes, status, headers, origin) => {
|
|
807
|
+
const out = new Headers({ [DOCUMENT_CACHE_HEADER]: origin });
|
|
808
|
+
if (headers.type !== void 0) out.set("content-type", headers.type);
|
|
809
|
+
if (headers.etag !== void 0) out.set("etag", headers.etag);
|
|
810
|
+
return new Response(bytes.slice(), {
|
|
811
|
+
status,
|
|
812
|
+
headers: out
|
|
813
|
+
});
|
|
814
|
+
};
|
|
815
|
+
return (async (input, init) => {
|
|
816
|
+
if (!cacheable(input, init)) return fetchImpl(input, init);
|
|
817
|
+
const url = requestUrl(input);
|
|
818
|
+
const key = documentKey(url);
|
|
819
|
+
const info = await store.getArchive(key);
|
|
820
|
+
const stored = info !== void 0 ? (await store.getBlocks(key, [0]))[0] : void 0;
|
|
821
|
+
if (info === void 0 || stored === void 0 || stored.length !== info.size) {
|
|
822
|
+
const response = await fetchImpl(input, init);
|
|
823
|
+
if (response.status !== 200) return response;
|
|
824
|
+
return respond(await save(url, response), 200, {
|
|
825
|
+
type: response.headers.get("content-type") ?? void 0,
|
|
826
|
+
etag: response.headers.get("etag") ?? void 0
|
|
827
|
+
}, "network");
|
|
828
|
+
}
|
|
829
|
+
const cached = (origin) => respond(stored, 200, {
|
|
830
|
+
type: info.type,
|
|
831
|
+
etag: info.etag
|
|
832
|
+
}, origin);
|
|
833
|
+
if (info.validated !== void 0 && now() - info.validated < maxAge(url)) return cached("hit");
|
|
834
|
+
const headers = new Headers(init?.headers);
|
|
835
|
+
if (info.etag !== void 0) headers.set("if-none-match", info.etag);
|
|
836
|
+
const absorb = async (response) => {
|
|
837
|
+
if (response.status === 304) {
|
|
838
|
+
await store.updateArchive({
|
|
839
|
+
...info,
|
|
840
|
+
validated: now()
|
|
841
|
+
});
|
|
842
|
+
return "revalidated";
|
|
843
|
+
}
|
|
844
|
+
if (response.status === 200) {
|
|
845
|
+
await save(url, response);
|
|
846
|
+
return "replaced";
|
|
847
|
+
}
|
|
848
|
+
if (response.status === 404 || response.status === 410) await store.deleteArchive(key);
|
|
849
|
+
return "kept";
|
|
850
|
+
};
|
|
851
|
+
const request = fetchImpl(url, {
|
|
852
|
+
...init,
|
|
853
|
+
headers
|
|
854
|
+
});
|
|
855
|
+
let timer;
|
|
856
|
+
const timeout = new Promise((resolve) => {
|
|
857
|
+
timer = setTimeout(() => resolve("timeout"), timeoutMs);
|
|
858
|
+
});
|
|
859
|
+
let response;
|
|
860
|
+
try {
|
|
861
|
+
response = await Promise.race([request, timeout]);
|
|
862
|
+
} catch {
|
|
863
|
+
clearTimeout(timer);
|
|
864
|
+
return cached("stale");
|
|
865
|
+
}
|
|
866
|
+
clearTimeout(timer);
|
|
867
|
+
if (response === "timeout") {
|
|
868
|
+
request.then(absorb).catch(() => void 0);
|
|
869
|
+
return cached("stale");
|
|
870
|
+
}
|
|
871
|
+
if (response.status === 304) {
|
|
872
|
+
await absorb(response);
|
|
873
|
+
return cached("revalidated");
|
|
874
|
+
}
|
|
875
|
+
if (response.status === 200) return respond(await save(url, response), 200, {
|
|
876
|
+
type: response.headers.get("content-type") ?? void 0,
|
|
877
|
+
etag: response.headers.get("etag") ?? void 0
|
|
878
|
+
}, "network");
|
|
879
|
+
if (response.status === 404 || response.status === 410) {
|
|
880
|
+
await store.deleteArchive(key);
|
|
881
|
+
return response;
|
|
882
|
+
}
|
|
883
|
+
return cached("stale");
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
//#endregion
|
|
887
|
+
//#region src/indexeddb-byte-store.ts
|
|
888
|
+
/**
|
|
889
|
+
* A `ByteStore` in IndexedDB, for a `BlockCache` that survives page loads.
|
|
890
|
+
*
|
|
891
|
+
* Three object stores: `blocks` holds each block's bytes under `[key, index]`; `touches` holds a
|
|
892
|
+
* small record per block (`length`, eviction `rank`) indexed by rank, so eviction walks one index
|
|
893
|
+
* oldest-first without loading any block; `archives` holds each archive's record and running
|
|
894
|
+
* totals, indexed by URL. Totals are kept in memory and updated when a transaction commits.
|
|
895
|
+
*
|
|
896
|
+
* Every failure resolves: an unavailable database (private browsing, a blocked upgrade) makes
|
|
897
|
+
* every read a miss and every write a no-op, and a quota failure evicts half the budget and
|
|
898
|
+
* retries once before the store stops writing (`degraded`).
|
|
899
|
+
*/
|
|
900
|
+
const BLOCKS = "blocks";
|
|
901
|
+
const TOUCHES = "touches";
|
|
902
|
+
const ARCHIVES = "archives";
|
|
903
|
+
/** Deletes per eviction transaction, so eviction never holds the database for long. */
|
|
904
|
+
const EVICT_SLICE = 256;
|
|
905
|
+
function toBuffer(bytes) {
|
|
906
|
+
return bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? bytes.buffer : bytes.slice().buffer;
|
|
907
|
+
}
|
|
908
|
+
const isQuotaError = (error) => error?.name === "QuotaExceededError";
|
|
909
|
+
const nextTick = () => new Promise((resolve) => {
|
|
910
|
+
const idle = globalThis.requestIdleCallback;
|
|
911
|
+
if (typeof idle === "function") idle(() => resolve());
|
|
912
|
+
else setTimeout(resolve, 0);
|
|
913
|
+
});
|
|
914
|
+
/**
|
|
915
|
+
* Open (lazily) a block store in IndexedDB. Undefined where IndexedDB does not exist (Node,
|
|
916
|
+
* restricted embedded browsers); a database that fails to open behaves as an empty store.
|
|
917
|
+
*/
|
|
918
|
+
function createIndexedDbByteStore(options = {}) {
|
|
919
|
+
const factory = options.indexedDB ?? globalThis.indexedDB ?? void 0;
|
|
920
|
+
if (factory === void 0) return void 0;
|
|
921
|
+
const name = options.name ?? "molen-bytes";
|
|
922
|
+
let requested = Math.max(0, options.budgetBytes ?? 512 * 1024 * 1024);
|
|
923
|
+
let quotaLimit = Number.POSITIVE_INFINITY;
|
|
924
|
+
let total = 0;
|
|
925
|
+
let blockCount = 0;
|
|
926
|
+
let archiveCount = 0;
|
|
927
|
+
let degraded = false;
|
|
928
|
+
const priorities = /* @__PURE__ */ new Map();
|
|
929
|
+
let database;
|
|
930
|
+
let evictChain = Promise.resolve();
|
|
931
|
+
const budget = () => Math.max(0, Math.min(requested, quotaLimit));
|
|
932
|
+
const loadTotals = (db) => new Promise((resolve) => {
|
|
933
|
+
try {
|
|
934
|
+
const request = db.transaction(ARCHIVES, "readonly").objectStore(ARCHIVES).getAll();
|
|
935
|
+
request.onsuccess = () => {
|
|
936
|
+
for (const archive of request.result) {
|
|
937
|
+
total += archive.bytes;
|
|
938
|
+
blockCount += archive.blocks;
|
|
939
|
+
archiveCount++;
|
|
940
|
+
priorities.set(archive.key, archive.priority);
|
|
941
|
+
}
|
|
942
|
+
resolve();
|
|
943
|
+
};
|
|
944
|
+
request.onerror = () => resolve();
|
|
945
|
+
} catch {
|
|
946
|
+
resolve();
|
|
947
|
+
}
|
|
948
|
+
});
|
|
949
|
+
const estimateQuota = async () => {
|
|
950
|
+
const storage = globalThis.navigator?.storage;
|
|
951
|
+
try {
|
|
952
|
+
if (options.persist === true) await storage?.persist?.();
|
|
953
|
+
const estimate = await storage?.estimate?.();
|
|
954
|
+
if (estimate?.quota !== void 0) {
|
|
955
|
+
const free = estimate.quota - (estimate.usage ?? 0) + total;
|
|
956
|
+
quotaLimit = Math.max(0, Math.floor(free * .5));
|
|
957
|
+
}
|
|
958
|
+
} catch {}
|
|
959
|
+
};
|
|
960
|
+
const open = () => {
|
|
961
|
+
database ??= new Promise((resolve) => {
|
|
962
|
+
let request;
|
|
963
|
+
try {
|
|
964
|
+
request = factory.open(name, 1);
|
|
965
|
+
} catch {
|
|
966
|
+
resolve(void 0);
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
request.onupgradeneeded = () => {
|
|
970
|
+
const db = request.result;
|
|
971
|
+
db.createObjectStore(BLOCKS);
|
|
972
|
+
const touches = db.createObjectStore(TOUCHES, { keyPath: ["key", "index"] });
|
|
973
|
+
touches.createIndex("rank", "rank");
|
|
974
|
+
touches.createIndex("key", "key");
|
|
975
|
+
db.createObjectStore(ARCHIVES, { keyPath: "key" }).createIndex("url", "url");
|
|
976
|
+
};
|
|
977
|
+
request.onsuccess = () => {
|
|
978
|
+
const db = request.result;
|
|
979
|
+
db.onversionchange = () => db.close();
|
|
980
|
+
resolve(db);
|
|
981
|
+
};
|
|
982
|
+
request.onerror = () => resolve(void 0);
|
|
983
|
+
request.onblocked = () => resolve(void 0);
|
|
984
|
+
}).then(async (db) => {
|
|
985
|
+
if (db === void 0) {
|
|
986
|
+
degraded = true;
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
await loadTotals(db);
|
|
990
|
+
await estimateQuota();
|
|
991
|
+
return db;
|
|
992
|
+
});
|
|
993
|
+
return database;
|
|
994
|
+
};
|
|
995
|
+
/** Run one transaction; resolves `fallback` when it fails or aborts. */
|
|
996
|
+
const transact = async (stores, mode, fallback, work, onAbort) => {
|
|
997
|
+
const db = await open();
|
|
998
|
+
if (db === void 0) return fallback;
|
|
999
|
+
return new Promise((resolve) => {
|
|
1000
|
+
let result = fallback;
|
|
1001
|
+
try {
|
|
1002
|
+
const tx = db.transaction(stores, mode);
|
|
1003
|
+
tx.oncomplete = () => resolve(result);
|
|
1004
|
+
tx.onabort = () => {
|
|
1005
|
+
onAbort?.(tx.error);
|
|
1006
|
+
resolve(fallback);
|
|
1007
|
+
};
|
|
1008
|
+
work(tx, (value) => {
|
|
1009
|
+
result = value;
|
|
1010
|
+
});
|
|
1011
|
+
} catch {
|
|
1012
|
+
resolve(fallback);
|
|
1013
|
+
}
|
|
1014
|
+
});
|
|
1015
|
+
};
|
|
1016
|
+
const getBlocks = async (key, indices) => {
|
|
1017
|
+
const misses = indices.map(() => void 0);
|
|
1018
|
+
if (indices.length === 0) return misses;
|
|
1019
|
+
return transact(BLOCKS, "readonly", misses, (tx, finish) => {
|
|
1020
|
+
const store = tx.objectStore(BLOCKS);
|
|
1021
|
+
const out = indices.map(() => void 0);
|
|
1022
|
+
finish(out);
|
|
1023
|
+
indices.forEach((index, n) => {
|
|
1024
|
+
const request = store.get([key, index]);
|
|
1025
|
+
request.onsuccess = () => {
|
|
1026
|
+
const value = request.result;
|
|
1027
|
+
if (value !== void 0 && value !== null) out[n] = new Uint8Array(value);
|
|
1028
|
+
};
|
|
1029
|
+
});
|
|
1030
|
+
});
|
|
1031
|
+
};
|
|
1032
|
+
const writeBlocks = (info, blocks, now) => {
|
|
1033
|
+
let addedBytes = 0;
|
|
1034
|
+
let addedBlocks = 0;
|
|
1035
|
+
let created = false;
|
|
1036
|
+
let quota = false;
|
|
1037
|
+
return transact([
|
|
1038
|
+
BLOCKS,
|
|
1039
|
+
TOUCHES,
|
|
1040
|
+
ARCHIVES
|
|
1041
|
+
], "readwrite", "failed", (tx, finish) => {
|
|
1042
|
+
const blockStore = tx.objectStore(BLOCKS);
|
|
1043
|
+
const touchStore = tx.objectStore(TOUCHES);
|
|
1044
|
+
const archiveStore = tx.objectStore(ARCHIVES);
|
|
1045
|
+
const rank = evictionRank(info.priority, now);
|
|
1046
|
+
for (const block of blocks) {
|
|
1047
|
+
const previous = touchStore.get([info.key, block.index]);
|
|
1048
|
+
previous.onsuccess = () => {
|
|
1049
|
+
const old = previous.result;
|
|
1050
|
+
addedBytes += block.bytes.length - (old?.length ?? 0);
|
|
1051
|
+
if (old === void 0) addedBlocks++;
|
|
1052
|
+
const record = {
|
|
1053
|
+
key: info.key,
|
|
1054
|
+
index: block.index,
|
|
1055
|
+
length: block.bytes.length,
|
|
1056
|
+
rank
|
|
1057
|
+
};
|
|
1058
|
+
try {
|
|
1059
|
+
touchStore.put(record);
|
|
1060
|
+
blockStore.put(toBuffer(block.bytes), [info.key, block.index]);
|
|
1061
|
+
} catch (error) {
|
|
1062
|
+
quota ||= isQuotaError(error);
|
|
1063
|
+
tx.abort();
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
}
|
|
1067
|
+
const existing = archiveStore.get(info.key);
|
|
1068
|
+
existing.onsuccess = () => {
|
|
1069
|
+
const old = existing.result;
|
|
1070
|
+
created = old === void 0;
|
|
1071
|
+
const record = {
|
|
1072
|
+
...info,
|
|
1073
|
+
bytes: (old?.bytes ?? 0) + addedBytes,
|
|
1074
|
+
blocks: (old?.blocks ?? 0) + addedBlocks
|
|
1075
|
+
};
|
|
1076
|
+
archiveStore.put(record);
|
|
1077
|
+
finish("ok");
|
|
1078
|
+
};
|
|
1079
|
+
}, (error) => {
|
|
1080
|
+
quota ||= isQuotaError(error);
|
|
1081
|
+
}).then((result) => {
|
|
1082
|
+
if (result === "ok") {
|
|
1083
|
+
total += addedBytes;
|
|
1084
|
+
blockCount += addedBlocks;
|
|
1085
|
+
if (created) archiveCount++;
|
|
1086
|
+
priorities.set(info.key, info.priority);
|
|
1087
|
+
return "ok";
|
|
1088
|
+
}
|
|
1089
|
+
return quota ? "quota" : "failed";
|
|
1090
|
+
});
|
|
1091
|
+
};
|
|
1092
|
+
const evictOnce = async (target) => {
|
|
1093
|
+
let freed = 0;
|
|
1094
|
+
for (;;) {
|
|
1095
|
+
if (total <= target) return freed;
|
|
1096
|
+
let sliceFreed = 0;
|
|
1097
|
+
let sliceBlocks = 0;
|
|
1098
|
+
let removedArchives = 0;
|
|
1099
|
+
let exhausted = false;
|
|
1100
|
+
if (!await transact([
|
|
1101
|
+
BLOCKS,
|
|
1102
|
+
TOUCHES,
|
|
1103
|
+
ARCHIVES
|
|
1104
|
+
], "readwrite", false, (tx, finish) => {
|
|
1105
|
+
const blockStore = tx.objectStore(BLOCKS);
|
|
1106
|
+
const touchStore = tx.objectStore(TOUCHES);
|
|
1107
|
+
const archiveStore = tx.objectStore(ARCHIVES);
|
|
1108
|
+
const decrements = /* @__PURE__ */ new Map();
|
|
1109
|
+
const request = touchStore.index("rank").openCursor();
|
|
1110
|
+
const settleArchives = () => {
|
|
1111
|
+
for (const [key, dec] of decrements) {
|
|
1112
|
+
const get = archiveStore.get(key);
|
|
1113
|
+
get.onsuccess = () => {
|
|
1114
|
+
const record = get.result;
|
|
1115
|
+
if (record === void 0) return;
|
|
1116
|
+
record.bytes -= dec.bytes;
|
|
1117
|
+
record.blocks -= dec.blocks;
|
|
1118
|
+
if (record.blocks <= 0) {
|
|
1119
|
+
archiveStore.delete(key);
|
|
1120
|
+
removedArchives++;
|
|
1121
|
+
} else archiveStore.put(record);
|
|
1122
|
+
};
|
|
1123
|
+
}
|
|
1124
|
+
finish(true);
|
|
1125
|
+
};
|
|
1126
|
+
request.onsuccess = () => {
|
|
1127
|
+
const cursor = request.result;
|
|
1128
|
+
if (cursor === null) {
|
|
1129
|
+
exhausted = true;
|
|
1130
|
+
settleArchives();
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
if (total - sliceFreed <= target || sliceBlocks >= EVICT_SLICE) {
|
|
1134
|
+
settleArchives();
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
const record = cursor.value;
|
|
1138
|
+
blockStore.delete([record.key, record.index]);
|
|
1139
|
+
cursor.delete();
|
|
1140
|
+
sliceFreed += record.length;
|
|
1141
|
+
sliceBlocks++;
|
|
1142
|
+
const dec = decrements.get(record.key) ?? {
|
|
1143
|
+
bytes: 0,
|
|
1144
|
+
blocks: 0
|
|
1145
|
+
};
|
|
1146
|
+
dec.bytes += record.length;
|
|
1147
|
+
dec.blocks++;
|
|
1148
|
+
decrements.set(record.key, dec);
|
|
1149
|
+
cursor.continue();
|
|
1150
|
+
};
|
|
1151
|
+
})) return freed;
|
|
1152
|
+
total -= sliceFreed;
|
|
1153
|
+
blockCount -= sliceBlocks;
|
|
1154
|
+
archiveCount -= removedArchives;
|
|
1155
|
+
freed += sliceFreed;
|
|
1156
|
+
if (exhausted || sliceBlocks === 0) return freed;
|
|
1157
|
+
await nextTick();
|
|
1158
|
+
}
|
|
1159
|
+
};
|
|
1160
|
+
/** Evictions run one at a time; each walks the rank index from the oldest block. */
|
|
1161
|
+
const evict = (target) => {
|
|
1162
|
+
const run = evictChain.then(async () => {
|
|
1163
|
+
await open();
|
|
1164
|
+
return evictOnce(Math.max(0, target));
|
|
1165
|
+
});
|
|
1166
|
+
evictChain = run.catch(() => 0);
|
|
1167
|
+
return run;
|
|
1168
|
+
};
|
|
1169
|
+
return {
|
|
1170
|
+
kind: "indexeddb",
|
|
1171
|
+
get budgetBytes() {
|
|
1172
|
+
return budget();
|
|
1173
|
+
},
|
|
1174
|
+
getBlocks,
|
|
1175
|
+
putBlocks: async (info, blocks, now) => {
|
|
1176
|
+
if (degraded || blocks.length === 0) return;
|
|
1177
|
+
let result = await writeBlocks(info, blocks, now);
|
|
1178
|
+
if (result === "quota") {
|
|
1179
|
+
await evict(Math.floor(budget() * .5));
|
|
1180
|
+
result = await writeBlocks(info, blocks, now);
|
|
1181
|
+
if (result === "quota") degraded = true;
|
|
1182
|
+
}
|
|
1183
|
+
},
|
|
1184
|
+
touch: async (key, indices, now) => {
|
|
1185
|
+
if (degraded || indices.length === 0) return;
|
|
1186
|
+
const rank = evictionRank(priorities.get(key) ?? "normal", now);
|
|
1187
|
+
await transact(TOUCHES, "readwrite", void 0, (tx) => {
|
|
1188
|
+
const store = tx.objectStore(TOUCHES);
|
|
1189
|
+
for (const index of indices) {
|
|
1190
|
+
const request = store.get([key, index]);
|
|
1191
|
+
request.onsuccess = () => {
|
|
1192
|
+
const record = request.result;
|
|
1193
|
+
if (record !== void 0) store.put({
|
|
1194
|
+
...record,
|
|
1195
|
+
rank
|
|
1196
|
+
});
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
});
|
|
1200
|
+
},
|
|
1201
|
+
getArchive: (key) => transact(ARCHIVES, "readonly", void 0, (tx, finish) => {
|
|
1202
|
+
const request = tx.objectStore(ARCHIVES).get(key);
|
|
1203
|
+
request.onsuccess = () => finish(request.result);
|
|
1204
|
+
}),
|
|
1205
|
+
updateArchive: async (info) => {
|
|
1206
|
+
let created = false;
|
|
1207
|
+
if (await transact(ARCHIVES, "readwrite", false, (tx, finish) => {
|
|
1208
|
+
const store = tx.objectStore(ARCHIVES);
|
|
1209
|
+
const request = store.get(info.key);
|
|
1210
|
+
request.onsuccess = () => {
|
|
1211
|
+
const old = request.result;
|
|
1212
|
+
created = old === void 0;
|
|
1213
|
+
store.put({
|
|
1214
|
+
...info,
|
|
1215
|
+
bytes: old?.bytes ?? 0,
|
|
1216
|
+
blocks: old?.blocks ?? 0
|
|
1217
|
+
});
|
|
1218
|
+
finish(true);
|
|
1219
|
+
};
|
|
1220
|
+
})) {
|
|
1221
|
+
if (created) archiveCount++;
|
|
1222
|
+
priorities.set(info.key, info.priority);
|
|
1223
|
+
}
|
|
1224
|
+
},
|
|
1225
|
+
archivesForUrl: (url) => transact(ARCHIVES, "readonly", [], (tx, finish) => {
|
|
1226
|
+
const request = tx.objectStore(ARCHIVES).index("url").getAll(url);
|
|
1227
|
+
request.onsuccess = () => finish(request.result);
|
|
1228
|
+
}),
|
|
1229
|
+
deleteArchive: async (key) => {
|
|
1230
|
+
let removed;
|
|
1231
|
+
if (await transact([
|
|
1232
|
+
BLOCKS,
|
|
1233
|
+
TOUCHES,
|
|
1234
|
+
ARCHIVES
|
|
1235
|
+
], "readwrite", false, (tx, finish) => {
|
|
1236
|
+
const range = IDBKeyRange.bound([key, Number.NEGATIVE_INFINITY], [key, Number.POSITIVE_INFINITY]);
|
|
1237
|
+
tx.objectStore(BLOCKS).delete(range);
|
|
1238
|
+
tx.objectStore(TOUCHES).delete(range);
|
|
1239
|
+
const archives = tx.objectStore(ARCHIVES);
|
|
1240
|
+
const request = archives.get(key);
|
|
1241
|
+
request.onsuccess = () => {
|
|
1242
|
+
removed = request.result;
|
|
1243
|
+
archives.delete(key);
|
|
1244
|
+
finish(true);
|
|
1245
|
+
};
|
|
1246
|
+
}) && removed !== void 0) {
|
|
1247
|
+
total -= removed.bytes;
|
|
1248
|
+
blockCount -= removed.blocks;
|
|
1249
|
+
archiveCount--;
|
|
1250
|
+
priorities.delete(key);
|
|
1251
|
+
}
|
|
1252
|
+
},
|
|
1253
|
+
evict,
|
|
1254
|
+
setBudget: (bytes) => {
|
|
1255
|
+
requested = Math.max(0, bytes);
|
|
1256
|
+
},
|
|
1257
|
+
stats: async () => {
|
|
1258
|
+
await open();
|
|
1259
|
+
return {
|
|
1260
|
+
backend: "indexeddb",
|
|
1261
|
+
bytes: total,
|
|
1262
|
+
blocks: blockCount,
|
|
1263
|
+
archives: archiveCount,
|
|
1264
|
+
budgetBytes: budget(),
|
|
1265
|
+
degraded
|
|
1266
|
+
};
|
|
1267
|
+
},
|
|
1268
|
+
clear: async () => {
|
|
1269
|
+
if (await transact([
|
|
1270
|
+
BLOCKS,
|
|
1271
|
+
TOUCHES,
|
|
1272
|
+
ARCHIVES
|
|
1273
|
+
], "readwrite", false, (tx, finish) => {
|
|
1274
|
+
tx.objectStore(BLOCKS).clear();
|
|
1275
|
+
tx.objectStore(TOUCHES).clear();
|
|
1276
|
+
tx.objectStore(ARCHIVES).clear();
|
|
1277
|
+
finish(true);
|
|
1278
|
+
})) {
|
|
1279
|
+
total = 0;
|
|
1280
|
+
blockCount = 0;
|
|
1281
|
+
archiveCount = 0;
|
|
1282
|
+
priorities.clear();
|
|
1283
|
+
degraded = false;
|
|
1284
|
+
}
|
|
1285
|
+
},
|
|
1286
|
+
close: () => {
|
|
1287
|
+
database?.then((db) => db?.close());
|
|
1288
|
+
}
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
//#endregion
|
|
1292
|
+
export { DOCUMENT_CACHE_HEADER, PRIORITY_BIAS_MS, cachingDocumentFetch, cachingRangeReader, cachingRangeSource, createBlockCache, createIndexedDbByteStore, createMemoryByteStore, evictionRank, isChangedError, urlRangeReader };
|