@bendyline/molen-pack 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/dist/cache.d.mts +263 -0
- package/dist/cache.mjs +1292 -0
- package/dist/index.d.mts +2 -1
- package/dist/index.mjs +2 -1
- package/dist/node.d.mts +1 -1
- package/dist/node.mjs +1 -1
- package/dist/{pack-DCQvXj1p.mjs → pack-BIXihu9D.mjs} +2 -268
- package/dist/{pack-DjufOvAS.d.mts → pack-CF6pPp9f.d.mts} +2 -26
- package/dist/source-7Pr2tntF.d.mts +41 -0
- package/dist/source-MDAzioCi.mjs +299 -0
- package/package.json +6 -1
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
//#region src/source.ts
|
|
2
|
+
/** The server replaced the pack while it was being read (its ETag changed). */
|
|
3
|
+
var PackChangedError = class extends Error {
|
|
4
|
+
constructor(url) {
|
|
5
|
+
super(`pack ${url} changed on the server while it was being read; open it again`);
|
|
6
|
+
this.name = "PackChangedError";
|
|
7
|
+
}
|
|
8
|
+
};
|
|
9
|
+
var HttpStatusError = class extends Error {
|
|
10
|
+
url;
|
|
11
|
+
status;
|
|
12
|
+
constructor(url, status) {
|
|
13
|
+
super(`HTTP ${status} for ${url}`);
|
|
14
|
+
this.url = url;
|
|
15
|
+
this.status = status;
|
|
16
|
+
this.name = "HttpStatusError";
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
function blobReader(blob) {
|
|
20
|
+
return {
|
|
21
|
+
size: blob.size,
|
|
22
|
+
read: async (offset, length) => new Uint8Array(await blob.slice(offset, offset + length).arrayBuffer())
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function isRetryable(error) {
|
|
26
|
+
if (error instanceof HttpStatusError) return error.status === 408 || error.status === 429 || error.status >= 500;
|
|
27
|
+
if (error instanceof PackChangedError) return false;
|
|
28
|
+
const name = error?.name;
|
|
29
|
+
return error instanceof TypeError || name === "TimeoutError";
|
|
30
|
+
}
|
|
31
|
+
function sleep(ms, signal) {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
if (signal.aborted) return reject(signal.reason);
|
|
34
|
+
const timer = setTimeout(() => {
|
|
35
|
+
signal.removeEventListener("abort", abort);
|
|
36
|
+
resolve();
|
|
37
|
+
}, ms);
|
|
38
|
+
const abort = () => {
|
|
39
|
+
clearTimeout(timer);
|
|
40
|
+
reject(signal.reason);
|
|
41
|
+
};
|
|
42
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
/** Run `task` with a per-attempt deadline, retrying transient failures with backoff. */
|
|
46
|
+
async function withRetry(task, retry, signal) {
|
|
47
|
+
const attempts = Math.max(1, retry.attempts ?? 3);
|
|
48
|
+
const base = retry.baseDelayMs ?? 250;
|
|
49
|
+
for (let attempt = 1;; attempt++) {
|
|
50
|
+
const deadline = AbortSignal.timeout(retry.timeoutMs ?? 3e4);
|
|
51
|
+
try {
|
|
52
|
+
return await task(AbortSignal.any([signal, deadline]));
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (signal.aborted) throw signal.reason;
|
|
55
|
+
if (attempt >= attempts || !isRetryable(error)) throw error;
|
|
56
|
+
await sleep(Math.min(base * 2 ** (attempt - 1), retry.maxDelayMs ?? 4e3), signal);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Bytes fetched when a remote pack is opened: the manifest and central directory live here. */
|
|
61
|
+
const TAIL_BYTES = 64 * 1024;
|
|
62
|
+
async function fetchBytes(response) {
|
|
63
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
64
|
+
}
|
|
65
|
+
/** Missing Content-Range is allowed for CORS hosts which do not expose this header. */
|
|
66
|
+
function contentRange(value, url) {
|
|
67
|
+
if (value === void 0) return void 0;
|
|
68
|
+
const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec(value.trim());
|
|
69
|
+
const start = Number(match?.[1]);
|
|
70
|
+
const end = Number(match?.[2]);
|
|
71
|
+
const size = match?.[3] === "*" ? void 0 : Number(match?.[3]);
|
|
72
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || size !== void 0 && (!Number.isSafeInteger(size) || size <= end)) throw new Error(`invalid Content-Range for ${url}: ${value}`);
|
|
73
|
+
return {
|
|
74
|
+
start,
|
|
75
|
+
end,
|
|
76
|
+
size
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function strongEtag(value) {
|
|
80
|
+
return value !== void 0 && !value.startsWith("W/") ? value : void 0;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Open a pack URL. A server that honours Range gets range reads; one that ignores it (200 to a
|
|
84
|
+
* range request) has already sent the whole file, which is used as is.
|
|
85
|
+
*/
|
|
86
|
+
async function openUrl(url, options) {
|
|
87
|
+
const { fetch: fetcher, retry, signal } = options;
|
|
88
|
+
if (options.mode === "whole" || options.mode === "auto" && options.sizeHint !== void 0 && options.sizeHint <= options.wholeThreshold) {
|
|
89
|
+
const bytes = await withRetry(async (attemptSignal) => {
|
|
90
|
+
const response = await fetcher(url, { signal: attemptSignal });
|
|
91
|
+
if (!response.ok) throw new HttpStatusError(url, response.status);
|
|
92
|
+
return fetchBytes(response);
|
|
93
|
+
}, retry, signal);
|
|
94
|
+
options.onResponse(bytes.length);
|
|
95
|
+
return {
|
|
96
|
+
kind: "whole",
|
|
97
|
+
bytes
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
let first = await withRetry(async (attemptSignal) => {
|
|
101
|
+
const response = await fetcher(url, {
|
|
102
|
+
headers: { Range: `bytes=-${TAIL_BYTES}` },
|
|
103
|
+
signal: attemptSignal
|
|
104
|
+
});
|
|
105
|
+
if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
106
|
+
return {
|
|
107
|
+
status: response.status,
|
|
108
|
+
bytes: await fetchBytes(response),
|
|
109
|
+
etag: response.headers.get("etag") ?? void 0,
|
|
110
|
+
range: response.headers.get("content-range") ?? void 0
|
|
111
|
+
};
|
|
112
|
+
}, retry, signal);
|
|
113
|
+
options.onResponse(first.bytes.length);
|
|
114
|
+
if (first.status === 200) return {
|
|
115
|
+
kind: "whole",
|
|
116
|
+
bytes: first.bytes
|
|
117
|
+
};
|
|
118
|
+
let range = contentRange(first.range, url);
|
|
119
|
+
const total = range?.size;
|
|
120
|
+
if (total !== void 0 && (range?.start !== Math.max(0, total - 65536) || range?.end !== total - 1 || first.bytes.length !== Math.min(total, 65536))) {
|
|
121
|
+
const validator = strongEtag(first.etag);
|
|
122
|
+
const start = Math.max(0, total - TAIL_BYTES);
|
|
123
|
+
first = await withRetry(async (attemptSignal) => {
|
|
124
|
+
const headers = { Range: `bytes=${start}-${total - 1}` };
|
|
125
|
+
if (validator !== void 0) headers["If-Range"] = validator;
|
|
126
|
+
const response = await fetcher(url, {
|
|
127
|
+
headers,
|
|
128
|
+
signal: attemptSignal
|
|
129
|
+
});
|
|
130
|
+
if (validator !== void 0 && (response.status === 200 || response.headers.has("etag") && response.headers.get("etag") !== validator)) throw new PackChangedError(url);
|
|
131
|
+
if (response.status !== 200 && response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
132
|
+
return {
|
|
133
|
+
status: response.status,
|
|
134
|
+
bytes: await fetchBytes(response),
|
|
135
|
+
etag: response.headers.get("etag") ?? validator,
|
|
136
|
+
range: response.headers.get("content-range") ?? void 0
|
|
137
|
+
};
|
|
138
|
+
}, retry, signal);
|
|
139
|
+
options.onResponse(first.bytes.length);
|
|
140
|
+
if (first.status === 200) return {
|
|
141
|
+
kind: "whole",
|
|
142
|
+
bytes: first.bytes
|
|
143
|
+
};
|
|
144
|
+
range = contentRange(first.range, url);
|
|
145
|
+
if (first.bytes.length !== total - start || range !== void 0 && (range.start !== start || range.end !== total - 1 || range.size !== void 0 && range.size !== total)) throw new Error(`tail range ${start}-${total - 1} of ${url} was not honoured`);
|
|
146
|
+
} else if (range !== void 0 && first.bytes.length !== range.end - range.start + 1) throw new Error(`Content-Range length does not match response bytes for ${url}`);
|
|
147
|
+
const etag = strongEtag(first.etag);
|
|
148
|
+
return {
|
|
149
|
+
kind: "range",
|
|
150
|
+
tail: first.bytes,
|
|
151
|
+
...total !== void 0 && Number.isFinite(total) ? { size: total } : {},
|
|
152
|
+
reader: (size) => httpRangeReader(url, {
|
|
153
|
+
fetch: fetcher,
|
|
154
|
+
retry,
|
|
155
|
+
signal,
|
|
156
|
+
size,
|
|
157
|
+
...etag !== void 0 ? { etag } : {},
|
|
158
|
+
learnEtag: false,
|
|
159
|
+
onResponse: options.onResponse
|
|
160
|
+
})
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
/** Range reads of one URL, validated with If-Range when a strong ETag is known. */
|
|
164
|
+
function httpRangeReader(url, options) {
|
|
165
|
+
const { fetch: fetcher, retry, signal, size } = options;
|
|
166
|
+
let etag = options.etag;
|
|
167
|
+
return {
|
|
168
|
+
size,
|
|
169
|
+
read: (offset, length, readSignal) => withRetry(async (attemptSignal) => {
|
|
170
|
+
const headers = { Range: `bytes=${offset}-${offset + length - 1}` };
|
|
171
|
+
if (etag !== void 0) headers["If-Range"] = etag;
|
|
172
|
+
const response = await fetcher(url, {
|
|
173
|
+
headers,
|
|
174
|
+
signal: readSignal === void 0 ? attemptSignal : AbortSignal.any([attemptSignal, readSignal])
|
|
175
|
+
});
|
|
176
|
+
if (response.status === 200) {
|
|
177
|
+
if (etag !== void 0) throw new PackChangedError(url);
|
|
178
|
+
const all = await fetchBytes(response);
|
|
179
|
+
options.onResponse(all.length);
|
|
180
|
+
return all.subarray(offset, offset + length);
|
|
181
|
+
}
|
|
182
|
+
if (response.status !== 206) throw new HttpStatusError(url, response.status);
|
|
183
|
+
const returnedRange = contentRange(response.headers.get("content-range") ?? void 0, url);
|
|
184
|
+
if (returnedRange !== void 0 && (returnedRange.start !== offset || returnedRange.end !== offset + length - 1 || returnedRange.size !== void 0 && returnedRange.size !== size)) throw new Error(`range ${offset}+${length} of ${url} was not honoured`);
|
|
185
|
+
const bytes = await fetchBytes(response);
|
|
186
|
+
options.onResponse(bytes.length);
|
|
187
|
+
if (bytes.length !== length) throw new Error(`range ${offset}+${length} of ${url} returned ${bytes.length} bytes`);
|
|
188
|
+
if (options.learnEtag && etag === void 0) etag = strongEtag(response.headers.get("etag") ?? void 0);
|
|
189
|
+
return bytes;
|
|
190
|
+
}, retry, signal)
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* A `RangeReader` over a URL whose size is already known, with no opening request: the first
|
|
195
|
+
* response's strong ETag becomes the If-Range validator for every later read, so a file replaced
|
|
196
|
+
* mid-read fails with `PackChangedError` instead of mixing versions. Pair it with
|
|
197
|
+
* `cachingRangeReader` to open a pack from cached bytes.
|
|
198
|
+
*/
|
|
199
|
+
function urlRangeReader(url, options) {
|
|
200
|
+
return httpRangeReader(url, {
|
|
201
|
+
fetch: options.fetch ?? ((input, init) => globalThis.fetch(input, init)),
|
|
202
|
+
retry: options.retry ?? {},
|
|
203
|
+
signal: options.signal ?? new AbortController().signal,
|
|
204
|
+
size: options.size,
|
|
205
|
+
learnEtag: true,
|
|
206
|
+
onResponse: () => void 0
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Queue reads for one source. Reads issued in the same tick are sorted, merged when close
|
|
211
|
+
* together, and run with bounded concurrency, high priority first.
|
|
212
|
+
*/
|
|
213
|
+
var ReadScheduler = class {
|
|
214
|
+
reader;
|
|
215
|
+
signal;
|
|
216
|
+
queue = [];
|
|
217
|
+
flushing = false;
|
|
218
|
+
running = 0;
|
|
219
|
+
batches = [];
|
|
220
|
+
concurrency;
|
|
221
|
+
mergeGap;
|
|
222
|
+
maxSpan;
|
|
223
|
+
constructor(reader, signal, options = {}) {
|
|
224
|
+
this.reader = reader;
|
|
225
|
+
this.signal = signal;
|
|
226
|
+
this.concurrency = Math.max(1, options.concurrency ?? 6);
|
|
227
|
+
this.mergeGap = options.mergeGap ?? 32 * 1024;
|
|
228
|
+
this.maxSpan = options.maxSpan ?? 4 * 1024 * 1024;
|
|
229
|
+
}
|
|
230
|
+
read(offset, length, high = true, signal) {
|
|
231
|
+
if (length === 0) return Promise.resolve(new Uint8Array(0));
|
|
232
|
+
return new Promise((resolve, reject) => {
|
|
233
|
+
if (signal?.aborted) return reject(signal.reason);
|
|
234
|
+
const abort = () => reject(signal?.reason);
|
|
235
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
236
|
+
this.queue.push({
|
|
237
|
+
offset,
|
|
238
|
+
length,
|
|
239
|
+
high,
|
|
240
|
+
resolve: (bytes) => {
|
|
241
|
+
signal?.removeEventListener("abort", abort);
|
|
242
|
+
resolve(bytes);
|
|
243
|
+
},
|
|
244
|
+
reject: (error) => {
|
|
245
|
+
signal?.removeEventListener("abort", abort);
|
|
246
|
+
reject(error);
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
if (!this.flushing) {
|
|
250
|
+
this.flushing = true;
|
|
251
|
+
queueMicrotask(() => this.flush());
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
flush() {
|
|
256
|
+
this.flushing = false;
|
|
257
|
+
const waiters = this.queue.sort((a, b) => a.offset - b.offset);
|
|
258
|
+
this.queue = [];
|
|
259
|
+
for (const waiter of waiters) {
|
|
260
|
+
const last = this.batches.at(-1);
|
|
261
|
+
const end = waiter.offset + waiter.length;
|
|
262
|
+
if (last !== void 0 && waiter.offset - last.end <= this.mergeGap && Math.max(last.end, end) - last.start <= this.maxSpan && last.start <= waiter.offset) {
|
|
263
|
+
last.end = Math.max(last.end, end);
|
|
264
|
+
last.high ||= waiter.high;
|
|
265
|
+
last.waiters.push(waiter);
|
|
266
|
+
} else this.batches.push({
|
|
267
|
+
start: waiter.offset,
|
|
268
|
+
end,
|
|
269
|
+
high: waiter.high,
|
|
270
|
+
waiters: [waiter]
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
this.batches.sort((a, b) => Number(b.high) - Number(a.high));
|
|
274
|
+
this.pump();
|
|
275
|
+
}
|
|
276
|
+
pump() {
|
|
277
|
+
if (this.signal.aborted) {
|
|
278
|
+
for (const batch of this.batches.splice(0)) for (const waiter of batch.waiters) waiter.reject(this.signal.reason);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
while (this.running < this.concurrency && this.batches.length > 0) {
|
|
282
|
+
const batch = this.batches.shift();
|
|
283
|
+
this.running++;
|
|
284
|
+
this.reader.read(batch.start, batch.end - batch.start, this.signal).then((bytes) => {
|
|
285
|
+
for (const waiter of batch.waiters) {
|
|
286
|
+
const from = waiter.offset - batch.start;
|
|
287
|
+
waiter.resolve(bytes.subarray(from, from + waiter.length));
|
|
288
|
+
}
|
|
289
|
+
}, (error) => {
|
|
290
|
+
for (const waiter of batch.waiters) waiter.reject(error);
|
|
291
|
+
}).finally(() => {
|
|
292
|
+
this.running--;
|
|
293
|
+
this.pump();
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
};
|
|
298
|
+
//#endregion
|
|
299
|
+
export { openUrl as a, blobReader as i, ReadScheduler as n, urlRangeReader as o, TAIL_BYTES as r, PackChangedError as t };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/molen-pack",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -32,6 +32,10 @@
|
|
|
32
32
|
"types": "./dist/index.d.mts",
|
|
33
33
|
"default": "./dist/index.mjs"
|
|
34
34
|
},
|
|
35
|
+
"./cache": {
|
|
36
|
+
"types": "./dist/cache.d.mts",
|
|
37
|
+
"default": "./dist/cache.mjs"
|
|
38
|
+
},
|
|
35
39
|
"./node": {
|
|
36
40
|
"types": "./dist/node.d.mts",
|
|
37
41
|
"default": "./dist/node.mjs"
|
|
@@ -47,6 +51,7 @@
|
|
|
47
51
|
"devDependencies": {
|
|
48
52
|
"@biomejs/biome": "^2.5.0",
|
|
49
53
|
"@types/node": "^24",
|
|
54
|
+
"fake-indexeddb": "6.2.5",
|
|
50
55
|
"tsdown": "^0.22.2",
|
|
51
56
|
"typescript": "^6.0.3",
|
|
52
57
|
"vitest": "^5.0.1"
|