@bendyline/molen-pack 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/dist/cache.d.mts +263 -0
- package/dist/cache.mjs +1292 -0
- package/dist/index.d.mts +4 -3
- package/dist/index.mjs +10 -9
- package/dist/node.d.mts +1 -1
- package/dist/node.mjs +1 -1
- package/dist/{pack-DCQvXj1p.mjs → pack-BIXihu9D.mjs} +2 -268
- package/dist/{pack-DjufOvAS.d.mts → pack-CF6pPp9f.d.mts} +2 -26
- package/dist/source-7Pr2tntF.d.mts +41 -0
- package/dist/source-MDAzioCi.mjs +299 -0
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -13,7 +13,8 @@ npm i @bendyline/molen-pack
|
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
Node >= 22.13, ESM only. The `.` export runs in browsers, Workers and Node; `./node` adds file and
|
|
16
|
-
directory helpers for Node
|
|
16
|
+
directory helpers for Node, and `./cache` a persistent byte cache with no dependencies at all. The
|
|
17
|
+
package's only dependencies are `@bendyline/molen-schema` and `fflate`.
|
|
17
18
|
|
|
18
19
|
## Use
|
|
19
20
|
|
|
@@ -61,6 +62,37 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
|
|
|
61
62
|
const pack = await openDirPack('content/my-props');
|
|
62
63
|
```
|
|
63
64
|
|
|
65
|
+
## Cache what you read
|
|
66
|
+
|
|
67
|
+
`./cache` keeps the bytes of range-read archives (packs, PMTiles) across visits. Each archive is
|
|
68
|
+
split into 64 KiB blocks held in memory and in IndexedDB; neighbouring misses become one range
|
|
69
|
+
request, concurrent reads share blocks, and the least recently used blocks go once the store passes
|
|
70
|
+
its budget.
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
import {
|
|
74
|
+
cachingRangeReader,
|
|
75
|
+
createBlockCache,
|
|
76
|
+
createIndexedDbByteStore,
|
|
77
|
+
createMemoryByteStore,
|
|
78
|
+
urlRangeReader,
|
|
79
|
+
} from '@bendyline/molen-pack/cache';
|
|
80
|
+
import { openPack } from '@bendyline/molen-pack';
|
|
81
|
+
|
|
82
|
+
const cache = createBlockCache(
|
|
83
|
+
createIndexedDbByteStore({ budgetBytes: 300 * 1024 * 1024 }) ?? createMemoryByteStore(),
|
|
84
|
+
);
|
|
85
|
+
// Key a pack by its content hash so a cached copy can never be another version.
|
|
86
|
+
const reader = urlRangeReader(url, { size: entry.size });
|
|
87
|
+
const pack = await openPack(cachingRangeReader(reader, cache, { key: entry.contentHash, url }));
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`cachingRangeSource(source, cache, archive)` does the same for a PMTiles `Source`, and
|
|
91
|
+
`cachingDocumentFetch(cache.store)` wraps `fetch` so small JSON documents are revalidated with
|
|
92
|
+
`If-None-Match` instead of refetched and still load offline. A stable URL whose file is replaced
|
|
93
|
+
in place is cached with `keyBy: 'etag'`: each server version is its own archive and older ones are
|
|
94
|
+
deleted. Store failures (private browsing, quota) only cost network, never a failed read.
|
|
95
|
+
|
|
64
96
|
## What's in it
|
|
65
97
|
|
|
66
98
|
- `openPack(input, options)` — a `Pack` from a URL, `Blob`, bytes, or a `RangeReader`, with
|
|
@@ -72,6 +104,8 @@ const pack = await openDirPack('content/my-props');
|
|
|
72
104
|
view of loose files, from bytes in any environment.
|
|
73
105
|
- `./node`: `buildPack`, `openFilePack`, `openDirPack`, `openPackAt`, `readPackSource`,
|
|
74
106
|
`extractPack`.
|
|
107
|
+
- `./cache`: `createBlockCache`, `createIndexedDbByteStore`, `createMemoryByteStore`,
|
|
108
|
+
`cachingRangeSource`, `cachingRangeReader`, `urlRangeReader`, `cachingDocumentFetch`.
|
|
75
109
|
|
|
76
110
|
A pack is a standard zip you can open with any zip tool. Binary files are their own members,
|
|
77
111
|
deflated when that helps. Small text files are concatenated into a few deflated solid blocks,
|
package/dist/cache.d.mts
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { a as urlRangeReader, i as UrlRangeReaderOptions, r as RangeReader } from "./source-7Pr2tntF.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/block-cache.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* A persistent cache for byte-range archives (PMTiles, zip packs, anything read by offset).
|
|
6
|
+
*
|
|
7
|
+
* Each archive is split into fixed-size blocks (64 KiB by default) held in two tiers: a small
|
|
8
|
+
* in-memory LRU and a `ByteStore` (IndexedDB in a browser, see `createIndexedDbByteStore`). A
|
|
9
|
+
* read takes what it can from either tier and fetches the rest with one request per run of
|
|
10
|
+
* contiguous missing blocks, so neighbouring reads share blocks and a revisited area costs no
|
|
11
|
+
* network at all. Concurrent reads of the same block share one request; a request is cancelled
|
|
12
|
+
* only when every read waiting on it has been aborted.
|
|
13
|
+
*
|
|
14
|
+
* Archives are identified by `key`: a content hash or a URL that never changes content. A stable
|
|
15
|
+
* URL whose bytes are replaced in place uses `keyBy: 'etag'` instead, so each server version is
|
|
16
|
+
* its own archive and older versions are deleted when a new one is seen. A server that changes
|
|
17
|
+
* the validator of a keyed archive mid-session makes the cache drop that archive's bytes and
|
|
18
|
+
* re-read, never mix versions.
|
|
19
|
+
*
|
|
20
|
+
* The store holds a byte budget. Blocks are evicted least recently used first, with a priority
|
|
21
|
+
* that biases recency: `high` blocks count as a week newer than they are, `low` ones as a day
|
|
22
|
+
* older, so a planet base layer outlives a street-level tile seen at the same time.
|
|
23
|
+
*/
|
|
24
|
+
/** How long an archive's blocks are kept relative to others when the store is full. */
|
|
25
|
+
type ArchivePriority = "high" | "normal" | "low";
|
|
26
|
+
/** What an archive is and where its bytes come from. */
|
|
27
|
+
interface ArchiveKey {
|
|
28
|
+
/** Identity of the bytes: a content hash or a URL whose content never changes. */
|
|
29
|
+
key?: string;
|
|
30
|
+
/** Key by the server's strong ETag instead, for a stable URL whose bytes are replaced. */
|
|
31
|
+
keyBy?: "etag";
|
|
32
|
+
/** Where the bytes come from; recorded with the archive (and the basis of `keyBy: 'etag'`). */
|
|
33
|
+
url: string;
|
|
34
|
+
/** Total size in bytes, when known. A read never asks for bytes past it. */
|
|
35
|
+
size?: number;
|
|
36
|
+
/** Default `'normal'`. */
|
|
37
|
+
priority?: ArchivePriority;
|
|
38
|
+
}
|
|
39
|
+
/** What a store records about one archive besides its blocks. */
|
|
40
|
+
interface StoredArchiveInfo {
|
|
41
|
+
key: string;
|
|
42
|
+
url: string;
|
|
43
|
+
priority: ArchivePriority;
|
|
44
|
+
size?: number;
|
|
45
|
+
/** The strong ETag the bytes were read with. */
|
|
46
|
+
etag?: string;
|
|
47
|
+
/** Media type, for documents. */
|
|
48
|
+
type?: string;
|
|
49
|
+
/** When a document was last confirmed current, in ms since the epoch. */
|
|
50
|
+
validated?: number;
|
|
51
|
+
}
|
|
52
|
+
/** A stored archive and the bytes it holds. */
|
|
53
|
+
interface StoredArchive extends StoredArchiveInfo {
|
|
54
|
+
bytes: number;
|
|
55
|
+
blocks: number;
|
|
56
|
+
}
|
|
57
|
+
/** Totals reported by a store. */
|
|
58
|
+
interface ByteStoreStats {
|
|
59
|
+
backend: "memory" | "indexeddb";
|
|
60
|
+
bytes: number;
|
|
61
|
+
blocks: number;
|
|
62
|
+
archives: number;
|
|
63
|
+
budgetBytes: number;
|
|
64
|
+
/** The store could not be opened or ran out of quota; nothing more is written. */
|
|
65
|
+
degraded: boolean;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Persistent block storage behind a `BlockCache`. Every method resolves, never rejects: a failure
|
|
69
|
+
* is a miss or a no-op, so a broken store only costs network.
|
|
70
|
+
*/
|
|
71
|
+
interface ByteStore {
|
|
72
|
+
readonly kind: "memory" | "indexeddb";
|
|
73
|
+
/** The budget in force (possibly clamped below the requested one by the origin's quota). */
|
|
74
|
+
readonly budgetBytes: number;
|
|
75
|
+
getBlocks(key: string, indices: readonly number[]): Promise<Array<Uint8Array | undefined>>;
|
|
76
|
+
/** Write blocks (replacing any with the same index) and the archive's record. */
|
|
77
|
+
putBlocks(info: StoredArchiveInfo, blocks: ReadonlyArray<{
|
|
78
|
+
index: number;
|
|
79
|
+
bytes: Uint8Array;
|
|
80
|
+
}>, now: number): Promise<void>;
|
|
81
|
+
/** Mark blocks as used now (they move to the back of the eviction order). */
|
|
82
|
+
touch(key: string, indices: readonly number[], now: number): Promise<void>;
|
|
83
|
+
getArchive(key: string): Promise<StoredArchive | undefined>;
|
|
84
|
+
/** Replace an archive's record without touching its blocks; creates an empty record if absent. */
|
|
85
|
+
updateArchive(info: StoredArchiveInfo): Promise<void>;
|
|
86
|
+
/** Every archive recorded with this URL. */
|
|
87
|
+
archivesForUrl(url: string): Promise<StoredArchive[]>;
|
|
88
|
+
deleteArchive(key: string): Promise<void>;
|
|
89
|
+
/** Evict blocks in priority-biased LRU order until at most `targetBytes` remain; returns bytes freed. */
|
|
90
|
+
evict(targetBytes: number): Promise<number>;
|
|
91
|
+
setBudget(bytes: number): void;
|
|
92
|
+
stats(): Promise<ByteStoreStats>;
|
|
93
|
+
clear(): Promise<void>;
|
|
94
|
+
close?(): void;
|
|
95
|
+
}
|
|
96
|
+
/** One read from the network (or wherever the archive lives) for the cache. */
|
|
97
|
+
type SpanReader = (offset: number, length: number, signal: AbortSignal, etag?: string) => Promise<{
|
|
98
|
+
bytes: Uint8Array;
|
|
99
|
+
etag?: string;
|
|
100
|
+
}>;
|
|
101
|
+
/** The result of a cached read. */
|
|
102
|
+
interface CachedRead {
|
|
103
|
+
bytes: Uint8Array;
|
|
104
|
+
/** The validator these bytes belong to, when known. */
|
|
105
|
+
etag?: string;
|
|
106
|
+
}
|
|
107
|
+
/** One archive opened through a `BlockCache`. */
|
|
108
|
+
interface CachedArchive {
|
|
109
|
+
/**
|
|
110
|
+
* Read `length` bytes at `offset`; shorter only at the end of the archive. With `etag`, the
|
|
111
|
+
* caller expects that version: bytes cached for another version are never returned.
|
|
112
|
+
*/
|
|
113
|
+
read(offset: number, length: number, signal?: AbortSignal, etag?: string): Promise<CachedRead>;
|
|
114
|
+
/** Total size, declared or learned from a short read at the end. */
|
|
115
|
+
size(): number | undefined;
|
|
116
|
+
/** The strong ETag of the cached bytes, when known. */
|
|
117
|
+
etag(): string | undefined;
|
|
118
|
+
/** Forget every cached block of this archive. */
|
|
119
|
+
drop(): Promise<void>;
|
|
120
|
+
}
|
|
121
|
+
interface BlockCacheOptions {
|
|
122
|
+
/** Block size in bytes (default 64 KiB). */
|
|
123
|
+
blockBytes?: number;
|
|
124
|
+
/** In-memory tier (default 32 MiB). */
|
|
125
|
+
memoryBytes?: number;
|
|
126
|
+
/** Largest single network request when merging missing blocks (default 4 MiB). */
|
|
127
|
+
maxSpanBytes?: number;
|
|
128
|
+
/** Delay before buffered writes go to the store (default 50 ms). */
|
|
129
|
+
flushDelayMs?: number;
|
|
130
|
+
/** A block's last-used time is written at most this often (default 60 s). */
|
|
131
|
+
touchIntervalMs?: number;
|
|
132
|
+
/** Evict when the store holds more than this times the budget (default 1.05)… */
|
|
133
|
+
evictHighWater?: number;
|
|
134
|
+
/** …down to this times the budget (default 0.9). */
|
|
135
|
+
evictLowWater?: number;
|
|
136
|
+
/** Clock, for tests (default `Date.now`). */
|
|
137
|
+
now?: () => number;
|
|
138
|
+
}
|
|
139
|
+
/** Counters and totals for a `BlockCache`. Block counts, not reads. */
|
|
140
|
+
interface BlockCacheStats {
|
|
141
|
+
backend: "memory" | "indexeddb";
|
|
142
|
+
degraded: boolean;
|
|
143
|
+
budgetBytes: number;
|
|
144
|
+
blockBytes: number;
|
|
145
|
+
memoryBytes: number;
|
|
146
|
+
memoryBlocks: number;
|
|
147
|
+
pendingBlocks: number;
|
|
148
|
+
storeBytes: number;
|
|
149
|
+
storeBlocks: number;
|
|
150
|
+
storeArchives: number;
|
|
151
|
+
memoryHits: number;
|
|
152
|
+
storeHits: number;
|
|
153
|
+
misses: number;
|
|
154
|
+
networkRequests: number;
|
|
155
|
+
networkBytes: number;
|
|
156
|
+
/** Reads passed straight through (no validator to key them, or a version mismatch). */
|
|
157
|
+
uncachedRequests: number;
|
|
158
|
+
evictedBytes: number;
|
|
159
|
+
}
|
|
160
|
+
/** A shared cache of archive blocks; open each archive through it. */
|
|
161
|
+
interface BlockCache {
|
|
162
|
+
readonly store: ByteStore;
|
|
163
|
+
readonly blockBytes: number;
|
|
164
|
+
open(archive: ArchiveKey, source: SpanReader): CachedArchive;
|
|
165
|
+
/** Forget an archive by key. */
|
|
166
|
+
drop(key: string): Promise<void>;
|
|
167
|
+
/** A snapshot; the store totals are as of the last `flush()`. */
|
|
168
|
+
stats(): BlockCacheStats;
|
|
169
|
+
/** Write buffered blocks, evict down to the budget and refresh the store totals. */
|
|
170
|
+
flush(): Promise<void>;
|
|
171
|
+
setBudget(bytes: number): void;
|
|
172
|
+
/** Delete everything, in memory and in the store. */
|
|
173
|
+
clear(): Promise<void>;
|
|
174
|
+
dispose(): void;
|
|
175
|
+
}
|
|
176
|
+
/** Recency bias per priority: added to a block's last-used time to order eviction. */
|
|
177
|
+
declare const PRIORITY_BIAS_MS: Readonly<Record<ArchivePriority, number>>;
|
|
178
|
+
/** A block's place in the eviction order: lower ranks go first. */
|
|
179
|
+
declare function evictionRank(priority: ArchivePriority, touched: number): number;
|
|
180
|
+
/** Whether an error says the server's bytes changed while they were being read. */
|
|
181
|
+
declare function isChangedError(error: unknown): boolean;
|
|
182
|
+
/** Create a cache over `store` (default: an in-memory store with a 64 MiB budget). */
|
|
183
|
+
declare function createBlockCache(store?: ByteStore, options?: BlockCacheOptions): BlockCache;
|
|
184
|
+
/** A `ByteStore` in memory: for tests, private browsing, and hosts without IndexedDB. */
|
|
185
|
+
declare function createMemoryByteStore(options?: {
|
|
186
|
+
budgetBytes?: number;
|
|
187
|
+
}): ByteStore;
|
|
188
|
+
//#endregion
|
|
189
|
+
//#region src/cache-adapters.d.ts
|
|
190
|
+
/** A PMTiles range response, typed structurally so this package never imports pmtiles. */
|
|
191
|
+
interface RangeSourceResponse {
|
|
192
|
+
data: ArrayBuffer;
|
|
193
|
+
etag?: string;
|
|
194
|
+
expires?: string;
|
|
195
|
+
cacheControl?: string;
|
|
196
|
+
}
|
|
197
|
+
/** The shape of a PMTiles `Source` (`FetchSource`, a retrying wrapper, a native reader). */
|
|
198
|
+
interface RangeSourceLike {
|
|
199
|
+
getKey(): string;
|
|
200
|
+
getBytes(offset: number, length: number, signal?: AbortSignal, etag?: string): Promise<RangeSourceResponse>;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* A PMTiles source whose bytes come from `cache` when it has them. The inner source must honour
|
|
204
|
+
* Range requests. Its validator handling is kept: a request carrying the archive's ETag still
|
|
205
|
+
* reaches the inner source on a miss, so a replaced file fails with the inner source's own
|
|
206
|
+
* mismatch error (which also drops the cached bytes). Use `keyBy: 'etag'` for a stable URL whose
|
|
207
|
+
* contents are replaced in place; then the first read of a session (the header) always goes to
|
|
208
|
+
* the network and names the version the rest are cached under.
|
|
209
|
+
*/
|
|
210
|
+
declare function cachingRangeSource(source: RangeSourceLike, cache: BlockCache, archive: ArchiveKey): RangeSourceLike;
|
|
211
|
+
interface CachingRangeReaderOptions {
|
|
212
|
+
/**
|
|
213
|
+
* Archives at most this size are fetched whole on the first read, in one request (default
|
|
214
|
+
* 4 MiB), so a small pack costs one request on its first visit and none after.
|
|
215
|
+
*/
|
|
216
|
+
wholeBelow?: number;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* A pack `RangeReader` whose bytes come from `cache` when it has them. Key it by the pack's
|
|
220
|
+
* content hash (`contentHash` in a pack index) so a cached copy can never be another version.
|
|
221
|
+
*/
|
|
222
|
+
declare function cachingRangeReader(reader: RangeReader, cache: BlockCache, archive: ArchiveKey, options?: CachingRangeReaderOptions): RangeReader;
|
|
223
|
+
//#endregion
|
|
224
|
+
//#region src/document-cache.d.ts
|
|
225
|
+
interface DocumentCacheOptions {
|
|
226
|
+
/**
|
|
227
|
+
* Serve a stored copy without asking the server while it is younger than this, in ms (default
|
|
228
|
+
* 0: always revalidate). A function picks a policy per URL.
|
|
229
|
+
*/
|
|
230
|
+
maxAgeMs?: number | ((url: string) => number);
|
|
231
|
+
/** Serve the stored copy when the server has not answered within this many ms (default 4000). */
|
|
232
|
+
timeoutMs?: number;
|
|
233
|
+
/** Largest document kept, in bytes (default 2 MiB). */
|
|
234
|
+
maxBytes?: number;
|
|
235
|
+
/** Clock, for tests (default `Date.now`). */
|
|
236
|
+
now?: () => number;
|
|
237
|
+
}
|
|
238
|
+
/** Response header naming where a cached document came from: `hit`, `revalidated` or `stale`. */
|
|
239
|
+
declare const DOCUMENT_CACHE_HEADER = "x-molen-cache";
|
|
240
|
+
/** Wrap `fetchImpl` so GETs of small documents are kept in `store`. */
|
|
241
|
+
declare function cachingDocumentFetch(store: ByteStore, fetchImpl?: typeof fetch, options?: DocumentCacheOptions): typeof fetch;
|
|
242
|
+
//#endregion
|
|
243
|
+
//#region src/indexeddb-byte-store.d.ts
|
|
244
|
+
interface IndexedDbByteStoreOptions {
|
|
245
|
+
/** Database name (default `'molen-bytes'`). */
|
|
246
|
+
name?: string;
|
|
247
|
+
/** Bytes kept before eviction (default 512 MiB); clamped to half the origin's free quota. */
|
|
248
|
+
budgetBytes?: number;
|
|
249
|
+
/**
|
|
250
|
+
* Ask the browser to make this origin's storage persistent (default false). Firefox shows a
|
|
251
|
+
* permission prompt for it, so only hosts that expect one (an installed app) should set it.
|
|
252
|
+
*/
|
|
253
|
+
persist?: boolean;
|
|
254
|
+
/** The IndexedDB factory (default `globalThis.indexedDB`). */
|
|
255
|
+
indexedDB?: IDBFactory;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Open (lazily) a block store in IndexedDB. Undefined where IndexedDB does not exist (Node,
|
|
259
|
+
* restricted embedded browsers); a database that fails to open behaves as an empty store.
|
|
260
|
+
*/
|
|
261
|
+
declare function createIndexedDbByteStore(options?: IndexedDbByteStoreOptions): ByteStore | undefined;
|
|
262
|
+
//#endregion
|
|
263
|
+
export { type ArchiveKey, type ArchivePriority, type BlockCache, type BlockCacheOptions, type BlockCacheStats, type ByteStore, type ByteStoreStats, type CachedArchive, type CachedRead, type CachingRangeReaderOptions, DOCUMENT_CACHE_HEADER, type DocumentCacheOptions, type IndexedDbByteStoreOptions, PRIORITY_BIAS_MS, type RangeReader, type RangeSourceLike, type RangeSourceResponse, type SpanReader, type StoredArchive, type StoredArchiveInfo, type UrlRangeReaderOptions, cachingDocumentFetch, cachingRangeReader, cachingRangeSource, createBlockCache, createIndexedDbByteStore, createMemoryByteStore, evictionRank, isChangedError, urlRangeReader };
|