@bendyline/molen-pack 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -13,7 +13,8 @@ npm i @bendyline/molen-pack
13
13
  ```
14
14
 
15
15
  Node >= 22.13, ESM only. The `.` export runs in browsers, Workers and Node; `./node` adds file and
16
- directory helpers for Node. Its only dependencies are `@bendyline/molen-schema` and `fflate`.
16
+ directory helpers for Node, and `./cache` a persistent byte cache with no dependencies at all. The
17
+ package's only dependencies are `@bendyline/molen-schema` and `fflate`.
17
18
 
18
19
  ## Use
19
20
 
@@ -61,6 +62,37 @@ built.file; // 'my.props-3f2a9c1e5b7d.zip', named by a hash of its bytes
61
62
  const pack = await openDirPack('content/my-props');
62
63
  ```
63
64
 
65
+ ## Cache what you read
66
+
67
+ `./cache` keeps the bytes of range-read archives (packs, PMTiles) across visits. Each archive is
68
+ split into 64 KiB blocks held in memory and in IndexedDB; neighbouring misses become one range
69
+ request, concurrent reads share blocks, and the least recently used blocks go once the store passes
70
+ its budget.
71
+
72
+ ```ts
73
+ import {
74
+ cachingRangeReader,
75
+ createBlockCache,
76
+ createIndexedDbByteStore,
77
+ createMemoryByteStore,
78
+ urlRangeReader,
79
+ } from '@bendyline/molen-pack/cache';
80
+ import { openPack } from '@bendyline/molen-pack';
81
+
82
+ const cache = createBlockCache(
83
+ createIndexedDbByteStore({ budgetBytes: 300 * 1024 * 1024 }) ?? createMemoryByteStore(),
84
+ );
85
+ // Key a pack by its content hash so a cached copy can never be another version.
86
+ const reader = urlRangeReader(url, { size: entry.size });
87
+ const pack = await openPack(cachingRangeReader(reader, cache, { key: entry.contentHash, url }));
88
+ ```
89
+
90
+ `cachingRangeSource(source, cache, archive)` does the same for a PMTiles `Source`, and
91
+ `cachingDocumentFetch(cache.store)` wraps `fetch` so small JSON documents are revalidated with
92
+ `If-None-Match` instead of refetched and still load offline. A stable URL whose file is replaced
93
+ in place is cached with `keyBy: 'etag'`: each server version is its own archive and older ones are
94
+ deleted. Store failures (private browsing, quota) only cost network, never a failed read.
95
+
64
96
  ## What's in it
65
97
 
66
98
  - `openPack(input, options)` — a `Pack` from a URL, `Blob`, bytes, or a `RangeReader`, with
@@ -72,6 +104,8 @@ const pack = await openDirPack('content/my-props');
72
104
  view of loose files, from bytes in any environment.
73
105
  - `./node`: `buildPack`, `openFilePack`, `openDirPack`, `openPackAt`, `readPackSource`,
74
106
  `extractPack`.
107
+ - `./cache`: `createBlockCache`, `createIndexedDbByteStore`, `createMemoryByteStore`,
108
+ `cachingRangeSource`, `cachingRangeReader`, `urlRangeReader`, `cachingDocumentFetch`.
75
109
 
76
110
  A pack is a standard zip you can open with any zip tool. Binary files are their own members,
77
111
  deflated when that helps. Small text files are concatenated into a few deflated solid blocks,
@@ -82,7 +116,7 @@ provides, and a `contentHash` over paths and contents that does not depend on th
82
116
 
83
117
  ## Status
84
118
 
85
- 0.x, on one fixed version line with every other `@bendyline/molen-*` package. The format is
119
+ 0.x, versioned independently of the other `@bendyline/molen-*` packages. The format is
86
120
  beta. Packs are written without zip64, so a pack must stay under 4 GiB and 65,535 members.
87
121
  Building the same files always produces the same bytes; that depends on the exact `fflate`
88
122
  version this package pins.
@@ -0,0 +1,263 @@
1
+ import { a as urlRangeReader, i as UrlRangeReaderOptions, r as RangeReader } from "./source-7Pr2tntF.mjs";
2
+
3
+ //#region src/block-cache.d.ts
4
+ /**
5
+ * A persistent cache for byte-range archives (PMTiles, zip packs, anything read by offset).
6
+ *
7
+ * Each archive is split into fixed-size blocks (64 KiB by default) held in two tiers: a small
8
+ * in-memory LRU and a `ByteStore` (IndexedDB in a browser, see `createIndexedDbByteStore`). A
9
+ * read takes what it can from either tier and fetches the rest with one request per run of
10
+ * contiguous missing blocks, so neighbouring reads share blocks and a revisited area costs no
11
+ * network at all. Concurrent reads of the same block share one request; a request is cancelled
12
+ * only when every read waiting on it has been aborted.
13
+ *
14
+ * Archives are identified by `key`: a content hash or a URL that never changes content. A stable
15
+ * URL whose bytes are replaced in place uses `keyBy: 'etag'` instead, so each server version is
16
+ * its own archive and older versions are deleted when a new one is seen. A server that changes
17
+ * the validator of a keyed archive mid-session makes the cache drop that archive's bytes and
18
+ * re-read, never mix versions.
19
+ *
20
+ * The store holds a byte budget. Blocks are evicted least recently used first, with a priority
21
+ * that biases recency: `high` blocks count as a week newer than they are, `low` ones as a day
22
+ * older, so a planet base layer outlives a street-level tile seen at the same time.
23
+ */
24
+ /** How long an archive's blocks are kept relative to others when the store is full. */
25
+ type ArchivePriority = "high" | "normal" | "low";
26
+ /** What an archive is and where its bytes come from. */
27
+ interface ArchiveKey {
28
+ /** Identity of the bytes: a content hash or a URL whose content never changes. */
29
+ key?: string;
30
+ /** Key by the server's strong ETag instead, for a stable URL whose bytes are replaced. */
31
+ keyBy?: "etag";
32
+ /** Where the bytes come from; recorded with the archive (and the basis of `keyBy: 'etag'`). */
33
+ url: string;
34
+ /** Total size in bytes, when known. A read never asks for bytes past it. */
35
+ size?: number;
36
+ /** Default `'normal'`. */
37
+ priority?: ArchivePriority;
38
+ }
39
+ /** What a store records about one archive besides its blocks. */
40
+ interface StoredArchiveInfo {
41
+ key: string;
42
+ url: string;
43
+ priority: ArchivePriority;
44
+ size?: number;
45
+ /** The strong ETag the bytes were read with. */
46
+ etag?: string;
47
+ /** Media type, for documents. */
48
+ type?: string;
49
+ /** When a document was last confirmed current, in ms since the epoch. */
50
+ validated?: number;
51
+ }
52
+ /** A stored archive and the bytes it holds. */
53
+ interface StoredArchive extends StoredArchiveInfo {
54
+ bytes: number;
55
+ blocks: number;
56
+ }
57
+ /** Totals reported by a store. */
58
+ interface ByteStoreStats {
59
+ backend: "memory" | "indexeddb";
60
+ bytes: number;
61
+ blocks: number;
62
+ archives: number;
63
+ budgetBytes: number;
64
+ /** The store could not be opened or ran out of quota; nothing more is written. */
65
+ degraded: boolean;
66
+ }
67
+ /**
68
+ * Persistent block storage behind a `BlockCache`. Every method resolves, never rejects: a failure
69
+ * is a miss or a no-op, so a broken store only costs network.
70
+ */
71
+ interface ByteStore {
72
+ readonly kind: "memory" | "indexeddb";
73
+ /** The budget in force (possibly clamped below the requested one by the origin's quota). */
74
+ readonly budgetBytes: number;
75
+ getBlocks(key: string, indices: readonly number[]): Promise<Array<Uint8Array | undefined>>;
76
+ /** Write blocks (replacing any with the same index) and the archive's record. */
77
+ putBlocks(info: StoredArchiveInfo, blocks: ReadonlyArray<{
78
+ index: number;
79
+ bytes: Uint8Array;
80
+ }>, now: number): Promise<void>;
81
+ /** Mark blocks as used now (they move to the back of the eviction order). */
82
+ touch(key: string, indices: readonly number[], now: number): Promise<void>;
83
+ getArchive(key: string): Promise<StoredArchive | undefined>;
84
+ /** Replace an archive's record without touching its blocks; creates an empty record if absent. */
85
+ updateArchive(info: StoredArchiveInfo): Promise<void>;
86
+ /** Every archive recorded with this URL. */
87
+ archivesForUrl(url: string): Promise<StoredArchive[]>;
88
+ deleteArchive(key: string): Promise<void>;
89
+ /** Evict blocks in priority-biased LRU order until at most `targetBytes` remain; returns bytes freed. */
90
+ evict(targetBytes: number): Promise<number>;
91
+ setBudget(bytes: number): void;
92
+ stats(): Promise<ByteStoreStats>;
93
+ clear(): Promise<void>;
94
+ close?(): void;
95
+ }
96
+ /** One read from the network (or wherever the archive lives) for the cache. */
97
+ type SpanReader = (offset: number, length: number, signal: AbortSignal, etag?: string) => Promise<{
98
+ bytes: Uint8Array;
99
+ etag?: string;
100
+ }>;
101
+ /** The result of a cached read. */
102
+ interface CachedRead {
103
+ bytes: Uint8Array;
104
+ /** The validator these bytes belong to, when known. */
105
+ etag?: string;
106
+ }
107
+ /** One archive opened through a `BlockCache`. */
108
+ interface CachedArchive {
109
+ /**
110
+ * Read `length` bytes at `offset`; shorter only at the end of the archive. With `etag`, the
111
+ * caller expects that version: bytes cached for another version are never returned.
112
+ */
113
+ read(offset: number, length: number, signal?: AbortSignal, etag?: string): Promise<CachedRead>;
114
+ /** Total size, declared or learned from a short read at the end. */
115
+ size(): number | undefined;
116
+ /** The strong ETag of the cached bytes, when known. */
117
+ etag(): string | undefined;
118
+ /** Forget every cached block of this archive. */
119
+ drop(): Promise<void>;
120
+ }
121
+ interface BlockCacheOptions {
122
+ /** Block size in bytes (default 64 KiB). */
123
+ blockBytes?: number;
124
+ /** In-memory tier (default 32 MiB). */
125
+ memoryBytes?: number;
126
+ /** Largest single network request when merging missing blocks (default 4 MiB). */
127
+ maxSpanBytes?: number;
128
+ /** Delay before buffered writes go to the store (default 50 ms). */
129
+ flushDelayMs?: number;
130
+ /** A block's last-used time is written at most this often (default 60 s). */
131
+ touchIntervalMs?: number;
132
+ /** Evict when the store holds more than this times the budget (default 1.05)… */
133
+ evictHighWater?: number;
134
+ /** …down to this times the budget (default 0.9). */
135
+ evictLowWater?: number;
136
+ /** Clock, for tests (default `Date.now`). */
137
+ now?: () => number;
138
+ }
139
+ /** Counters and totals for a `BlockCache`. Block counts, not reads. */
140
+ interface BlockCacheStats {
141
+ backend: "memory" | "indexeddb";
142
+ degraded: boolean;
143
+ budgetBytes: number;
144
+ blockBytes: number;
145
+ memoryBytes: number;
146
+ memoryBlocks: number;
147
+ pendingBlocks: number;
148
+ storeBytes: number;
149
+ storeBlocks: number;
150
+ storeArchives: number;
151
+ memoryHits: number;
152
+ storeHits: number;
153
+ misses: number;
154
+ networkRequests: number;
155
+ networkBytes: number;
156
+ /** Reads passed straight through (no validator to key them, or a version mismatch). */
157
+ uncachedRequests: number;
158
+ evictedBytes: number;
159
+ }
160
+ /** A shared cache of archive blocks; open each archive through it. */
161
+ interface BlockCache {
162
+ readonly store: ByteStore;
163
+ readonly blockBytes: number;
164
+ open(archive: ArchiveKey, source: SpanReader): CachedArchive;
165
+ /** Forget an archive by key. */
166
+ drop(key: string): Promise<void>;
167
+ /** A snapshot; the store totals are as of the last `flush()`. */
168
+ stats(): BlockCacheStats;
169
+ /** Write buffered blocks, evict down to the budget and refresh the store totals. */
170
+ flush(): Promise<void>;
171
+ setBudget(bytes: number): void;
172
+ /** Delete everything, in memory and in the store. */
173
+ clear(): Promise<void>;
174
+ dispose(): void;
175
+ }
176
+ /** Recency bias per priority: added to a block's last-used time to order eviction. */
177
+ declare const PRIORITY_BIAS_MS: Readonly<Record<ArchivePriority, number>>;
178
+ /** A block's place in the eviction order: lower ranks go first. */
179
+ declare function evictionRank(priority: ArchivePriority, touched: number): number;
180
+ /** Whether an error says the server's bytes changed while they were being read. */
181
+ declare function isChangedError(error: unknown): boolean;
182
+ /** Create a cache over `store` (default: an in-memory store with a 64 MiB budget). */
183
+ declare function createBlockCache(store?: ByteStore, options?: BlockCacheOptions): BlockCache;
184
+ /** A `ByteStore` in memory: for tests, private browsing, and hosts without IndexedDB. */
185
+ declare function createMemoryByteStore(options?: {
186
+ budgetBytes?: number;
187
+ }): ByteStore;
188
+ //#endregion
189
+ //#region src/cache-adapters.d.ts
190
+ /** A PMTiles range response, typed structurally so this package never imports pmtiles. */
191
+ interface RangeSourceResponse {
192
+ data: ArrayBuffer;
193
+ etag?: string;
194
+ expires?: string;
195
+ cacheControl?: string;
196
+ }
197
+ /** The shape of a PMTiles `Source` (`FetchSource`, a retrying wrapper, a native reader). */
198
+ interface RangeSourceLike {
199
+ getKey(): string;
200
+ getBytes(offset: number, length: number, signal?: AbortSignal, etag?: string): Promise<RangeSourceResponse>;
201
+ }
202
+ /**
203
+ * A PMTiles source whose bytes come from `cache` when it has them. The inner source must honour
204
+ * Range requests. Its validator handling is kept: a request carrying the archive's ETag still
205
+ * reaches the inner source on a miss, so a replaced file fails with the inner source's own
206
+ * mismatch error (which also drops the cached bytes). Use `keyBy: 'etag'` for a stable URL whose
207
+ * contents are replaced in place; then the first read of a session (the header) always goes to
208
+ * the network and names the version the rest are cached under.
209
+ */
210
+ declare function cachingRangeSource(source: RangeSourceLike, cache: BlockCache, archive: ArchiveKey): RangeSourceLike;
211
+ interface CachingRangeReaderOptions {
212
+ /**
213
+ * Archives at most this size are fetched whole on the first read, in one request (default
214
+ * 4 MiB), so a small pack costs one request on its first visit and none after.
215
+ */
216
+ wholeBelow?: number;
217
+ }
218
+ /**
219
+ * A pack `RangeReader` whose bytes come from `cache` when it has them. Key it by the pack's
220
+ * content hash (`contentHash` in a pack index) so a cached copy can never be another version.
221
+ */
222
+ declare function cachingRangeReader(reader: RangeReader, cache: BlockCache, archive: ArchiveKey, options?: CachingRangeReaderOptions): RangeReader;
223
+ //#endregion
224
+ //#region src/document-cache.d.ts
225
+ interface DocumentCacheOptions {
226
+ /**
227
+ * Serve a stored copy without asking the server while it is younger than this, in ms (default
228
+ * 0: always revalidate). A function picks a policy per URL.
229
+ */
230
+ maxAgeMs?: number | ((url: string) => number);
231
+ /** Serve the stored copy when the server has not answered within this many ms (default 4000). */
232
+ timeoutMs?: number;
233
+ /** Largest document kept, in bytes (default 2 MiB). */
234
+ maxBytes?: number;
235
+ /** Clock, for tests (default `Date.now`). */
236
+ now?: () => number;
237
+ }
238
+ /** Response header naming where a cached document came from: `hit`, `revalidated` or `stale`. */
239
+ declare const DOCUMENT_CACHE_HEADER = "x-molen-cache";
240
+ /** Wrap `fetchImpl` so GETs of small documents are kept in `store`. */
241
+ declare function cachingDocumentFetch(store: ByteStore, fetchImpl?: typeof fetch, options?: DocumentCacheOptions): typeof fetch;
242
+ //#endregion
243
+ //#region src/indexeddb-byte-store.d.ts
244
+ interface IndexedDbByteStoreOptions {
245
+ /** Database name (default `'molen-bytes'`). */
246
+ name?: string;
247
+ /** Bytes kept before eviction (default 512 MiB); clamped to half the origin's free quota. */
248
+ budgetBytes?: number;
249
+ /**
250
+ * Ask the browser to make this origin's storage persistent (default false). Firefox shows a
251
+ * permission prompt for it, so only hosts that expect one (an installed app) should set it.
252
+ */
253
+ persist?: boolean;
254
+ /** The IndexedDB factory (default `globalThis.indexedDB`). */
255
+ indexedDB?: IDBFactory;
256
+ }
257
+ /**
258
+ * Open (lazily) a block store in IndexedDB. Undefined where IndexedDB does not exist (Node,
259
+ * restricted embedded browsers); a database that fails to open behaves as an empty store.
260
+ */
261
+ declare function createIndexedDbByteStore(options?: IndexedDbByteStoreOptions): ByteStore | undefined;
262
+ //#endregion
263
+ export { type ArchiveKey, type ArchivePriority, type BlockCache, type BlockCacheOptions, type BlockCacheStats, type ByteStore, type ByteStoreStats, type CachedArchive, type CachedRead, type CachingRangeReaderOptions, DOCUMENT_CACHE_HEADER, type DocumentCacheOptions, type IndexedDbByteStoreOptions, PRIORITY_BIAS_MS, type RangeReader, type RangeSourceLike, type RangeSourceResponse, type SpanReader, type StoredArchive, type StoredArchiveInfo, type UrlRangeReaderOptions, cachingDocumentFetch, cachingRangeReader, cachingRangeSource, createBlockCache, createIndexedDbByteStore, createMemoryByteStore, evictionRank, isChangedError, urlRangeReader };