@drupflare/spool 0.3.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +279 -1607
  2. package/dist/angular/index.d.ts +13 -0
  3. package/dist/angular/index.js +25 -0
  4. package/dist/angular/pages.d.ts +28 -0
  5. package/dist/angular/pages.js +85 -0
  6. package/dist/cdn/client.js +1 -1
  7. package/dist/cdn/location.d.ts +6 -0
  8. package/dist/cdn/location.js +8 -0
  9. package/dist/cdn/manifest.json +7 -7
  10. package/dist/cdn/spool-sw.js +1 -1
  11. package/dist/client/bootstrap.d.ts +57 -0
  12. package/dist/client/bootstrap.js +307 -0
  13. package/dist/client/contact.d.ts +24 -0
  14. package/dist/client/contact.js +66 -0
  15. package/dist/client/declared.d.ts +23 -0
  16. package/dist/client/declared.js +68 -0
  17. package/dist/client/entry.js +123 -2
  18. package/dist/client/gate.d.ts +1 -0
  19. package/dist/client/gate.js +148 -0
  20. package/dist/client/index.d.ts +25 -0
  21. package/dist/client/index.js +1 -0
  22. package/dist/client/live.d.ts +66 -0
  23. package/dist/client/live.js +225 -0
  24. package/dist/client/navigate-entry.d.ts +1 -0
  25. package/dist/client/navigate-entry.js +4 -0
  26. package/dist/client/navigate.d.ts +54 -0
  27. package/dist/client/navigate.js +272 -0
  28. package/dist/client/persist.d.ts +2 -0
  29. package/dist/client/persist.js +9 -0
  30. package/dist/client/register.d.ts +34 -2
  31. package/dist/client/register.js +64 -7
  32. package/dist/express/index.d.ts +11 -0
  33. package/dist/express/index.js +209 -0
  34. package/dist/hono/index.d.ts +19 -0
  35. package/dist/hono/index.js +49 -0
  36. package/dist/next/index.d.ts +33 -0
  37. package/dist/next/index.js +41 -0
  38. package/dist/next/pages.d.ts +22 -0
  39. package/dist/next/pages.js +39 -0
  40. package/dist/node/listener.d.ts +3 -0
  41. package/dist/node/listener.js +57 -2
  42. package/dist/nuxt/index.d.ts +22 -0
  43. package/dist/nuxt/index.js +104 -0
  44. package/dist/nuxt/pages.d.ts +26 -0
  45. package/dist/nuxt/pages.js +48 -0
  46. package/dist/nuxt/runtime/index.d.ts +24 -0
  47. package/dist/nuxt/runtime/index.js +58 -0
  48. package/dist/plugins/coverage.d.ts +14 -0
  49. package/dist/plugins/coverage.js +27 -0
  50. package/dist/plugins/mode.d.ts +19 -0
  51. package/dist/plugins/mode.js +20 -0
  52. package/dist/plugins/path.d.ts +16 -0
  53. package/dist/plugins/path.js +139 -0
  54. package/dist/plugins/report.d.ts +22 -0
  55. package/dist/plugins/report.js +18 -0
  56. package/dist/plugins/routes.d.ts +38 -0
  57. package/dist/plugins/routes.js +52 -0
  58. package/dist/plugins/static.d.ts +42 -0
  59. package/dist/plugins/static.js +97 -0
  60. package/dist/protocol/bundle.d.ts +111 -0
  61. package/dist/protocol/bundle.js +428 -0
  62. package/dist/protocol/config.d.ts +80 -1
  63. package/dist/protocol/config.js +31 -1
  64. package/dist/protocol/dataset.d.ts +189 -0
  65. package/dist/protocol/dataset.js +212 -0
  66. package/dist/protocol/envelope.d.ts +34 -1
  67. package/dist/protocol/envelope.js +16 -1
  68. package/dist/protocol/errors.d.ts +1 -1
  69. package/dist/protocol/graph.d.ts +23 -4
  70. package/dist/protocol/graph.js +34 -3
  71. package/dist/protocol/index-rows.d.ts +22 -0
  72. package/dist/protocol/index-rows.js +53 -0
  73. package/dist/protocol/index.d.ts +5 -0
  74. package/dist/protocol/index.js +5 -0
  75. package/dist/protocol/normalise.d.ts +4 -4
  76. package/dist/protocol/normalise.js +9 -5
  77. package/dist/protocol/plan.d.ts +100 -0
  78. package/dist/protocol/plan.js +661 -0
  79. package/dist/protocol/provision.d.ts +46 -0
  80. package/dist/protocol/provision.js +38 -0
  81. package/dist/protocol/query.d.ts +66 -5
  82. package/dist/protocol/query.js +103 -4
  83. package/dist/protocol/ring.d.ts +58 -0
  84. package/dist/protocol/ring.js +6 -0
  85. package/dist/protocol/route.d.ts +63 -13
  86. package/dist/protocol/route.js +126 -6
  87. package/dist/protocol/tag.d.ts +22 -2
  88. package/dist/protocol/tag.js +25 -4
  89. package/dist/protocol/view.d.ts +2 -0
  90. package/dist/protocol/view.js +8 -2
  91. package/dist/protocol/views.d.ts +6 -0
  92. package/dist/protocol/views.js +15 -0
  93. package/dist/protocol/write.d.ts +234 -0
  94. package/dist/protocol/write.js +343 -0
  95. package/dist/proxy/eligible.d.ts +22 -1
  96. package/dist/proxy/eligible.js +62 -15
  97. package/dist/proxy/proxy.js +28 -18
  98. package/dist/react-router/index.d.ts +20 -0
  99. package/dist/react-router/index.js +60 -0
  100. package/dist/react-router/pages.d.ts +34 -0
  101. package/dist/react-router/pages.js +83 -0
  102. package/dist/report/dataset.d.ts +48 -0
  103. package/dist/report/dataset.js +93 -0
  104. package/dist/report/demand.d.ts +49 -0
  105. package/dist/report/demand.js +60 -0
  106. package/dist/report/derive.d.ts +285 -0
  107. package/dist/report/derive.js +676 -1
  108. package/dist/report/economics.d.ts +66 -1
  109. package/dist/report/economics.js +208 -1
  110. package/dist/report/headline.d.ts +158 -0
  111. package/dist/report/headline.js +473 -0
  112. package/dist/report/index.d.ts +4 -0
  113. package/dist/report/index.js +4 -0
  114. package/dist/report/inputs.d.ts +176 -0
  115. package/dist/report/inputs.js +248 -0
  116. package/dist/report/institution.d.ts +188 -1
  117. package/dist/report/institution.js +146 -0
  118. package/dist/report/loops.d.ts +73 -0
  119. package/dist/report/loops.js +101 -0
  120. package/dist/report/measurement.d.ts +1 -1
  121. package/dist/report/measurement.js +7 -1
  122. package/dist/report/model.d.ts +2 -0
  123. package/dist/report/model.js +4 -0
  124. package/dist/report/thirdparty.d.ts +57 -0
  125. package/dist/report/thirdparty.js +111 -0
  126. package/dist/report/traffic.d.ts +8 -0
  127. package/dist/report/traffic.js +7 -0
  128. package/dist/report/trim.d.ts +56 -0
  129. package/dist/report/trim.js +83 -1
  130. package/dist/server/bundle.d.ts +55 -0
  131. package/dist/server/bundle.js +172 -0
  132. package/dist/server/dataset.d.ts +54 -0
  133. package/dist/server/dataset.js +355 -0
  134. package/dist/server/embedded.d.ts +7 -3
  135. package/dist/server/embedded.js +7 -3
  136. package/dist/server/index.d.ts +6 -0
  137. package/dist/server/index.js +6 -0
  138. package/dist/server/live-sockets.d.ts +54 -0
  139. package/dist/server/live-sockets.js +131 -0
  140. package/dist/server/live.d.ts +146 -0
  141. package/dist/server/live.js +228 -0
  142. package/dist/server/ring.d.ts +77 -0
  143. package/dist/server/ring.js +311 -0
  144. package/dist/server/spool.d.ts +158 -3
  145. package/dist/server/spool.js +578 -41
  146. package/dist/server/static.d.ts +42 -0
  147. package/dist/server/static.js +29 -0
  148. package/dist/server/stub.d.ts +43 -5
  149. package/dist/server/stub.js +126 -8
  150. package/dist/server/write.d.ts +49 -0
  151. package/dist/server/write.js +223 -0
  152. package/dist/stores/durable-object.d.ts +46 -0
  153. package/dist/stores/durable-object.js +114 -0
  154. package/dist/sw/analytics.d.ts +7 -1
  155. package/dist/sw/analytics.js +16 -2
  156. package/dist/sw/coverage.d.ts +6 -0
  157. package/dist/sw/coverage.js +34 -0
  158. package/dist/sw/dataset-sync.d.ts +76 -0
  159. package/dist/sw/dataset-sync.js +149 -0
  160. package/dist/sw/dataset.d.ts +91 -0
  161. package/dist/sw/dataset.js +318 -0
  162. package/dist/sw/family.d.ts +15 -0
  163. package/dist/sw/family.js +48 -0
  164. package/dist/sw/graph/index.d.ts +31 -1
  165. package/dist/sw/graph/index.js +57 -5
  166. package/dist/sw/graph/invalidate.d.ts +15 -3
  167. package/dist/sw/graph/invalidate.js +42 -4
  168. package/dist/sw/idb.d.ts +2 -0
  169. package/dist/sw/idb.js +80 -11
  170. package/dist/sw/index.d.ts +2 -0
  171. package/dist/sw/index.js +1 -0
  172. package/dist/sw/pack/store.d.ts +128 -0
  173. package/dist/sw/pack/store.js +737 -0
  174. package/dist/sw/partition.d.ts +6 -1
  175. package/dist/sw/partition.js +21 -2
  176. package/dist/sw/policy.d.ts +2 -0
  177. package/dist/sw/policy.js +11 -0
  178. package/dist/sw/principal-dataset.d.ts +64 -0
  179. package/dist/sw/principal-dataset.js +254 -0
  180. package/dist/sw/probe.d.ts +16 -0
  181. package/dist/sw/probe.js +25 -0
  182. package/dist/sw/render.d.ts +7 -2
  183. package/dist/sw/render.js +19 -5
  184. package/dist/sw/replica.d.ts +12 -3
  185. package/dist/sw/replica.js +40 -5
  186. package/dist/sw/router.d.ts +3 -2
  187. package/dist/sw/router.js +15 -2
  188. package/dist/sw/runtime.d.ts +90 -2
  189. package/dist/sw/runtime.js +857 -61
  190. package/dist/sw/scheduler.d.ts +6 -1
  191. package/dist/sw/scheduler.js +22 -5
  192. package/dist/sw/seal.d.ts +13 -0
  193. package/dist/sw/seal.js +13 -4
  194. package/dist/sw/static.d.ts +18 -0
  195. package/dist/sw/static.js +84 -1
  196. package/dist/sw/storage.d.ts +51 -1
  197. package/dist/sw/storage.js +33 -1
  198. package/dist/sw/write.d.ts +147 -0
  199. package/dist/sw/write.js +625 -0
  200. package/dist/version.d.ts +1 -1
  201. package/dist/version.js +1 -2
  202. package/dist/vite/index.d.ts +26 -0
  203. package/dist/vite/index.js +66 -0
  204. package/dist/vue/index.d.ts +36 -0
  205. package/dist/vue/index.js +53 -0
  206. package/package.json +65 -4
@@ -0,0 +1,737 @@
1
+ import { BLOCK_ROWS, buildPack, bundleDigest, compactionDue, readBlock, readFrames } from '../../protocol/bundle.js';
2
+ import { ProtocolError } from '../../protocol/errors.js';
3
+ import { canonicalJson, digestAdd, digestSub, EMPTY_DIGEST, rowHash } from '../../protocol/query.js';
4
+ import { done, wrap } from '../idb.js';
5
+ const STORES = ['blocks', 'index', 'overlay', 'head', 'templates', 'predicted'];
6
+ const ACTIVE = 'active';
7
+ const INDEX_FRAME = -1;
8
+ const DIGEST_FORM = /^[0-9a-f]{64}$/;
9
+ /** bytes kept free past an install's own size; a default, not measured */
10
+ export const QUOTA_MARGIN_BYTES = 8 * 1024 * 1024;
11
+ const encoder = new TextEncoder();
12
+ const decoder = new TextDecoder();
13
+ function refuse(message) {
14
+ throw new ProtocolError('bad-sync', message);
15
+ }
16
+ const isCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
17
+ function parseIndex(payload) {
18
+ let value;
19
+ try {
20
+ value = JSON.parse(decoder.decode(payload));
21
+ }
22
+ catch {
23
+ refuse('the index is not JSON');
24
+ }
25
+ if (typeof value !== 'object' || value === null)
26
+ refuse('the index is not an object');
27
+ const index = value;
28
+ if (!isCount(index.version) ||
29
+ index.codec !== 'gzip' ||
30
+ !isCount(index.blockRows) ||
31
+ index.blockRows < 1 ||
32
+ typeof index.digest !== 'string' ||
33
+ !DIGEST_FORM.test(index.digest) ||
34
+ !Array.isArray(index.tables)) {
35
+ refuse('the index has a bad version, codec, block size or digest');
36
+ }
37
+ const names = new Set();
38
+ for (const entry of index.tables) {
39
+ if (typeof entry.name !== 'string' ||
40
+ names.has(entry.name) ||
41
+ !isCount(entry.rows) ||
42
+ entry.rows < 1 ||
43
+ typeof entry.first !== 'string' ||
44
+ !Array.isArray(entry.blocks) ||
45
+ entry.blocks.length !== Math.ceil(entry.rows / index.blockRows) ||
46
+ !entry.blocks.every((hash) => typeof hash === 'string')) {
47
+ refuse('an index table does not match its rows and blocks');
48
+ }
49
+ names.add(entry.name);
50
+ }
51
+ return index;
52
+ }
53
+ /** the block whose first and last keys hold `key`, or -1 */
54
+ function blockFor(record, key) {
55
+ let low = 0;
56
+ let high = record.first.length - 1;
57
+ let found = -1;
58
+ while (low <= high) {
59
+ const mid = (low + high) >> 1;
60
+ if ((record.first[mid] ?? '') <= key) {
61
+ found = mid;
62
+ low = mid + 1;
63
+ }
64
+ else
65
+ high = mid - 1;
66
+ }
67
+ return found >= 0 && key <= (record.last[found] ?? '') ? found : -1;
68
+ }
69
+ function find(rows, key) {
70
+ let low = 0;
71
+ let high = rows.length - 1;
72
+ while (low <= high) {
73
+ const mid = (low + high) >> 1;
74
+ const entry = rows[mid];
75
+ if (entry?.[0] === key)
76
+ return entry[1];
77
+ if ((entry?.[0] ?? '') < key)
78
+ low = mid + 1;
79
+ else
80
+ high = mid - 1;
81
+ }
82
+ return undefined;
83
+ }
84
+ async function sha256Url(bytes) {
85
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
86
+ let binary = '';
87
+ for (const byte of digest)
88
+ binary += String.fromCharCode(byte);
89
+ return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
90
+ }
91
+ async function gzip(bytes) {
92
+ const piped = new Response(bytes).body?.pipeThrough(new CompressionStream('gzip'));
93
+ if (!piped)
94
+ refuse('no body to compress');
95
+ return new Uint8Array(await new Response(piped).arrayBuffer());
96
+ }
97
+ async function* once(bytes) {
98
+ yield bytes;
99
+ }
100
+ /** the overlay's changes grouped by table; each list stays in key order */
101
+ function byTable(entries) {
102
+ const out = new Map();
103
+ for (const [table, key, row] of entries) {
104
+ const list = out.get(table) ?? [];
105
+ list.push([key, row]);
106
+ out.set(table, list);
107
+ }
108
+ return out;
109
+ }
110
+ /** `above` laid over `below`, both in key order; a key in both takes the `above` change */
111
+ function laid(below, above) {
112
+ const out = new Map(below);
113
+ for (const [key, row] of above)
114
+ out.set(key, row);
115
+ return [...out].sort(([a], [b]) => (a < b ? -1 : 1));
116
+ }
117
+ function deltaTables(changes) {
118
+ const tables = {};
119
+ for (const [table, list] of changes) {
120
+ tables[table] = {
121
+ puts: list.flatMap(([key, row]) => (row === null ? [] : [[key, row]])),
122
+ deletes: list.flatMap(([key, row]) => (row === null ? [key] : []))
123
+ };
124
+ }
125
+ return tables;
126
+ }
127
+ async function browserEstimate() {
128
+ const browser = globalThis.navigator;
129
+ return browser?.storage?.estimate();
130
+ }
131
+ /**
132
+ * A device's site pack in IndexedDB `spool-bundle`: compressed blocks of sorted rows, an overlay of
133
+ * changed rows, and one head record that names the active pack. A read inflates at most one block.
134
+ */
135
+ export function createPackStore(factory = indexedDB, options = {}) {
136
+ const name = options.name ?? 'spool-bundle';
137
+ const estimate = options.estimate ?? browserEstimate;
138
+ const counters = { blockReads: 0, blockInflates: 0, overlayReads: 0 };
139
+ const cache = new Map();
140
+ let opened;
141
+ let queue = Promise.resolve();
142
+ const open = () => (opened ??= new Promise((resolve, reject) => {
143
+ const request = factory.open(name, 2);
144
+ request.onupgradeneeded = () => {
145
+ for (const store of STORES) {
146
+ if (!request.result.objectStoreNames.contains(store)) {
147
+ request.result.createObjectStore(store);
148
+ }
149
+ }
150
+ };
151
+ request.onsuccess = () => {
152
+ const db = request.result;
153
+ db.onversionchange = () => {
154
+ db.close();
155
+ opened = undefined;
156
+ };
157
+ resolve(db);
158
+ };
159
+ request.onerror = () => reject(request.error);
160
+ }));
161
+ async function run(stores, mode, body) {
162
+ const db = await open();
163
+ const t = db.transaction(stores, mode);
164
+ const finished = done(t);
165
+ finished.catch(() => undefined);
166
+ try {
167
+ const result = await body(t);
168
+ await finished;
169
+ return result;
170
+ }
171
+ catch (error) {
172
+ try {
173
+ t.abort();
174
+ }
175
+ catch {
176
+ // already finished
177
+ }
178
+ throw error;
179
+ }
180
+ }
181
+ // installs, deltas and compactions run one at a time; reads do not wait on them
182
+ const exclusive = (work) => {
183
+ const next = queue.then(work);
184
+ queue = next.catch(() => undefined);
185
+ return next;
186
+ };
187
+ const readHead = () => run(['head', 'predicted'], 'readonly', async (t) => {
188
+ const head = await wrap(t.objectStore('head').get(ACTIVE));
189
+ if (!head)
190
+ return undefined;
191
+ return { ...head, unconfirmed: await wrap(t.objectStore('predicted').count()) };
192
+ });
193
+ const readIndex = (generation, table) => run(['index'], 'readonly', (t) => wrap(t.objectStore('index').get([generation, table])));
194
+ async function overlayIn(t, range, layer = 'overlay') {
195
+ const store = t.objectStore(layer);
196
+ const [keys, values] = await Promise.all([
197
+ wrap(store.getAllKeys(range)),
198
+ wrap(store.getAll(range))
199
+ ]);
200
+ if (layer === 'overlay')
201
+ counters.overlayReads += keys.length;
202
+ return keys.map((key, i) => {
203
+ const [table, id] = key;
204
+ return [table, id, values[i]];
205
+ });
206
+ }
207
+ function inflate(head, table, record, ordinal, payload) {
208
+ counters.blockInflates++;
209
+ return readBlock({
210
+ version: head.version,
211
+ codec: 'gzip',
212
+ blockRows: head.blockRows,
213
+ digest: head.digest,
214
+ tables: [
215
+ {
216
+ name: table,
217
+ rows: record.rows,
218
+ first: record.first[0] ?? null,
219
+ blocks: record.hashes
220
+ }
221
+ ]
222
+ }, { table: 0, ordinal, payload });
223
+ }
224
+ async function blockAt(head, table, record, ordinal) {
225
+ const payload = await run(['head', 'blocks'], 'readonly', async (t) => {
226
+ const now = await wrap(t.objectStore('head').get(ACTIVE));
227
+ if (now?.generation !== head.generation)
228
+ refuse('the pack changed during a read');
229
+ counters.blockReads++;
230
+ return wrap(t.objectStore('blocks').get([head.generation, table, ordinal]));
231
+ });
232
+ if (!payload)
233
+ refuse(`block ${table}/${ordinal} is missing`);
234
+ return inflate(head, table, record, ordinal, payload);
235
+ }
236
+ /** a table's rows in key order: each block with the changes laid over it */
237
+ async function* rowsOf(head, record, table, changes) {
238
+ const count = record?.hashes.length ?? 0;
239
+ let at = 0;
240
+ for (let ordinal = 0; ordinal <= count; ordinal++) {
241
+ const last = record && ordinal < count;
242
+ const base = last ? await blockAt(head, table, record, ordinal) : [];
243
+ const end = last ? record.last[ordinal] : undefined;
244
+ let b = 0;
245
+ for (;;) {
246
+ const change = changes[at];
247
+ const next = base[b];
248
+ if (change &&
249
+ (end === undefined || change[0] <= end) &&
250
+ (!next || change[0] <= next[0])) {
251
+ at++;
252
+ if (next && next[0] === change[0])
253
+ b++;
254
+ if (change[1] !== null)
255
+ yield [change[0], change[1]];
256
+ }
257
+ else if (next) {
258
+ b++;
259
+ yield next;
260
+ }
261
+ else
262
+ break;
263
+ }
264
+ }
265
+ }
266
+ async function lookup(table, key, predicted) {
267
+ return run(['head', 'index', 'overlay', 'predicted', 'blocks'], 'readonly', async (t) => {
268
+ const head = await wrap(t.objectStore('head').get(ACTIVE));
269
+ if (!head)
270
+ return undefined;
271
+ if (predicted) {
272
+ const guess = await wrap(t.objectStore('predicted').get([table, key]));
273
+ if (guess !== undefined)
274
+ return guess ?? undefined;
275
+ }
276
+ counters.overlayReads++;
277
+ const held = await wrap(t.objectStore('overlay').get([table, key]));
278
+ if (held !== undefined)
279
+ return held ?? undefined;
280
+ if (!head.tables.includes(table))
281
+ return undefined;
282
+ const record = await wrap(t.objectStore('index').get([head.generation, table]));
283
+ const ordinal = record ? blockFor(record, key) : -1;
284
+ if (!record || ordinal < 0)
285
+ return undefined;
286
+ const hit = cache.get(table);
287
+ if (hit?.generation === head.generation && hit.ordinal === ordinal) {
288
+ return find(hit.rows, key);
289
+ }
290
+ const payload = await wrap(t.objectStore('blocks').get([head.generation, table, ordinal]));
291
+ counters.blockReads++;
292
+ if (!payload)
293
+ refuse(`block ${table}/${ordinal} is missing`);
294
+ const rows = await inflate(head, table, record, ordinal, payload);
295
+ cache.set(table, { generation: head.generation, ordinal, rows });
296
+ return find(rows, key);
297
+ });
298
+ }
299
+ const getRow = (table, key) => lookup(table, key, true);
300
+ async function scanTable(table) {
301
+ const snap = await run(['head', 'index', 'overlay', 'predicted'], 'readonly', async (t) => {
302
+ const head = await wrap(t.objectStore('head').get(ACTIVE));
303
+ if (!head)
304
+ return undefined;
305
+ const record = head.tables.includes(table)
306
+ ? await wrap(t.objectStore('index').get([head.generation, table]))
307
+ : undefined;
308
+ const range = IDBKeyRange.bound([table], [table, []]);
309
+ const entries = await overlayIn(t, range);
310
+ const predicted = await overlayIn(t, range, 'predicted');
311
+ const changes = laid(byTable(entries).get(table) ?? [], byTable(predicted).get(table) ?? []);
312
+ return { head, record, changes };
313
+ });
314
+ if (!snap)
315
+ return [];
316
+ const out = [];
317
+ for await (const row of rowsOf(snap.head, snap.record, table, snap.changes))
318
+ out.push(row);
319
+ return out;
320
+ }
321
+ function dropStaged(generation) {
322
+ return run(['blocks', 'index'], 'readwrite', (t) => {
323
+ const range = IDBKeyRange.lowerBound([generation]);
324
+ t.objectStore('blocks').delete(range);
325
+ t.objectStore('index').delete(range);
326
+ });
327
+ }
328
+ /**
329
+ * Makes `next` the active head and drops the records of `previous`. The head must still be
330
+ * `previous` when the swap commits.
331
+ */
332
+ function swap(previous, next) {
333
+ cache.clear();
334
+ return run(['head', 'blocks', 'index', 'overlay', 'predicted'], 'readwrite', async (t) => {
335
+ const current = await wrap(t.objectStore('head').get(ACTIVE));
336
+ if (current?.generation !== previous?.generation)
337
+ refuse('the head moved during a swap');
338
+ if (previous) {
339
+ const old = IDBKeyRange.bound([previous.generation], [previous.generation, []]);
340
+ t.objectStore('blocks').delete(old);
341
+ t.objectStore('index').delete(old);
342
+ }
343
+ t.objectStore('overlay').clear();
344
+ t.objectStore('predicted').clear();
345
+ t.objectStore('head').put(next, ACTIVE);
346
+ });
347
+ }
348
+ /** writes a pack's blocks under a new generation; the head does not point at them yet */
349
+ async function stage(source, generation, held) {
350
+ let index;
351
+ const records = [];
352
+ let table = 0;
353
+ let ordinal = 0;
354
+ let seen = 0;
355
+ let rows = 0;
356
+ let blocks = 0;
357
+ let previous;
358
+ let digest = EMPTY_DIGEST;
359
+ for await (const frame of readFrames(source)) {
360
+ if (!index) {
361
+ if (frame.table !== INDEX_FRAME)
362
+ refuse('the pack does not start with its index');
363
+ index = parseIndex(frame.payload);
364
+ if (held && index.version < held.version) {
365
+ refuse(`the pack is version ${index.version}, the head holds ${held.version}`);
366
+ }
367
+ for (const entry of index.tables) {
368
+ records.push({ rows: entry.rows, first: [], last: [], hashes: [] });
369
+ }
370
+ continue;
371
+ }
372
+ const entry = index.tables[table];
373
+ if (!entry || frame.table !== table || frame.ordinal !== ordinal) {
374
+ refuse(`expected block ${table}/${ordinal}, got ${frame.table}/${frame.ordinal}`);
375
+ }
376
+ const block = await readBlock(index, frame);
377
+ const size = Math.min(index.blockRows, entry.rows - ordinal * index.blockRows);
378
+ if (block.length !== size)
379
+ refuse(`block ${table}/${ordinal} holds ${block.length} rows`);
380
+ if (ordinal === 0)
381
+ previous = undefined;
382
+ for (const [key, row] of block) {
383
+ if (previous !== undefined && previous >= key) {
384
+ refuse(`${entry.name}: key ${key} is out of order`);
385
+ }
386
+ previous = key;
387
+ digest = digestAdd(digest, await rowHash(JSON.stringify([entry.name, key]), row));
388
+ }
389
+ const first = block[0]?.[0] ?? '';
390
+ if (ordinal === 0 && first !== entry.first)
391
+ refuse(`${entry.name}: wrong first key`);
392
+ records[table]?.first.push(first);
393
+ records[table]?.last.push(block[block.length - 1]?.[0] ?? '');
394
+ records[table]?.hashes.push(entry.blocks[ordinal] ?? '');
395
+ await run(['blocks'], 'readwrite', (t) => wrap(t.objectStore('blocks').put(frame.payload, [generation, entry.name, ordinal])));
396
+ seen += block.length;
397
+ rows += block.length;
398
+ blocks++;
399
+ if (ordinal + 1 < entry.blocks.length) {
400
+ ordinal++;
401
+ continue;
402
+ }
403
+ if (seen !== entry.rows)
404
+ refuse(`${entry.name}: ${seen} rows, the index says ${entry.rows}`);
405
+ table++;
406
+ ordinal = 0;
407
+ seen = 0;
408
+ }
409
+ if (!index)
410
+ refuse('the pack is empty');
411
+ if (table !== index.tables.length)
412
+ refuse('the pack ends before its last block');
413
+ if (digest !== index.digest)
414
+ refuse('the set digest does not match');
415
+ const parsed = index;
416
+ await run(['index'], 'readwrite', (t) => {
417
+ parsed.tables.forEach((entry, i) => t.objectStore('index').put(records[i], [generation, entry.name]));
418
+ });
419
+ return { index: parsed, rows, blocks };
420
+ }
421
+ async function estimateBeforeInstall(bytes) {
422
+ const storage = await estimate();
423
+ if (storage?.quota === undefined)
424
+ return { checked: false, fits: true };
425
+ const usage = storage.usage ?? 0;
426
+ if (usage + bytes + QUOTA_MARGIN_BYTES <= storage.quota)
427
+ return { checked: true, fits: true };
428
+ return { checked: true, fits: false, bytes, usage, quota: storage.quota };
429
+ }
430
+ /** stages a pack under a new generation and swaps it in; the caller holds the queue */
431
+ async function installSource(source) {
432
+ const head = await readHead();
433
+ const generation = (head?.generation ?? 0) + 1;
434
+ await dropStaged(generation);
435
+ try {
436
+ const { index, rows, blocks } = await stage(source, generation, head);
437
+ await swap(head, {
438
+ version: index.version,
439
+ digest: index.digest,
440
+ generation,
441
+ blockRows: index.blockRows,
442
+ codec: 'gzip',
443
+ tables: index.tables.map((entry) => entry.name)
444
+ });
445
+ return {
446
+ installed: true,
447
+ version: index.version,
448
+ tables: index.tables.length,
449
+ rows,
450
+ blocks
451
+ };
452
+ }
453
+ catch (error) {
454
+ await dropStaged(generation);
455
+ throw error;
456
+ }
457
+ }
458
+ function installPack(source, options = {}) {
459
+ return exclusive(async () => {
460
+ if (options.bytes !== undefined) {
461
+ const fit = await estimateBeforeInstall(options.bytes);
462
+ if (!fit.fits)
463
+ return { installed: false, fit };
464
+ }
465
+ return installSource(source);
466
+ });
467
+ }
468
+ /**
469
+ * A ring of the same version as a complete head is `held`, one older is `older`, and neither
470
+ * touches the store. A partial ring replaces the head and its overlay in one transaction, and a
471
+ * complete ring is checked against its own rows and installed as a pack.
472
+ */
473
+ function installRing(ring) {
474
+ return exclusive(async () => {
475
+ const head = await readHead();
476
+ if (head && ring.version < head.version)
477
+ return { installed: false, reason: 'older' };
478
+ if (head && !head.partial && ring.version === head.version) {
479
+ return { installed: false, reason: 'held' };
480
+ }
481
+ const rows = Object.values(ring.tables).reduce((sum, list) => sum + list.length, 0);
482
+ if (ring.complete) {
483
+ if ((await bundleDigest(ring.tables)) !== ring.digest) {
484
+ refuse('the ring digest does not match its rows');
485
+ }
486
+ const { bytes } = await buildPack({ version: ring.version, tables: ring.tables });
487
+ if (ring.routes)
488
+ await putRoutes(ring.routes, ring.templates);
489
+ const { version, rows: installed } = await installSource(once(bytes));
490
+ return { installed: true, version, rows: installed, complete: true };
491
+ }
492
+ if (ring.routes)
493
+ await putRoutes(ring.routes, ring.templates);
494
+ const generation = (head?.generation ?? 0) + 1;
495
+ await run(['head', 'blocks', 'index', 'overlay', 'predicted'], 'readwrite', async (t) => {
496
+ const current = await wrap(t.objectStore('head').get(ACTIVE));
497
+ if (current?.generation !== head?.generation)
498
+ refuse('the head moved during a ring');
499
+ if (head) {
500
+ const old = IDBKeyRange.bound([head.generation], [head.generation, []]);
501
+ t.objectStore('blocks').delete(old);
502
+ t.objectStore('index').delete(old);
503
+ }
504
+ t.objectStore('overlay').clear();
505
+ t.objectStore('predicted').clear();
506
+ for (const [table, list] of Object.entries(ring.tables)) {
507
+ for (const [key, row] of list)
508
+ t.objectStore('overlay').put(row, [table, key]);
509
+ }
510
+ t.objectStore('head').put({
511
+ version: ring.version,
512
+ digest: ring.digest,
513
+ generation,
514
+ blockRows: BLOCK_ROWS,
515
+ codec: 'gzip',
516
+ tables: Object.keys(ring.tables),
517
+ partial: true
518
+ }, ACTIVE);
519
+ });
520
+ cache.clear();
521
+ return { installed: true, version: ring.version, rows, complete: false };
522
+ });
523
+ }
524
+ function applyDelta(delta) {
525
+ return exclusive(() => applyDeltaNow(delta));
526
+ }
527
+ /** the caller holds the queue */
528
+ async function applyDeltaNow(delta) {
529
+ const head = await readHead();
530
+ if (!head)
531
+ refuse('no pack is installed');
532
+ if (head.partial)
533
+ refuse('the held pack is partial; install the pack first');
534
+ if (head.version !== delta.from) {
535
+ refuse(`the delta starts at ${delta.from}, the store holds ${head.version}`);
536
+ }
537
+ if (delta.to < delta.from)
538
+ refuse(`the delta ends at ${delta.to}, before it starts at ${delta.from}`);
539
+ // the set digest moves by the rows that leave and the rows that arrive
540
+ const without = async (digest, table, key) => {
541
+ const old = await lookup(table, key, false);
542
+ if (old === undefined)
543
+ return digest;
544
+ return digestSub(digest, await rowHash(JSON.stringify([table, key]), old));
545
+ };
546
+ let digest = head.digest;
547
+ for (const [table, change] of Object.entries(delta.tables)) {
548
+ for (const key of change.deletes)
549
+ digest = await without(digest, table, key);
550
+ for (const [key, row] of change.puts) {
551
+ digest = digestAdd(await without(digest, table, key), await rowHash(JSON.stringify([table, key]), row));
552
+ }
553
+ }
554
+ if (digest !== delta.digest)
555
+ refuse('the candidate digest does not match the delta');
556
+ await run(['head', 'overlay', 'predicted'], 'readwrite', async (t) => {
557
+ // a verified delta settles the rows it covers; one that moves the version drops the rest
558
+ const predicted = t.objectStore('predicted');
559
+ for (const [table, change] of Object.entries(delta.tables)) {
560
+ for (const key of change.deletes)
561
+ predicted.delete([table, key]);
562
+ for (const [key] of change.puts)
563
+ predicted.delete([table, key]);
564
+ }
565
+ if (delta.to !== delta.from)
566
+ predicted.clear();
567
+ const current = await wrap(t.objectStore('head').get(ACTIVE));
568
+ if (!current || current.version !== head.version || current.digest !== head.digest) {
569
+ refuse('the base moved during a delta');
570
+ }
571
+ for (const [table, change] of Object.entries(delta.tables)) {
572
+ for (const key of change.deletes)
573
+ t.objectStore('overlay').put(null, [table, key]);
574
+ for (const [key, row] of change.puts)
575
+ t.objectStore('overlay').put(row, [table, key]);
576
+ }
577
+ t.objectStore('head').put({ ...current, version: delta.to, digest: delta.digest }, ACTIVE);
578
+ });
579
+ }
580
+ /**
581
+ * Lays predicted rows over the held pack. Nothing is checked: the rows wait until the next verified
582
+ * delta confirms or drops them, and a compaction waits for them too.
583
+ */
584
+ function applyPredicted(predicted) {
585
+ return exclusive(async () => {
586
+ if (!(await readHead()))
587
+ return;
588
+ await run(['predicted'], 'readwrite', (t) => {
589
+ const store = t.objectStore('predicted');
590
+ for (const [table, key, row] of predicted.puts)
591
+ store.put(row, [table, key]);
592
+ for (const [table, key] of predicted.deletes)
593
+ store.put(null, [table, key]);
594
+ });
595
+ });
596
+ }
597
+ /**
598
+ * Confirms the predicted rows as the change from the held version to `to`. The change is checked like a
599
+ * verified delta: when the digest of the held rows with the predicted rows laid over them is not `digest`,
600
+ * it throws and the rows stay.
601
+ */
602
+ function confirmPredicted(to, digest) {
603
+ return exclusive(async () => {
604
+ const head = await readHead();
605
+ if (!head)
606
+ refuse('no pack is installed');
607
+ const changes = await run(['predicted'], 'readonly', (t) => overlayIn(t, IDBKeyRange.lowerBound(['']), 'predicted'));
608
+ await applyDeltaNow({
609
+ from: head.version,
610
+ to,
611
+ digest,
612
+ tables: deltaTables(byTable(changes))
613
+ });
614
+ });
615
+ }
616
+ /** removes every predicted row and keeps the version */
617
+ function dropPredicted() {
618
+ return exclusive(() => run(['predicted'], 'readwrite', (t) => {
619
+ t.objectStore('predicted').clear();
620
+ }));
621
+ }
622
+ function compact() {
623
+ return exclusive(async () => {
624
+ const head = await readHead();
625
+ if (!head || head.partial || head.unconfirmed > 0)
626
+ return false;
627
+ const changes = byTable(await run(['overlay'], 'readonly', (t) => overlayIn(t, IDBKeyRange.lowerBound(['']))));
628
+ const delta = {
629
+ from: head.version,
630
+ to: head.version,
631
+ digest: head.digest,
632
+ tables: deltaTables(changes)
633
+ };
634
+ if (!compactionDue(delta))
635
+ return false;
636
+ const generation = head.generation + 1;
637
+ await dropStaged(generation);
638
+ try {
639
+ const tables = [];
640
+ for (const table of [...new Set([...head.tables, ...changes.keys()])].sort()) {
641
+ const record = head.tables.includes(table)
642
+ ? await readIndex(head.generation, table)
643
+ : undefined;
644
+ const out = { rows: 0, first: [], last: [], hashes: [] };
645
+ let chunk = [];
646
+ const flush = async () => {
647
+ const payload = await gzip(new Uint8Array(encoder.encode(canonicalJson(chunk))));
648
+ const ordinal = out.hashes.length;
649
+ out.first.push(chunk[0]?.[0] ?? '');
650
+ out.last.push(chunk[chunk.length - 1]?.[0] ?? '');
651
+ out.hashes.push(await sha256Url(payload));
652
+ out.rows += chunk.length;
653
+ await run(['blocks'], 'readwrite', (t) => wrap(t.objectStore('blocks').put(payload, [generation, table, ordinal])));
654
+ chunk = [];
655
+ };
656
+ for await (const row of rowsOf(head, record, table, changes.get(table) ?? [])) {
657
+ chunk.push(row);
658
+ if (chunk.length === head.blockRows)
659
+ await flush();
660
+ }
661
+ if (chunk.length > 0)
662
+ await flush();
663
+ if (out.rows > 0) {
664
+ tables.push(table);
665
+ await run(['index'], 'readwrite', (t) => wrap(t.objectStore('index').put(out, [generation, table])));
666
+ }
667
+ }
668
+ await swap(head, {
669
+ version: head.version,
670
+ digest: head.digest,
671
+ generation,
672
+ blockRows: head.blockRows,
673
+ codec: 'gzip',
674
+ tables
675
+ });
676
+ return true;
677
+ }
678
+ catch (error) {
679
+ await dropStaged(generation);
680
+ throw error;
681
+ }
682
+ });
683
+ }
684
+ // getAll returns records by pattern, but routes match first-come
685
+ function putRoutes(routes, templates = {}) {
686
+ return run(['templates'], 'readwrite', (t) => {
687
+ const store = t.objectStore('templates');
688
+ store.clear();
689
+ routes.forEach((route, order) => {
690
+ const template = templates[route.pattern];
691
+ store.put(template ? { order, route, template } : { order, route }, route.pattern);
692
+ });
693
+ });
694
+ }
695
+ async function routeRecords() {
696
+ const records = await run(['templates'], 'readonly', (t) => wrap(t.objectStore('templates').getAll()));
697
+ return records.sort((a, b) => a.order - b.order);
698
+ }
699
+ async function getRoutes() {
700
+ return (await routeRecords()).map((record) => record.route);
701
+ }
702
+ async function getTemplates() {
703
+ const out = {};
704
+ for (const { route, template } of await routeRecords()) {
705
+ if (template)
706
+ out[route.pattern] = template;
707
+ }
708
+ return out;
709
+ }
710
+ return {
711
+ counters,
712
+ head: readHead,
713
+ recordCounts: () => run(['blocks', 'index', 'overlay', 'head'], 'readonly', async (t) => {
714
+ const count = (store) => wrap(t.objectStore(store).count());
715
+ const [blocks, index, overlay, head] = await Promise.all([
716
+ count('blocks'),
717
+ count('index'),
718
+ count('overlay'),
719
+ count('head')
720
+ ]);
721
+ return { blocks, index, overlay, head };
722
+ }),
723
+ installPack,
724
+ installRing,
725
+ estimateBeforeInstall,
726
+ applyDelta,
727
+ applyPredicted,
728
+ confirmPredicted,
729
+ dropPredicted,
730
+ getRow,
731
+ scanTable,
732
+ compact,
733
+ putRoutes,
734
+ getRoutes,
735
+ getTemplates
736
+ };
737
+ }