@drupflare/spool 0.4.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +206 -1756
  2. package/dist/angular/index.d.ts +13 -0
  3. package/dist/angular/index.js +25 -0
  4. package/dist/angular/pages.d.ts +28 -0
  5. package/dist/angular/pages.js +85 -0
  6. package/dist/cdn/client.js +1 -1
  7. package/dist/cdn/location.d.ts +6 -0
  8. package/dist/cdn/location.js +8 -0
  9. package/dist/cdn/manifest.json +7 -7
  10. package/dist/cdn/spool-sw.js +1 -1
  11. package/dist/client/bootstrap.d.ts +57 -0
  12. package/dist/client/bootstrap.js +307 -0
  13. package/dist/client/contact.d.ts +24 -0
  14. package/dist/client/contact.js +66 -0
  15. package/dist/client/declared.d.ts +23 -0
  16. package/dist/client/declared.js +68 -0
  17. package/dist/client/entry.js +91 -2
  18. package/dist/client/gate.d.ts +1 -0
  19. package/dist/client/gate.js +148 -0
  20. package/dist/client/index.d.ts +25 -0
  21. package/dist/client/index.js +1 -0
  22. package/dist/client/live.d.ts +66 -0
  23. package/dist/client/live.js +225 -0
  24. package/dist/client/navigate-entry.d.ts +1 -0
  25. package/dist/client/navigate-entry.js +4 -0
  26. package/dist/client/navigate.d.ts +54 -0
  27. package/dist/client/navigate.js +272 -0
  28. package/dist/client/persist.d.ts +2 -0
  29. package/dist/client/persist.js +9 -0
  30. package/dist/client/register.d.ts +11 -1
  31. package/dist/client/register.js +19 -8
  32. package/dist/express/index.d.ts +11 -0
  33. package/dist/express/index.js +209 -0
  34. package/dist/hono/index.d.ts +19 -0
  35. package/dist/hono/index.js +49 -0
  36. package/dist/next/index.d.ts +33 -0
  37. package/dist/next/index.js +41 -0
  38. package/dist/next/pages.d.ts +22 -0
  39. package/dist/next/pages.js +39 -0
  40. package/dist/node/listener.d.ts +3 -0
  41. package/dist/node/listener.js +57 -2
  42. package/dist/nuxt/index.d.ts +22 -0
  43. package/dist/nuxt/index.js +104 -0
  44. package/dist/nuxt/pages.d.ts +26 -0
  45. package/dist/nuxt/pages.js +48 -0
  46. package/dist/nuxt/runtime/index.d.ts +24 -0
  47. package/dist/nuxt/runtime/index.js +58 -0
  48. package/dist/plugins/coverage.d.ts +14 -0
  49. package/dist/plugins/coverage.js +27 -0
  50. package/dist/plugins/mode.d.ts +19 -0
  51. package/dist/plugins/mode.js +20 -0
  52. package/dist/plugins/path.d.ts +16 -0
  53. package/dist/plugins/path.js +139 -0
  54. package/dist/plugins/report.d.ts +22 -0
  55. package/dist/plugins/report.js +18 -0
  56. package/dist/plugins/routes.d.ts +38 -0
  57. package/dist/plugins/routes.js +52 -0
  58. package/dist/plugins/static.d.ts +42 -0
  59. package/dist/plugins/static.js +97 -0
  60. package/dist/protocol/bundle.d.ts +111 -0
  61. package/dist/protocol/bundle.js +428 -0
  62. package/dist/protocol/config.d.ts +65 -1
  63. package/dist/protocol/config.js +28 -0
  64. package/dist/protocol/dataset.d.ts +6 -0
  65. package/dist/protocol/dataset.js +6 -0
  66. package/dist/protocol/envelope.d.ts +33 -1
  67. package/dist/protocol/envelope.js +15 -1
  68. package/dist/protocol/errors.d.ts +1 -1
  69. package/dist/protocol/index-rows.d.ts +22 -0
  70. package/dist/protocol/index-rows.js +53 -0
  71. package/dist/protocol/index.d.ts +2 -0
  72. package/dist/protocol/index.js +2 -0
  73. package/dist/protocol/normalise.d.ts +4 -4
  74. package/dist/protocol/normalise.js +9 -5
  75. package/dist/protocol/ring.d.ts +58 -0
  76. package/dist/protocol/ring.js +6 -0
  77. package/dist/protocol/route.d.ts +57 -11
  78. package/dist/protocol/route.js +95 -3
  79. package/dist/protocol/tag.d.ts +22 -2
  80. package/dist/protocol/tag.js +25 -4
  81. package/dist/protocol/view.d.ts +2 -0
  82. package/dist/protocol/view.js +8 -2
  83. package/dist/protocol/views.d.ts +6 -0
  84. package/dist/protocol/views.js +15 -0
  85. package/dist/protocol/write.d.ts +92 -4
  86. package/dist/protocol/write.js +251 -9
  87. package/dist/proxy/eligible.d.ts +22 -1
  88. package/dist/proxy/eligible.js +62 -15
  89. package/dist/proxy/proxy.js +28 -18
  90. package/dist/react-router/index.d.ts +20 -0
  91. package/dist/react-router/index.js +60 -0
  92. package/dist/react-router/pages.d.ts +34 -0
  93. package/dist/react-router/pages.js +83 -0
  94. package/dist/report/demand.d.ts +49 -0
  95. package/dist/report/demand.js +60 -0
  96. package/dist/report/economics.d.ts +55 -0
  97. package/dist/report/economics.js +189 -0
  98. package/dist/report/headline.d.ts +158 -0
  99. package/dist/report/headline.js +473 -0
  100. package/dist/report/index.d.ts +2 -0
  101. package/dist/report/index.js +2 -0
  102. package/dist/report/inputs.d.ts +79 -0
  103. package/dist/report/inputs.js +115 -0
  104. package/dist/report/measurement.d.ts +1 -1
  105. package/dist/report/measurement.js +5 -1
  106. package/dist/report/traffic.d.ts +8 -0
  107. package/dist/report/traffic.js +7 -0
  108. package/dist/server/bundle.d.ts +55 -0
  109. package/dist/server/bundle.js +172 -0
  110. package/dist/server/dataset.d.ts +8 -2
  111. package/dist/server/dataset.js +51 -4
  112. package/dist/server/embedded.d.ts +7 -3
  113. package/dist/server/embedded.js +7 -3
  114. package/dist/server/index.d.ts +5 -1
  115. package/dist/server/index.js +5 -1
  116. package/dist/server/live-sockets.d.ts +54 -0
  117. package/dist/server/live-sockets.js +131 -0
  118. package/dist/server/live.d.ts +146 -0
  119. package/dist/server/live.js +228 -0
  120. package/dist/server/ring.d.ts +77 -0
  121. package/dist/server/ring.js +311 -0
  122. package/dist/server/spool.d.ts +139 -4
  123. package/dist/server/spool.js +457 -28
  124. package/dist/server/static.d.ts +42 -0
  125. package/dist/server/static.js +29 -0
  126. package/dist/server/stub.d.ts +43 -5
  127. package/dist/server/stub.js +126 -8
  128. package/dist/server/write.d.ts +23 -1
  129. package/dist/server/write.js +96 -0
  130. package/dist/stores/durable-object.d.ts +46 -0
  131. package/dist/stores/durable-object.js +114 -0
  132. package/dist/sw/analytics.d.ts +1 -1
  133. package/dist/sw/analytics.js +2 -2
  134. package/dist/sw/coverage.d.ts +6 -0
  135. package/dist/sw/coverage.js +34 -0
  136. package/dist/sw/family.d.ts +15 -0
  137. package/dist/sw/family.js +48 -0
  138. package/dist/sw/idb.d.ts +2 -0
  139. package/dist/sw/idb.js +2 -2
  140. package/dist/sw/index.d.ts +2 -0
  141. package/dist/sw/index.js +1 -0
  142. package/dist/sw/pack/store.d.ts +128 -0
  143. package/dist/sw/pack/store.js +737 -0
  144. package/dist/sw/policy.d.ts +2 -0
  145. package/dist/sw/policy.js +11 -0
  146. package/dist/sw/probe.d.ts +16 -0
  147. package/dist/sw/probe.js +25 -0
  148. package/dist/sw/router.d.ts +3 -2
  149. package/dist/sw/router.js +15 -2
  150. package/dist/sw/runtime.d.ts +48 -1
  151. package/dist/sw/runtime.js +548 -68
  152. package/dist/sw/scheduler.d.ts +6 -1
  153. package/dist/sw/scheduler.js +22 -5
  154. package/dist/sw/write.d.ts +41 -1
  155. package/dist/sw/write.js +188 -5
  156. package/dist/version.d.ts +1 -1
  157. package/dist/version.js +1 -2
  158. package/dist/vite/index.d.ts +26 -0
  159. package/dist/vite/index.js +66 -0
  160. package/dist/vue/index.d.ts +36 -0
  161. package/dist/vue/index.js +53 -0
  162. package/package.json +64 -4
@@ -0,0 +1,428 @@
1
+ import { ProtocolError } from './errors.js';
2
+ import { canonicalJson, digestAdd, EMPTY_DIGEST, rowHash } from './query.js';
3
+ /** rows per block when the caller names no size */
4
+ export const BLOCK_ROWS = 128;
5
+ /** overlay rows past which a base pack is compacted; a default, not measured */
6
+ export const COMPACT_OVERLAY_ROWS = 1000;
7
+ /** overlay bytes past which a base pack is compacted; a default, not measured */
8
+ export const COMPACT_OVERLAY_BYTES = 65_536;
9
+ /** the largest payload a frame may declare; a reader refuses a larger header before the payload */
10
+ export const MAX_BUNDLE_FRAME_BYTES = 8_388_608;
11
+ /** the most one block may inflate to; a reader refuses a larger block before it holds the bytes */
12
+ export const MAX_BUNDLE_BLOCK_BYTES = 1_048_576;
13
+ const INDEX_FRAME = -1;
14
+ const HEADER_LIMIT = 256;
15
+ const DIGEST_FORM = /^[0-9a-f]{64}$/;
16
+ const encoder = new TextEncoder();
17
+ const decoder = new TextDecoder();
18
+ const encode = (text) => new Uint8Array(encoder.encode(text));
19
+ function refuse(message) {
20
+ throw new ProtocolError('bad-bundle', message);
21
+ }
22
+ const isCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
23
+ const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
24
+ function concatBytes(parts) {
25
+ const out = new Uint8Array(parts.reduce((size, part) => size + part.length, 0));
26
+ let at = 0;
27
+ for (const part of parts) {
28
+ out.set(part, at);
29
+ at += part.length;
30
+ }
31
+ return out;
32
+ }
33
+ async function sha256Url(bytes) {
34
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
35
+ let binary = '';
36
+ for (const byte of digest)
37
+ binary += String.fromCharCode(byte);
38
+ return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
39
+ }
40
+ async function transform(bytes, stream) {
41
+ const piped = new Response(bytes).body?.pipeThrough(stream);
42
+ if (!piped)
43
+ refuse('no body to transform');
44
+ return new Uint8Array(await new Response(piped).arrayBuffer());
45
+ }
46
+ const gzip = (bytes) => transform(bytes, new CompressionStream('gzip'));
47
+ /**
48
+ * Inflates a gzip block. The trailer's size is checked before any output, and the output is counted
49
+ * as it streams, because a trailer can lie. A block over `MAX_BUNDLE_BLOCK_BYTES` is cancelled there.
50
+ */
51
+ async function gunzipCapped(payload, name) {
52
+ const trailer = payload.length - 4;
53
+ if (trailer >= 0) {
54
+ const declared = new DataView(payload.buffer, payload.byteOffset + trailer, 4).getUint32(0, true);
55
+ if (declared > MAX_BUNDLE_BLOCK_BYTES) {
56
+ refuse(`${name} declares ${declared} bytes, over the block cap`);
57
+ }
58
+ }
59
+ const piped = new Response(payload).body?.pipeThrough(new DecompressionStream('gzip'));
60
+ if (!piped)
61
+ refuse('no body to transform');
62
+ const reader = piped.getReader();
63
+ const parts = [];
64
+ let size = 0;
65
+ for (;;) {
66
+ const next = await reader.read().catch(() => refuse(`${name} does not decode`));
67
+ if (next.done)
68
+ break;
69
+ size += next.value.byteLength;
70
+ if (size > MAX_BUNDLE_BLOCK_BYTES) {
71
+ await reader.cancel().catch(() => undefined);
72
+ refuse(`${name} is over the block cap`);
73
+ }
74
+ parts.push(new Uint8Array(next.value));
75
+ }
76
+ return new Uint8Array(await new Blob(parts).arrayBuffer());
77
+ }
78
+ function compareKeys(a, b) {
79
+ return a < b ? -1 : a > b ? 1 : 0;
80
+ }
81
+ /** the rows of one table sorted by key; a repeated key is refused */
82
+ function sortRows(name, rows) {
83
+ const sorted = [...rows].sort(([a], [b]) => compareKeys(a, b));
84
+ for (let i = 1; i < sorted.length; i++) {
85
+ if (sorted[i - 1]?.[0] === sorted[i]?.[0])
86
+ refuse(`${name}: key ${sorted[i]?.[0]} repeats`);
87
+ }
88
+ return sorted;
89
+ }
90
+ function frame(table, ordinal, payload) {
91
+ const header = encode(JSON.stringify({ table, ordinal, bytes: payload.length }));
92
+ const prefix = new Uint8Array(4);
93
+ new DataView(prefix.buffer).setUint32(0, header.length);
94
+ return concatBytes([prefix, header, payload]);
95
+ }
96
+ /**
97
+ * The set digest of every table: the sum of each row's hash modulo 2^256, where a row's hash covers
98
+ * its table name and its key as well as its value. An empty table adds nothing.
99
+ */
100
+ export async function bundleDigest(tables) {
101
+ let sum = EMPTY_DIGEST;
102
+ for (const name of Object.keys(tables)) {
103
+ for (const [key, row] of tables[name] ?? []) {
104
+ sum = digestAdd(sum, await rowHash(JSON.stringify([name, key]), row));
105
+ }
106
+ }
107
+ return sum;
108
+ }
109
+ /**
110
+ * Encodes a state as a pack: an index frame, then one frame per block in table order. Empty tables
111
+ * are left out. Throws `bad-bundle` for a repeated key, a block size under one, or a block over
112
+ * `maxFrameBytes` or `MAX_BUNDLE_BLOCK_BYTES`, which a reader would refuse.
113
+ */
114
+ export async function buildPack(state, blockRows = BLOCK_ROWS, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
115
+ if (!Number.isSafeInteger(blockRows) || blockRows < 1)
116
+ refuse('blockRows must be 1 or more');
117
+ const frames = [];
118
+ const tables = [];
119
+ const names = Object.keys(state.tables)
120
+ .filter((name) => (state.tables[name]?.length ?? 0) > 0)
121
+ .sort(compareKeys);
122
+ for (const [table, name] of names.entries()) {
123
+ const rows = sortRows(name, state.tables[name] ?? []);
124
+ const blocks = [];
125
+ for (let ordinal = 0; ordinal * blockRows < rows.length; ordinal++) {
126
+ const chunk = rows.slice(ordinal * blockRows, (ordinal + 1) * blockRows);
127
+ const raw = encode(canonicalJson(chunk));
128
+ if (raw.length > MAX_BUNDLE_BLOCK_BYTES) {
129
+ refuse(`block ${name}/${ordinal} is over the block cap`);
130
+ }
131
+ const payload = await gzip(raw);
132
+ if (payload.length > maxFrameBytes) {
133
+ refuse(`block ${name}/${ordinal} is over the frame cap`);
134
+ }
135
+ blocks.push(await sha256Url(payload));
136
+ frames.push(frame(table, ordinal, payload));
137
+ }
138
+ tables.push({ name, rows: rows.length, first: rows[0]?.[0] ?? null, blocks });
139
+ }
140
+ const index = {
141
+ version: state.version,
142
+ codec: 'gzip',
143
+ blockRows,
144
+ digest: await bundleDigest(state.tables),
145
+ tables
146
+ };
147
+ const head = frame(INDEX_FRAME, 0, encode(canonicalJson(index)));
148
+ return { index, bytes: concatBytes([head, ...frames]) };
149
+ }
150
+ /**
151
+ * Yields the frames of a pack as their bytes arrive, so a reader can verify and keep each block
152
+ * before the next one lands. Throws `bad-bundle` for a frame that is cut short, has a bad header,
153
+ * or declares a payload over `maxFrameBytes`; that check runs before the payload is read.
154
+ */
155
+ export async function* readFrames(source, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
156
+ const chunks = source[Symbol.asyncIterator]();
157
+ let buffer = new Uint8Array(0);
158
+ const fill = async (size) => {
159
+ while (buffer.length < size) {
160
+ const next = await chunks.next();
161
+ if (next.done)
162
+ return false;
163
+ buffer = concatBytes([buffer, next.value]);
164
+ }
165
+ return true;
166
+ };
167
+ const take = (size) => {
168
+ const out = buffer.slice(0, size);
169
+ buffer = buffer.subarray(size);
170
+ return out;
171
+ };
172
+ while (await fill(1)) {
173
+ if (!(await fill(4)))
174
+ refuse('a frame is cut short');
175
+ const headerSize = new DataView(buffer.buffer, buffer.byteOffset, 4).getUint32(0);
176
+ if (headerSize === 0 || headerSize > HEADER_LIMIT)
177
+ refuse('a frame header has a bad size');
178
+ if (!(await fill(4 + headerSize)))
179
+ refuse('a frame is cut short');
180
+ const header = JSON.parse(decoder.decode(buffer.subarray(4, 4 + headerSize)));
181
+ take(4 + headerSize);
182
+ if (!isPlain(header) ||
183
+ typeof header['table'] !== 'number' ||
184
+ !Number.isSafeInteger(header['table']) ||
185
+ header['table'] < INDEX_FRAME ||
186
+ !isCount(header['ordinal']) ||
187
+ !isCount(header['bytes'])) {
188
+ refuse('a frame header is not { table, ordinal, bytes }');
189
+ }
190
+ const table = header['table'];
191
+ const ordinal = header['ordinal'];
192
+ const bytes = header['bytes'];
193
+ if (bytes > maxFrameBytes)
194
+ refuse('a frame is over the frame cap');
195
+ if (!(await fill(bytes)))
196
+ refuse('a frame is cut short');
197
+ yield { table, ordinal, payload: take(bytes) };
198
+ }
199
+ }
200
+ function parseIndex(value) {
201
+ if (!isPlain(value))
202
+ refuse('the index is not an object');
203
+ const { version, codec, blockRows, digest, tables } = value;
204
+ if (!isCount(version) ||
205
+ codec !== 'gzip' ||
206
+ typeof blockRows !== 'number' ||
207
+ !Number.isSafeInteger(blockRows) ||
208
+ blockRows < 1 ||
209
+ typeof digest !== 'string' ||
210
+ !DIGEST_FORM.test(digest) ||
211
+ !Array.isArray(tables)) {
212
+ refuse('the index has a bad version, codec, block size or digest');
213
+ }
214
+ const names = new Set();
215
+ const out = tables.map((table) => {
216
+ if (!isPlain(table) ||
217
+ typeof table['name'] !== 'string' ||
218
+ names.has(table['name']) ||
219
+ !isCount(table['rows']) ||
220
+ table['rows'] < 1 ||
221
+ typeof table['first'] !== 'string' ||
222
+ !Array.isArray(table['blocks']) ||
223
+ !table['blocks'].every((hash) => typeof hash === 'string') ||
224
+ table['blocks'].length !== Math.ceil(table['rows'] / blockRows)) {
225
+ refuse('an index table does not match its rows and blocks');
226
+ }
227
+ names.add(table['name']);
228
+ return {
229
+ name: table['name'],
230
+ rows: table['rows'],
231
+ first: table['first'],
232
+ blocks: table['blocks']
233
+ };
234
+ });
235
+ return { version, codec, blockRows, digest, tables: out };
236
+ }
237
+ /**
238
+ * Checks one block frame against the index and returns its rows. The hash is checked before the
239
+ * block is decompressed, so a corrupt block is refused without being parsed.
240
+ */
241
+ export async function readBlock(index, block) {
242
+ const table = index.tables[block.table];
243
+ const expected = table?.blocks[block.ordinal];
244
+ if (expected === undefined)
245
+ refuse(`no block ${block.table}/${block.ordinal} in the index`);
246
+ if ((await sha256Url(block.payload)) !== expected) {
247
+ refuse(`block ${block.table}/${block.ordinal} does not match its hash`);
248
+ }
249
+ const raw = await gunzipCapped(block.payload, `block ${block.table}/${block.ordinal}`);
250
+ let rows;
251
+ try {
252
+ rows = JSON.parse(decoder.decode(raw));
253
+ }
254
+ catch {
255
+ refuse(`block ${block.table}/${block.ordinal} does not decode`);
256
+ }
257
+ if (!Array.isArray(rows))
258
+ refuse(`block ${block.table}/${block.ordinal} is not a row list`);
259
+ for (const entry of rows) {
260
+ if (!Array.isArray(entry) || typeof entry[0] !== 'string' || !isPlain(entry[1])) {
261
+ refuse(`block ${block.table}/${block.ordinal} has a row that is not [key, row]`);
262
+ }
263
+ }
264
+ return rows;
265
+ }
266
+ /**
267
+ * Reads a whole pack, or a pack that arrives in chunks, and returns its index and state. Every
268
+ * block is checked as it lands, and the set digest is checked at the end. Throws `bad-bundle`.
269
+ */
270
+ export async function readBundle(source, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
271
+ async function* whole(bytes) {
272
+ yield bytes;
273
+ }
274
+ const frames = readFrames(source instanceof Uint8Array ? whole(source) : source, maxFrameBytes);
275
+ let index;
276
+ const tables = {};
277
+ let table = 0;
278
+ let ordinal = 0;
279
+ for await (const block of frames) {
280
+ if (!index) {
281
+ if (block.table !== INDEX_FRAME)
282
+ refuse('the pack does not start with its index');
283
+ index = parseIndex(parseJson(block.payload));
284
+ continue;
285
+ }
286
+ const entry = index.tables[table];
287
+ if (!entry || block.table !== table || block.ordinal !== ordinal) {
288
+ refuse(`expected block ${table}/${ordinal}, got ${block.table}/${block.ordinal}`);
289
+ }
290
+ const size = Math.min(index.blockRows, entry.rows - ordinal * index.blockRows);
291
+ const rows = await readBlock(index, block);
292
+ if (rows.length !== size)
293
+ refuse(`block ${table}/${ordinal} holds ${rows.length} rows`);
294
+ const held = (tables[entry.name] ??= []);
295
+ for (const [key, row] of rows) {
296
+ if (held.length > 0 && (held[held.length - 1]?.[0] ?? '') >= key) {
297
+ refuse(`${entry.name}: key ${key} is out of order`);
298
+ }
299
+ held.push([key, row]);
300
+ }
301
+ if (ordinal === 0 && rows[0]?.[0] !== entry.first)
302
+ refuse(`${entry.name}: wrong first key`);
303
+ if (ordinal + 1 < entry.blocks.length)
304
+ ordinal++;
305
+ else {
306
+ table++;
307
+ ordinal = 0;
308
+ }
309
+ }
310
+ if (!index)
311
+ refuse('the pack is empty');
312
+ if (table !== index.tables.length)
313
+ refuse('the pack ends before its last block');
314
+ if ((await bundleDigest(tables)) !== index.digest)
315
+ refuse('the set digest does not match');
316
+ return { index, state: { version: index.version, tables } };
317
+ }
318
+ function parseJson(payload) {
319
+ try {
320
+ return JSON.parse(decoder.decode(payload));
321
+ }
322
+ catch {
323
+ return refuse('the index is not JSON');
324
+ }
325
+ }
326
+ /** the change from `from` to `to`: puts for new or changed rows, deletes for rows that left */
327
+ export async function makeDelta(from, to) {
328
+ const names = [...new Set([...Object.keys(from.tables), ...Object.keys(to.tables)])].sort(compareKeys);
329
+ const tables = {};
330
+ for (const name of names) {
331
+ const before = new Map(from.tables[name] ?? []);
332
+ const after = sortRows(name, to.tables[name] ?? []);
333
+ const kept = new Set(after.map(([key]) => key));
334
+ const puts = after.filter(([key, row]) => {
335
+ const old = before.get(key);
336
+ return old === undefined || canonicalJson(old) !== canonicalJson(row);
337
+ });
338
+ const deletes = [...before.keys()].filter((key) => !kept.has(key)).sort(compareKeys);
339
+ if (puts.length > 0 || deletes.length > 0)
340
+ tables[name] = { puts, deletes };
341
+ }
342
+ return { from: from.version, to: to.version, digest: await bundleDigest(to.tables), tables };
343
+ }
344
+ /**
345
+ * Applies a delta to the state at its `from` version and returns the state at `to`. Refuses a base
346
+ * at another version, and a result whose set digest is not the delta's.
347
+ */
348
+ export async function applyDelta(base, delta) {
349
+ if (base.version !== delta.from) {
350
+ refuse(`the delta starts at ${delta.from}, the base holds ${base.version}`);
351
+ }
352
+ const names = new Set([...Object.keys(base.tables), ...Object.keys(delta.tables)]);
353
+ const tables = {};
354
+ for (const name of names) {
355
+ const change = delta.tables[name];
356
+ const rows = new Map(base.tables[name] ?? []);
357
+ for (const key of change?.deletes ?? [])
358
+ rows.delete(key);
359
+ for (const [key, row] of change?.puts ?? [])
360
+ rows.set(key, row);
361
+ if (rows.size > 0)
362
+ tables[name] = sortRows(name, [...rows]);
363
+ }
364
+ if ((await bundleDigest(tables)) !== delta.digest) {
365
+ refuse('the applied state does not match the delta digest');
366
+ }
367
+ return { version: delta.to, tables };
368
+ }
369
+ /**
370
+ * One delta equal to applying `deltas` in order. The deltas must chain: each `from` is the previous
371
+ * `to`. A key put and then removed becomes a tombstone, which a holder without the key ignores.
372
+ */
373
+ export function mergeDeltas(deltas) {
374
+ const first = deltas[0];
375
+ const last = deltas[deltas.length - 1];
376
+ if (!first || !last)
377
+ refuse('there is no delta to merge');
378
+ for (let i = 1; i < deltas.length; i++) {
379
+ if (deltas[i - 1]?.to !== deltas[i]?.from)
380
+ refuse('the deltas do not chain');
381
+ }
382
+ const ops = new Map();
383
+ for (const delta of deltas) {
384
+ for (const [name, change] of Object.entries(delta.tables)) {
385
+ const table = ops.get(name) ?? new Map();
386
+ ops.set(name, table);
387
+ for (const key of change.deletes)
388
+ table.set(key, null);
389
+ for (const [key, row] of change.puts)
390
+ table.set(key, row);
391
+ }
392
+ }
393
+ const tables = {};
394
+ for (const name of [...ops.keys()].sort(compareKeys)) {
395
+ const entries = [...(ops.get(name) ?? [])].sort(([a], [b]) => compareKeys(a, b));
396
+ const puts = [];
397
+ const deletes = [];
398
+ for (const [key, row] of entries) {
399
+ if (row === null)
400
+ deletes.push(key);
401
+ else
402
+ puts.push([key, row]);
403
+ }
404
+ if (puts.length > 0 || deletes.length > 0)
405
+ tables[name] = { puts, deletes };
406
+ }
407
+ return { from: first.from, to: last.to, digest: last.digest, tables };
408
+ }
409
+ /**
410
+ * whether a delta has more than {@link COMPACT_OVERLAY_ROWS} rows or
411
+ * {@link COMPACT_OVERLAY_BYTES} bytes of JSON
412
+ */
413
+ export function compactionDue(delta) {
414
+ let rows = 0;
415
+ for (const change of Object.values(delta.tables)) {
416
+ rows += change.puts.length + change.deletes.length;
417
+ }
418
+ return (rows > COMPACT_OVERLAY_ROWS ||
419
+ encoder.encode(JSON.stringify(delta)).length > COMPACT_OVERLAY_BYTES);
420
+ }
421
+ /**
422
+ * Folds a delta into a base pack and returns the new pack, built with the base's block size. Its
423
+ * bytes equal {@link buildPack} run on the state the delta reaches.
424
+ */
425
+ export async function compactBundle(base, overlay) {
426
+ const { index, state } = await readBundle(base);
427
+ return buildPack(await applyDelta(state, overlay), index.blockRows);
428
+ }
@@ -1,11 +1,16 @@
1
1
  import type { PROTOCOL_VERSION } from './artifact.js';
2
+ import type { BundleDelta } from './bundle.js';
2
3
  import type { CollectorConfig } from './collector.js';
3
4
  import type { SerializedRoute } from './route.js';
4
5
  import type { CompiledTemplate } from './template/compile.js';
5
6
  import type { Mode } from './view.js';
6
- import type { WriteConfig } from './write.js';
7
+ import type { ResourceWrite, WriteConfig } from './write.js';
7
8
  /** every path under this prefix belongs to spool */
8
9
  export declare const SPOOL_PREFIX = "/.spool/";
10
+ /** the prefix of the site packs, one path per version: `/.spool/bundle/<version>` */
11
+ export declare const BUNDLE_PREFIX = "/.spool/bundle/";
12
+ /** the prefix of the deltas between two packs: `/.spool/bundle/delta/<from>/<to>` */
13
+ export declare const BUNDLE_DELTA_PREFIX = "/.spool/bundle/delta/";
9
14
  /** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
10
15
  export declare const PATHS: {
11
16
  readonly sync: "/.spool/sync";
@@ -15,8 +20,30 @@ export declare const PATHS: {
15
20
  readonly write: "/.spool/write";
16
21
  readonly worker: "/spool-sw.js";
17
22
  };
23
+ /**
24
+ * where a visible page holds one server-sent event stream open: the principal lease, a `revoked`
25
+ * event, and public tag changes where the site opts in. The request takes `partition` (the principal
26
+ * partition the page holds) and `have` (the cursor the page holds); the worker never handles it.
27
+ */
28
+ export declare const LIVE_PATH = "/.spool/live";
29
+ /**
30
+ * the text a page sends over a live socket to renew its lease. The server answers it with a `lease`
31
+ * event, which a hibernating Durable Object can do without waking.
32
+ */
33
+ export declare const LIVE_PING = "ping";
34
+ /** the cookie a no-worker page writes with the version it holds; the server reads it, nothing else does */
35
+ export declare const COOKIES: {
36
+ readonly held: "spool-held";
37
+ };
18
38
  /** the same-origin path of the page runtime script for a given version */
19
39
  export declare function clientPath(version: string): string;
40
+ /** the same-origin path of the in-page navigation script, loaded once the page holds a head */
41
+ export declare function navigatePath(version: string): string;
42
+ /**
43
+ * the same-origin path of the boot script for a runtime version and a pack version: the page
44
+ * runtime, the navigation script and that pack in one file
45
+ */
46
+ export declare function bootPath(version: string, pack: number): string;
20
47
  /**
21
48
  * Priors and the hot list for provisioning. Every number is a scenario assumption the operator
22
49
  * replaces with its own telemetry; `hot` names paths to warm in priority order, riding the next
@@ -57,10 +84,25 @@ export type ThirdPartyConfig = {
57
84
  /** keep opaque (no-cors) responses, which Chromium counts as about 7 MB each against quota; default false */
58
85
  allowOpaque?: boolean;
59
86
  };
87
+ /**
88
+ * The pack a worker script carries: the current version and set digest, and the deltas from the
89
+ * versions just before it, each straight to the current one.
90
+ */
91
+ export type StubPack = {
92
+ version: number;
93
+ digest: string;
94
+ deltas: BundleDelta[];
95
+ };
60
96
  /** what the server compiles into the worker script: routes, templates and the tag to inject */
61
97
  export type SwConfig = {
62
98
  protocol: typeof PROTOCOL_VERSION;
63
99
  version: string;
100
+ /** the hash of the worker's own code; a different one empties the replica, a content version does not */
101
+ runtime?: string;
102
+ /** the pack the script carries, when the server holds one for its current version */
103
+ pack?: StubPack;
104
+ /** the site's security headers, as name and value pairs, for a worker that holds no site policy yet */
105
+ policy?: Array<[string, string]>;
64
106
  routes: SerializedRoute[];
65
107
  /** compiled templates keyed by route pattern */
66
108
  templates: Record<string, CompiledTemplate>;
@@ -74,6 +116,13 @@ export type SwConfig = {
74
116
  shadowSample?: number;
75
117
  /** whether the worker queues a `spool.view` per page view; true when absent */
76
118
  views?: boolean;
119
+ /** seconds the browser may hold the worker script: the lease after which the page checks it again */
120
+ lease?: number;
121
+ /**
122
+ * The operator's counter for the stub. Its bytes change when the stamp does, so a static host that
123
+ * republishes each lease period makes a forced check install a new script, and the install is a contact.
124
+ */
125
+ stamp?: number;
77
126
  /** what the worker assumes before telemetry says otherwise, and which pages to warm */
78
127
  provision?: ProvisionConfig;
79
128
  /**
@@ -96,9 +145,24 @@ export type SwConfig = {
96
145
  datasetShare?: number;
97
146
  /** the declared write classes; absent while declared writes are off, which is the default */
98
147
  writes?: WriteConfig[];
148
+ /** the resource writes a request is sent to the server for, each applied from its `Spool-Delta` */
149
+ resourceWrites?: readonly ResourceWrite[];
99
150
  /** the principal datasets and how long each may answer after its last confirmed sync; absent while the flag is off */
100
151
  principalDatasets?: Array<{
101
152
  name: string;
102
153
  maxStaleMs: number;
103
154
  }>;
155
+ /**
156
+ * Present while the server answers `/.spool/live`. `public` is true when the stream also carries
157
+ * public freshness, so the worker treats its public replica as current while a lease is open.
158
+ * `mode` is `always` when the page opens the stream whenever it is visible, and `adaptive` when the
159
+ * worker asks for it at the second lease renewal of a page's visible session. `socket` is true
160
+ * when the hub also accepts a WebSocket upgrade at the same path; a page then opens a socket
161
+ * and falls back to server-sent events if the upgrade fails.
162
+ */
163
+ live?: {
164
+ public: boolean;
165
+ mode: 'adaptive' | 'always';
166
+ socket?: true;
167
+ };
104
168
  };
@@ -1,5 +1,9 @@
1
1
  /** every path under this prefix belongs to spool */
2
2
  export const SPOOL_PREFIX = '/.spool/';
3
+ /** the prefix of the site packs, one path per version: `/.spool/bundle/<version>` */
4
+ export const BUNDLE_PREFIX = '/.spool/bundle/';
5
+ /** the prefix of the deltas between two packs: `/.spool/bundle/delta/<from>/<to>` */
6
+ export const BUNDLE_DELTA_PREFIX = '/.spool/bundle/delta/';
3
7
  /** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
4
8
  export const PATHS = {
5
9
  sync: '/.spool/sync',
@@ -9,7 +13,31 @@ export const PATHS = {
9
13
  write: '/.spool/write',
10
14
  worker: '/spool-sw.js'
11
15
  };
16
+ /**
17
+ * where a visible page holds one server-sent event stream open: the principal lease, a `revoked`
18
+ * event, and public tag changes where the site opts in. The request takes `partition` (the principal
19
+ * partition the page holds) and `have` (the cursor the page holds); the worker never handles it.
20
+ */
21
+ export const LIVE_PATH = '/.spool/live';
22
+ /**
23
+ * the text a page sends over a live socket to renew its lease. The server answers it with a `lease`
24
+ * event, which a hibernating Durable Object can do without waking.
25
+ */
26
+ export const LIVE_PING = 'ping';
27
+ /** the cookie a no-worker page writes with the version it holds; the server reads it, nothing else does */
28
+ export const COOKIES = { held: 'spool-held' };
12
29
  /** the same-origin path of the page runtime script for a given version */
13
30
  export function clientPath(version) {
14
31
  return `${SPOOL_PREFIX}${encodeURIComponent(version)}/client.js`;
15
32
  }
33
+ /** the same-origin path of the in-page navigation script, loaded once the page holds a head */
34
+ export function navigatePath(version) {
35
+ return `${SPOOL_PREFIX}${encodeURIComponent(version)}/navigate.js`;
36
+ }
37
+ /**
38
+ * the same-origin path of the boot script for a runtime version and a pack version: the page
39
+ * runtime, the navigation script and that pack in one file
40
+ */
41
+ export function bootPath(version, pack) {
42
+ return `${SPOOL_PREFIX}${encodeURIComponent(version)}/boot/${pack}.js`;
43
+ }
@@ -81,6 +81,12 @@ export type DatasetDeclaration<Row extends DatasetRow = DatasetRow> = {
81
81
  maxRows: number;
82
82
  /** most bytes of row JSON a seed may hold */
83
83
  maxBytes: number;
84
+ /**
85
+ * Principal datasets only: the field names of `row` that `viewer` may hold, from `null` for an
86
+ * anonymous request. The server sends each viewer only those fields, so the device never holds
87
+ * the others. Absent, every field goes to every viewer.
88
+ */
89
+ fields?(viewer: Identity | null, row: Row): readonly string[];
84
90
  /**
85
91
  * `always` (the default) answers every seed. `auto` answers a seed only while `datasetBenefit`
86
92
  * says the dataset beats both v0.3 and native HTTP; otherwise the seed is refused with `409`.
@@ -20,6 +20,12 @@ export function defineDataset(name, declaration, options = {}) {
20
20
  }
21
21
  if (scope !== 'public' && scope !== 'principal')
22
22
  return bad(name, `unknown scope "${scope}"`);
23
+ if (declaration.fields !== undefined) {
24
+ if (scope !== 'principal')
25
+ return bad(name, 'fields applies to a principal dataset');
26
+ if (typeof declaration.fields !== 'function')
27
+ return bad(name, 'fields must be a function');
28
+ }
23
29
  if (scope === 'principal') {
24
30
  const stale = declaration.maxStaleMs;
25
31
  if (typeof stale !== 'number' || !Number.isSafeInteger(stale) || stale < 1) {
@@ -1,4 +1,5 @@
1
- import { type Scope } from './artifact.js';
1
+ import { PROTOCOL_VERSION, type Scope } from './artifact.js';
2
+ import { type SyncResponse } from './sync.js';
2
3
  /** the header names, lowercase as `Headers` reports them */
3
4
  export declare const HEADERS: {
4
5
  readonly protocol: "spool-protocol";
@@ -10,6 +11,8 @@ export declare const HEADERS: {
10
11
  readonly vectorSeen: "spool-vector-seen";
11
12
  readonly provision: "spool-provision";
12
13
  readonly provisionReason: "spool-provision-reason";
14
+ readonly delta: "spool-delta";
15
+ readonly have: "spool-have";
13
16
  };
14
17
  /** `Spool-Clear` token that wipes everything the worker holds */
15
18
  export declare const CLEAR_ALL = "*";
@@ -48,3 +51,32 @@ export declare function writeEnvelope(headers: Headers, envelope: Envelope): voi
48
51
  export declare function decodeEnvelopeOrClear(headers: Headers): Envelope | undefined;
49
52
  /** reads the spool headers; throws `ProtocolError` on a malformed one */
50
53
  export declare function decodeEnvelope(headers: Headers): Envelope;
54
+ /** the names of the events `/.spool/live` sends, each as the SSE `event` field with one JSON `data` line */
55
+ export declare const LIVE_EVENTS: readonly ["hello", "lease", "delta", "revoked"];
56
+ /** the name of a live event */
57
+ export type LiveEventName = (typeof LIVE_EVENTS)[number];
58
+ /** what each live event carries */
59
+ export type LiveEvents = {
60
+ /** first on every stream: the protocol and the cursor the stream's own deltas follow */
61
+ hello: {
62
+ protocol: typeof PROTOCOL_VERSION;
63
+ cursor: number;
64
+ };
65
+ /**
66
+ * The session is good for `leaseMs` from the moment the page reads the event, and again after the
67
+ * next one. It is a duration so a device clock that is wrong does not change it.
68
+ */
69
+ lease: {
70
+ leaseMs: number;
71
+ };
72
+ /** the tags changed after `from` up to `cursor`, as a `/.spool/sync` answer; public information only */
73
+ delta: LiveDelta;
74
+ /** the session behind the stream ended; the stream closes after this event */
75
+ revoked: Record<string, never>;
76
+ };
77
+ /** a `/.spool/sync` answer for the tags changed after `from`, pushed */
78
+ export type LiveDelta = SyncResponse & {
79
+ from: number;
80
+ };
81
+ /** reads the `data` of a `delta` event; throws `ProtocolError` on a malformed one */
82
+ export declare function parseLiveDelta(value: unknown): LiveDelta;