@drupflare/spool 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -1607
- package/dist/angular/index.d.ts +13 -0
- package/dist/angular/index.js +25 -0
- package/dist/angular/pages.d.ts +28 -0
- package/dist/angular/pages.js +85 -0
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/location.d.ts +6 -0
- package/dist/cdn/location.js +8 -0
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/bootstrap.d.ts +57 -0
- package/dist/client/bootstrap.js +307 -0
- package/dist/client/contact.d.ts +24 -0
- package/dist/client/contact.js +66 -0
- package/dist/client/declared.d.ts +23 -0
- package/dist/client/declared.js +68 -0
- package/dist/client/entry.js +123 -2
- package/dist/client/gate.d.ts +1 -0
- package/dist/client/gate.js +148 -0
- package/dist/client/index.d.ts +25 -0
- package/dist/client/index.js +1 -0
- package/dist/client/live.d.ts +66 -0
- package/dist/client/live.js +225 -0
- package/dist/client/navigate-entry.d.ts +1 -0
- package/dist/client/navigate-entry.js +4 -0
- package/dist/client/navigate.d.ts +54 -0
- package/dist/client/navigate.js +272 -0
- package/dist/client/persist.d.ts +2 -0
- package/dist/client/persist.js +9 -0
- package/dist/client/register.d.ts +34 -2
- package/dist/client/register.js +64 -7
- package/dist/express/index.d.ts +11 -0
- package/dist/express/index.js +209 -0
- package/dist/hono/index.d.ts +19 -0
- package/dist/hono/index.js +49 -0
- package/dist/next/index.d.ts +33 -0
- package/dist/next/index.js +41 -0
- package/dist/next/pages.d.ts +22 -0
- package/dist/next/pages.js +39 -0
- package/dist/node/listener.d.ts +3 -0
- package/dist/node/listener.js +57 -2
- package/dist/nuxt/index.d.ts +22 -0
- package/dist/nuxt/index.js +104 -0
- package/dist/nuxt/pages.d.ts +26 -0
- package/dist/nuxt/pages.js +48 -0
- package/dist/nuxt/runtime/index.d.ts +24 -0
- package/dist/nuxt/runtime/index.js +58 -0
- package/dist/plugins/coverage.d.ts +14 -0
- package/dist/plugins/coverage.js +27 -0
- package/dist/plugins/mode.d.ts +19 -0
- package/dist/plugins/mode.js +20 -0
- package/dist/plugins/path.d.ts +16 -0
- package/dist/plugins/path.js +139 -0
- package/dist/plugins/report.d.ts +22 -0
- package/dist/plugins/report.js +18 -0
- package/dist/plugins/routes.d.ts +38 -0
- package/dist/plugins/routes.js +52 -0
- package/dist/plugins/static.d.ts +42 -0
- package/dist/plugins/static.js +97 -0
- package/dist/protocol/bundle.d.ts +111 -0
- package/dist/protocol/bundle.js +428 -0
- package/dist/protocol/config.d.ts +80 -1
- package/dist/protocol/config.js +31 -1
- package/dist/protocol/dataset.d.ts +189 -0
- package/dist/protocol/dataset.js +212 -0
- package/dist/protocol/envelope.d.ts +34 -1
- package/dist/protocol/envelope.js +16 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index-rows.d.ts +22 -0
- package/dist/protocol/index-rows.js +53 -0
- package/dist/protocol/index.d.ts +5 -0
- package/dist/protocol/index.js +5 -0
- package/dist/protocol/normalise.d.ts +4 -4
- package/dist/protocol/normalise.js +9 -5
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/ring.d.ts +58 -0
- package/dist/protocol/ring.js +6 -0
- package/dist/protocol/route.d.ts +63 -13
- package/dist/protocol/route.js +126 -6
- package/dist/protocol/tag.d.ts +22 -2
- package/dist/protocol/tag.js +25 -4
- package/dist/protocol/view.d.ts +2 -0
- package/dist/protocol/view.js +8 -2
- package/dist/protocol/views.d.ts +6 -0
- package/dist/protocol/views.js +15 -0
- package/dist/protocol/write.d.ts +234 -0
- package/dist/protocol/write.js +343 -0
- package/dist/proxy/eligible.d.ts +22 -1
- package/dist/proxy/eligible.js +62 -15
- package/dist/proxy/proxy.js +28 -18
- package/dist/react-router/index.d.ts +20 -0
- package/dist/react-router/index.js +60 -0
- package/dist/react-router/pages.d.ts +34 -0
- package/dist/react-router/pages.js +83 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/demand.d.ts +49 -0
- package/dist/report/demand.js +60 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +66 -1
- package/dist/report/economics.js +208 -1
- package/dist/report/headline.d.ts +158 -0
- package/dist/report/headline.js +473 -0
- package/dist/report/index.d.ts +4 -0
- package/dist/report/index.js +4 -0
- package/dist/report/inputs.d.ts +176 -0
- package/dist/report/inputs.js +248 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +7 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +8 -0
- package/dist/report/traffic.js +7 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/bundle.d.ts +55 -0
- package/dist/server/bundle.js +172 -0
- package/dist/server/dataset.d.ts +54 -0
- package/dist/server/dataset.js +355 -0
- package/dist/server/embedded.d.ts +7 -3
- package/dist/server/embedded.js +7 -3
- package/dist/server/index.d.ts +6 -0
- package/dist/server/index.js +6 -0
- package/dist/server/live-sockets.d.ts +54 -0
- package/dist/server/live-sockets.js +131 -0
- package/dist/server/live.d.ts +146 -0
- package/dist/server/live.js +228 -0
- package/dist/server/ring.d.ts +77 -0
- package/dist/server/ring.js +311 -0
- package/dist/server/spool.d.ts +158 -3
- package/dist/server/spool.js +578 -41
- package/dist/server/static.d.ts +42 -0
- package/dist/server/static.js +29 -0
- package/dist/server/stub.d.ts +43 -5
- package/dist/server/stub.js +126 -8
- package/dist/server/write.d.ts +49 -0
- package/dist/server/write.js +223 -0
- package/dist/stores/durable-object.d.ts +46 -0
- package/dist/stores/durable-object.js +114 -0
- package/dist/sw/analytics.d.ts +7 -1
- package/dist/sw/analytics.js +16 -2
- package/dist/sw/coverage.d.ts +6 -0
- package/dist/sw/coverage.js +34 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/family.d.ts +15 -0
- package/dist/sw/family.js +48 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.d.ts +2 -0
- package/dist/sw/idb.js +80 -11
- package/dist/sw/index.d.ts +2 -0
- package/dist/sw/index.js +1 -0
- package/dist/sw/pack/store.d.ts +128 -0
- package/dist/sw/pack/store.js +737 -0
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/policy.d.ts +2 -0
- package/dist/sw/policy.js +11 -0
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/probe.d.ts +16 -0
- package/dist/sw/probe.js +25 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/router.d.ts +3 -2
- package/dist/sw/router.js +15 -2
- package/dist/sw/runtime.d.ts +90 -2
- package/dist/sw/runtime.js +857 -61
- package/dist/sw/scheduler.d.ts +6 -1
- package/dist/sw/scheduler.js +22 -5
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +147 -0
- package/dist/sw/write.js +625 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -2
- package/dist/vite/index.d.ts +26 -0
- package/dist/vite/index.js +66 -0
- package/dist/vue/index.d.ts +36 -0
- package/dist/vue/index.js +53 -0
- package/package.json +65 -4
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { ProtocolError } from './errors.js';
|
|
2
|
+
import { canonicalJson, digestAdd, EMPTY_DIGEST, rowHash } from './query.js';
|
|
3
|
+
/** rows per block when the caller names no size */
|
|
4
|
+
export const BLOCK_ROWS = 128;
|
|
5
|
+
/** overlay rows past which a base pack is compacted; a default, not measured */
|
|
6
|
+
export const COMPACT_OVERLAY_ROWS = 1000;
|
|
7
|
+
/** overlay bytes past which a base pack is compacted; a default, not measured */
|
|
8
|
+
export const COMPACT_OVERLAY_BYTES = 65_536;
|
|
9
|
+
/** the largest payload a frame may declare; a reader refuses a larger header before the payload */
|
|
10
|
+
export const MAX_BUNDLE_FRAME_BYTES = 8_388_608;
|
|
11
|
+
/** the most one block may inflate to; a reader refuses a larger block before it holds the bytes */
|
|
12
|
+
export const MAX_BUNDLE_BLOCK_BYTES = 1_048_576;
|
|
13
|
+
const INDEX_FRAME = -1;
|
|
14
|
+
const HEADER_LIMIT = 256;
|
|
15
|
+
const DIGEST_FORM = /^[0-9a-f]{64}$/;
|
|
16
|
+
const encoder = new TextEncoder();
|
|
17
|
+
const decoder = new TextDecoder();
|
|
18
|
+
const encode = (text) => new Uint8Array(encoder.encode(text));
|
|
19
|
+
function refuse(message) {
|
|
20
|
+
throw new ProtocolError('bad-bundle', message);
|
|
21
|
+
}
|
|
22
|
+
const isCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
23
|
+
const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
24
|
+
function concatBytes(parts) {
|
|
25
|
+
const out = new Uint8Array(parts.reduce((size, part) => size + part.length, 0));
|
|
26
|
+
let at = 0;
|
|
27
|
+
for (const part of parts) {
|
|
28
|
+
out.set(part, at);
|
|
29
|
+
at += part.length;
|
|
30
|
+
}
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
async function sha256Url(bytes) {
|
|
34
|
+
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
|
|
35
|
+
let binary = '';
|
|
36
|
+
for (const byte of digest)
|
|
37
|
+
binary += String.fromCharCode(byte);
|
|
38
|
+
return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
|
|
39
|
+
}
|
|
40
|
+
async function transform(bytes, stream) {
|
|
41
|
+
const piped = new Response(bytes).body?.pipeThrough(stream);
|
|
42
|
+
if (!piped)
|
|
43
|
+
refuse('no body to transform');
|
|
44
|
+
return new Uint8Array(await new Response(piped).arrayBuffer());
|
|
45
|
+
}
|
|
46
|
+
const gzip = (bytes) => transform(bytes, new CompressionStream('gzip'));
|
|
47
|
+
/**
|
|
48
|
+
* Inflates a gzip block. The trailer's size is checked before any output, and the output is counted
|
|
49
|
+
* as it streams, because a trailer can lie. A block over `MAX_BUNDLE_BLOCK_BYTES` is cancelled there.
|
|
50
|
+
*/
|
|
51
|
+
async function gunzipCapped(payload, name) {
|
|
52
|
+
const trailer = payload.length - 4;
|
|
53
|
+
if (trailer >= 0) {
|
|
54
|
+
const declared = new DataView(payload.buffer, payload.byteOffset + trailer, 4).getUint32(0, true);
|
|
55
|
+
if (declared > MAX_BUNDLE_BLOCK_BYTES) {
|
|
56
|
+
refuse(`${name} declares ${declared} bytes, over the block cap`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
const piped = new Response(payload).body?.pipeThrough(new DecompressionStream('gzip'));
|
|
60
|
+
if (!piped)
|
|
61
|
+
refuse('no body to transform');
|
|
62
|
+
const reader = piped.getReader();
|
|
63
|
+
const parts = [];
|
|
64
|
+
let size = 0;
|
|
65
|
+
for (;;) {
|
|
66
|
+
const next = await reader.read().catch(() => refuse(`${name} does not decode`));
|
|
67
|
+
if (next.done)
|
|
68
|
+
break;
|
|
69
|
+
size += next.value.byteLength;
|
|
70
|
+
if (size > MAX_BUNDLE_BLOCK_BYTES) {
|
|
71
|
+
await reader.cancel().catch(() => undefined);
|
|
72
|
+
refuse(`${name} is over the block cap`);
|
|
73
|
+
}
|
|
74
|
+
parts.push(new Uint8Array(next.value));
|
|
75
|
+
}
|
|
76
|
+
return new Uint8Array(await new Blob(parts).arrayBuffer());
|
|
77
|
+
}
|
|
78
|
+
function compareKeys(a, b) {
|
|
79
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
80
|
+
}
|
|
81
|
+
/** the rows of one table sorted by key; a repeated key is refused */
|
|
82
|
+
function sortRows(name, rows) {
|
|
83
|
+
const sorted = [...rows].sort(([a], [b]) => compareKeys(a, b));
|
|
84
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
85
|
+
if (sorted[i - 1]?.[0] === sorted[i]?.[0])
|
|
86
|
+
refuse(`${name}: key ${sorted[i]?.[0]} repeats`);
|
|
87
|
+
}
|
|
88
|
+
return sorted;
|
|
89
|
+
}
|
|
90
|
+
function frame(table, ordinal, payload) {
|
|
91
|
+
const header = encode(JSON.stringify({ table, ordinal, bytes: payload.length }));
|
|
92
|
+
const prefix = new Uint8Array(4);
|
|
93
|
+
new DataView(prefix.buffer).setUint32(0, header.length);
|
|
94
|
+
return concatBytes([prefix, header, payload]);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The set digest of every table: the sum of each row's hash modulo 2^256, where a row's hash covers
|
|
98
|
+
* its table name and its key as well as its value. An empty table adds nothing.
|
|
99
|
+
*/
|
|
100
|
+
export async function bundleDigest(tables) {
|
|
101
|
+
let sum = EMPTY_DIGEST;
|
|
102
|
+
for (const name of Object.keys(tables)) {
|
|
103
|
+
for (const [key, row] of tables[name] ?? []) {
|
|
104
|
+
sum = digestAdd(sum, await rowHash(JSON.stringify([name, key]), row));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return sum;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Encodes a state as a pack: an index frame, then one frame per block in table order. Empty tables
|
|
111
|
+
* are left out. Throws `bad-bundle` for a repeated key, a block size under one, or a block over
|
|
112
|
+
* `maxFrameBytes` or `MAX_BUNDLE_BLOCK_BYTES`, which a reader would refuse.
|
|
113
|
+
*/
|
|
114
|
+
export async function buildPack(state, blockRows = BLOCK_ROWS, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
|
|
115
|
+
if (!Number.isSafeInteger(blockRows) || blockRows < 1)
|
|
116
|
+
refuse('blockRows must be 1 or more');
|
|
117
|
+
const frames = [];
|
|
118
|
+
const tables = [];
|
|
119
|
+
const names = Object.keys(state.tables)
|
|
120
|
+
.filter((name) => (state.tables[name]?.length ?? 0) > 0)
|
|
121
|
+
.sort(compareKeys);
|
|
122
|
+
for (const [table, name] of names.entries()) {
|
|
123
|
+
const rows = sortRows(name, state.tables[name] ?? []);
|
|
124
|
+
const blocks = [];
|
|
125
|
+
for (let ordinal = 0; ordinal * blockRows < rows.length; ordinal++) {
|
|
126
|
+
const chunk = rows.slice(ordinal * blockRows, (ordinal + 1) * blockRows);
|
|
127
|
+
const raw = encode(canonicalJson(chunk));
|
|
128
|
+
if (raw.length > MAX_BUNDLE_BLOCK_BYTES) {
|
|
129
|
+
refuse(`block ${name}/${ordinal} is over the block cap`);
|
|
130
|
+
}
|
|
131
|
+
const payload = await gzip(raw);
|
|
132
|
+
if (payload.length > maxFrameBytes) {
|
|
133
|
+
refuse(`block ${name}/${ordinal} is over the frame cap`);
|
|
134
|
+
}
|
|
135
|
+
blocks.push(await sha256Url(payload));
|
|
136
|
+
frames.push(frame(table, ordinal, payload));
|
|
137
|
+
}
|
|
138
|
+
tables.push({ name, rows: rows.length, first: rows[0]?.[0] ?? null, blocks });
|
|
139
|
+
}
|
|
140
|
+
const index = {
|
|
141
|
+
version: state.version,
|
|
142
|
+
codec: 'gzip',
|
|
143
|
+
blockRows,
|
|
144
|
+
digest: await bundleDigest(state.tables),
|
|
145
|
+
tables
|
|
146
|
+
};
|
|
147
|
+
const head = frame(INDEX_FRAME, 0, encode(canonicalJson(index)));
|
|
148
|
+
return { index, bytes: concatBytes([head, ...frames]) };
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Yields the frames of a pack as their bytes arrive, so a reader can verify and keep each block
|
|
152
|
+
* before the next one lands. Throws `bad-bundle` for a frame that is cut short, has a bad header,
|
|
153
|
+
* or declares a payload over `maxFrameBytes`; that check runs before the payload is read.
|
|
154
|
+
*/
|
|
155
|
+
export async function* readFrames(source, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
|
|
156
|
+
const chunks = source[Symbol.asyncIterator]();
|
|
157
|
+
let buffer = new Uint8Array(0);
|
|
158
|
+
const fill = async (size) => {
|
|
159
|
+
while (buffer.length < size) {
|
|
160
|
+
const next = await chunks.next();
|
|
161
|
+
if (next.done)
|
|
162
|
+
return false;
|
|
163
|
+
buffer = concatBytes([buffer, next.value]);
|
|
164
|
+
}
|
|
165
|
+
return true;
|
|
166
|
+
};
|
|
167
|
+
const take = (size) => {
|
|
168
|
+
const out = buffer.slice(0, size);
|
|
169
|
+
buffer = buffer.subarray(size);
|
|
170
|
+
return out;
|
|
171
|
+
};
|
|
172
|
+
while (await fill(1)) {
|
|
173
|
+
if (!(await fill(4)))
|
|
174
|
+
refuse('a frame is cut short');
|
|
175
|
+
const headerSize = new DataView(buffer.buffer, buffer.byteOffset, 4).getUint32(0);
|
|
176
|
+
if (headerSize === 0 || headerSize > HEADER_LIMIT)
|
|
177
|
+
refuse('a frame header has a bad size');
|
|
178
|
+
if (!(await fill(4 + headerSize)))
|
|
179
|
+
refuse('a frame is cut short');
|
|
180
|
+
const header = JSON.parse(decoder.decode(buffer.subarray(4, 4 + headerSize)));
|
|
181
|
+
take(4 + headerSize);
|
|
182
|
+
if (!isPlain(header) ||
|
|
183
|
+
typeof header['table'] !== 'number' ||
|
|
184
|
+
!Number.isSafeInteger(header['table']) ||
|
|
185
|
+
header['table'] < INDEX_FRAME ||
|
|
186
|
+
!isCount(header['ordinal']) ||
|
|
187
|
+
!isCount(header['bytes'])) {
|
|
188
|
+
refuse('a frame header is not { table, ordinal, bytes }');
|
|
189
|
+
}
|
|
190
|
+
const table = header['table'];
|
|
191
|
+
const ordinal = header['ordinal'];
|
|
192
|
+
const bytes = header['bytes'];
|
|
193
|
+
if (bytes > maxFrameBytes)
|
|
194
|
+
refuse('a frame is over the frame cap');
|
|
195
|
+
if (!(await fill(bytes)))
|
|
196
|
+
refuse('a frame is cut short');
|
|
197
|
+
yield { table, ordinal, payload: take(bytes) };
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function parseIndex(value) {
|
|
201
|
+
if (!isPlain(value))
|
|
202
|
+
refuse('the index is not an object');
|
|
203
|
+
const { version, codec, blockRows, digest, tables } = value;
|
|
204
|
+
if (!isCount(version) ||
|
|
205
|
+
codec !== 'gzip' ||
|
|
206
|
+
typeof blockRows !== 'number' ||
|
|
207
|
+
!Number.isSafeInteger(blockRows) ||
|
|
208
|
+
blockRows < 1 ||
|
|
209
|
+
typeof digest !== 'string' ||
|
|
210
|
+
!DIGEST_FORM.test(digest) ||
|
|
211
|
+
!Array.isArray(tables)) {
|
|
212
|
+
refuse('the index has a bad version, codec, block size or digest');
|
|
213
|
+
}
|
|
214
|
+
const names = new Set();
|
|
215
|
+
const out = tables.map((table) => {
|
|
216
|
+
if (!isPlain(table) ||
|
|
217
|
+
typeof table['name'] !== 'string' ||
|
|
218
|
+
names.has(table['name']) ||
|
|
219
|
+
!isCount(table['rows']) ||
|
|
220
|
+
table['rows'] < 1 ||
|
|
221
|
+
typeof table['first'] !== 'string' ||
|
|
222
|
+
!Array.isArray(table['blocks']) ||
|
|
223
|
+
!table['blocks'].every((hash) => typeof hash === 'string') ||
|
|
224
|
+
table['blocks'].length !== Math.ceil(table['rows'] / blockRows)) {
|
|
225
|
+
refuse('an index table does not match its rows and blocks');
|
|
226
|
+
}
|
|
227
|
+
names.add(table['name']);
|
|
228
|
+
return {
|
|
229
|
+
name: table['name'],
|
|
230
|
+
rows: table['rows'],
|
|
231
|
+
first: table['first'],
|
|
232
|
+
blocks: table['blocks']
|
|
233
|
+
};
|
|
234
|
+
});
|
|
235
|
+
return { version, codec, blockRows, digest, tables: out };
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Checks one block frame against the index and returns its rows. The hash is checked before the
|
|
239
|
+
* block is decompressed, so a corrupt block is refused without being parsed.
|
|
240
|
+
*/
|
|
241
|
+
export async function readBlock(index, block) {
|
|
242
|
+
const table = index.tables[block.table];
|
|
243
|
+
const expected = table?.blocks[block.ordinal];
|
|
244
|
+
if (expected === undefined)
|
|
245
|
+
refuse(`no block ${block.table}/${block.ordinal} in the index`);
|
|
246
|
+
if ((await sha256Url(block.payload)) !== expected) {
|
|
247
|
+
refuse(`block ${block.table}/${block.ordinal} does not match its hash`);
|
|
248
|
+
}
|
|
249
|
+
const raw = await gunzipCapped(block.payload, `block ${block.table}/${block.ordinal}`);
|
|
250
|
+
let rows;
|
|
251
|
+
try {
|
|
252
|
+
rows = JSON.parse(decoder.decode(raw));
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
refuse(`block ${block.table}/${block.ordinal} does not decode`);
|
|
256
|
+
}
|
|
257
|
+
if (!Array.isArray(rows))
|
|
258
|
+
refuse(`block ${block.table}/${block.ordinal} is not a row list`);
|
|
259
|
+
for (const entry of rows) {
|
|
260
|
+
if (!Array.isArray(entry) || typeof entry[0] !== 'string' || !isPlain(entry[1])) {
|
|
261
|
+
refuse(`block ${block.table}/${block.ordinal} has a row that is not [key, row]`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return rows;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Reads a whole pack, or a pack that arrives in chunks, and returns its index and state. Every
|
|
268
|
+
* block is checked as it lands, and the set digest is checked at the end. Throws `bad-bundle`.
|
|
269
|
+
*/
|
|
270
|
+
export async function readBundle(source, maxFrameBytes = MAX_BUNDLE_FRAME_BYTES) {
|
|
271
|
+
async function* whole(bytes) {
|
|
272
|
+
yield bytes;
|
|
273
|
+
}
|
|
274
|
+
const frames = readFrames(source instanceof Uint8Array ? whole(source) : source, maxFrameBytes);
|
|
275
|
+
let index;
|
|
276
|
+
const tables = {};
|
|
277
|
+
let table = 0;
|
|
278
|
+
let ordinal = 0;
|
|
279
|
+
for await (const block of frames) {
|
|
280
|
+
if (!index) {
|
|
281
|
+
if (block.table !== INDEX_FRAME)
|
|
282
|
+
refuse('the pack does not start with its index');
|
|
283
|
+
index = parseIndex(parseJson(block.payload));
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
const entry = index.tables[table];
|
|
287
|
+
if (!entry || block.table !== table || block.ordinal !== ordinal) {
|
|
288
|
+
refuse(`expected block ${table}/${ordinal}, got ${block.table}/${block.ordinal}`);
|
|
289
|
+
}
|
|
290
|
+
const size = Math.min(index.blockRows, entry.rows - ordinal * index.blockRows);
|
|
291
|
+
const rows = await readBlock(index, block);
|
|
292
|
+
if (rows.length !== size)
|
|
293
|
+
refuse(`block ${table}/${ordinal} holds ${rows.length} rows`);
|
|
294
|
+
const held = (tables[entry.name] ??= []);
|
|
295
|
+
for (const [key, row] of rows) {
|
|
296
|
+
if (held.length > 0 && (held[held.length - 1]?.[0] ?? '') >= key) {
|
|
297
|
+
refuse(`${entry.name}: key ${key} is out of order`);
|
|
298
|
+
}
|
|
299
|
+
held.push([key, row]);
|
|
300
|
+
}
|
|
301
|
+
if (ordinal === 0 && rows[0]?.[0] !== entry.first)
|
|
302
|
+
refuse(`${entry.name}: wrong first key`);
|
|
303
|
+
if (ordinal + 1 < entry.blocks.length)
|
|
304
|
+
ordinal++;
|
|
305
|
+
else {
|
|
306
|
+
table++;
|
|
307
|
+
ordinal = 0;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (!index)
|
|
311
|
+
refuse('the pack is empty');
|
|
312
|
+
if (table !== index.tables.length)
|
|
313
|
+
refuse('the pack ends before its last block');
|
|
314
|
+
if ((await bundleDigest(tables)) !== index.digest)
|
|
315
|
+
refuse('the set digest does not match');
|
|
316
|
+
return { index, state: { version: index.version, tables } };
|
|
317
|
+
}
|
|
318
|
+
function parseJson(payload) {
|
|
319
|
+
try {
|
|
320
|
+
return JSON.parse(decoder.decode(payload));
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return refuse('the index is not JSON');
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
/** the change from `from` to `to`: puts for new or changed rows, deletes for rows that left */
|
|
327
|
+
export async function makeDelta(from, to) {
|
|
328
|
+
const names = [...new Set([...Object.keys(from.tables), ...Object.keys(to.tables)])].sort(compareKeys);
|
|
329
|
+
const tables = {};
|
|
330
|
+
for (const name of names) {
|
|
331
|
+
const before = new Map(from.tables[name] ?? []);
|
|
332
|
+
const after = sortRows(name, to.tables[name] ?? []);
|
|
333
|
+
const kept = new Set(after.map(([key]) => key));
|
|
334
|
+
const puts = after.filter(([key, row]) => {
|
|
335
|
+
const old = before.get(key);
|
|
336
|
+
return old === undefined || canonicalJson(old) !== canonicalJson(row);
|
|
337
|
+
});
|
|
338
|
+
const deletes = [...before.keys()].filter((key) => !kept.has(key)).sort(compareKeys);
|
|
339
|
+
if (puts.length > 0 || deletes.length > 0)
|
|
340
|
+
tables[name] = { puts, deletes };
|
|
341
|
+
}
|
|
342
|
+
return { from: from.version, to: to.version, digest: await bundleDigest(to.tables), tables };
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Applies a delta to the state at its `from` version and returns the state at `to`. Refuses a base
|
|
346
|
+
* at another version, and a result whose set digest is not the delta's.
|
|
347
|
+
*/
|
|
348
|
+
export async function applyDelta(base, delta) {
|
|
349
|
+
if (base.version !== delta.from) {
|
|
350
|
+
refuse(`the delta starts at ${delta.from}, the base holds ${base.version}`);
|
|
351
|
+
}
|
|
352
|
+
const names = new Set([...Object.keys(base.tables), ...Object.keys(delta.tables)]);
|
|
353
|
+
const tables = {};
|
|
354
|
+
for (const name of names) {
|
|
355
|
+
const change = delta.tables[name];
|
|
356
|
+
const rows = new Map(base.tables[name] ?? []);
|
|
357
|
+
for (const key of change?.deletes ?? [])
|
|
358
|
+
rows.delete(key);
|
|
359
|
+
for (const [key, row] of change?.puts ?? [])
|
|
360
|
+
rows.set(key, row);
|
|
361
|
+
if (rows.size > 0)
|
|
362
|
+
tables[name] = sortRows(name, [...rows]);
|
|
363
|
+
}
|
|
364
|
+
if ((await bundleDigest(tables)) !== delta.digest) {
|
|
365
|
+
refuse('the applied state does not match the delta digest');
|
|
366
|
+
}
|
|
367
|
+
return { version: delta.to, tables };
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* One delta equal to applying `deltas` in order. The deltas must chain: each `from` is the previous
|
|
371
|
+
* `to`. A key put and then removed becomes a tombstone, which a holder without the key ignores.
|
|
372
|
+
*/
|
|
373
|
+
export function mergeDeltas(deltas) {
|
|
374
|
+
const first = deltas[0];
|
|
375
|
+
const last = deltas[deltas.length - 1];
|
|
376
|
+
if (!first || !last)
|
|
377
|
+
refuse('there is no delta to merge');
|
|
378
|
+
for (let i = 1; i < deltas.length; i++) {
|
|
379
|
+
if (deltas[i - 1]?.to !== deltas[i]?.from)
|
|
380
|
+
refuse('the deltas do not chain');
|
|
381
|
+
}
|
|
382
|
+
const ops = new Map();
|
|
383
|
+
for (const delta of deltas) {
|
|
384
|
+
for (const [name, change] of Object.entries(delta.tables)) {
|
|
385
|
+
const table = ops.get(name) ?? new Map();
|
|
386
|
+
ops.set(name, table);
|
|
387
|
+
for (const key of change.deletes)
|
|
388
|
+
table.set(key, null);
|
|
389
|
+
for (const [key, row] of change.puts)
|
|
390
|
+
table.set(key, row);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
const tables = {};
|
|
394
|
+
for (const name of [...ops.keys()].sort(compareKeys)) {
|
|
395
|
+
const entries = [...(ops.get(name) ?? [])].sort(([a], [b]) => compareKeys(a, b));
|
|
396
|
+
const puts = [];
|
|
397
|
+
const deletes = [];
|
|
398
|
+
for (const [key, row] of entries) {
|
|
399
|
+
if (row === null)
|
|
400
|
+
deletes.push(key);
|
|
401
|
+
else
|
|
402
|
+
puts.push([key, row]);
|
|
403
|
+
}
|
|
404
|
+
if (puts.length > 0 || deletes.length > 0)
|
|
405
|
+
tables[name] = { puts, deletes };
|
|
406
|
+
}
|
|
407
|
+
return { from: first.from, to: last.to, digest: last.digest, tables };
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* whether a delta has more than {@link COMPACT_OVERLAY_ROWS} rows or
|
|
411
|
+
* {@link COMPACT_OVERLAY_BYTES} bytes of JSON
|
|
412
|
+
*/
|
|
413
|
+
export function compactionDue(delta) {
|
|
414
|
+
let rows = 0;
|
|
415
|
+
for (const change of Object.values(delta.tables)) {
|
|
416
|
+
rows += change.puts.length + change.deletes.length;
|
|
417
|
+
}
|
|
418
|
+
return (rows > COMPACT_OVERLAY_ROWS ||
|
|
419
|
+
encoder.encode(JSON.stringify(delta)).length > COMPACT_OVERLAY_BYTES);
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Folds a delta into a base pack and returns the new pack, built with the base's block size. Its
|
|
423
|
+
* bytes equal {@link buildPack} run on the state the delta reaches.
|
|
424
|
+
*/
|
|
425
|
+
export async function compactBundle(base, overlay) {
|
|
426
|
+
const { index, state } = await readBundle(base);
|
|
427
|
+
return buildPack(await applyDelta(state, overlay), index.blockRows);
|
|
428
|
+
}
|
|
@@ -1,19 +1,49 @@
|
|
|
1
1
|
import type { PROTOCOL_VERSION } from './artifact.js';
|
|
2
|
+
import type { BundleDelta } from './bundle.js';
|
|
2
3
|
import type { CollectorConfig } from './collector.js';
|
|
3
4
|
import type { SerializedRoute } from './route.js';
|
|
4
5
|
import type { CompiledTemplate } from './template/compile.js';
|
|
5
6
|
import type { Mode } from './view.js';
|
|
7
|
+
import type { ResourceWrite, WriteConfig } from './write.js';
|
|
6
8
|
/** every path under this prefix belongs to spool */
|
|
7
9
|
export declare const SPOOL_PREFIX = "/.spool/";
|
|
8
|
-
/**
|
|
10
|
+
/** the prefix of the site packs, one path per version: `/.spool/bundle/<version>` */
|
|
11
|
+
export declare const BUNDLE_PREFIX = "/.spool/bundle/";
|
|
12
|
+
/** the prefix of the deltas between two packs: `/.spool/bundle/delta/<from>/<to>` */
|
|
13
|
+
export declare const BUNDLE_DELTA_PREFIX = "/.spool/bundle/delta/";
|
|
14
|
+
/** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
|
|
9
15
|
export declare const PATHS: {
|
|
10
16
|
readonly sync: "/.spool/sync";
|
|
11
17
|
readonly artifact: "/.spool/artifact";
|
|
18
|
+
readonly dataset: "/.spool/dataset";
|
|
12
19
|
readonly beacon: "/.spool/beacon";
|
|
20
|
+
readonly write: "/.spool/write";
|
|
13
21
|
readonly worker: "/spool-sw.js";
|
|
14
22
|
};
|
|
23
|
+
/**
|
|
24
|
+
* where a visible page holds one server-sent event stream open: the principal lease, a `revoked`
|
|
25
|
+
* event, and public tag changes where the site opts in. The request takes `partition` (the principal
|
|
26
|
+
* partition the page holds) and `have` (the cursor the page holds); the worker never handles it.
|
|
27
|
+
*/
|
|
28
|
+
export declare const LIVE_PATH = "/.spool/live";
|
|
29
|
+
/**
|
|
30
|
+
* the text a page sends over a live socket to renew its lease. The server answers it with a `lease`
|
|
31
|
+
* event, which a hibernating Durable Object can do without waking.
|
|
32
|
+
*/
|
|
33
|
+
export declare const LIVE_PING = "ping";
|
|
34
|
+
/** the cookie a no-worker page writes with the version it holds; the server reads it, nothing else does */
|
|
35
|
+
export declare const COOKIES: {
|
|
36
|
+
readonly held: "spool-held";
|
|
37
|
+
};
|
|
15
38
|
/** the same-origin path of the page runtime script for a given version */
|
|
16
39
|
export declare function clientPath(version: string): string;
|
|
40
|
+
/** the same-origin path of the in-page navigation script, loaded once the page holds a head */
|
|
41
|
+
export declare function navigatePath(version: string): string;
|
|
42
|
+
/**
|
|
43
|
+
* the same-origin path of the boot script for a runtime version and a pack version: the page
|
|
44
|
+
* runtime, the navigation script and that pack in one file
|
|
45
|
+
*/
|
|
46
|
+
export declare function bootPath(version: string, pack: number): string;
|
|
17
47
|
/**
|
|
18
48
|
* Priors and the hot list for provisioning. Every number is a scenario assumption the operator
|
|
19
49
|
* replaces with its own telemetry; `hot` names paths to warm in priority order, riding the next
|
|
@@ -54,10 +84,25 @@ export type ThirdPartyConfig = {
|
|
|
54
84
|
/** keep opaque (no-cors) responses, which Chromium counts as about 7 MB each against quota; default false */
|
|
55
85
|
allowOpaque?: boolean;
|
|
56
86
|
};
|
|
87
|
+
/**
|
|
88
|
+
* The pack a worker script carries: the current version and set digest, and the deltas from the
|
|
89
|
+
* versions just before it, each straight to the current one.
|
|
90
|
+
*/
|
|
91
|
+
export type StubPack = {
|
|
92
|
+
version: number;
|
|
93
|
+
digest: string;
|
|
94
|
+
deltas: BundleDelta[];
|
|
95
|
+
};
|
|
57
96
|
/** what the server compiles into the worker script: routes, templates and the tag to inject */
|
|
58
97
|
export type SwConfig = {
|
|
59
98
|
protocol: typeof PROTOCOL_VERSION;
|
|
60
99
|
version: string;
|
|
100
|
+
/** the hash of the worker's own code; a different one empties the replica, a content version does not */
|
|
101
|
+
runtime?: string;
|
|
102
|
+
/** the pack the script carries, when the server holds one for its current version */
|
|
103
|
+
pack?: StubPack;
|
|
104
|
+
/** the site's security headers, as name and value pairs, for a worker that holds no site policy yet */
|
|
105
|
+
policy?: Array<[string, string]>;
|
|
61
106
|
routes: SerializedRoute[];
|
|
62
107
|
/** compiled templates keyed by route pattern */
|
|
63
108
|
templates: Record<string, CompiledTemplate>;
|
|
@@ -71,6 +116,13 @@ export type SwConfig = {
|
|
|
71
116
|
shadowSample?: number;
|
|
72
117
|
/** whether the worker queues a `spool.view` per page view; true when absent */
|
|
73
118
|
views?: boolean;
|
|
119
|
+
/** seconds the browser may hold the worker script: the lease after which the page checks it again */
|
|
120
|
+
lease?: number;
|
|
121
|
+
/**
|
|
122
|
+
* The operator's counter for the stub. Its bytes change when the stamp does, so a static host that
|
|
123
|
+
* republishes each lease period makes a forced check install a new script, and the install is a contact.
|
|
124
|
+
*/
|
|
125
|
+
stamp?: number;
|
|
74
126
|
/** what the worker assumes before telemetry says otherwise, and which pages to warm */
|
|
75
127
|
provision?: ProvisionConfig;
|
|
76
128
|
/**
|
|
@@ -86,4 +138,31 @@ export type SwConfig = {
|
|
|
86
138
|
collectors?: CollectorConfig[];
|
|
87
139
|
/** the opt-in cache for versioned cross-origin files */
|
|
88
140
|
thirdParty?: ThirdPartyConfig;
|
|
141
|
+
/**
|
|
142
|
+
* The share, above 0 and at most 1, of the storage headroom one dataset may use. The worker
|
|
143
|
+
* refuses a seed whose rows would use more; 1 when absent.
|
|
144
|
+
*/
|
|
145
|
+
datasetShare?: number;
|
|
146
|
+
/** the declared write classes; absent while declared writes are off, which is the default */
|
|
147
|
+
writes?: WriteConfig[];
|
|
148
|
+
/** the resource writes a request is sent to the server for, each applied from its `Spool-Delta` */
|
|
149
|
+
resourceWrites?: readonly ResourceWrite[];
|
|
150
|
+
/** the principal datasets and how long each may answer after its last confirmed sync; absent while the flag is off */
|
|
151
|
+
principalDatasets?: Array<{
|
|
152
|
+
name: string;
|
|
153
|
+
maxStaleMs: number;
|
|
154
|
+
}>;
|
|
155
|
+
/**
|
|
156
|
+
* Present while the server answers `/.spool/live`. `public` is true when the stream also carries
|
|
157
|
+
* public freshness, so the worker treats its public replica as current while a lease is open.
|
|
158
|
+
* `mode` is `always` when the page opens the stream whenever it is visible, and `adaptive` when the
|
|
159
|
+
* worker asks for it at the second lease renewal of a page's visible session. `socket` is true
|
|
160
|
+
* when the hub also accepts a WebSocket upgrade at the same path; a page then opens a socket
|
|
161
|
+
* and falls back to server-sent events if the upgrade fails.
|
|
162
|
+
*/
|
|
163
|
+
live?: {
|
|
164
|
+
public: boolean;
|
|
165
|
+
mode: 'adaptive' | 'always';
|
|
166
|
+
socket?: true;
|
|
167
|
+
};
|
|
89
168
|
};
|
package/dist/protocol/config.js
CHANGED
|
@@ -1,13 +1,43 @@
|
|
|
1
1
|
/** every path under this prefix belongs to spool */
|
|
2
2
|
export const SPOOL_PREFIX = '/.spool/';
|
|
3
|
-
/**
|
|
3
|
+
/** the prefix of the site packs, one path per version: `/.spool/bundle/<version>` */
|
|
4
|
+
export const BUNDLE_PREFIX = '/.spool/bundle/';
|
|
5
|
+
/** the prefix of the deltas between two packs: `/.spool/bundle/delta/<from>/<to>` */
|
|
6
|
+
export const BUNDLE_DELTA_PREFIX = '/.spool/bundle/delta/';
|
|
7
|
+
/** where the server answers sync, artifact, dataset and beacon requests, and where the worker script lives */
|
|
4
8
|
export const PATHS = {
|
|
5
9
|
sync: '/.spool/sync',
|
|
6
10
|
artifact: '/.spool/artifact',
|
|
11
|
+
dataset: '/.spool/dataset',
|
|
7
12
|
beacon: '/.spool/beacon',
|
|
13
|
+
write: '/.spool/write',
|
|
8
14
|
worker: '/spool-sw.js'
|
|
9
15
|
};
|
|
16
|
+
/**
|
|
17
|
+
* where a visible page holds one server-sent event stream open: the principal lease, a `revoked`
|
|
18
|
+
* event, and public tag changes where the site opts in. The request takes `partition` (the principal
|
|
19
|
+
* partition the page holds) and `have` (the cursor the page holds); the worker never handles it.
|
|
20
|
+
*/
|
|
21
|
+
export const LIVE_PATH = '/.spool/live';
|
|
22
|
+
/**
|
|
23
|
+
* the text a page sends over a live socket to renew its lease. The server answers it with a `lease`
|
|
24
|
+
* event, which a hibernating Durable Object can do without waking.
|
|
25
|
+
*/
|
|
26
|
+
export const LIVE_PING = 'ping';
|
|
27
|
+
/** the cookie a no-worker page writes with the version it holds; the server reads it, nothing else does */
|
|
28
|
+
export const COOKIES = { held: 'spool-held' };
|
|
10
29
|
/** the same-origin path of the page runtime script for a given version */
|
|
11
30
|
export function clientPath(version) {
|
|
12
31
|
return `${SPOOL_PREFIX}${encodeURIComponent(version)}/client.js`;
|
|
13
32
|
}
|
|
33
|
+
/** the same-origin path of the in-page navigation script, loaded once the page holds a head */
|
|
34
|
+
export function navigatePath(version) {
|
|
35
|
+
return `${SPOOL_PREFIX}${encodeURIComponent(version)}/navigate.js`;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* the same-origin path of the boot script for a runtime version and a pack version: the page
|
|
39
|
+
* runtime, the navigation script and that pack in one file
|
|
40
|
+
*/
|
|
41
|
+
export function bootPath(version, pack) {
|
|
42
|
+
return `${SPOOL_PREFIX}${encodeURIComponent(version)}/boot/${pack}.js`;
|
|
43
|
+
}
|