@drupflare/spool 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -1607
- package/dist/angular/index.d.ts +13 -0
- package/dist/angular/index.js +25 -0
- package/dist/angular/pages.d.ts +28 -0
- package/dist/angular/pages.js +85 -0
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/location.d.ts +6 -0
- package/dist/cdn/location.js +8 -0
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/bootstrap.d.ts +57 -0
- package/dist/client/bootstrap.js +307 -0
- package/dist/client/contact.d.ts +24 -0
- package/dist/client/contact.js +66 -0
- package/dist/client/declared.d.ts +23 -0
- package/dist/client/declared.js +68 -0
- package/dist/client/entry.js +123 -2
- package/dist/client/gate.d.ts +1 -0
- package/dist/client/gate.js +148 -0
- package/dist/client/index.d.ts +25 -0
- package/dist/client/index.js +1 -0
- package/dist/client/live.d.ts +66 -0
- package/dist/client/live.js +225 -0
- package/dist/client/navigate-entry.d.ts +1 -0
- package/dist/client/navigate-entry.js +4 -0
- package/dist/client/navigate.d.ts +54 -0
- package/dist/client/navigate.js +272 -0
- package/dist/client/persist.d.ts +2 -0
- package/dist/client/persist.js +9 -0
- package/dist/client/register.d.ts +34 -2
- package/dist/client/register.js +64 -7
- package/dist/express/index.d.ts +11 -0
- package/dist/express/index.js +209 -0
- package/dist/hono/index.d.ts +19 -0
- package/dist/hono/index.js +49 -0
- package/dist/next/index.d.ts +33 -0
- package/dist/next/index.js +41 -0
- package/dist/next/pages.d.ts +22 -0
- package/dist/next/pages.js +39 -0
- package/dist/node/listener.d.ts +3 -0
- package/dist/node/listener.js +57 -2
- package/dist/nuxt/index.d.ts +22 -0
- package/dist/nuxt/index.js +104 -0
- package/dist/nuxt/pages.d.ts +26 -0
- package/dist/nuxt/pages.js +48 -0
- package/dist/nuxt/runtime/index.d.ts +24 -0
- package/dist/nuxt/runtime/index.js +58 -0
- package/dist/plugins/coverage.d.ts +14 -0
- package/dist/plugins/coverage.js +27 -0
- package/dist/plugins/mode.d.ts +19 -0
- package/dist/plugins/mode.js +20 -0
- package/dist/plugins/path.d.ts +16 -0
- package/dist/plugins/path.js +139 -0
- package/dist/plugins/report.d.ts +22 -0
- package/dist/plugins/report.js +18 -0
- package/dist/plugins/routes.d.ts +38 -0
- package/dist/plugins/routes.js +52 -0
- package/dist/plugins/static.d.ts +42 -0
- package/dist/plugins/static.js +97 -0
- package/dist/protocol/bundle.d.ts +111 -0
- package/dist/protocol/bundle.js +428 -0
- package/dist/protocol/config.d.ts +80 -1
- package/dist/protocol/config.js +31 -1
- package/dist/protocol/dataset.d.ts +189 -0
- package/dist/protocol/dataset.js +212 -0
- package/dist/protocol/envelope.d.ts +34 -1
- package/dist/protocol/envelope.js +16 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index-rows.d.ts +22 -0
- package/dist/protocol/index-rows.js +53 -0
- package/dist/protocol/index.d.ts +5 -0
- package/dist/protocol/index.js +5 -0
- package/dist/protocol/normalise.d.ts +4 -4
- package/dist/protocol/normalise.js +9 -5
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/ring.d.ts +58 -0
- package/dist/protocol/ring.js +6 -0
- package/dist/protocol/route.d.ts +63 -13
- package/dist/protocol/route.js +126 -6
- package/dist/protocol/tag.d.ts +22 -2
- package/dist/protocol/tag.js +25 -4
- package/dist/protocol/view.d.ts +2 -0
- package/dist/protocol/view.js +8 -2
- package/dist/protocol/views.d.ts +6 -0
- package/dist/protocol/views.js +15 -0
- package/dist/protocol/write.d.ts +234 -0
- package/dist/protocol/write.js +343 -0
- package/dist/proxy/eligible.d.ts +22 -1
- package/dist/proxy/eligible.js +62 -15
- package/dist/proxy/proxy.js +28 -18
- package/dist/react-router/index.d.ts +20 -0
- package/dist/react-router/index.js +60 -0
- package/dist/react-router/pages.d.ts +34 -0
- package/dist/react-router/pages.js +83 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/demand.d.ts +49 -0
- package/dist/report/demand.js +60 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +66 -1
- package/dist/report/economics.js +208 -1
- package/dist/report/headline.d.ts +158 -0
- package/dist/report/headline.js +473 -0
- package/dist/report/index.d.ts +4 -0
- package/dist/report/index.js +4 -0
- package/dist/report/inputs.d.ts +176 -0
- package/dist/report/inputs.js +248 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +7 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +8 -0
- package/dist/report/traffic.js +7 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/bundle.d.ts +55 -0
- package/dist/server/bundle.js +172 -0
- package/dist/server/dataset.d.ts +54 -0
- package/dist/server/dataset.js +355 -0
- package/dist/server/embedded.d.ts +7 -3
- package/dist/server/embedded.js +7 -3
- package/dist/server/index.d.ts +6 -0
- package/dist/server/index.js +6 -0
- package/dist/server/live-sockets.d.ts +54 -0
- package/dist/server/live-sockets.js +131 -0
- package/dist/server/live.d.ts +146 -0
- package/dist/server/live.js +228 -0
- package/dist/server/ring.d.ts +77 -0
- package/dist/server/ring.js +311 -0
- package/dist/server/spool.d.ts +158 -3
- package/dist/server/spool.js +578 -41
- package/dist/server/static.d.ts +42 -0
- package/dist/server/static.js +29 -0
- package/dist/server/stub.d.ts +43 -5
- package/dist/server/stub.js +126 -8
- package/dist/server/write.d.ts +49 -0
- package/dist/server/write.js +223 -0
- package/dist/stores/durable-object.d.ts +46 -0
- package/dist/stores/durable-object.js +114 -0
- package/dist/sw/analytics.d.ts +7 -1
- package/dist/sw/analytics.js +16 -2
- package/dist/sw/coverage.d.ts +6 -0
- package/dist/sw/coverage.js +34 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/family.d.ts +15 -0
- package/dist/sw/family.js +48 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.d.ts +2 -0
- package/dist/sw/idb.js +80 -11
- package/dist/sw/index.d.ts +2 -0
- package/dist/sw/index.js +1 -0
- package/dist/sw/pack/store.d.ts +128 -0
- package/dist/sw/pack/store.js +737 -0
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/policy.d.ts +2 -0
- package/dist/sw/policy.js +11 -0
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/probe.d.ts +16 -0
- package/dist/sw/probe.js +25 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/router.d.ts +3 -2
- package/dist/sw/router.js +15 -2
- package/dist/sw/runtime.d.ts +90 -2
- package/dist/sw/runtime.js +857 -61
- package/dist/sw/scheduler.d.ts +6 -1
- package/dist/sw/scheduler.js +22 -5
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +147 -0
- package/dist/sw/write.js +625 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -2
- package/dist/vite/index.d.ts +26 -0
- package/dist/vite/index.js +66 -0
- package/dist/vue/index.d.ts +36 -0
- package/dist/vue/index.js +53 -0
- package/package.json +65 -4
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { buildPack } from '../protocol/bundle.js';
|
|
2
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
3
|
+
import { indexPlan, indexRowsOf } from '../protocol/index-rows.js';
|
|
4
|
+
import { canonicalJson } from '../protocol/query.js';
|
|
5
|
+
import { json } from './sync.js';
|
|
6
|
+
/** a store that lives as long as the process; fine for tests and single-instance Node */
|
|
7
|
+
export function memoryBundleStore() {
|
|
8
|
+
const held = new Map();
|
|
9
|
+
return {
|
|
10
|
+
put: async (version, bytes) => {
|
|
11
|
+
held.set(version, bytes);
|
|
12
|
+
},
|
|
13
|
+
get: async (version) => held.get(version)
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const encoder = new TextEncoder();
|
|
17
|
+
async function seedRows(dataset, ctx) {
|
|
18
|
+
const rows = [];
|
|
19
|
+
let bytes = 0;
|
|
20
|
+
const take = (row) => {
|
|
21
|
+
const key = dataset.key(row);
|
|
22
|
+
if (typeof key !== 'string') {
|
|
23
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: a row key must be a string`);
|
|
24
|
+
}
|
|
25
|
+
rows.push([key, row]);
|
|
26
|
+
bytes += encoder.encode(canonicalJson(row)).length;
|
|
27
|
+
if (rows.length > dataset.maxRows || bytes > dataset.maxBytes) {
|
|
28
|
+
throw new ProtocolError('dataset-limit', `${dataset.name}: over the seed limit`);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
const add = (item) => {
|
|
32
|
+
if (Array.isArray(item))
|
|
33
|
+
for (const row of item)
|
|
34
|
+
take(row);
|
|
35
|
+
else
|
|
36
|
+
take(item);
|
|
37
|
+
};
|
|
38
|
+
const result = await dataset.seed(ctx);
|
|
39
|
+
if (Array.isArray(result))
|
|
40
|
+
for (const item of result)
|
|
41
|
+
add(item);
|
|
42
|
+
else if (Symbol.asyncIterator in Object(result)) {
|
|
43
|
+
for await (const item of result)
|
|
44
|
+
add(item);
|
|
45
|
+
}
|
|
46
|
+
else
|
|
47
|
+
for (const item of result)
|
|
48
|
+
add(item);
|
|
49
|
+
return rows;
|
|
50
|
+
}
|
|
51
|
+
function sameBytes(a, b) {
|
|
52
|
+
if (a.length !== b.length)
|
|
53
|
+
return false;
|
|
54
|
+
for (let i = 0; i < a.length; i++)
|
|
55
|
+
if (a[i] !== b[i])
|
|
56
|
+
return false;
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* The index tables the public routes need. A family route adds `path:<dataset>`, each row's
|
|
61
|
+
* concrete path mapped to its key. A query route adds `query:<dataset>`, each row's filter and sort
|
|
62
|
+
* fields. A plan, a principal route and a pattern with a param other than the key add nothing.
|
|
63
|
+
*/
|
|
64
|
+
export function indexTables(routes, tables) {
|
|
65
|
+
const held = (dataset, pattern) => {
|
|
66
|
+
if (!Object.hasOwn(tables, dataset)) {
|
|
67
|
+
throw new ProtocolError('bad-bundle', `${pattern}: ${dataset} is not a public dataset`);
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
for (const route of routes) {
|
|
71
|
+
if (route.scope !== 'public')
|
|
72
|
+
continue;
|
|
73
|
+
if (route.family)
|
|
74
|
+
held(route.family.dataset, route.pattern);
|
|
75
|
+
if (route.query?.dataset !== undefined && route.query.plan === undefined) {
|
|
76
|
+
held(route.query.dataset, route.pattern);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const plan = indexPlan(routes);
|
|
80
|
+
const pathDatasets = [...new Set(plan.paths.map((path) => path.dataset))];
|
|
81
|
+
const datasets = new Set([...pathDatasets, ...plan.fields.keys()]);
|
|
82
|
+
const out = {};
|
|
83
|
+
for (const dataset of pathDatasets)
|
|
84
|
+
out[`path:${dataset}`] = [];
|
|
85
|
+
for (const dataset of plan.fields.keys())
|
|
86
|
+
out[`query:${dataset}`] = [];
|
|
87
|
+
for (const name of Object.keys(out)) {
|
|
88
|
+
if (Object.hasOwn(tables, name)) {
|
|
89
|
+
throw new ProtocolError('bad-bundle', `${name} is already a table of the pack`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
for (const dataset of datasets) {
|
|
93
|
+
for (const [key, row] of tables[dataset] ?? []) {
|
|
94
|
+
for (const [table, rowKey, value] of indexRowsOf(plan, dataset, key, row)) {
|
|
95
|
+
if (value !== null)
|
|
96
|
+
out[table]?.push([rowKey, value]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Builds the pack of the public datasets at `version` and keeps it in `store`. A version that
|
|
104
|
+
* already holds other bytes is refused, since its URL is served as immutable. A site with no
|
|
105
|
+
* dataset builds an empty pack.
|
|
106
|
+
*/
|
|
107
|
+
export async function buildBundle(options) {
|
|
108
|
+
const ctx = { request: new Request(options.url), url: options.url };
|
|
109
|
+
const tables = {};
|
|
110
|
+
for (const dataset of options.datasets) {
|
|
111
|
+
if (dataset.scope !== 'public')
|
|
112
|
+
continue;
|
|
113
|
+
if (Object.hasOwn(tables, dataset.name)) {
|
|
114
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: declared twice`);
|
|
115
|
+
}
|
|
116
|
+
tables[dataset.name] = await seedRows(dataset, ctx);
|
|
117
|
+
}
|
|
118
|
+
const { index, bytes } = await buildPack({
|
|
119
|
+
version: options.version,
|
|
120
|
+
tables: { ...tables, ...indexTables(options.routes ?? [], tables) }
|
|
121
|
+
}, options.blockRows, options.maxBytes);
|
|
122
|
+
const held = await options.store.get(options.version);
|
|
123
|
+
if (held === undefined)
|
|
124
|
+
await options.store.put(options.version, bytes);
|
|
125
|
+
else if (!sameBytes(held, bytes)) {
|
|
126
|
+
throw new ProtocolError('bad-bundle', `bundle ${options.version} already holds other bytes`);
|
|
127
|
+
}
|
|
128
|
+
return { version: options.version, index, bytes };
|
|
129
|
+
}
|
|
130
|
+
/** the response to `GET /.spool/bundle/<version>`: the pack bytes, or 404 for a version not held */
|
|
131
|
+
export async function respondBundle(store, version) {
|
|
132
|
+
const number = /^\d{1,15}$/.test(version) ? Number(version) : undefined;
|
|
133
|
+
const bytes = store && number !== undefined ? await store.get(number) : undefined;
|
|
134
|
+
if (bytes === undefined)
|
|
135
|
+
return json({ error: 'no bundle' }, 404);
|
|
136
|
+
return new Response(bytes, {
|
|
137
|
+
headers: {
|
|
138
|
+
'content-type': 'application/octet-stream',
|
|
139
|
+
'cache-control': 'public, max-age=31536000, immutable'
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
/** base64 of `bytes`, built in slices so the argument list of `fromCharCode` stays small */
|
|
144
|
+
function base64(bytes) {
|
|
145
|
+
let text = '';
|
|
146
|
+
for (let at = 0; at < bytes.length; at += 0x8000) {
|
|
147
|
+
text += String.fromCharCode(...bytes.subarray(at, at + 0x8000));
|
|
148
|
+
}
|
|
149
|
+
return btoa(text);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* The response to `GET /.spool/<runtime>/boot/<version>.js`: `code`, the page runtime and the
|
|
153
|
+
* navigation script, after a line that hands the pack at `version` to the runtime as base64. One
|
|
154
|
+
* script tag then installs the runtime and the pack in one request. With a `shape` the same line
|
|
155
|
+
* also hands over the routes and templates, so no document has to carry them. 404 for a version
|
|
156
|
+
* not held.
|
|
157
|
+
*/
|
|
158
|
+
export async function respondBoot(store, version, code, shape) {
|
|
159
|
+
const number = /^\d{1,15}$/.test(version) ? Number(version) : undefined;
|
|
160
|
+
const bytes = store && number !== undefined ? await store.get(number) : undefined;
|
|
161
|
+
if (bytes === undefined || number === undefined)
|
|
162
|
+
return json({ error: 'no bundle' }, 404);
|
|
163
|
+
const boot = '(globalThis.__SPOOL_BOOT__=globalThis.__SPOOL_BOOT__||{})';
|
|
164
|
+
const given = shape ? `${boot}.shape=${JSON.stringify(shape)};` : '';
|
|
165
|
+
const hand = `${given}${boot}.pack={version:${number},data:"${base64(bytes)}"};\n`;
|
|
166
|
+
return new Response(hand + code, {
|
|
167
|
+
headers: {
|
|
168
|
+
'content-type': 'text/javascript; charset=utf-8',
|
|
169
|
+
'cache-control': 'public, max-age=31536000, immutable'
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { type Dataset, type DatasetContext, type DatasetRequest, type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
|
|
2
|
+
import { type DatasetInputs, type ProvisionPrior } from '../protocol/provision.js';
|
|
3
|
+
import type { Identity } from '../protocol/route.js';
|
|
4
|
+
/**
|
|
5
|
+
* The body for a request. With `since` and the current schema it is a patch of the rows that
|
|
6
|
+
* changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
|
|
7
|
+
* `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
|
|
8
|
+
* seed page, with the proof on the last.
|
|
9
|
+
*/
|
|
10
|
+
export declare function answerDataset(dataset: Dataset, ask: DatasetRequest, ctx: DatasetContext): Promise<DatasetResponse>;
|
|
11
|
+
/** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
|
|
12
|
+
export declare const NATIVE_LOOK_DAYS = 1;
|
|
13
|
+
/** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
|
|
14
|
+
export declare const NOT_MODIFIED_BYTES = 220;
|
|
15
|
+
/** what the server has seen of one `provision: 'auto'` dataset, in memory */
|
|
16
|
+
export type DatasetStats = {
|
|
17
|
+
/** compressed bytes of a full seed, once one has been built */
|
|
18
|
+
seedBytes?: number;
|
|
19
|
+
/** compressed bytes of a patch: the server's own one-update patch first, then an exponentially weighted mean of the patches answered */
|
|
20
|
+
patchBytes?: number;
|
|
21
|
+
/** the first and latest cursor the server answered, with the time */
|
|
22
|
+
first?: {
|
|
23
|
+
cursor: number;
|
|
24
|
+
at: number;
|
|
25
|
+
};
|
|
26
|
+
last?: {
|
|
27
|
+
cursor: number;
|
|
28
|
+
at: number;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
/** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
|
|
32
|
+
export declare function datasetInputs(seed: number, stats: DatasetStats, prior: ProvisionPrior): DatasetInputs;
|
|
33
|
+
/** the admission inputs `respondDataset` needs from the server: its prior and what it has seen, by dataset name */
|
|
34
|
+
export type Admission = {
|
|
35
|
+
prior: ProvisionPrior;
|
|
36
|
+
stats: Map<string, DatasetStats>;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* The fields of a row one viewer may hold, from the dataset's `fields`. A family dataset with no
|
|
40
|
+
* `fields` gives the viewer its own row whole and refuses any other row; a dataset no family reads
|
|
41
|
+
* has no view and every row goes out whole.
|
|
42
|
+
*/
|
|
43
|
+
export declare function viewOf(dataset: Dataset, viewer: Identity, family?: boolean): ((row: DatasetRow) => DatasetRow) | undefined;
|
|
44
|
+
/** who is asking a principal dataset, and the `Spool-Partition` value that names them */
|
|
45
|
+
export type DatasetPrincipal = {
|
|
46
|
+
identity: Identity;
|
|
47
|
+
partition: string;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
|
|
51
|
+
* first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
|
|
52
|
+
* with the control that bound.
|
|
53
|
+
*/
|
|
54
|
+
export declare function respondDataset(request: Request, url: URL, datasets: ReadonlyMap<string, Dataset>, admission: Admission, who: (() => Promise<DatasetPrincipal | null>) | undefined, families: ReadonlySet<string>): Promise<Response>;
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { DATASET_PAGE_ROWS, parseDatasetRequest } from '../protocol/dataset.js';
|
|
2
|
+
import { HEADERS } from '../protocol/envelope.js';
|
|
3
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
4
|
+
import { datasetBenefit } from '../protocol/provision.js';
|
|
5
|
+
import { isSetProof, setProof } from '../protocol/query.js';
|
|
6
|
+
import { json } from './sync.js';
|
|
7
|
+
const encoder = new TextEncoder();
|
|
8
|
+
function bad(dataset, message) {
|
|
9
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
function limit(dataset, message) {
|
|
12
|
+
throw new ProtocolError('dataset-limit', `${dataset.name}: ${message}`);
|
|
13
|
+
}
|
|
14
|
+
function checkCursor(dataset, value, what) {
|
|
15
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
|
16
|
+
return bad(dataset, `${what} must be a non-negative integer`);
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
function checkProof(dataset, proof) {
|
|
21
|
+
if (proof === undefined)
|
|
22
|
+
return undefined;
|
|
23
|
+
if (!isSetProof(proof))
|
|
24
|
+
return bad(dataset, 'a proof must be a set proof');
|
|
25
|
+
return proof;
|
|
26
|
+
}
|
|
27
|
+
/** the key of a row, checked: a plain object with a non-empty string key */
|
|
28
|
+
function keyed(dataset, row) {
|
|
29
|
+
if (typeof row !== 'object' || row === null || Array.isArray(row)) {
|
|
30
|
+
return bad(dataset, 'a row must be an object');
|
|
31
|
+
}
|
|
32
|
+
const key = dataset.key(row);
|
|
33
|
+
if (typeof key !== 'string' || key === '')
|
|
34
|
+
return bad(dataset, 'key(row) must be a non-empty string');
|
|
35
|
+
return [key, row];
|
|
36
|
+
}
|
|
37
|
+
async function* rowsOf(dataset, ctx, after) {
|
|
38
|
+
const source = await dataset.seed(after === undefined ? ctx : { ...ctx, after });
|
|
39
|
+
if (Array.isArray(source)) {
|
|
40
|
+
for (const row of source)
|
|
41
|
+
yield row;
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
for await (const page of source) {
|
|
45
|
+
if (!Array.isArray(page))
|
|
46
|
+
return bad(dataset, 'seed must give rows or pages of rows');
|
|
47
|
+
for (const row of page)
|
|
48
|
+
yield row;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* One pass over the seed in key order. Keeps the first `size` rows after `after`, notes whether
|
|
53
|
+
* another follows, and with `admit` keeps going to count rows and bytes against the dataset's limits.
|
|
54
|
+
*/
|
|
55
|
+
async function scan(dataset, ctx, size, after, admit) {
|
|
56
|
+
const page = new Map();
|
|
57
|
+
let more = false;
|
|
58
|
+
let count = 0;
|
|
59
|
+
let bytes = 0;
|
|
60
|
+
let previous;
|
|
61
|
+
for await (const raw of rowsOf(dataset, ctx, after)) {
|
|
62
|
+
const [key, row] = keyed(dataset, raw);
|
|
63
|
+
if (after !== undefined && key <= after)
|
|
64
|
+
continue;
|
|
65
|
+
if (previous !== undefined && key <= previous) {
|
|
66
|
+
return bad(dataset, 'seed rows must come in strictly ascending key order');
|
|
67
|
+
}
|
|
68
|
+
previous = key;
|
|
69
|
+
count += 1;
|
|
70
|
+
if (page.size < size)
|
|
71
|
+
page.set(key, row);
|
|
72
|
+
else {
|
|
73
|
+
more = true;
|
|
74
|
+
if (!admit)
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
if (admit) {
|
|
78
|
+
bytes += encoder.encode(JSON.stringify(row)).length;
|
|
79
|
+
if (count > dataset.maxRows)
|
|
80
|
+
limit(dataset, `a seed over ${dataset.maxRows} rows`);
|
|
81
|
+
if (bytes > dataset.maxBytes)
|
|
82
|
+
limit(dataset, `a seed over ${dataset.maxBytes} bytes`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return { page, more, count };
|
|
86
|
+
}
|
|
87
|
+
/** the proof from reading every row; the cost a backend avoids by returning one */
|
|
88
|
+
async function readProof(dataset, ctx) {
|
|
89
|
+
const all = new Map();
|
|
90
|
+
let previous;
|
|
91
|
+
for await (const raw of rowsOf(dataset, ctx)) {
|
|
92
|
+
const [key, row] = keyed(dataset, raw);
|
|
93
|
+
if (previous !== undefined && key <= previous) {
|
|
94
|
+
return bad(dataset, 'seed rows must come in strictly ascending key order');
|
|
95
|
+
}
|
|
96
|
+
previous = key;
|
|
97
|
+
all.set(key, row);
|
|
98
|
+
}
|
|
99
|
+
return setProof(all);
|
|
100
|
+
}
|
|
101
|
+
function response(dataset, cursor, body) {
|
|
102
|
+
return { protocol: 1, name: dataset.name, schema: dataset.schema, cursor, ...body };
|
|
103
|
+
}
|
|
104
|
+
async function seedOf(dataset, ask, ctx) {
|
|
105
|
+
const head = await dataset.head(ctx);
|
|
106
|
+
const cursor = checkCursor(dataset, head?.cursor, 'head().cursor');
|
|
107
|
+
const known = checkProof(dataset, head.proof);
|
|
108
|
+
if (known && known.count > dataset.maxRows) {
|
|
109
|
+
limit(dataset, `a seed over ${dataset.maxRows} rows`);
|
|
110
|
+
}
|
|
111
|
+
const size = ask.limit ?? DATASET_PAGE_ROWS;
|
|
112
|
+
const { page, more } = await scan(dataset, ctx, size, ask.after, ask.after === undefined);
|
|
113
|
+
const puts = [...page].map(([key, row]) => ({ key, row }));
|
|
114
|
+
const paging = ask.after === undefined ? {} : { after: ask.after };
|
|
115
|
+
if (more) {
|
|
116
|
+
const next = puts[puts.length - 1]?.key;
|
|
117
|
+
return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, next });
|
|
118
|
+
}
|
|
119
|
+
const proof = known ?? (ask.after === undefined ? await setProof(page) : await readProof(dataset, ctx));
|
|
120
|
+
return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, proof });
|
|
121
|
+
}
|
|
122
|
+
async function patchOf(dataset, since, ctx) {
|
|
123
|
+
const oldest = checkCursor(dataset, await dataset.horizon(ctx), 'horizon()');
|
|
124
|
+
if (since < oldest)
|
|
125
|
+
return null;
|
|
126
|
+
const found = await dataset.changes(since, ctx);
|
|
127
|
+
if (found === 'reset')
|
|
128
|
+
return null;
|
|
129
|
+
const cursor = checkCursor(dataset, found?.cursor, 'changes().cursor');
|
|
130
|
+
if (since > cursor)
|
|
131
|
+
return null;
|
|
132
|
+
const wanted = [...new Set(found.puts)].sort();
|
|
133
|
+
if (wanted.some((key) => typeof key !== 'string' || key === '')) {
|
|
134
|
+
return bad(dataset, 'changes().puts must be non-empty strings');
|
|
135
|
+
}
|
|
136
|
+
const gone = new Map();
|
|
137
|
+
for (const del of found.deletes) {
|
|
138
|
+
if (typeof del?.key !== 'string' || del.key === '') {
|
|
139
|
+
return bad(dataset, 'changes().deletes need a key');
|
|
140
|
+
}
|
|
141
|
+
gone.set(del.key, {
|
|
142
|
+
key: del.key,
|
|
143
|
+
cursor: checkCursor(dataset, del.cursor, 'a delete cursor')
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const asked = new Set(wanted);
|
|
147
|
+
for (const key of gone.keys()) {
|
|
148
|
+
if (asked.has(key))
|
|
149
|
+
return bad(dataset, `${key} is both a put and a delete`);
|
|
150
|
+
}
|
|
151
|
+
const read = new Map();
|
|
152
|
+
if (wanted.length > 0) {
|
|
153
|
+
for (const raw of await dataset.read(wanted, ctx)) {
|
|
154
|
+
const [key, row] = keyed(dataset, raw);
|
|
155
|
+
if (!asked.has(key))
|
|
156
|
+
return bad(dataset, `read returned ${key}, which was not asked for`);
|
|
157
|
+
read.set(key, row);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for (const key of wanted) {
|
|
161
|
+
if (!read.has(key))
|
|
162
|
+
gone.set(key, { key, cursor });
|
|
163
|
+
}
|
|
164
|
+
const proof = checkProof(dataset, found.proof) ?? (await readProof(dataset, ctx));
|
|
165
|
+
return response(dataset, cursor, {
|
|
166
|
+
reset: false,
|
|
167
|
+
puts: wanted.flatMap((key) => read.has(key) ? [{ key, row: read.get(key) }] : []),
|
|
168
|
+
deletes: [...gone.values()].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)),
|
|
169
|
+
proof
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The body for a request. With `since` and the current schema it is a patch of the rows that
|
|
174
|
+
* changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
|
|
175
|
+
* `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
|
|
176
|
+
* seed page, with the proof on the last.
|
|
177
|
+
*/
|
|
178
|
+
export async function answerDataset(dataset, ask, ctx) {
|
|
179
|
+
if (ask.since !== undefined && ask.schema === dataset.schema) {
|
|
180
|
+
const patch = await patchOf(dataset, ask.since, ctx);
|
|
181
|
+
if (patch)
|
|
182
|
+
return patch;
|
|
183
|
+
}
|
|
184
|
+
return seedOf(dataset, ask, ctx);
|
|
185
|
+
}
|
|
186
|
+
/** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
|
|
187
|
+
export const NATIVE_LOOK_DAYS = 1;
|
|
188
|
+
/** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
|
|
189
|
+
export const NOT_MODIFIED_BYTES = 220;
|
|
190
|
+
const DAY_MS = 86_400_000;
|
|
191
|
+
const PATCH_WEIGHT = 0.2;
|
|
192
|
+
/** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
|
|
193
|
+
export function datasetInputs(seed, stats, prior) {
|
|
194
|
+
const perDay = updatesPerDay(stats) ?? prior.changesPerDay;
|
|
195
|
+
const changed = 1 - Math.exp(-perDay * NATIVE_LOOK_DAYS);
|
|
196
|
+
return {
|
|
197
|
+
seedBytes: seed,
|
|
198
|
+
patchBytes: stats.patchBytes ?? seed,
|
|
199
|
+
updatesPerDay: perDay,
|
|
200
|
+
v03: { seedBytes: seed, refetchBytes: seed },
|
|
201
|
+
nativeBytesPerView: changed * seed + (1 - changed) * NOT_MODIFIED_BYTES
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
function updatesPerDay(stats) {
|
|
205
|
+
const { first, last } = stats;
|
|
206
|
+
if (!first || !last || last.at <= first.at || last.cursor < first.cursor)
|
|
207
|
+
return undefined;
|
|
208
|
+
return ((last.cursor - first.cursor) * DAY_MS) / (last.at - first.at);
|
|
209
|
+
}
|
|
210
|
+
async function gzipBytes(text) {
|
|
211
|
+
const packed = new Blob([text]).stream().pipeThrough(new CompressionStream('gzip'));
|
|
212
|
+
return (await new Response(packed).arrayBuffer()).byteLength;
|
|
213
|
+
}
|
|
214
|
+
// walks every page of a seed without sending it, so the server knows what a device would fetch
|
|
215
|
+
async function measureSeed(dataset, stats, ctx) {
|
|
216
|
+
let total = 0;
|
|
217
|
+
let after;
|
|
218
|
+
for (;;) {
|
|
219
|
+
const ask = after === undefined ? { name: dataset.name } : { name: dataset.name, after };
|
|
220
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
221
|
+
total += await gzipBytes(JSON.stringify(body));
|
|
222
|
+
if (body.next === undefined)
|
|
223
|
+
break;
|
|
224
|
+
after = body.next;
|
|
225
|
+
}
|
|
226
|
+
stats.seedBytes = total;
|
|
227
|
+
return total;
|
|
228
|
+
}
|
|
229
|
+
// built before any device holds the dataset, so the first decision has a patch to weigh
|
|
230
|
+
async function measurePatch(dataset, stats, ctx) {
|
|
231
|
+
const { cursor } = await dataset.head(ctx);
|
|
232
|
+
if (cursor < 1)
|
|
233
|
+
return;
|
|
234
|
+
const body = await answerDataset(dataset, { name: dataset.name, schema: dataset.schema, since: cursor - 1 }, ctx);
|
|
235
|
+
if (!body.reset)
|
|
236
|
+
stats.patchBytes = await gzipBytes(JSON.stringify(body));
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* The fields of a row one viewer may hold, from the dataset's `fields`. A family dataset with no
|
|
240
|
+
* `fields` gives the viewer its own row whole and refuses any other row; a dataset no family reads
|
|
241
|
+
* has no view and every row goes out whole.
|
|
242
|
+
*/
|
|
243
|
+
export function viewOf(dataset, viewer, family = false) {
|
|
244
|
+
const { fields } = dataset;
|
|
245
|
+
if (fields === undefined) {
|
|
246
|
+
if (!family)
|
|
247
|
+
return undefined;
|
|
248
|
+
return (row) => {
|
|
249
|
+
if (dataset.key(row) === viewer.principal)
|
|
250
|
+
return row;
|
|
251
|
+
return bad(dataset, 'a row of another principal needs a fields function');
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
return (row) => {
|
|
255
|
+
const names = fields.call(dataset, viewer, row);
|
|
256
|
+
if (!Array.isArray(names) || names.some((name) => typeof name !== 'string')) {
|
|
257
|
+
return bad(dataset, 'fields must return a list of field names');
|
|
258
|
+
}
|
|
259
|
+
const out = {};
|
|
260
|
+
for (const name of names)
|
|
261
|
+
if (Object.hasOwn(row, name))
|
|
262
|
+
out[name] = row[name];
|
|
263
|
+
return out;
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* The body with each row cut to the viewer's fields. A proof is taken again over the cut rows, since
|
|
268
|
+
* the backend's proof covers the whole row and the device checks the rows it holds against it.
|
|
269
|
+
*/
|
|
270
|
+
async function viewed(dataset, body, ctx, view) {
|
|
271
|
+
const puts = body.puts.map(({ key, row }) => ({ key, row: view(row) }));
|
|
272
|
+
if (body.proof === undefined)
|
|
273
|
+
return { ...body, puts };
|
|
274
|
+
const all = new Map();
|
|
275
|
+
for await (const raw of rowsOf(dataset, ctx)) {
|
|
276
|
+
const [key, row] = keyed(dataset, raw);
|
|
277
|
+
all.set(key, view(row));
|
|
278
|
+
}
|
|
279
|
+
return { ...body, puts, proof: await setProof(all) };
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Answers a principal dataset. Everything the dataset's functions see comes from the identity the
|
|
283
|
+
* server resolved, never from the URL, and a request with no identity gets `401`. The answer is
|
|
284
|
+
* never shared-cacheable and names the partition it belongs to, which the device checks before it
|
|
285
|
+
* applies a row.
|
|
286
|
+
*/
|
|
287
|
+
async function respondPrincipalDataset(request, url, dataset, ask, who, families) {
|
|
288
|
+
const caller = who ? await who() : null;
|
|
289
|
+
if (!caller)
|
|
290
|
+
return json({ error: 'sign in' }, 401, { 'cache-control': 'private, no-store' });
|
|
291
|
+
let body;
|
|
292
|
+
try {
|
|
293
|
+
const ctx = { request, url, principal: caller.identity };
|
|
294
|
+
body = await answerDataset(dataset, ask, ctx);
|
|
295
|
+
const view = viewOf(dataset, caller.identity, families.has(dataset.name));
|
|
296
|
+
if (view)
|
|
297
|
+
body = await viewed(dataset, body, ctx, view);
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
if (error instanceof ProtocolError && error.code === 'dataset-forbidden') {
|
|
301
|
+
return json({ error: 'forbidden' }, 403, { 'cache-control': 'private, no-store' });
|
|
302
|
+
}
|
|
303
|
+
throw error;
|
|
304
|
+
}
|
|
305
|
+
return json(body, 200, {
|
|
306
|
+
[HEADERS.protocol]: '1',
|
|
307
|
+
[HEADERS.partition]: caller.partition,
|
|
308
|
+
'cache-control': 'private, no-store'
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
|
|
313
|
+
* first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
|
|
314
|
+
* with the control that bound.
|
|
315
|
+
*/
|
|
316
|
+
export async function respondDataset(request, url, datasets, admission, who, families) {
|
|
317
|
+
const ask = parseDatasetRequest(url.searchParams);
|
|
318
|
+
const dataset = datasets.get(ask.name);
|
|
319
|
+
if (!dataset)
|
|
320
|
+
return json({ error: 'no dataset' }, 404);
|
|
321
|
+
if (dataset.scope === 'principal')
|
|
322
|
+
return respondPrincipalDataset(request, url, dataset, ask, who, families);
|
|
323
|
+
const ctx = { request, url };
|
|
324
|
+
if (dataset.provision !== 'auto') {
|
|
325
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
326
|
+
return json(body, 200, { [HEADERS.protocol]: '1' });
|
|
327
|
+
}
|
|
328
|
+
const stats = admission.stats.get(dataset.name) ?? {};
|
|
329
|
+
admission.stats.set(dataset.name, stats);
|
|
330
|
+
if (ask.since === undefined && ask.after === undefined) {
|
|
331
|
+
const seedBytes = stats.seedBytes ?? (await measureSeed(dataset, stats, ctx));
|
|
332
|
+
if (stats.patchBytes === undefined)
|
|
333
|
+
await measurePatch(dataset, stats, ctx);
|
|
334
|
+
const inputs = datasetInputs(seedBytes, stats, admission.prior);
|
|
335
|
+
const decision = datasetBenefit(inputs, admission.prior);
|
|
336
|
+
if (!decision.ship) {
|
|
337
|
+
return json({ error: 'declined', controls: decision.controls }, 409, {
|
|
338
|
+
[HEADERS.provision]: 'declined',
|
|
339
|
+
[HEADERS.provisionReason]: decision.controls
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
344
|
+
const now = Date.now();
|
|
345
|
+
stats.first ??= { cursor: body.cursor, at: now };
|
|
346
|
+
stats.last = { cursor: body.cursor, at: now };
|
|
347
|
+
if (!body.reset) {
|
|
348
|
+
const bytes = await gzipBytes(JSON.stringify(body));
|
|
349
|
+
stats.patchBytes =
|
|
350
|
+
stats.patchBytes === undefined
|
|
351
|
+
? bytes
|
|
352
|
+
: (1 - PATCH_WEIGHT) * stats.patchBytes + PATCH_WEIGHT * bytes;
|
|
353
|
+
}
|
|
354
|
+
return json(body, 200, { [HEADERS.protocol]: '1' });
|
|
355
|
+
}
|