@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/dist/server/spool.js
CHANGED
|
@@ -4,22 +4,25 @@ import { checkCollectors } from '../protocol/collector.js';
|
|
|
4
4
|
import { clientPath, PATHS, SPOOL_PREFIX } from '../protocol/config.js';
|
|
5
5
|
import { HEADERS, writeEnvelope } from '../protocol/envelope.js';
|
|
6
6
|
import { ProtocolError } from '../protocol/errors.js';
|
|
7
|
-
import { collectRefs, createCollector } from '../protocol/graph.js';
|
|
7
|
+
import { collectRefs, createCollector, resolveRefs, splitRowId } from '../protocol/graph.js';
|
|
8
8
|
import { shellHash } from '../protocol/hash.js';
|
|
9
9
|
import { escapeInline, inlineElement, placeInline } from '../protocol/inline.js';
|
|
10
10
|
import { checkPattern } from '../protocol/pattern.js';
|
|
11
11
|
import { benefit, chooseEntries, DEFAULT_CATALOGUE_BYTES, priorFrom } from '../protocol/provision.js';
|
|
12
|
-
import { verifyProof } from '../protocol/query.js';
|
|
12
|
+
import { keyField, verifyProof } from '../protocol/query.js';
|
|
13
13
|
import { matchRoute, serializeRoute } from '../protocol/route.js';
|
|
14
14
|
import { artifactTag, MAX_HAVE, parseHeld } from '../protocol/sync.js';
|
|
15
15
|
import { clientTag, injectClientTag } from '../protocol/tag.js';
|
|
16
|
+
import { serializeWrite } from '../protocol/write.js';
|
|
16
17
|
import { VERSION } from '../version.js';
|
|
17
18
|
import { respondBeacon } from './beacon.js';
|
|
19
|
+
import { respondDataset } from './dataset.js';
|
|
18
20
|
import { CLIENT_CORE } from './embedded.js';
|
|
19
21
|
import { partitionId } from './identity.js';
|
|
20
22
|
import { compressedSize } from './size.js';
|
|
21
23
|
import { workerResponse, workerScript } from './stub.js';
|
|
22
24
|
import { json, respondSync } from './sync.js';
|
|
25
|
+
import { createWriteServer } from './write.js';
|
|
23
26
|
/** most changed tags a response carries inline before the worker is left to ask `/.spool/sync` */
|
|
24
27
|
export const MAX_PIGGYBACK = 32;
|
|
25
28
|
/** most pages one catalogue request makes the server build, whatever its route suggests */
|
|
@@ -40,6 +43,63 @@ export function createSpool(options) {
|
|
|
40
43
|
}
|
|
41
44
|
seen.add(route.pattern);
|
|
42
45
|
}
|
|
46
|
+
const datasets = new Map();
|
|
47
|
+
const principalOn = options.principalDatasets === true;
|
|
48
|
+
for (const dataset of options.datasets ?? []) {
|
|
49
|
+
if (datasets.has(dataset.name)) {
|
|
50
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: declared twice`);
|
|
51
|
+
}
|
|
52
|
+
if (dataset.scope === 'principal') {
|
|
53
|
+
if (!principalOn) {
|
|
54
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs the principal datasets flag: set the principalDatasets option of createSpool`);
|
|
55
|
+
}
|
|
56
|
+
if (!options.identity) {
|
|
57
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs an identity option`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
datasets.set(dataset.name, dataset);
|
|
61
|
+
}
|
|
62
|
+
for (const route of routes) {
|
|
63
|
+
const name = route.query?.dataset;
|
|
64
|
+
if (name !== undefined) {
|
|
65
|
+
const dataset = datasets.get(name);
|
|
66
|
+
if (!dataset) {
|
|
67
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.dataset "${name}" is not declared`);
|
|
68
|
+
}
|
|
69
|
+
if (dataset.scope !== route.scope) {
|
|
70
|
+
throw new ProtocolError('bad-route', `${route.pattern}: a ${route.scope} route cannot query the ${dataset.scope} dataset "${name}"`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const join of route.query?.plan?.join ?? []) {
|
|
74
|
+
if (!datasets.has(join.dataset)) {
|
|
75
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" is not declared`);
|
|
76
|
+
}
|
|
77
|
+
if (datasets.get(join.dataset)?.scope !== route.scope) {
|
|
78
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" must be ${route.scope}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const writes = [...(options.writes?.classes ?? [])];
|
|
83
|
+
const writeNames = new Set();
|
|
84
|
+
for (const write of writes) {
|
|
85
|
+
if (writeNames.has(write.name)) {
|
|
86
|
+
throw new ProtocolError('bad-write', `${write.name}: declared twice`);
|
|
87
|
+
}
|
|
88
|
+
writeNames.add(write.name);
|
|
89
|
+
const target = routes.find((route) => route.pattern === write.path);
|
|
90
|
+
if (!target || target.scope !== 'principal') {
|
|
91
|
+
throw new ProtocolError('bad-write', `${write.name}: path ${write.path} must be a declared principal route`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (writes.length > 0 && !options.identity) {
|
|
95
|
+
throw new ProtocolError('bad-write', 'writes need an identity option');
|
|
96
|
+
}
|
|
97
|
+
const writesOn = options.writes?.enabled === true && writes.length > 0;
|
|
98
|
+
const writeServer = createWriteServer({
|
|
99
|
+
enabled: writesOn,
|
|
100
|
+
writes,
|
|
101
|
+
...(options.writes?.remember !== undefined ? { remember: options.writes.remember } : {})
|
|
102
|
+
});
|
|
43
103
|
const mode = options.mode ?? 'serve';
|
|
44
104
|
if (mode !== 'serve' && mode !== 'shadow') {
|
|
45
105
|
throw new ProtocolError('bad-route', `unknown mode "${String(mode)}"`);
|
|
@@ -66,6 +126,7 @@ export function createSpool(options) {
|
|
|
66
126
|
const collectors = [...(options.collectors ?? [])];
|
|
67
127
|
checkCollectors(collectors);
|
|
68
128
|
const prior = priorFrom(options.provision);
|
|
129
|
+
const datasetStats = new Map();
|
|
69
130
|
const catalogueBytes = options.catalogueBytes ?? DEFAULT_CATALOGUE_BYTES;
|
|
70
131
|
if (!(Number.isFinite(catalogueBytes) && catalogueBytes >= 0)) {
|
|
71
132
|
throw new ProtocolError('bad-route', 'catalogueBytes must be a non-negative number');
|
|
@@ -104,6 +165,11 @@ export function createSpool(options) {
|
|
|
104
165
|
config.collectors = collectors;
|
|
105
166
|
if (options.thirdParty)
|
|
106
167
|
config.thirdParty = options.thirdParty;
|
|
168
|
+
if (writesOn)
|
|
169
|
+
config.writes = writes.map(serializeWrite);
|
|
170
|
+
const principalSets = [...datasets.values()].flatMap((d) => d.scope === 'principal' ? [{ name: d.name, maxStaleMs: d.maxStaleMs }] : []);
|
|
171
|
+
if (principalOn && principalSets.length > 0)
|
|
172
|
+
config.principalDatasets = principalSets;
|
|
107
173
|
if (options.views === false)
|
|
108
174
|
config.views = false;
|
|
109
175
|
let script;
|
|
@@ -115,7 +181,8 @@ export function createSpool(options) {
|
|
|
115
181
|
params,
|
|
116
182
|
identity,
|
|
117
183
|
record: collector.record,
|
|
118
|
-
projection: collector.projection
|
|
184
|
+
projection: collector.projection,
|
|
185
|
+
row: collector.row
|
|
119
186
|
});
|
|
120
187
|
/** drops the records and projections the worker already holds at a version no change has passed */
|
|
121
188
|
async function trim(artifact, have) {
|
|
@@ -154,9 +221,6 @@ export function createSpool(options) {
|
|
|
154
221
|
const ctx = context(request, new URL(path, url), hit.params, identity, collector);
|
|
155
222
|
const dependsOn = [...(await hit.route.depends(ctx))];
|
|
156
223
|
const loaded = await hit.route.load(ctx);
|
|
157
|
-
if (hit.route.query && !(await verifyProof(hit.route.query, loaded))) {
|
|
158
|
-
throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof`);
|
|
159
|
-
}
|
|
160
224
|
const body = hit.route.strategy === 'assembly' ? loaded : { data: loaded };
|
|
161
225
|
const collected = collector.collect(vector);
|
|
162
226
|
const stored = new Set([...collected.records.map((r) => r.id)]);
|
|
@@ -172,6 +236,26 @@ export function createSpool(options) {
|
|
|
172
236
|
throw new ProtocolError('bad-artifact', `${path} references projection ${id} it did not register`);
|
|
173
237
|
}
|
|
174
238
|
}
|
|
239
|
+
for (const id of refs.rows ?? []) {
|
|
240
|
+
const at = splitRowId(id);
|
|
241
|
+
if (!at || datasets.get(at.dataset)?.scope !== hit.route.scope) {
|
|
242
|
+
throw new ProtocolError('bad-artifact', `${path} references row ${id} of no dataset here`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (hit.route.query && hit.route.query.dataset === undefined) {
|
|
246
|
+
// the worker checks the proof after it resolves refs, so the server checks the same text
|
|
247
|
+
const recordsById = new Map(collected.records.map((r) => [r.id, r]));
|
|
248
|
+
const projectionsById = new Map(collected.projections.map((p) => [p.id, p]));
|
|
249
|
+
const lookup = {
|
|
250
|
+
record: (id) => recordsById.get(id),
|
|
251
|
+
projection: (id) => projectionsById.get(id)
|
|
252
|
+
};
|
|
253
|
+
const resolved = resolveRefs(loaded, lookup);
|
|
254
|
+
if (!resolved.ok ||
|
|
255
|
+
!(await verifyProof(hit.route.query, resolved.value, keyField(hit.route.query)))) {
|
|
256
|
+
throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof over its resolved rows`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
175
259
|
const artifact = parseArtifact({
|
|
176
260
|
protocol: PROTOCOL_VERSION,
|
|
177
261
|
id: path,
|
|
@@ -184,18 +268,24 @@ export function createSpool(options) {
|
|
|
184
268
|
...(collected.projections.length > 0 ? { projections: collected.projections } : {})
|
|
185
269
|
});
|
|
186
270
|
const trimmed = await trim(artifact, have);
|
|
187
|
-
if (weighed
|
|
271
|
+
if (weighed) {
|
|
188
272
|
const bytes = new TextEncoder().encode(JSON.stringify(trimmed)).length;
|
|
189
|
-
|
|
190
|
-
const
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
273
|
+
// a query.dataset route's rows are bounded by the dataset's maxBytes at its seed
|
|
274
|
+
const budget = hit.route.query?.dataset === undefined ? hit.route.provision?.maxBytes : undefined;
|
|
275
|
+
if (budget !== undefined && bytes > budget)
|
|
276
|
+
return { declined: 'budget' };
|
|
277
|
+
if (hit.route.scope === 'public') {
|
|
278
|
+
const mine = { ...prior, ...hit.route.provision };
|
|
279
|
+
const data = hit.route.strategy === 'data';
|
|
280
|
+
const worth = benefit({
|
|
281
|
+
id: path,
|
|
282
|
+
kind: 'route',
|
|
283
|
+
provision: { bytes, requests: 1 },
|
|
284
|
+
avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
|
|
285
|
+
}, mine);
|
|
286
|
+
if (worth <= 0)
|
|
287
|
+
return { declined: 'benefit' };
|
|
288
|
+
}
|
|
199
289
|
}
|
|
200
290
|
const out = { artifact: trimmed };
|
|
201
291
|
if (identity && hit.route.scope === 'principal') {
|
|
@@ -226,7 +316,11 @@ export function createSpool(options) {
|
|
|
226
316
|
if ('declined' in built) {
|
|
227
317
|
return new Response(null, {
|
|
228
318
|
status: 204,
|
|
229
|
-
headers: {
|
|
319
|
+
headers: {
|
|
320
|
+
'cache-control': 'no-store',
|
|
321
|
+
[HEADERS.provision]: 'declined',
|
|
322
|
+
[HEADERS.provisionReason]: built.declined
|
|
323
|
+
}
|
|
230
324
|
});
|
|
231
325
|
}
|
|
232
326
|
const headers = { [HEADERS.protocol]: String(PROTOCOL_VERSION) };
|
|
@@ -364,6 +458,20 @@ export function createSpool(options) {
|
|
|
364
458
|
}
|
|
365
459
|
if (url.pathname === PATHS.artifact && get)
|
|
366
460
|
return await respondArtifact(request, url);
|
|
461
|
+
if (url.pathname === PATHS.dataset && get) {
|
|
462
|
+
return await respondDataset(request, url, datasets, { prior, stats: datasetStats }, async () => {
|
|
463
|
+
const identity = await who(request);
|
|
464
|
+
if (!identity || !options.identity)
|
|
465
|
+
return null;
|
|
466
|
+
return {
|
|
467
|
+
identity,
|
|
468
|
+
partition: await partitionId(options.identity.secret, identity.principal)
|
|
469
|
+
};
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
if (url.pathname === PATHS.write) {
|
|
473
|
+
return await writeServer.respond(request, writesOn ? await who(request) : null);
|
|
474
|
+
}
|
|
367
475
|
if (url.pathname === PATHS.beacon) {
|
|
368
476
|
return await respondBeacon(request, sink, provide(request), catalogue(request));
|
|
369
477
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Identity } from '../protocol/route.js';
|
|
2
|
+
import { type Write } from '../protocol/write.js';
|
|
3
|
+
/** how many `(class, principal, key)` slots the server remembers an applied write id for */
|
|
4
|
+
export declare const DEFAULT_REMEMBER = 10000;
|
|
5
|
+
/** what {@link createWriteServer} takes */
|
|
6
|
+
export type WriteServerOptions = {
|
|
7
|
+
/** false answers every request with `writes-off` */
|
|
8
|
+
enabled: boolean;
|
|
9
|
+
writes: readonly Write[];
|
|
10
|
+
/** slots whose last applied id is remembered, oldest forgotten first; default {@link DEFAULT_REMEMBER} */
|
|
11
|
+
remember?: number;
|
|
12
|
+
};
|
|
13
|
+
/** the handler `createSpool` routes `PUT /.spool/write` to */
|
|
14
|
+
export type WriteServer = {
|
|
15
|
+
respond(request: Request, identity: Identity | null): Promise<Response>;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
|
|
19
|
+
* It checks the declaration, the principal, the key the site derives for that principal, the version
|
|
20
|
+
* header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
|
|
21
|
+
* a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
|
|
22
|
+
* answers 200 with the current version and applies nothing, and the same id with another body is
|
|
23
|
+
* refused. Memory is per server instance, so a deployment with several instances needs the site's
|
|
24
|
+
* `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
|
|
25
|
+
* reading as a conflict.
|
|
26
|
+
*/
|
|
27
|
+
export declare function createWriteServer(options: WriteServerOptions): WriteServer;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { shellHash } from '../protocol/hash.js';
|
|
2
|
+
import { isWriteId, isWriteKey, readIfMatch, WRITE_HEADERS, WRITE_STATUS } from '../protocol/write.js';
|
|
3
|
+
import { json } from './sync.js';
|
|
4
|
+
/** how many `(class, principal, key)` slots the server remembers an applied write id for */
|
|
5
|
+
export const DEFAULT_REMEMBER = 10_000;
|
|
6
|
+
const refuse = (code, error, headers = {}) => json({ code, error }, WRITE_STATUS[code], headers);
|
|
7
|
+
/** the body as text, or null when it is longer than `max` bytes; stops reading once it is */
|
|
8
|
+
async function readBounded(request, max) {
|
|
9
|
+
const declared = Number(request.headers.get('content-length') ?? Number.NaN);
|
|
10
|
+
if (declared > max)
|
|
11
|
+
return null;
|
|
12
|
+
if (!request.body)
|
|
13
|
+
return '';
|
|
14
|
+
const reader = request.body.getReader();
|
|
15
|
+
const chunks = [];
|
|
16
|
+
let total = 0;
|
|
17
|
+
for (;;) {
|
|
18
|
+
const { done, value } = await reader.read();
|
|
19
|
+
if (done)
|
|
20
|
+
break;
|
|
21
|
+
total += value.byteLength;
|
|
22
|
+
if (total > max) {
|
|
23
|
+
await reader.cancel();
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
chunks.push(value);
|
|
27
|
+
}
|
|
28
|
+
const whole = new Uint8Array(total);
|
|
29
|
+
let at = 0;
|
|
30
|
+
for (const chunk of chunks) {
|
|
31
|
+
whole.set(chunk, at);
|
|
32
|
+
at += chunk.byteLength;
|
|
33
|
+
}
|
|
34
|
+
return new TextDecoder().decode(whole);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
|
|
38
|
+
* It checks the declaration, the principal, the key the site derives for that principal, the version
|
|
39
|
+
* header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
|
|
40
|
+
* a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
|
|
41
|
+
* answers 200 with the current version and applies nothing, and the same id with another body is
|
|
42
|
+
* refused. Memory is per server instance, so a deployment with several instances needs the site's
|
|
43
|
+
* `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
|
|
44
|
+
* reading as a conflict.
|
|
45
|
+
*/
|
|
46
|
+
export function createWriteServer(options) {
|
|
47
|
+
const byName = new Map(options.writes.map((write) => [write.name, write]));
|
|
48
|
+
const limit = options.remember ?? DEFAULT_REMEMBER;
|
|
49
|
+
const applied = new Map();
|
|
50
|
+
const locks = new Map();
|
|
51
|
+
const remember = (slot, entry) => {
|
|
52
|
+
applied.delete(slot);
|
|
53
|
+
applied.set(slot, entry);
|
|
54
|
+
while (applied.size > limit)
|
|
55
|
+
applied.delete(applied.keys().next().value);
|
|
56
|
+
};
|
|
57
|
+
/** runs `fn` after every earlier request for the same slot */
|
|
58
|
+
function exclusive(slot, fn) {
|
|
59
|
+
const run = (locks.get(slot) ?? Promise.resolve()).then(fn, fn);
|
|
60
|
+
const tail = run.catch(() => undefined);
|
|
61
|
+
locks.set(slot, tail);
|
|
62
|
+
void tail.then(() => {
|
|
63
|
+
if (locks.get(slot) === tail)
|
|
64
|
+
locks.delete(slot);
|
|
65
|
+
});
|
|
66
|
+
return run;
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
async respond(request, identity) {
|
|
70
|
+
if (!options.enabled)
|
|
71
|
+
return refuse('writes-off', 'declared writes are off');
|
|
72
|
+
if (request.method !== 'PUT') {
|
|
73
|
+
return refuse('bad-method', 'use PUT', { allow: 'PUT' });
|
|
74
|
+
}
|
|
75
|
+
const url = new URL(request.url);
|
|
76
|
+
const write = byName.get(url.searchParams.get('name') ?? '');
|
|
77
|
+
if (!write)
|
|
78
|
+
return refuse('unknown-write', 'no such write class');
|
|
79
|
+
if (!identity)
|
|
80
|
+
return refuse('sign-in', 'sign in to write');
|
|
81
|
+
const key = url.searchParams.get('key');
|
|
82
|
+
if (!isWriteKey(key))
|
|
83
|
+
return refuse('bad-key', 'key is missing or malformed');
|
|
84
|
+
const id = request.headers.get(WRITE_HEADERS.id);
|
|
85
|
+
if (!isWriteId(id))
|
|
86
|
+
return refuse('bad-id', 'Spool-Write-Id is missing or malformed');
|
|
87
|
+
const header = request.headers.get('if-match');
|
|
88
|
+
if (header === null)
|
|
89
|
+
return refuse('missing-version', 'If-Match is required');
|
|
90
|
+
const version = readIfMatch(header);
|
|
91
|
+
if (version === null)
|
|
92
|
+
return refuse('bad-version', 'If-Match is not one version');
|
|
93
|
+
const ctx = { identity, writeId: id };
|
|
94
|
+
if ((await write.key(ctx, request)) !== key) {
|
|
95
|
+
return refuse('forbidden', 'that key is not yours');
|
|
96
|
+
}
|
|
97
|
+
const text = await readBounded(request, write.maxBytes);
|
|
98
|
+
if (text === null)
|
|
99
|
+
return refuse('too-large', `the body is over ${write.maxBytes} bytes`);
|
|
100
|
+
let value;
|
|
101
|
+
try {
|
|
102
|
+
const body = JSON.parse(text);
|
|
103
|
+
if (typeof body !== 'object' || body === null || !('value' in body))
|
|
104
|
+
throw new Error();
|
|
105
|
+
value = body.value;
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return refuse('bad-body', 'the body must be JSON of the form {"value": ...}');
|
|
109
|
+
}
|
|
110
|
+
const slot = `${write.name}\u0000${identity.principal}\u0000${key}`;
|
|
111
|
+
return exclusive(slot, async () => {
|
|
112
|
+
const hash = await shellHash(text);
|
|
113
|
+
const last = applied.get(slot);
|
|
114
|
+
if (last?.id === id) {
|
|
115
|
+
if (last.hash !== hash)
|
|
116
|
+
return refuse('id-reused', 'that id was used for another body');
|
|
117
|
+
return json({ version: (await write.read(ctx, key)).version });
|
|
118
|
+
}
|
|
119
|
+
const result = await write.apply(ctx, key, value, version);
|
|
120
|
+
if ('conflict' in result)
|
|
121
|
+
return json({ code: 'conflict', conflict: result.conflict }, 412);
|
|
122
|
+
remember(slot, { id, hash });
|
|
123
|
+
return json({ version: result.version });
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
package/dist/sw/analytics.d.ts
CHANGED
|
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
|
|
|
78
78
|
* browser's. A request that fails, or a network that is down, loses the event for good.
|
|
79
79
|
*/
|
|
80
80
|
sendSameOrigin(timeoutMs: number): Promise<void>;
|
|
81
|
+
/**
|
|
82
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
83
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
84
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
85
|
+
*/
|
|
86
|
+
dropSameOrigin(): Promise<void>;
|
|
81
87
|
}
|
package/dist/sw/analytics.js
CHANGED
|
@@ -199,6 +199,20 @@ export class BeaconQueue {
|
|
|
199
199
|
}
|
|
200
200
|
await this.#report(lost);
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
204
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
205
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
206
|
+
*/
|
|
207
|
+
async dropSameOrigin() {
|
|
208
|
+
if (this.#flushing)
|
|
209
|
+
await this.#flushing.catch(() => false);
|
|
210
|
+
const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
|
|
211
|
+
if (mine.length === 0)
|
|
212
|
+
return;
|
|
213
|
+
await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
|
|
214
|
+
await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
|
|
215
|
+
}
|
|
202
216
|
async #run() {
|
|
203
217
|
const { storage, send, replay, want, catalogue } = this.#deps;
|
|
204
218
|
await this.#expire();
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { DatasetStore } from './dataset.js';
|
|
2
|
+
/** why a dataset was not held or not brought current */
|
|
3
|
+
export type DatasetRefusal =
|
|
4
|
+
/** the rows would use more than the storage the device has to give */
|
|
5
|
+
'budget'
|
|
6
|
+
/** the proof failed again after a patch and one reseed */
|
|
7
|
+
| 'proof'
|
|
8
|
+
/** the server has no such dataset, or refused the request */
|
|
9
|
+
| 'unavailable'
|
|
10
|
+
/** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
|
|
11
|
+
| 'declined'
|
|
12
|
+
/** the server's answer was not a dataset response, or its paging did not advance */
|
|
13
|
+
| 'bad-response'
|
|
14
|
+
/** the replica was wiped while the apply ran */
|
|
15
|
+
| 'epoch'
|
|
16
|
+
/** a principal dataset's answer named another partition than the one held, or none */
|
|
17
|
+
| 'partition'
|
|
18
|
+
/** the server answered a principal dataset with a sign-in request: the session ended */
|
|
19
|
+
| 'ended'
|
|
20
|
+
/** the server forbids this principal the dataset: its held rows were dropped */
|
|
21
|
+
| 'revoked'
|
|
22
|
+
/** the storage transaction failed, for example on quota */
|
|
23
|
+
| 'storage';
|
|
24
|
+
/**
|
|
25
|
+
* What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
|
|
26
|
+
* changed them, `current` found nothing to change. `offline` means no answer came and the held
|
|
27
|
+
* rows, if any, are as they were; `refused` names why a dataset is not held.
|
|
28
|
+
*/
|
|
29
|
+
export type DatasetOutcome = {
|
|
30
|
+
name: string;
|
|
31
|
+
requests: number;
|
|
32
|
+
} & ({
|
|
33
|
+
status: 'seeded' | 'reseeded' | 'patched' | 'current';
|
|
34
|
+
cursor: number;
|
|
35
|
+
rows: number;
|
|
36
|
+
} | {
|
|
37
|
+
status: 'offline';
|
|
38
|
+
} | {
|
|
39
|
+
status: 'refused';
|
|
40
|
+
reason: DatasetRefusal;
|
|
41
|
+
by?: string | null;
|
|
42
|
+
});
|
|
43
|
+
/** what a sync needs from the worker around it */
|
|
44
|
+
export type DatasetSyncDeps = {
|
|
45
|
+
store: DatasetStore;
|
|
46
|
+
fetch(url: string, init?: RequestInit): Promise<Response>;
|
|
47
|
+
origin: string;
|
|
48
|
+
/** the replica's wipe count */
|
|
49
|
+
epoch(): number;
|
|
50
|
+
/** the most bytes of rows one dataset may use; undefined when the device cannot tell */
|
|
51
|
+
budget(): Promise<number | undefined>;
|
|
52
|
+
/**
|
|
53
|
+
* Set for a principal dataset: reads what a response says about whose rows it carries before
|
|
54
|
+
* anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
|
|
55
|
+
*/
|
|
56
|
+
guard?(response: Response): 'ok' | DatasetGuardVerdict;
|
|
57
|
+
};
|
|
58
|
+
/** why a guard stopped a response */
|
|
59
|
+
export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
|
|
60
|
+
/**
|
|
61
|
+
* The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
|
|
62
|
+
* dataset, and a success that does not name exactly this partition is another principal's or none.
|
|
63
|
+
*/
|
|
64
|
+
export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
|
|
65
|
+
/** seed pages followed before a sync gives up on a server whose paging does not end */
|
|
66
|
+
export declare const DATASET_MAX_PAGES = 1000;
|
|
67
|
+
/**
|
|
68
|
+
* Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
|
|
69
|
+
* is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
|
|
70
|
+
* rows are applied to a staging area and written in one transaction after the collection proof
|
|
71
|
+
* matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
|
|
72
|
+
* which covers rows that moved while the pages were read; a proof that still fails clears the
|
|
73
|
+
* dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
|
|
74
|
+
* an outcome.
|
|
75
|
+
*/
|
|
76
|
+
export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { PATHS } from '../protocol/config.js';
|
|
2
|
+
import { encodeDatasetRequest, parseDatasetResponse } from '../protocol/dataset.js';
|
|
3
|
+
import { HEADERS } from '../protocol/envelope.js';
|
|
4
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
5
|
+
/**
|
|
6
|
+
* The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
|
|
7
|
+
* dataset, and a success that does not name exactly this partition is another principal's or none.
|
|
8
|
+
*/
|
|
9
|
+
export function principalGuard(partition) {
|
|
10
|
+
return (response) => {
|
|
11
|
+
if (response.status === 401)
|
|
12
|
+
return 'ended';
|
|
13
|
+
if (response.status === 403)
|
|
14
|
+
return 'revoked';
|
|
15
|
+
if (response.ok && response.headers.get(HEADERS.partition) !== partition)
|
|
16
|
+
return 'partition';
|
|
17
|
+
return 'ok';
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/** seed pages followed before a sync gives up on a server whose paging does not end */
|
|
21
|
+
export const DATASET_MAX_PAGES = 1000;
|
|
22
|
+
class Refused extends Error {
|
|
23
|
+
reason;
|
|
24
|
+
by;
|
|
25
|
+
constructor(reason, by) {
|
|
26
|
+
super(reason);
|
|
27
|
+
this.reason = reason;
|
|
28
|
+
this.by = by;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
|
|
33
|
+
* is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
|
|
34
|
+
* rows are applied to a staging area and written in one transaction after the collection proof
|
|
35
|
+
* matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
|
|
36
|
+
* which covers rows that moved while the pages were read; a proof that still fails clears the
|
|
37
|
+
* dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
|
|
38
|
+
* an outcome.
|
|
39
|
+
*/
|
|
40
|
+
export async function syncDataset(deps, name) {
|
|
41
|
+
const { store, origin } = deps;
|
|
42
|
+
const epoch = deps.epoch();
|
|
43
|
+
let requests = 0;
|
|
44
|
+
const get = async (request) => {
|
|
45
|
+
requests += 1;
|
|
46
|
+
const response = await deps.fetch(`${origin}${PATHS.dataset}?${encodeDatasetRequest(request)}`, { cache: 'no-store', credentials: 'same-origin' });
|
|
47
|
+
const verdict = deps.guard?.(response) ?? 'ok';
|
|
48
|
+
if (verdict !== 'ok')
|
|
49
|
+
throw new Refused(verdict);
|
|
50
|
+
if (response.status === 409)
|
|
51
|
+
throw new Refused('declined', response.headers.get(HEADERS.provisionReason));
|
|
52
|
+
if (response.status === 400 || response.status === 404)
|
|
53
|
+
throw new Refused('unavailable');
|
|
54
|
+
if (!response.ok)
|
|
55
|
+
throw new TypeError(`dataset answered ${response.status}`);
|
|
56
|
+
const parsed = parseDatasetResponse(await response.json());
|
|
57
|
+
if (parsed.name !== name)
|
|
58
|
+
throw new Refused('bad-response');
|
|
59
|
+
return parsed;
|
|
60
|
+
};
|
|
61
|
+
const write = async (staging, at, status) => {
|
|
62
|
+
try {
|
|
63
|
+
if (deps.epoch() !== epoch)
|
|
64
|
+
throw new Refused('epoch');
|
|
65
|
+
if (!(await store.commit(staging, at, epoch)))
|
|
66
|
+
throw new Refused('epoch');
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (error instanceof Refused)
|
|
70
|
+
throw error;
|
|
71
|
+
throw new Refused('storage');
|
|
72
|
+
}
|
|
73
|
+
return { name, requests, status, cursor: at.cursor, rows: staging.rows.size };
|
|
74
|
+
};
|
|
75
|
+
const seed = async (first, status, retry) => {
|
|
76
|
+
let page = first ?? (await get({ name }));
|
|
77
|
+
const { cursor, schema } = page;
|
|
78
|
+
const budget = await deps.budget();
|
|
79
|
+
const staging = store.stage(name);
|
|
80
|
+
let matches;
|
|
81
|
+
let after;
|
|
82
|
+
for (let pages = 1;; pages += 1) {
|
|
83
|
+
if (deps.epoch() !== epoch)
|
|
84
|
+
throw new Refused('epoch');
|
|
85
|
+
matches = await staging.add(page);
|
|
86
|
+
if (budget !== undefined && staging.bytes > budget)
|
|
87
|
+
throw new Refused('budget');
|
|
88
|
+
if (page.next === undefined)
|
|
89
|
+
break;
|
|
90
|
+
if (pages >= DATASET_MAX_PAGES || (after !== undefined && page.next <= after)) {
|
|
91
|
+
throw new Refused('bad-response');
|
|
92
|
+
}
|
|
93
|
+
after = page.next;
|
|
94
|
+
page = await get({ name, after });
|
|
95
|
+
}
|
|
96
|
+
if (matches === true)
|
|
97
|
+
return write(staging, { cursor, schema }, status);
|
|
98
|
+
// rows moved while the pages were read: the changes since the first page repair the copy
|
|
99
|
+
const patch = await get({ name, since: cursor, schema });
|
|
100
|
+
if (!patch.reset && (await staging.add(patch)) === true) {
|
|
101
|
+
return write(staging, { cursor: patch.cursor, schema }, status);
|
|
102
|
+
}
|
|
103
|
+
if (retry)
|
|
104
|
+
return seed(undefined, 'reseeded', false);
|
|
105
|
+
throw new Refused('proof');
|
|
106
|
+
};
|
|
107
|
+
try {
|
|
108
|
+
const held = await store.get(name);
|
|
109
|
+
if (!held)
|
|
110
|
+
return await seed(undefined, 'seeded', true);
|
|
111
|
+
const { cursor, schema, digest, count } = held.meta;
|
|
112
|
+
const response = await get({ name, since: cursor, schema });
|
|
113
|
+
if (response.reset)
|
|
114
|
+
return await seed(response, 'reseeded', true);
|
|
115
|
+
if (response.cursor < cursor)
|
|
116
|
+
return { name, requests, status: 'current', cursor, rows: count };
|
|
117
|
+
const proof = response.proof;
|
|
118
|
+
if (response.cursor === cursor) {
|
|
119
|
+
// nothing moved; a proof that disagrees with the held rows means the copy is damaged
|
|
120
|
+
if (proof && (proof.count !== count || proof.digest !== digest)) {
|
|
121
|
+
await store.drop(name);
|
|
122
|
+
return await seed(undefined, 'reseeded', false);
|
|
123
|
+
}
|
|
124
|
+
if (!(await store.touch(name, epoch)))
|
|
125
|
+
throw new Refused('epoch');
|
|
126
|
+
return { name, requests, status: 'current', cursor, rows: count };
|
|
127
|
+
}
|
|
128
|
+
const staging = store.stage(name, held);
|
|
129
|
+
if ((await staging.add(response)) === true) {
|
|
130
|
+
return await write(staging, { cursor: response.cursor, schema }, 'patched');
|
|
131
|
+
}
|
|
132
|
+
// the patched rows do not match the server's proof: nothing wrong is kept
|
|
133
|
+
await store.drop(name);
|
|
134
|
+
return await seed(undefined, 'reseeded', false);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (error instanceof Refused) {
|
|
138
|
+
// the server stopped admitting this principal to the dataset, so no row of it stays
|
|
139
|
+
if (error.reason === 'revoked' && deps.epoch() === epoch) {
|
|
140
|
+
await store.drop(name).catch(() => undefined);
|
|
141
|
+
}
|
|
142
|
+
return { name, requests, status: 'refused', reason: error.reason, by: error.by };
|
|
143
|
+
}
|
|
144
|
+
if (error instanceof ProtocolError) {
|
|
145
|
+
return { name, requests, status: 'refused', reason: 'bad-response' };
|
|
146
|
+
}
|
|
147
|
+
return { name, requests, status: 'offline' };
|
|
148
|
+
}
|
|
149
|
+
}
|