@drupflare/spool 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +499 -43
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +34 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/media/embeds.d.ts +3 -2
- package/dist/media/embeds.js +16 -1
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +3 -0
- package/dist/protocol/envelope.js +14 -3
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +7 -0
- package/dist/protocol/index.js +7 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +52 -1
- package/dist/protocol/provision.js +53 -5
- package/dist/protocol/query.d.ts +141 -0
- package/dist/protocol/query.js +252 -0
- package/dist/protocol/route.d.ts +85 -5
- package/dist/protocol/route.js +91 -0
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/index.d.ts +1 -0
- package/dist/proxy/index.js +1 -0
- package/dist/proxy/proxy.js +8 -2
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +5 -0
- package/dist/report/index.js +5 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +284 -0
- package/dist/report/institution.js +225 -0
- package/dist/report/ledger.d.ts +28 -0
- package/dist/report/ledger.js +30 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +6 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.js +3 -0
- package/dist/server/spool.d.ts +40 -0
- package/dist/server/spool.js +184 -23
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +8 -1
- package/dist/sw/render.js +28 -1
- package/dist/sw/replica.d.ts +14 -3
- package/dist/sw/replica.js +46 -5
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +440 -29
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +5 -2
package/dist/server/spool.js
CHANGED
|
@@ -4,19 +4,25 @@ import { checkCollectors } from '../protocol/collector.js';
|
|
|
4
4
|
import { clientPath, PATHS, SPOOL_PREFIX } from '../protocol/config.js';
|
|
5
5
|
import { HEADERS, writeEnvelope } from '../protocol/envelope.js';
|
|
6
6
|
import { ProtocolError } from '../protocol/errors.js';
|
|
7
|
-
import { collectRefs, createCollector } from '../protocol/graph.js';
|
|
7
|
+
import { collectRefs, createCollector, resolveRefs, splitRowId } from '../protocol/graph.js';
|
|
8
|
+
import { shellHash } from '../protocol/hash.js';
|
|
9
|
+
import { escapeInline, inlineElement, placeInline } from '../protocol/inline.js';
|
|
8
10
|
import { checkPattern } from '../protocol/pattern.js';
|
|
9
11
|
import { benefit, chooseEntries, DEFAULT_CATALOGUE_BYTES, priorFrom } from '../protocol/provision.js';
|
|
12
|
+
import { keyField, verifyProof } from '../protocol/query.js';
|
|
10
13
|
import { matchRoute, serializeRoute } from '../protocol/route.js';
|
|
11
14
|
import { artifactTag, MAX_HAVE, parseHeld } from '../protocol/sync.js';
|
|
12
15
|
import { clientTag, injectClientTag } from '../protocol/tag.js';
|
|
16
|
+
import { serializeWrite } from '../protocol/write.js';
|
|
13
17
|
import { VERSION } from '../version.js';
|
|
14
18
|
import { respondBeacon } from './beacon.js';
|
|
19
|
+
import { respondDataset } from './dataset.js';
|
|
15
20
|
import { CLIENT_CORE } from './embedded.js';
|
|
16
21
|
import { partitionId } from './identity.js';
|
|
17
22
|
import { compressedSize } from './size.js';
|
|
18
23
|
import { workerResponse, workerScript } from './stub.js';
|
|
19
24
|
import { json, respondSync } from './sync.js';
|
|
25
|
+
import { createWriteServer } from './write.js';
|
|
20
26
|
/** most changed tags a response carries inline before the worker is left to ask `/.spool/sync` */
|
|
21
27
|
export const MAX_PIGGYBACK = 32;
|
|
22
28
|
/** most pages one catalogue request makes the server build, whatever its route suggests */
|
|
@@ -37,6 +43,63 @@ export function createSpool(options) {
|
|
|
37
43
|
}
|
|
38
44
|
seen.add(route.pattern);
|
|
39
45
|
}
|
|
46
|
+
const datasets = new Map();
|
|
47
|
+
const principalOn = options.principalDatasets === true;
|
|
48
|
+
for (const dataset of options.datasets ?? []) {
|
|
49
|
+
if (datasets.has(dataset.name)) {
|
|
50
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: declared twice`);
|
|
51
|
+
}
|
|
52
|
+
if (dataset.scope === 'principal') {
|
|
53
|
+
if (!principalOn) {
|
|
54
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs the principal datasets flag: set the principalDatasets option of createSpool`);
|
|
55
|
+
}
|
|
56
|
+
if (!options.identity) {
|
|
57
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs an identity option`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
datasets.set(dataset.name, dataset);
|
|
61
|
+
}
|
|
62
|
+
for (const route of routes) {
|
|
63
|
+
const name = route.query?.dataset;
|
|
64
|
+
if (name !== undefined) {
|
|
65
|
+
const dataset = datasets.get(name);
|
|
66
|
+
if (!dataset) {
|
|
67
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.dataset "${name}" is not declared`);
|
|
68
|
+
}
|
|
69
|
+
if (dataset.scope !== route.scope) {
|
|
70
|
+
throw new ProtocolError('bad-route', `${route.pattern}: a ${route.scope} route cannot query the ${dataset.scope} dataset "${name}"`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const join of route.query?.plan?.join ?? []) {
|
|
74
|
+
if (!datasets.has(join.dataset)) {
|
|
75
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" is not declared`);
|
|
76
|
+
}
|
|
77
|
+
if (datasets.get(join.dataset)?.scope !== route.scope) {
|
|
78
|
+
throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" must be ${route.scope}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const writes = [...(options.writes?.classes ?? [])];
|
|
83
|
+
const writeNames = new Set();
|
|
84
|
+
for (const write of writes) {
|
|
85
|
+
if (writeNames.has(write.name)) {
|
|
86
|
+
throw new ProtocolError('bad-write', `${write.name}: declared twice`);
|
|
87
|
+
}
|
|
88
|
+
writeNames.add(write.name);
|
|
89
|
+
const target = routes.find((route) => route.pattern === write.path);
|
|
90
|
+
if (!target || target.scope !== 'principal') {
|
|
91
|
+
throw new ProtocolError('bad-write', `${write.name}: path ${write.path} must be a declared principal route`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (writes.length > 0 && !options.identity) {
|
|
95
|
+
throw new ProtocolError('bad-write', 'writes need an identity option');
|
|
96
|
+
}
|
|
97
|
+
const writesOn = options.writes?.enabled === true && writes.length > 0;
|
|
98
|
+
const writeServer = createWriteServer({
|
|
99
|
+
enabled: writesOn,
|
|
100
|
+
writes,
|
|
101
|
+
...(options.writes?.remember !== undefined ? { remember: options.writes.remember } : {})
|
|
102
|
+
});
|
|
40
103
|
const mode = options.mode ?? 'serve';
|
|
41
104
|
if (mode !== 'serve' && mode !== 'shadow') {
|
|
42
105
|
throw new ProtocolError('bad-route', `unknown mode "${String(mode)}"`);
|
|
@@ -63,11 +126,20 @@ export function createSpool(options) {
|
|
|
63
126
|
const collectors = [...(options.collectors ?? [])];
|
|
64
127
|
checkCollectors(collectors);
|
|
65
128
|
const prior = priorFrom(options.provision);
|
|
129
|
+
const datasetStats = new Map();
|
|
66
130
|
const catalogueBytes = options.catalogueBytes ?? DEFAULT_CATALOGUE_BYTES;
|
|
67
131
|
if (!(Number.isFinite(catalogueBytes) && catalogueBytes >= 0)) {
|
|
68
132
|
throw new ProtocolError('bad-route', 'catalogueBytes must be a non-negative number');
|
|
69
133
|
}
|
|
134
|
+
const catalogueConfidence = options.catalogueConfidence ?? 0;
|
|
135
|
+
if (!(Number.isFinite(catalogueConfidence) && catalogueConfidence >= 0)) {
|
|
136
|
+
throw new ProtocolError('bad-route', 'catalogueConfidence must be a non-negative number');
|
|
137
|
+
}
|
|
70
138
|
const measure = options.catalogueSize ?? compressedSize;
|
|
139
|
+
const inlineMax = options.inlineMaxBytes ?? 0;
|
|
140
|
+
if (!(Number.isFinite(inlineMax) && inlineMax >= 0)) {
|
|
141
|
+
throw new ProtocolError('bad-route', 'inlineMaxBytes must be a non-negative number');
|
|
142
|
+
}
|
|
71
143
|
const config = {
|
|
72
144
|
protocol: PROTOCOL_VERSION,
|
|
73
145
|
version: VERSION,
|
|
@@ -93,6 +165,11 @@ export function createSpool(options) {
|
|
|
93
165
|
config.collectors = collectors;
|
|
94
166
|
if (options.thirdParty)
|
|
95
167
|
config.thirdParty = options.thirdParty;
|
|
168
|
+
if (writesOn)
|
|
169
|
+
config.writes = writes.map(serializeWrite);
|
|
170
|
+
const principalSets = [...datasets.values()].flatMap((d) => d.scope === 'principal' ? [{ name: d.name, maxStaleMs: d.maxStaleMs }] : []);
|
|
171
|
+
if (principalOn && principalSets.length > 0)
|
|
172
|
+
config.principalDatasets = principalSets;
|
|
96
173
|
if (options.views === false)
|
|
97
174
|
config.views = false;
|
|
98
175
|
let script;
|
|
@@ -104,7 +181,8 @@ export function createSpool(options) {
|
|
|
104
181
|
params,
|
|
105
182
|
identity,
|
|
106
183
|
record: collector.record,
|
|
107
|
-
projection: collector.projection
|
|
184
|
+
projection: collector.projection,
|
|
185
|
+
row: collector.row
|
|
108
186
|
});
|
|
109
187
|
/** drops the records and projections the worker already holds at a version no change has passed */
|
|
110
188
|
async function trim(artifact, have) {
|
|
@@ -158,6 +236,26 @@ export function createSpool(options) {
|
|
|
158
236
|
throw new ProtocolError('bad-artifact', `${path} references projection ${id} it did not register`);
|
|
159
237
|
}
|
|
160
238
|
}
|
|
239
|
+
for (const id of refs.rows ?? []) {
|
|
240
|
+
const at = splitRowId(id);
|
|
241
|
+
if (!at || datasets.get(at.dataset)?.scope !== hit.route.scope) {
|
|
242
|
+
throw new ProtocolError('bad-artifact', `${path} references row ${id} of no dataset here`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (hit.route.query && hit.route.query.dataset === undefined) {
|
|
246
|
+
// the worker checks the proof after it resolves refs, so the server checks the same text
|
|
247
|
+
const recordsById = new Map(collected.records.map((r) => [r.id, r]));
|
|
248
|
+
const projectionsById = new Map(collected.projections.map((p) => [p.id, p]));
|
|
249
|
+
const lookup = {
|
|
250
|
+
record: (id) => recordsById.get(id),
|
|
251
|
+
projection: (id) => projectionsById.get(id)
|
|
252
|
+
};
|
|
253
|
+
const resolved = resolveRefs(loaded, lookup);
|
|
254
|
+
if (!resolved.ok ||
|
|
255
|
+
!(await verifyProof(hit.route.query, resolved.value, keyField(hit.route.query)))) {
|
|
256
|
+
throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof over its resolved rows`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
161
259
|
const artifact = parseArtifact({
|
|
162
260
|
protocol: PROTOCOL_VERSION,
|
|
163
261
|
id: path,
|
|
@@ -170,18 +268,24 @@ export function createSpool(options) {
|
|
|
170
268
|
...(collected.projections.length > 0 ? { projections: collected.projections } : {})
|
|
171
269
|
});
|
|
172
270
|
const trimmed = await trim(artifact, have);
|
|
173
|
-
if (weighed
|
|
271
|
+
if (weighed) {
|
|
174
272
|
const bytes = new TextEncoder().encode(JSON.stringify(trimmed)).length;
|
|
175
|
-
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
273
|
+
// a query.dataset route's rows are bounded by the dataset's maxBytes at its seed
|
|
274
|
+
const budget = hit.route.query?.dataset === undefined ? hit.route.provision?.maxBytes : undefined;
|
|
275
|
+
if (budget !== undefined && bytes > budget)
|
|
276
|
+
return { declined: 'budget' };
|
|
277
|
+
if (hit.route.scope === 'public') {
|
|
278
|
+
const mine = { ...prior, ...hit.route.provision };
|
|
279
|
+
const data = hit.route.strategy === 'data';
|
|
280
|
+
const worth = benefit({
|
|
281
|
+
id: path,
|
|
282
|
+
kind: 'route',
|
|
283
|
+
provision: { bytes, requests: 1 },
|
|
284
|
+
avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
|
|
285
|
+
}, mine);
|
|
286
|
+
if (worth <= 0)
|
|
287
|
+
return { declined: 'benefit' };
|
|
288
|
+
}
|
|
185
289
|
}
|
|
186
290
|
const out = { artifact: trimmed };
|
|
187
291
|
if (identity && hit.route.scope === 'principal') {
|
|
@@ -189,6 +293,20 @@ export function createSpool(options) {
|
|
|
189
293
|
}
|
|
190
294
|
return out;
|
|
191
295
|
}
|
|
296
|
+
/** the escaped artifact JSON of a public page, or null when it is over the limit or fails to build */
|
|
297
|
+
async function inlineText(url) {
|
|
298
|
+
try {
|
|
299
|
+
const probe = new Request(url.href, { headers: { accept: 'application/json' } });
|
|
300
|
+
const built = await buildArtifact(probe, url, url.pathname, {}, true);
|
|
301
|
+
if (!('artifact' in built) || built.artifact.scope !== 'public')
|
|
302
|
+
return null;
|
|
303
|
+
const text = escapeInline(JSON.stringify(built.artifact));
|
|
304
|
+
return new TextEncoder().encode(text).length <= inlineMax ? text : null;
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
return null;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
192
310
|
async function respondArtifact(request, url) {
|
|
193
311
|
const path = url.searchParams.get('path') ?? '';
|
|
194
312
|
const raw = url.searchParams.get('have');
|
|
@@ -198,7 +316,11 @@ export function createSpool(options) {
|
|
|
198
316
|
if ('declined' in built) {
|
|
199
317
|
return new Response(null, {
|
|
200
318
|
status: 204,
|
|
201
|
-
headers: {
|
|
319
|
+
headers: {
|
|
320
|
+
'cache-control': 'no-store',
|
|
321
|
+
[HEADERS.provision]: 'declined',
|
|
322
|
+
[HEADERS.provisionReason]: built.declined
|
|
323
|
+
}
|
|
202
324
|
});
|
|
203
325
|
}
|
|
204
326
|
const headers = { [HEADERS.protocol]: String(PROTOCOL_VERSION) };
|
|
@@ -284,9 +406,20 @@ export function createSpool(options) {
|
|
|
284
406
|
const documentBytes = target.route.documentBytes ??
|
|
285
407
|
(target.route.strategy === 'data' ? entryBytes : prior.avoidedBytes);
|
|
286
408
|
built.set(guess.path, result);
|
|
287
|
-
pool.push({
|
|
409
|
+
pool.push({
|
|
410
|
+
path: guess.path,
|
|
411
|
+
p: guess.p,
|
|
412
|
+
...(typeof guess.n === 'number' && guess.n > 0 && Number.isFinite(guess.n)
|
|
413
|
+
? { weight: guess.n }
|
|
414
|
+
: {}),
|
|
415
|
+
entryBytes,
|
|
416
|
+
documentBytes
|
|
417
|
+
});
|
|
288
418
|
}
|
|
289
|
-
const chosen = chooseEntries(pool, {
|
|
419
|
+
const chosen = chooseEntries(pool, {
|
|
420
|
+
capBytes: catalogueBytes,
|
|
421
|
+
z: catalogueConfidence
|
|
422
|
+
});
|
|
290
423
|
const entries = chosen.flatMap((c) => built.get(c.path) ?? []);
|
|
291
424
|
const partition = entries.find((e) => e.partition !== undefined)?.partition;
|
|
292
425
|
return {
|
|
@@ -325,6 +458,20 @@ export function createSpool(options) {
|
|
|
325
458
|
}
|
|
326
459
|
if (url.pathname === PATHS.artifact && get)
|
|
327
460
|
return await respondArtifact(request, url);
|
|
461
|
+
if (url.pathname === PATHS.dataset && get) {
|
|
462
|
+
return await respondDataset(request, url, datasets, { prior, stats: datasetStats }, async () => {
|
|
463
|
+
const identity = await who(request);
|
|
464
|
+
if (!identity || !options.identity)
|
|
465
|
+
return null;
|
|
466
|
+
return {
|
|
467
|
+
identity,
|
|
468
|
+
partition: await partitionId(options.identity.secret, identity.principal)
|
|
469
|
+
};
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
if (url.pathname === PATHS.write) {
|
|
473
|
+
return await writeServer.respond(request, writesOn ? await who(request) : null);
|
|
474
|
+
}
|
|
328
475
|
if (url.pathname === PATHS.beacon) {
|
|
329
476
|
return await respondBeacon(request, sink, provide(request), catalogue(request));
|
|
330
477
|
}
|
|
@@ -394,6 +541,12 @@ export function createSpool(options) {
|
|
|
394
541
|
headers.set('x-spool-mode', hit.route.mode ?? mode);
|
|
395
542
|
}
|
|
396
543
|
const kept = hit?.route.strategy === 'static' || hit?.route.strategy === 'data' || isHtml(response);
|
|
544
|
+
const tag = request.method === 'GET' &&
|
|
545
|
+
response.status === 200 &&
|
|
546
|
+
isHtml(response) &&
|
|
547
|
+
!response.headers.has('content-encoding');
|
|
548
|
+
const html = tag ? await response.text() : null;
|
|
549
|
+
let inline = null;
|
|
397
550
|
if (hit && kept && (hit.route.scope === 'public' || identity)) {
|
|
398
551
|
const ctx = context(request, url, hit.params, identity);
|
|
399
552
|
envelope.artifact = {
|
|
@@ -402,23 +555,31 @@ export function createSpool(options) {
|
|
|
402
555
|
deps: [...(await hit.route.depends(ctx))],
|
|
403
556
|
version: before
|
|
404
557
|
};
|
|
558
|
+
const { assembled } = decorateOptions;
|
|
559
|
+
if (tag && hit.route.scope === 'public') {
|
|
560
|
+
if (hit.route.strategy === 'assembly' && assembled !== undefined) {
|
|
561
|
+
envelope.artifact.hash = await shellHash(assembled);
|
|
562
|
+
}
|
|
563
|
+
else if (hit.route.strategy === 'template' && inlineMax > 0) {
|
|
564
|
+
inline = await inlineText(url);
|
|
565
|
+
if (inline !== null)
|
|
566
|
+
envelope.artifact.hash = await shellHash(inline);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
405
569
|
}
|
|
406
570
|
writeEnvelope(headers, envelope);
|
|
407
571
|
if (decorateOptions.hardReset)
|
|
408
572
|
headers.set('clear-site-data', '"storage"');
|
|
409
|
-
|
|
410
|
-
response.status === 200 &&
|
|
411
|
-
isHtml(response) &&
|
|
412
|
-
!response.headers.has('content-encoding');
|
|
413
|
-
if (!tag) {
|
|
573
|
+
if (html === null) {
|
|
414
574
|
return new Response(response.body, {
|
|
415
575
|
status: response.status,
|
|
416
576
|
statusText: response.statusText,
|
|
417
577
|
headers
|
|
418
578
|
});
|
|
419
579
|
}
|
|
420
|
-
|
|
421
|
-
|
|
580
|
+
let tagged = injectClientTag(html, config.clientTag);
|
|
581
|
+
if (inline !== null)
|
|
582
|
+
tagged = placeInline(tagged, inlineElement(inline));
|
|
422
583
|
if (tagged !== html) {
|
|
423
584
|
headers.delete('content-length');
|
|
424
585
|
const etag = headers.get('etag');
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Identity } from '../protocol/route.js';
|
|
2
|
+
import { type Write } from '../protocol/write.js';
|
|
3
|
+
/** how many `(class, principal, key)` slots the server remembers an applied write id for */
|
|
4
|
+
export declare const DEFAULT_REMEMBER = 10000;
|
|
5
|
+
/** what {@link createWriteServer} takes */
|
|
6
|
+
export type WriteServerOptions = {
|
|
7
|
+
/** false answers every request with `writes-off` */
|
|
8
|
+
enabled: boolean;
|
|
9
|
+
writes: readonly Write[];
|
|
10
|
+
/** slots whose last applied id is remembered, oldest forgotten first; default {@link DEFAULT_REMEMBER} */
|
|
11
|
+
remember?: number;
|
|
12
|
+
};
|
|
13
|
+
/** the handler `createSpool` routes `PUT /.spool/write` to */
|
|
14
|
+
export type WriteServer = {
|
|
15
|
+
respond(request: Request, identity: Identity | null): Promise<Response>;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
|
|
19
|
+
* It checks the declaration, the principal, the key the site derives for that principal, the version
|
|
20
|
+
* header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
|
|
21
|
+
* a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
|
|
22
|
+
* answers 200 with the current version and applies nothing, and the same id with another body is
|
|
23
|
+
* refused. Memory is per server instance, so a deployment with several instances needs the site's
|
|
24
|
+
* `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
|
|
25
|
+
* reading as a conflict.
|
|
26
|
+
*/
|
|
27
|
+
export declare function createWriteServer(options: WriteServerOptions): WriteServer;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { shellHash } from '../protocol/hash.js';
|
|
2
|
+
import { isWriteId, isWriteKey, readIfMatch, WRITE_HEADERS, WRITE_STATUS } from '../protocol/write.js';
|
|
3
|
+
import { json } from './sync.js';
|
|
4
|
+
/** how many `(class, principal, key)` slots the server remembers an applied write id for */
|
|
5
|
+
export const DEFAULT_REMEMBER = 10_000;
|
|
6
|
+
const refuse = (code, error, headers = {}) => json({ code, error }, WRITE_STATUS[code], headers);
|
|
7
|
+
/** the body as text, or null when it is longer than `max` bytes; stops reading once it is */
|
|
8
|
+
async function readBounded(request, max) {
|
|
9
|
+
const declared = Number(request.headers.get('content-length') ?? Number.NaN);
|
|
10
|
+
if (declared > max)
|
|
11
|
+
return null;
|
|
12
|
+
if (!request.body)
|
|
13
|
+
return '';
|
|
14
|
+
const reader = request.body.getReader();
|
|
15
|
+
const chunks = [];
|
|
16
|
+
let total = 0;
|
|
17
|
+
for (;;) {
|
|
18
|
+
const { done, value } = await reader.read();
|
|
19
|
+
if (done)
|
|
20
|
+
break;
|
|
21
|
+
total += value.byteLength;
|
|
22
|
+
if (total > max) {
|
|
23
|
+
await reader.cancel();
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
chunks.push(value);
|
|
27
|
+
}
|
|
28
|
+
const whole = new Uint8Array(total);
|
|
29
|
+
let at = 0;
|
|
30
|
+
for (const chunk of chunks) {
|
|
31
|
+
whole.set(chunk, at);
|
|
32
|
+
at += chunk.byteLength;
|
|
33
|
+
}
|
|
34
|
+
return new TextDecoder().decode(whole);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
|
|
38
|
+
* It checks the declaration, the principal, the key the site derives for that principal, the version
|
|
39
|
+
* header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
|
|
40
|
+
* a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
|
|
41
|
+
* answers 200 with the current version and applies nothing, and the same id with another body is
|
|
42
|
+
* refused. Memory is per server instance, so a deployment with several instances needs the site's
|
|
43
|
+
* `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
|
|
44
|
+
* reading as a conflict.
|
|
45
|
+
*/
|
|
46
|
+
export function createWriteServer(options) {
|
|
47
|
+
const byName = new Map(options.writes.map((write) => [write.name, write]));
|
|
48
|
+
const limit = options.remember ?? DEFAULT_REMEMBER;
|
|
49
|
+
const applied = new Map();
|
|
50
|
+
const locks = new Map();
|
|
51
|
+
const remember = (slot, entry) => {
|
|
52
|
+
applied.delete(slot);
|
|
53
|
+
applied.set(slot, entry);
|
|
54
|
+
while (applied.size > limit)
|
|
55
|
+
applied.delete(applied.keys().next().value);
|
|
56
|
+
};
|
|
57
|
+
/** runs `fn` after every earlier request for the same slot */
|
|
58
|
+
function exclusive(slot, fn) {
|
|
59
|
+
const run = (locks.get(slot) ?? Promise.resolve()).then(fn, fn);
|
|
60
|
+
const tail = run.catch(() => undefined);
|
|
61
|
+
locks.set(slot, tail);
|
|
62
|
+
void tail.then(() => {
|
|
63
|
+
if (locks.get(slot) === tail)
|
|
64
|
+
locks.delete(slot);
|
|
65
|
+
});
|
|
66
|
+
return run;
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
async respond(request, identity) {
|
|
70
|
+
if (!options.enabled)
|
|
71
|
+
return refuse('writes-off', 'declared writes are off');
|
|
72
|
+
if (request.method !== 'PUT') {
|
|
73
|
+
return refuse('bad-method', 'use PUT', { allow: 'PUT' });
|
|
74
|
+
}
|
|
75
|
+
const url = new URL(request.url);
|
|
76
|
+
const write = byName.get(url.searchParams.get('name') ?? '');
|
|
77
|
+
if (!write)
|
|
78
|
+
return refuse('unknown-write', 'no such write class');
|
|
79
|
+
if (!identity)
|
|
80
|
+
return refuse('sign-in', 'sign in to write');
|
|
81
|
+
const key = url.searchParams.get('key');
|
|
82
|
+
if (!isWriteKey(key))
|
|
83
|
+
return refuse('bad-key', 'key is missing or malformed');
|
|
84
|
+
const id = request.headers.get(WRITE_HEADERS.id);
|
|
85
|
+
if (!isWriteId(id))
|
|
86
|
+
return refuse('bad-id', 'Spool-Write-Id is missing or malformed');
|
|
87
|
+
const header = request.headers.get('if-match');
|
|
88
|
+
if (header === null)
|
|
89
|
+
return refuse('missing-version', 'If-Match is required');
|
|
90
|
+
const version = readIfMatch(header);
|
|
91
|
+
if (version === null)
|
|
92
|
+
return refuse('bad-version', 'If-Match is not one version');
|
|
93
|
+
const ctx = { identity, writeId: id };
|
|
94
|
+
if ((await write.key(ctx, request)) !== key) {
|
|
95
|
+
return refuse('forbidden', 'that key is not yours');
|
|
96
|
+
}
|
|
97
|
+
const text = await readBounded(request, write.maxBytes);
|
|
98
|
+
if (text === null)
|
|
99
|
+
return refuse('too-large', `the body is over ${write.maxBytes} bytes`);
|
|
100
|
+
let value;
|
|
101
|
+
try {
|
|
102
|
+
const body = JSON.parse(text);
|
|
103
|
+
if (typeof body !== 'object' || body === null || !('value' in body))
|
|
104
|
+
throw new Error();
|
|
105
|
+
value = body.value;
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return refuse('bad-body', 'the body must be JSON of the form {"value": ...}');
|
|
109
|
+
}
|
|
110
|
+
const slot = `${write.name}\u0000${identity.principal}\u0000${key}`;
|
|
111
|
+
return exclusive(slot, async () => {
|
|
112
|
+
const hash = await shellHash(text);
|
|
113
|
+
const last = applied.get(slot);
|
|
114
|
+
if (last?.id === id) {
|
|
115
|
+
if (last.hash !== hash)
|
|
116
|
+
return refuse('id-reused', 'that id was used for another body');
|
|
117
|
+
return json({ version: (await write.read(ctx, key)).version });
|
|
118
|
+
}
|
|
119
|
+
const result = await write.apply(ctx, key, value, version);
|
|
120
|
+
if ('conflict' in result)
|
|
121
|
+
return json({ code: 'conflict', conflict: result.conflict }, 412);
|
|
122
|
+
remember(slot, { id, hash });
|
|
123
|
+
return json({ version: result.version });
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
package/dist/sw/analytics.d.ts
CHANGED
|
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
|
|
|
78
78
|
* browser's. A request that fails, or a network that is down, loses the event for good.
|
|
79
79
|
*/
|
|
80
80
|
sendSameOrigin(timeoutMs: number): Promise<void>;
|
|
81
|
+
/**
|
|
82
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
83
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
84
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
85
|
+
*/
|
|
86
|
+
dropSameOrigin(): Promise<void>;
|
|
81
87
|
}
|
package/dist/sw/analytics.js
CHANGED
|
@@ -199,6 +199,20 @@ export class BeaconQueue {
|
|
|
199
199
|
}
|
|
200
200
|
await this.#report(lost);
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
204
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
205
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
206
|
+
*/
|
|
207
|
+
async dropSameOrigin() {
|
|
208
|
+
if (this.#flushing)
|
|
209
|
+
await this.#flushing.catch(() => false);
|
|
210
|
+
const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
|
|
211
|
+
if (mine.length === 0)
|
|
212
|
+
return;
|
|
213
|
+
await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
|
|
214
|
+
await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
|
|
215
|
+
}
|
|
202
216
|
async #run() {
|
|
203
217
|
const { storage, send, replay, want, catalogue } = this.#deps;
|
|
204
218
|
await this.#expire();
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { DatasetStore } from './dataset.js';
|
|
2
|
+
/** why a dataset was not held or not brought current */
|
|
3
|
+
export type DatasetRefusal =
|
|
4
|
+
/** the rows would use more than the storage the device has to give */
|
|
5
|
+
'budget'
|
|
6
|
+
/** the proof failed again after a patch and one reseed */
|
|
7
|
+
| 'proof'
|
|
8
|
+
/** the server has no such dataset, or refused the request */
|
|
9
|
+
| 'unavailable'
|
|
10
|
+
/** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
|
|
11
|
+
| 'declined'
|
|
12
|
+
/** the server's answer was not a dataset response, or its paging did not advance */
|
|
13
|
+
| 'bad-response'
|
|
14
|
+
/** the replica was wiped while the apply ran */
|
|
15
|
+
| 'epoch'
|
|
16
|
+
/** a principal dataset's answer named another partition than the one held, or none */
|
|
17
|
+
| 'partition'
|
|
18
|
+
/** the server answered a principal dataset with a sign-in request: the session ended */
|
|
19
|
+
| 'ended'
|
|
20
|
+
/** the server forbids this principal the dataset: its held rows were dropped */
|
|
21
|
+
| 'revoked'
|
|
22
|
+
/** the storage transaction failed, for example on quota */
|
|
23
|
+
| 'storage';
|
|
24
|
+
/**
|
|
25
|
+
* What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
|
|
26
|
+
* changed them, `current` found nothing to change. `offline` means no answer came and the held
|
|
27
|
+
* rows, if any, are as they were; `refused` names why a dataset is not held.
|
|
28
|
+
*/
|
|
29
|
+
export type DatasetOutcome = {
|
|
30
|
+
name: string;
|
|
31
|
+
requests: number;
|
|
32
|
+
} & ({
|
|
33
|
+
status: 'seeded' | 'reseeded' | 'patched' | 'current';
|
|
34
|
+
cursor: number;
|
|
35
|
+
rows: number;
|
|
36
|
+
} | {
|
|
37
|
+
status: 'offline';
|
|
38
|
+
} | {
|
|
39
|
+
status: 'refused';
|
|
40
|
+
reason: DatasetRefusal;
|
|
41
|
+
by?: string | null;
|
|
42
|
+
});
|
|
43
|
+
/** what a sync needs from the worker around it */
|
|
44
|
+
export type DatasetSyncDeps = {
|
|
45
|
+
store: DatasetStore;
|
|
46
|
+
fetch(url: string, init?: RequestInit): Promise<Response>;
|
|
47
|
+
origin: string;
|
|
48
|
+
/** the replica's wipe count */
|
|
49
|
+
epoch(): number;
|
|
50
|
+
/** the most bytes of rows one dataset may use; undefined when the device cannot tell */
|
|
51
|
+
budget(): Promise<number | undefined>;
|
|
52
|
+
/**
|
|
53
|
+
* Set for a principal dataset: reads what a response says about whose rows it carries before
|
|
54
|
+
* anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
|
|
55
|
+
*/
|
|
56
|
+
guard?(response: Response): 'ok' | DatasetGuardVerdict;
|
|
57
|
+
};
|
|
58
|
+
/** why a guard stopped a response */
|
|
59
|
+
export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
|
|
60
|
+
/**
|
|
61
|
+
* The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
|
|
62
|
+
* dataset, and a success that does not name exactly this partition is another principal's or none.
|
|
63
|
+
*/
|
|
64
|
+
export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
|
|
65
|
+
/** seed pages followed before a sync gives up on a server whose paging does not end */
|
|
66
|
+
export declare const DATASET_MAX_PAGES = 1000;
|
|
67
|
+
/**
|
|
68
|
+
* Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
|
|
69
|
+
* is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
|
|
70
|
+
* rows are applied to a staging area and written in one transaction after the collection proof
|
|
71
|
+
* matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
|
|
72
|
+
* which covers rows that moved while the pages were read; a proof that still fails clears the
|
|
73
|
+
* dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
|
|
74
|
+
* an outcome.
|
|
75
|
+
*/
|
|
76
|
+
export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;
|