@drupflare/spool 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +499 -43
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +34 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/client/register.d.ts +23 -1
  11. package/dist/client/register.js +53 -7
  12. package/dist/media/embeds.d.ts +3 -2
  13. package/dist/media/embeds.js +16 -1
  14. package/dist/protocol/config.d.ts +16 -1
  15. package/dist/protocol/config.js +3 -1
  16. package/dist/protocol/dataset.d.ts +183 -0
  17. package/dist/protocol/dataset.js +206 -0
  18. package/dist/protocol/envelope.d.ts +3 -0
  19. package/dist/protocol/envelope.js +14 -3
  20. package/dist/protocol/errors.d.ts +1 -1
  21. package/dist/protocol/graph.d.ts +23 -4
  22. package/dist/protocol/graph.js +34 -3
  23. package/dist/protocol/hash.d.ts +2 -0
  24. package/dist/protocol/hash.js +8 -0
  25. package/dist/protocol/index.d.ts +7 -0
  26. package/dist/protocol/index.js +7 -0
  27. package/dist/protocol/inline.d.ts +14 -0
  28. package/dist/protocol/inline.js +43 -0
  29. package/dist/protocol/navigation.d.ts +81 -0
  30. package/dist/protocol/navigation.js +249 -0
  31. package/dist/protocol/normalise.d.ts +4 -3
  32. package/dist/protocol/normalise.js +6 -4
  33. package/dist/protocol/plan.d.ts +100 -0
  34. package/dist/protocol/plan.js +661 -0
  35. package/dist/protocol/provision.d.ts +52 -1
  36. package/dist/protocol/provision.js +53 -5
  37. package/dist/protocol/query.d.ts +141 -0
  38. package/dist/protocol/query.js +252 -0
  39. package/dist/protocol/route.d.ts +85 -5
  40. package/dist/protocol/route.js +91 -0
  41. package/dist/protocol/write.d.ts +146 -0
  42. package/dist/protocol/write.js +101 -0
  43. package/dist/proxy/bases.d.ts +53 -0
  44. package/dist/proxy/bases.js +76 -0
  45. package/dist/proxy/index.d.ts +1 -0
  46. package/dist/proxy/index.js +1 -0
  47. package/dist/proxy/proxy.js +8 -2
  48. package/dist/report/capacity.d.ts +59 -0
  49. package/dist/report/capacity.js +76 -0
  50. package/dist/report/channels.d.ts +60 -0
  51. package/dist/report/channels.js +65 -0
  52. package/dist/report/dataset.d.ts +48 -0
  53. package/dist/report/dataset.js +93 -0
  54. package/dist/report/derive.d.ts +285 -0
  55. package/dist/report/derive.js +676 -1
  56. package/dist/report/economics.d.ts +11 -1
  57. package/dist/report/economics.js +19 -1
  58. package/dist/report/index.d.ts +5 -0
  59. package/dist/report/index.js +5 -0
  60. package/dist/report/inputs.d.ts +97 -0
  61. package/dist/report/inputs.js +133 -0
  62. package/dist/report/institution.d.ts +284 -0
  63. package/dist/report/institution.js +225 -0
  64. package/dist/report/ledger.d.ts +28 -0
  65. package/dist/report/ledger.js +30 -0
  66. package/dist/report/loops.d.ts +73 -0
  67. package/dist/report/loops.js +101 -0
  68. package/dist/report/measurement.d.ts +1 -1
  69. package/dist/report/measurement.js +6 -1
  70. package/dist/report/model.d.ts +2 -0
  71. package/dist/report/model.js +4 -0
  72. package/dist/report/thirdparty.d.ts +57 -0
  73. package/dist/report/thirdparty.js +111 -0
  74. package/dist/report/traffic.d.ts +11 -2
  75. package/dist/report/traffic.js +13 -3
  76. package/dist/report/trim.d.ts +56 -0
  77. package/dist/report/trim.js +83 -1
  78. package/dist/server/dataset.d.ts +48 -0
  79. package/dist/server/dataset.js +308 -0
  80. package/dist/server/embedded.d.ts +3 -3
  81. package/dist/server/embedded.js +3 -3
  82. package/dist/server/index.d.ts +3 -0
  83. package/dist/server/index.js +3 -0
  84. package/dist/server/spool.d.ts +40 -0
  85. package/dist/server/spool.js +184 -23
  86. package/dist/server/write.d.ts +27 -0
  87. package/dist/server/write.js +127 -0
  88. package/dist/sw/analytics.d.ts +6 -0
  89. package/dist/sw/analytics.js +14 -0
  90. package/dist/sw/dataset-sync.d.ts +76 -0
  91. package/dist/sw/dataset-sync.js +149 -0
  92. package/dist/sw/dataset.d.ts +91 -0
  93. package/dist/sw/dataset.js +318 -0
  94. package/dist/sw/graph/index.d.ts +31 -1
  95. package/dist/sw/graph/index.js +57 -5
  96. package/dist/sw/graph/invalidate.d.ts +15 -3
  97. package/dist/sw/graph/invalidate.js +42 -4
  98. package/dist/sw/idb.js +78 -9
  99. package/dist/sw/partition.d.ts +6 -1
  100. package/dist/sw/partition.js +21 -2
  101. package/dist/sw/principal-dataset.d.ts +64 -0
  102. package/dist/sw/principal-dataset.js +254 -0
  103. package/dist/sw/render.d.ts +8 -1
  104. package/dist/sw/render.js +28 -1
  105. package/dist/sw/replica.d.ts +14 -3
  106. package/dist/sw/replica.js +46 -5
  107. package/dist/sw/router.js +2 -2
  108. package/dist/sw/runtime.d.ts +43 -2
  109. package/dist/sw/runtime.js +440 -29
  110. package/dist/sw/seal.d.ts +13 -0
  111. package/dist/sw/seal.js +13 -4
  112. package/dist/sw/static.d.ts +18 -0
  113. package/dist/sw/static.js +84 -1
  114. package/dist/sw/storage.d.ts +51 -1
  115. package/dist/sw/storage.js +33 -1
  116. package/dist/sw/write.d.ts +107 -0
  117. package/dist/sw/write.js +442 -0
  118. package/package.json +5 -2
@@ -4,19 +4,25 @@ import { checkCollectors } from '../protocol/collector.js';
4
4
  import { clientPath, PATHS, SPOOL_PREFIX } from '../protocol/config.js';
5
5
  import { HEADERS, writeEnvelope } from '../protocol/envelope.js';
6
6
  import { ProtocolError } from '../protocol/errors.js';
7
- import { collectRefs, createCollector } from '../protocol/graph.js';
7
+ import { collectRefs, createCollector, resolveRefs, splitRowId } from '../protocol/graph.js';
8
+ import { shellHash } from '../protocol/hash.js';
9
+ import { escapeInline, inlineElement, placeInline } from '../protocol/inline.js';
8
10
  import { checkPattern } from '../protocol/pattern.js';
9
11
  import { benefit, chooseEntries, DEFAULT_CATALOGUE_BYTES, priorFrom } from '../protocol/provision.js';
12
+ import { keyField, verifyProof } from '../protocol/query.js';
10
13
  import { matchRoute, serializeRoute } from '../protocol/route.js';
11
14
  import { artifactTag, MAX_HAVE, parseHeld } from '../protocol/sync.js';
12
15
  import { clientTag, injectClientTag } from '../protocol/tag.js';
16
+ import { serializeWrite } from '../protocol/write.js';
13
17
  import { VERSION } from '../version.js';
14
18
  import { respondBeacon } from './beacon.js';
19
+ import { respondDataset } from './dataset.js';
15
20
  import { CLIENT_CORE } from './embedded.js';
16
21
  import { partitionId } from './identity.js';
17
22
  import { compressedSize } from './size.js';
18
23
  import { workerResponse, workerScript } from './stub.js';
19
24
  import { json, respondSync } from './sync.js';
25
+ import { createWriteServer } from './write.js';
20
26
  /** most changed tags a response carries inline before the worker is left to ask `/.spool/sync` */
21
27
  export const MAX_PIGGYBACK = 32;
22
28
  /** most pages one catalogue request makes the server build, whatever its route suggests */
@@ -37,6 +43,63 @@ export function createSpool(options) {
37
43
  }
38
44
  seen.add(route.pattern);
39
45
  }
46
+ const datasets = new Map();
47
+ const principalOn = options.principalDatasets === true;
48
+ for (const dataset of options.datasets ?? []) {
49
+ if (datasets.has(dataset.name)) {
50
+ throw new ProtocolError('bad-dataset', `${dataset.name}: declared twice`);
51
+ }
52
+ if (dataset.scope === 'principal') {
53
+ if (!principalOn) {
54
+ throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs the principal datasets flag: set the principalDatasets option of createSpool`);
55
+ }
56
+ if (!options.identity) {
57
+ throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs an identity option`);
58
+ }
59
+ }
60
+ datasets.set(dataset.name, dataset);
61
+ }
62
+ for (const route of routes) {
63
+ const name = route.query?.dataset;
64
+ if (name !== undefined) {
65
+ const dataset = datasets.get(name);
66
+ if (!dataset) {
67
+ throw new ProtocolError('bad-route', `${route.pattern}: query.dataset "${name}" is not declared`);
68
+ }
69
+ if (dataset.scope !== route.scope) {
70
+ throw new ProtocolError('bad-route', `${route.pattern}: a ${route.scope} route cannot query the ${dataset.scope} dataset "${name}"`);
71
+ }
72
+ }
73
+ for (const join of route.query?.plan?.join ?? []) {
74
+ if (!datasets.has(join.dataset)) {
75
+ throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" is not declared`);
76
+ }
77
+ if (datasets.get(join.dataset)?.scope !== route.scope) {
78
+ throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" must be ${route.scope}`);
79
+ }
80
+ }
81
+ }
82
+ const writes = [...(options.writes?.classes ?? [])];
83
+ const writeNames = new Set();
84
+ for (const write of writes) {
85
+ if (writeNames.has(write.name)) {
86
+ throw new ProtocolError('bad-write', `${write.name}: declared twice`);
87
+ }
88
+ writeNames.add(write.name);
89
+ const target = routes.find((route) => route.pattern === write.path);
90
+ if (!target || target.scope !== 'principal') {
91
+ throw new ProtocolError('bad-write', `${write.name}: path ${write.path} must be a declared principal route`);
92
+ }
93
+ }
94
+ if (writes.length > 0 && !options.identity) {
95
+ throw new ProtocolError('bad-write', 'writes need an identity option');
96
+ }
97
+ const writesOn = options.writes?.enabled === true && writes.length > 0;
98
+ const writeServer = createWriteServer({
99
+ enabled: writesOn,
100
+ writes,
101
+ ...(options.writes?.remember !== undefined ? { remember: options.writes.remember } : {})
102
+ });
40
103
  const mode = options.mode ?? 'serve';
41
104
  if (mode !== 'serve' && mode !== 'shadow') {
42
105
  throw new ProtocolError('bad-route', `unknown mode "${String(mode)}"`);
@@ -63,11 +126,20 @@ export function createSpool(options) {
63
126
  const collectors = [...(options.collectors ?? [])];
64
127
  checkCollectors(collectors);
65
128
  const prior = priorFrom(options.provision);
129
+ const datasetStats = new Map();
66
130
  const catalogueBytes = options.catalogueBytes ?? DEFAULT_CATALOGUE_BYTES;
67
131
  if (!(Number.isFinite(catalogueBytes) && catalogueBytes >= 0)) {
68
132
  throw new ProtocolError('bad-route', 'catalogueBytes must be a non-negative number');
69
133
  }
134
+ const catalogueConfidence = options.catalogueConfidence ?? 0;
135
+ if (!(Number.isFinite(catalogueConfidence) && catalogueConfidence >= 0)) {
136
+ throw new ProtocolError('bad-route', 'catalogueConfidence must be a non-negative number');
137
+ }
70
138
  const measure = options.catalogueSize ?? compressedSize;
139
+ const inlineMax = options.inlineMaxBytes ?? 0;
140
+ if (!(Number.isFinite(inlineMax) && inlineMax >= 0)) {
141
+ throw new ProtocolError('bad-route', 'inlineMaxBytes must be a non-negative number');
142
+ }
71
143
  const config = {
72
144
  protocol: PROTOCOL_VERSION,
73
145
  version: VERSION,
@@ -93,6 +165,11 @@ export function createSpool(options) {
93
165
  config.collectors = collectors;
94
166
  if (options.thirdParty)
95
167
  config.thirdParty = options.thirdParty;
168
+ if (writesOn)
169
+ config.writes = writes.map(serializeWrite);
170
+ const principalSets = [...datasets.values()].flatMap((d) => d.scope === 'principal' ? [{ name: d.name, maxStaleMs: d.maxStaleMs }] : []);
171
+ if (principalOn && principalSets.length > 0)
172
+ config.principalDatasets = principalSets;
96
173
  if (options.views === false)
97
174
  config.views = false;
98
175
  let script;
@@ -104,7 +181,8 @@ export function createSpool(options) {
104
181
  params,
105
182
  identity,
106
183
  record: collector.record,
107
- projection: collector.projection
184
+ projection: collector.projection,
185
+ row: collector.row
108
186
  });
109
187
  /** drops the records and projections the worker already holds at a version no change has passed */
110
188
  async function trim(artifact, have) {
@@ -158,6 +236,26 @@ export function createSpool(options) {
158
236
  throw new ProtocolError('bad-artifact', `${path} references projection ${id} it did not register`);
159
237
  }
160
238
  }
239
+ for (const id of refs.rows ?? []) {
240
+ const at = splitRowId(id);
241
+ if (!at || datasets.get(at.dataset)?.scope !== hit.route.scope) {
242
+ throw new ProtocolError('bad-artifact', `${path} references row ${id} of no dataset here`);
243
+ }
244
+ }
245
+ if (hit.route.query && hit.route.query.dataset === undefined) {
246
+ // the worker checks the proof after it resolves refs, so the server checks the same text
247
+ const recordsById = new Map(collected.records.map((r) => [r.id, r]));
248
+ const projectionsById = new Map(collected.projections.map((p) => [p.id, p]));
249
+ const lookup = {
250
+ record: (id) => recordsById.get(id),
251
+ projection: (id) => projectionsById.get(id)
252
+ };
253
+ const resolved = resolveRefs(loaded, lookup);
254
+ if (!resolved.ok ||
255
+ !(await verifyProof(hit.route.query, resolved.value, keyField(hit.route.query)))) {
256
+ throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof over its resolved rows`);
257
+ }
258
+ }
161
259
  const artifact = parseArtifact({
162
260
  protocol: PROTOCOL_VERSION,
163
261
  id: path,
@@ -170,18 +268,24 @@ export function createSpool(options) {
170
268
  ...(collected.projections.length > 0 ? { projections: collected.projections } : {})
171
269
  });
172
270
  const trimmed = await trim(artifact, have);
173
- if (weighed && hit.route.scope === 'public') {
271
+ if (weighed) {
174
272
  const bytes = new TextEncoder().encode(JSON.stringify(trimmed)).length;
175
- const mine = { ...prior, ...hit.route.provision };
176
- const data = hit.route.strategy === 'data';
177
- const worth = benefit({
178
- id: path,
179
- kind: 'route',
180
- provision: { bytes, requests: 1 },
181
- avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
182
- }, mine);
183
- if (worth <= 0)
184
- return { declined: true };
273
+ // a query.dataset route's rows are bounded by the dataset's maxBytes at its seed
274
+ const budget = hit.route.query?.dataset === undefined ? hit.route.provision?.maxBytes : undefined;
275
+ if (budget !== undefined && bytes > budget)
276
+ return { declined: 'budget' };
277
+ if (hit.route.scope === 'public') {
278
+ const mine = { ...prior, ...hit.route.provision };
279
+ const data = hit.route.strategy === 'data';
280
+ const worth = benefit({
281
+ id: path,
282
+ kind: 'route',
283
+ provision: { bytes, requests: 1 },
284
+ avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
285
+ }, mine);
286
+ if (worth <= 0)
287
+ return { declined: 'benefit' };
288
+ }
185
289
  }
186
290
  const out = { artifact: trimmed };
187
291
  if (identity && hit.route.scope === 'principal') {
@@ -189,6 +293,20 @@ export function createSpool(options) {
189
293
  }
190
294
  return out;
191
295
  }
296
+ /** the escaped artifact JSON of a public page, or null when it is over the limit or fails to build */
297
+ async function inlineText(url) {
298
+ try {
299
+ const probe = new Request(url.href, { headers: { accept: 'application/json' } });
300
+ const built = await buildArtifact(probe, url, url.pathname, {}, true);
301
+ if (!('artifact' in built) || built.artifact.scope !== 'public')
302
+ return null;
303
+ const text = escapeInline(JSON.stringify(built.artifact));
304
+ return new TextEncoder().encode(text).length <= inlineMax ? text : null;
305
+ }
306
+ catch {
307
+ return null;
308
+ }
309
+ }
192
310
  async function respondArtifact(request, url) {
193
311
  const path = url.searchParams.get('path') ?? '';
194
312
  const raw = url.searchParams.get('have');
@@ -198,7 +316,11 @@ export function createSpool(options) {
198
316
  if ('declined' in built) {
199
317
  return new Response(null, {
200
318
  status: 204,
201
- headers: { 'cache-control': 'no-store', [HEADERS.provision]: 'declined' }
319
+ headers: {
320
+ 'cache-control': 'no-store',
321
+ [HEADERS.provision]: 'declined',
322
+ [HEADERS.provisionReason]: built.declined
323
+ }
202
324
  });
203
325
  }
204
326
  const headers = { [HEADERS.protocol]: String(PROTOCOL_VERSION) };
@@ -284,9 +406,20 @@ export function createSpool(options) {
284
406
  const documentBytes = target.route.documentBytes ??
285
407
  (target.route.strategy === 'data' ? entryBytes : prior.avoidedBytes);
286
408
  built.set(guess.path, result);
287
- pool.push({ path: guess.path, p: guess.p, entryBytes, documentBytes });
409
+ pool.push({
410
+ path: guess.path,
411
+ p: guess.p,
412
+ ...(typeof guess.n === 'number' && guess.n > 0 && Number.isFinite(guess.n)
413
+ ? { weight: guess.n }
414
+ : {}),
415
+ entryBytes,
416
+ documentBytes
417
+ });
288
418
  }
289
- const chosen = chooseEntries(pool, { capBytes: catalogueBytes });
419
+ const chosen = chooseEntries(pool, {
420
+ capBytes: catalogueBytes,
421
+ z: catalogueConfidence
422
+ });
290
423
  const entries = chosen.flatMap((c) => built.get(c.path) ?? []);
291
424
  const partition = entries.find((e) => e.partition !== undefined)?.partition;
292
425
  return {
@@ -325,6 +458,20 @@ export function createSpool(options) {
325
458
  }
326
459
  if (url.pathname === PATHS.artifact && get)
327
460
  return await respondArtifact(request, url);
461
+ if (url.pathname === PATHS.dataset && get) {
462
+ return await respondDataset(request, url, datasets, { prior, stats: datasetStats }, async () => {
463
+ const identity = await who(request);
464
+ if (!identity || !options.identity)
465
+ return null;
466
+ return {
467
+ identity,
468
+ partition: await partitionId(options.identity.secret, identity.principal)
469
+ };
470
+ });
471
+ }
472
+ if (url.pathname === PATHS.write) {
473
+ return await writeServer.respond(request, writesOn ? await who(request) : null);
474
+ }
328
475
  if (url.pathname === PATHS.beacon) {
329
476
  return await respondBeacon(request, sink, provide(request), catalogue(request));
330
477
  }
@@ -394,6 +541,12 @@ export function createSpool(options) {
394
541
  headers.set('x-spool-mode', hit.route.mode ?? mode);
395
542
  }
396
543
  const kept = hit?.route.strategy === 'static' || hit?.route.strategy === 'data' || isHtml(response);
544
+ const tag = request.method === 'GET' &&
545
+ response.status === 200 &&
546
+ isHtml(response) &&
547
+ !response.headers.has('content-encoding');
548
+ const html = tag ? await response.text() : null;
549
+ let inline = null;
397
550
  if (hit && kept && (hit.route.scope === 'public' || identity)) {
398
551
  const ctx = context(request, url, hit.params, identity);
399
552
  envelope.artifact = {
@@ -402,23 +555,31 @@ export function createSpool(options) {
402
555
  deps: [...(await hit.route.depends(ctx))],
403
556
  version: before
404
557
  };
558
+ const { assembled } = decorateOptions;
559
+ if (tag && hit.route.scope === 'public') {
560
+ if (hit.route.strategy === 'assembly' && assembled !== undefined) {
561
+ envelope.artifact.hash = await shellHash(assembled);
562
+ }
563
+ else if (hit.route.strategy === 'template' && inlineMax > 0) {
564
+ inline = await inlineText(url);
565
+ if (inline !== null)
566
+ envelope.artifact.hash = await shellHash(inline);
567
+ }
568
+ }
405
569
  }
406
570
  writeEnvelope(headers, envelope);
407
571
  if (decorateOptions.hardReset)
408
572
  headers.set('clear-site-data', '"storage"');
409
- const tag = request.method === 'GET' &&
410
- response.status === 200 &&
411
- isHtml(response) &&
412
- !response.headers.has('content-encoding');
413
- if (!tag) {
573
+ if (html === null) {
414
574
  return new Response(response.body, {
415
575
  status: response.status,
416
576
  statusText: response.statusText,
417
577
  headers
418
578
  });
419
579
  }
420
- const html = await response.text();
421
- const tagged = injectClientTag(html, config.clientTag);
580
+ let tagged = injectClientTag(html, config.clientTag);
581
+ if (inline !== null)
582
+ tagged = placeInline(tagged, inlineElement(inline));
422
583
  if (tagged !== html) {
423
584
  headers.delete('content-length');
424
585
  const etag = headers.get('etag');
@@ -0,0 +1,27 @@
1
+ import type { Identity } from '../protocol/route.js';
2
+ import { type Write } from '../protocol/write.js';
3
+ /** how many `(class, principal, key)` slots the server remembers an applied write id for */
4
+ export declare const DEFAULT_REMEMBER = 10000;
5
+ /** what {@link createWriteServer} takes */
6
+ export type WriteServerOptions = {
7
+ /** false answers every request with `writes-off` */
8
+ enabled: boolean;
9
+ writes: readonly Write[];
10
+ /** slots whose last applied id is remembered, oldest forgotten first; default {@link DEFAULT_REMEMBER} */
11
+ remember?: number;
12
+ };
13
+ /** the handler `createSpool` routes `PUT /.spool/write` to */
14
+ export type WriteServer = {
15
+ respond(request: Request, identity: Identity | null): Promise<Response>;
16
+ };
17
+ /**
18
+ * Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
19
+ * It checks the declaration, the principal, the key the site derives for that principal, the version
20
+ * header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
21
+ * a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
22
+ * answers 200 with the current version and applies nothing, and the same id with another body is
23
+ * refused. Memory is per server instance, so a deployment with several instances needs the site's
24
+ * `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
25
+ * reading as a conflict.
26
+ */
27
+ export declare function createWriteServer(options: WriteServerOptions): WriteServer;
@@ -0,0 +1,127 @@
1
+ import { shellHash } from '../protocol/hash.js';
2
+ import { isWriteId, isWriteKey, readIfMatch, WRITE_HEADERS, WRITE_STATUS } from '../protocol/write.js';
3
+ import { json } from './sync.js';
4
+ /** how many `(class, principal, key)` slots the server remembers an applied write id for */
5
+ export const DEFAULT_REMEMBER = 10_000;
6
+ const refuse = (code, error, headers = {}) => json({ code, error }, WRITE_STATUS[code], headers);
7
+ /** the body as text, or null when it is longer than `max` bytes; stops reading once it is */
8
+ async function readBounded(request, max) {
9
+ const declared = Number(request.headers.get('content-length') ?? Number.NaN);
10
+ if (declared > max)
11
+ return null;
12
+ if (!request.body)
13
+ return '';
14
+ const reader = request.body.getReader();
15
+ const chunks = [];
16
+ let total = 0;
17
+ for (;;) {
18
+ const { done, value } = await reader.read();
19
+ if (done)
20
+ break;
21
+ total += value.byteLength;
22
+ if (total > max) {
23
+ await reader.cancel();
24
+ return null;
25
+ }
26
+ chunks.push(value);
27
+ }
28
+ const whole = new Uint8Array(total);
29
+ let at = 0;
30
+ for (const chunk of chunks) {
31
+ whole.set(chunk, at);
32
+ at += chunk.byteLength;
33
+ }
34
+ return new TextDecoder().decode(whole);
35
+ }
36
+ /**
37
+ * Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
38
+ * It checks the declaration, the principal, the key the site derives for that principal, the version
39
+ * header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
40
+ * a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
41
+ * answers 200 with the current version and applies nothing, and the same id with another body is
42
+ * refused. Memory is per server instance, so a deployment with several instances needs the site's
43
+ * `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
44
+ * reading as a conflict.
45
+ */
46
+ export function createWriteServer(options) {
47
+ const byName = new Map(options.writes.map((write) => [write.name, write]));
48
+ const limit = options.remember ?? DEFAULT_REMEMBER;
49
+ const applied = new Map();
50
+ const locks = new Map();
51
+ const remember = (slot, entry) => {
52
+ applied.delete(slot);
53
+ applied.set(slot, entry);
54
+ while (applied.size > limit)
55
+ applied.delete(applied.keys().next().value);
56
+ };
57
+ /** runs `fn` after every earlier request for the same slot */
58
+ function exclusive(slot, fn) {
59
+ const run = (locks.get(slot) ?? Promise.resolve()).then(fn, fn);
60
+ const tail = run.catch(() => undefined);
61
+ locks.set(slot, tail);
62
+ void tail.then(() => {
63
+ if (locks.get(slot) === tail)
64
+ locks.delete(slot);
65
+ });
66
+ return run;
67
+ }
68
+ return {
69
+ async respond(request, identity) {
70
+ if (!options.enabled)
71
+ return refuse('writes-off', 'declared writes are off');
72
+ if (request.method !== 'PUT') {
73
+ return refuse('bad-method', 'use PUT', { allow: 'PUT' });
74
+ }
75
+ const url = new URL(request.url);
76
+ const write = byName.get(url.searchParams.get('name') ?? '');
77
+ if (!write)
78
+ return refuse('unknown-write', 'no such write class');
79
+ if (!identity)
80
+ return refuse('sign-in', 'sign in to write');
81
+ const key = url.searchParams.get('key');
82
+ if (!isWriteKey(key))
83
+ return refuse('bad-key', 'key is missing or malformed');
84
+ const id = request.headers.get(WRITE_HEADERS.id);
85
+ if (!isWriteId(id))
86
+ return refuse('bad-id', 'Spool-Write-Id is missing or malformed');
87
+ const header = request.headers.get('if-match');
88
+ if (header === null)
89
+ return refuse('missing-version', 'If-Match is required');
90
+ const version = readIfMatch(header);
91
+ if (version === null)
92
+ return refuse('bad-version', 'If-Match is not one version');
93
+ const ctx = { identity, writeId: id };
94
+ if ((await write.key(ctx, request)) !== key) {
95
+ return refuse('forbidden', 'that key is not yours');
96
+ }
97
+ const text = await readBounded(request, write.maxBytes);
98
+ if (text === null)
99
+ return refuse('too-large', `the body is over ${write.maxBytes} bytes`);
100
+ let value;
101
+ try {
102
+ const body = JSON.parse(text);
103
+ if (typeof body !== 'object' || body === null || !('value' in body))
104
+ throw new Error();
105
+ value = body.value;
106
+ }
107
+ catch {
108
+ return refuse('bad-body', 'the body must be JSON of the form {"value": ...}');
109
+ }
110
+ const slot = `${write.name}\u0000${identity.principal}\u0000${key}`;
111
+ return exclusive(slot, async () => {
112
+ const hash = await shellHash(text);
113
+ const last = applied.get(slot);
114
+ if (last?.id === id) {
115
+ if (last.hash !== hash)
116
+ return refuse('id-reused', 'that id was used for another body');
117
+ return json({ version: (await write.read(ctx, key)).version });
118
+ }
119
+ const result = await write.apply(ctx, key, value, version);
120
+ if ('conflict' in result)
121
+ return json({ code: 'conflict', conflict: result.conflict }, 412);
122
+ remember(slot, { id, hash });
123
+ return json({ version: result.version });
124
+ });
125
+ }
126
+ };
127
+ }
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
78
78
  * browser's. A request that fails, or a network that is down, loses the event for good.
79
79
  */
80
80
  sendSameOrigin(timeoutMs: number): Promise<void>;
81
+ /**
82
+ * Drops the held same-origin requests without sending them, for a response that names another
83
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
84
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
85
+ */
86
+ dropSameOrigin(): Promise<void>;
81
87
  }
@@ -199,6 +199,20 @@ export class BeaconQueue {
199
199
  }
200
200
  await this.#report(lost);
201
201
  }
202
+ /**
203
+ * Drops the held same-origin requests without sending them, for a response that names another
204
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
205
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
206
+ */
207
+ async dropSameOrigin() {
208
+ if (this.#flushing)
209
+ await this.#flushing.catch(() => false);
210
+ const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
211
+ if (mine.length === 0)
212
+ return;
213
+ await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
214
+ await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
215
+ }
202
216
  async #run() {
203
217
  const { storage, send, replay, want, catalogue } = this.#deps;
204
218
  await this.#expire();
@@ -0,0 +1,76 @@
1
+ import type { DatasetStore } from './dataset.js';
2
+ /** why a dataset was not held or not brought current */
3
+ export type DatasetRefusal =
4
+ /** the rows would use more than the storage the device has to give */
5
+ 'budget'
6
+ /** the proof failed again after a patch and one reseed */
7
+ | 'proof'
8
+ /** the server has no such dataset, or refused the request */
9
+ | 'unavailable'
10
+ /** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
11
+ | 'declined'
12
+ /** the server's answer was not a dataset response, or its paging did not advance */
13
+ | 'bad-response'
14
+ /** the replica was wiped while the apply ran */
15
+ | 'epoch'
16
+ /** a principal dataset's answer named another partition than the one held, or none */
17
+ | 'partition'
18
+ /** the server answered a principal dataset with a sign-in request: the session ended */
19
+ | 'ended'
20
+ /** the server forbids this principal the dataset: its held rows were dropped */
21
+ | 'revoked'
22
+ /** the storage transaction failed, for example on quota */
23
+ | 'storage';
24
+ /**
25
+ * What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
26
+ * changed them, `current` found nothing to change. `offline` means no answer came and the held
27
+ * rows, if any, are as they were; `refused` names why a dataset is not held.
28
+ */
29
+ export type DatasetOutcome = {
30
+ name: string;
31
+ requests: number;
32
+ } & ({
33
+ status: 'seeded' | 'reseeded' | 'patched' | 'current';
34
+ cursor: number;
35
+ rows: number;
36
+ } | {
37
+ status: 'offline';
38
+ } | {
39
+ status: 'refused';
40
+ reason: DatasetRefusal;
41
+ by?: string | null;
42
+ });
43
+ /** what a sync needs from the worker around it */
44
+ export type DatasetSyncDeps = {
45
+ store: DatasetStore;
46
+ fetch(url: string, init?: RequestInit): Promise<Response>;
47
+ origin: string;
48
+ /** the replica's wipe count */
49
+ epoch(): number;
50
+ /** the most bytes of rows one dataset may use; undefined when the device cannot tell */
51
+ budget(): Promise<number | undefined>;
52
+ /**
53
+ * Set for a principal dataset: reads what a response says about whose rows it carries before
54
+ * anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
55
+ */
56
+ guard?(response: Response): 'ok' | DatasetGuardVerdict;
57
+ };
58
+ /** why a guard stopped a response */
59
+ export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
60
+ /**
61
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
62
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
63
+ */
64
+ export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
65
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
66
+ export declare const DATASET_MAX_PAGES = 1000;
67
+ /**
68
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
69
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
70
+ * rows are applied to a staging area and written in one transaction after the collection proof
71
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
72
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
73
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
74
+ * an outcome.
75
+ */
76
+ export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;