@drupflare/spool 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +263 -41
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/entry.js +33 -1
  6. package/dist/client/register.d.ts +23 -1
  7. package/dist/client/register.js +53 -7
  8. package/dist/protocol/config.d.ts +16 -1
  9. package/dist/protocol/config.js +3 -1
  10. package/dist/protocol/dataset.d.ts +183 -0
  11. package/dist/protocol/dataset.js +206 -0
  12. package/dist/protocol/envelope.d.ts +1 -0
  13. package/dist/protocol/envelope.js +2 -1
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/graph.d.ts +23 -4
  16. package/dist/protocol/graph.js +34 -3
  17. package/dist/protocol/index.d.ts +3 -0
  18. package/dist/protocol/index.js +3 -0
  19. package/dist/protocol/plan.d.ts +100 -0
  20. package/dist/protocol/plan.js +661 -0
  21. package/dist/protocol/provision.d.ts +46 -0
  22. package/dist/protocol/provision.js +38 -0
  23. package/dist/protocol/query.d.ts +66 -5
  24. package/dist/protocol/query.js +103 -4
  25. package/dist/protocol/route.d.ts +6 -2
  26. package/dist/protocol/route.js +31 -3
  27. package/dist/protocol/write.d.ts +146 -0
  28. package/dist/protocol/write.js +101 -0
  29. package/dist/report/dataset.d.ts +48 -0
  30. package/dist/report/dataset.js +93 -0
  31. package/dist/report/derive.d.ts +285 -0
  32. package/dist/report/derive.js +676 -1
  33. package/dist/report/economics.d.ts +11 -1
  34. package/dist/report/economics.js +19 -1
  35. package/dist/report/index.d.ts +2 -0
  36. package/dist/report/index.js +2 -0
  37. package/dist/report/inputs.d.ts +97 -0
  38. package/dist/report/inputs.js +133 -0
  39. package/dist/report/institution.d.ts +188 -1
  40. package/dist/report/institution.js +146 -0
  41. package/dist/report/loops.d.ts +73 -0
  42. package/dist/report/loops.js +101 -0
  43. package/dist/report/measurement.d.ts +1 -1
  44. package/dist/report/measurement.js +3 -1
  45. package/dist/report/model.d.ts +2 -0
  46. package/dist/report/model.js +4 -0
  47. package/dist/report/thirdparty.d.ts +57 -0
  48. package/dist/report/thirdparty.js +111 -0
  49. package/dist/report/trim.d.ts +56 -0
  50. package/dist/report/trim.js +83 -1
  51. package/dist/server/dataset.d.ts +48 -0
  52. package/dist/server/dataset.js +308 -0
  53. package/dist/server/embedded.d.ts +3 -3
  54. package/dist/server/embedded.js +3 -3
  55. package/dist/server/index.d.ts +2 -0
  56. package/dist/server/index.js +2 -0
  57. package/dist/server/spool.d.ts +20 -0
  58. package/dist/server/spool.js +126 -18
  59. package/dist/server/write.d.ts +27 -0
  60. package/dist/server/write.js +127 -0
  61. package/dist/sw/analytics.d.ts +6 -0
  62. package/dist/sw/analytics.js +14 -0
  63. package/dist/sw/dataset-sync.d.ts +76 -0
  64. package/dist/sw/dataset-sync.js +149 -0
  65. package/dist/sw/dataset.d.ts +91 -0
  66. package/dist/sw/dataset.js +318 -0
  67. package/dist/sw/graph/index.d.ts +31 -1
  68. package/dist/sw/graph/index.js +57 -5
  69. package/dist/sw/graph/invalidate.d.ts +15 -3
  70. package/dist/sw/graph/invalidate.js +42 -4
  71. package/dist/sw/idb.js +78 -9
  72. package/dist/sw/partition.d.ts +6 -1
  73. package/dist/sw/partition.js +21 -2
  74. package/dist/sw/principal-dataset.d.ts +64 -0
  75. package/dist/sw/principal-dataset.js +254 -0
  76. package/dist/sw/render.d.ts +7 -2
  77. package/dist/sw/render.js +19 -5
  78. package/dist/sw/replica.d.ts +12 -3
  79. package/dist/sw/replica.js +40 -5
  80. package/dist/sw/runtime.d.ts +43 -2
  81. package/dist/sw/runtime.js +340 -24
  82. package/dist/sw/seal.d.ts +13 -0
  83. package/dist/sw/seal.js +13 -4
  84. package/dist/sw/static.d.ts +18 -0
  85. package/dist/sw/static.js +84 -1
  86. package/dist/sw/storage.d.ts +51 -1
  87. package/dist/sw/storage.js +33 -1
  88. package/dist/sw/write.d.ts +107 -0
  89. package/dist/sw/write.js +442 -0
  90. package/package.json +3 -2
@@ -4,22 +4,25 @@ import { checkCollectors } from '../protocol/collector.js';
4
4
  import { clientPath, PATHS, SPOOL_PREFIX } from '../protocol/config.js';
5
5
  import { HEADERS, writeEnvelope } from '../protocol/envelope.js';
6
6
  import { ProtocolError } from '../protocol/errors.js';
7
- import { collectRefs, createCollector } from '../protocol/graph.js';
7
+ import { collectRefs, createCollector, resolveRefs, splitRowId } from '../protocol/graph.js';
8
8
  import { shellHash } from '../protocol/hash.js';
9
9
  import { escapeInline, inlineElement, placeInline } from '../protocol/inline.js';
10
10
  import { checkPattern } from '../protocol/pattern.js';
11
11
  import { benefit, chooseEntries, DEFAULT_CATALOGUE_BYTES, priorFrom } from '../protocol/provision.js';
12
- import { verifyProof } from '../protocol/query.js';
12
+ import { keyField, verifyProof } from '../protocol/query.js';
13
13
  import { matchRoute, serializeRoute } from '../protocol/route.js';
14
14
  import { artifactTag, MAX_HAVE, parseHeld } from '../protocol/sync.js';
15
15
  import { clientTag, injectClientTag } from '../protocol/tag.js';
16
+ import { serializeWrite } from '../protocol/write.js';
16
17
  import { VERSION } from '../version.js';
17
18
  import { respondBeacon } from './beacon.js';
19
+ import { respondDataset } from './dataset.js';
18
20
  import { CLIENT_CORE } from './embedded.js';
19
21
  import { partitionId } from './identity.js';
20
22
  import { compressedSize } from './size.js';
21
23
  import { workerResponse, workerScript } from './stub.js';
22
24
  import { json, respondSync } from './sync.js';
25
+ import { createWriteServer } from './write.js';
23
26
  /** most changed tags a response carries inline before the worker is left to ask `/.spool/sync` */
24
27
  export const MAX_PIGGYBACK = 32;
25
28
  /** most pages one catalogue request makes the server build, whatever its route suggests */
@@ -40,6 +43,63 @@ export function createSpool(options) {
40
43
  }
41
44
  seen.add(route.pattern);
42
45
  }
46
+ const datasets = new Map();
47
+ const principalOn = options.principalDatasets === true;
48
+ for (const dataset of options.datasets ?? []) {
49
+ if (datasets.has(dataset.name)) {
50
+ throw new ProtocolError('bad-dataset', `${dataset.name}: declared twice`);
51
+ }
52
+ if (dataset.scope === 'principal') {
53
+ if (!principalOn) {
54
+ throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs the principal datasets flag: set the principalDatasets option of createSpool`);
55
+ }
56
+ if (!options.identity) {
57
+ throw new ProtocolError('bad-dataset', `${dataset.name}: a principal dataset needs an identity option`);
58
+ }
59
+ }
60
+ datasets.set(dataset.name, dataset);
61
+ }
62
+ for (const route of routes) {
63
+ const name = route.query?.dataset;
64
+ if (name !== undefined) {
65
+ const dataset = datasets.get(name);
66
+ if (!dataset) {
67
+ throw new ProtocolError('bad-route', `${route.pattern}: query.dataset "${name}" is not declared`);
68
+ }
69
+ if (dataset.scope !== route.scope) {
70
+ throw new ProtocolError('bad-route', `${route.pattern}: a ${route.scope} route cannot query the ${dataset.scope} dataset "${name}"`);
71
+ }
72
+ }
73
+ for (const join of route.query?.plan?.join ?? []) {
74
+ if (!datasets.has(join.dataset)) {
75
+ throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" is not declared`);
76
+ }
77
+ if (datasets.get(join.dataset)?.scope !== route.scope) {
78
+ throw new ProtocolError('bad-route', `${route.pattern}: query.plan join dataset "${join.dataset}" must be ${route.scope}`);
79
+ }
80
+ }
81
+ }
82
+ const writes = [...(options.writes?.classes ?? [])];
83
+ const writeNames = new Set();
84
+ for (const write of writes) {
85
+ if (writeNames.has(write.name)) {
86
+ throw new ProtocolError('bad-write', `${write.name}: declared twice`);
87
+ }
88
+ writeNames.add(write.name);
89
+ const target = routes.find((route) => route.pattern === write.path);
90
+ if (!target || target.scope !== 'principal') {
91
+ throw new ProtocolError('bad-write', `${write.name}: path ${write.path} must be a declared principal route`);
92
+ }
93
+ }
94
+ if (writes.length > 0 && !options.identity) {
95
+ throw new ProtocolError('bad-write', 'writes need an identity option');
96
+ }
97
+ const writesOn = options.writes?.enabled === true && writes.length > 0;
98
+ const writeServer = createWriteServer({
99
+ enabled: writesOn,
100
+ writes,
101
+ ...(options.writes?.remember !== undefined ? { remember: options.writes.remember } : {})
102
+ });
43
103
  const mode = options.mode ?? 'serve';
44
104
  if (mode !== 'serve' && mode !== 'shadow') {
45
105
  throw new ProtocolError('bad-route', `unknown mode "${String(mode)}"`);
@@ -66,6 +126,7 @@ export function createSpool(options) {
66
126
  const collectors = [...(options.collectors ?? [])];
67
127
  checkCollectors(collectors);
68
128
  const prior = priorFrom(options.provision);
129
+ const datasetStats = new Map();
69
130
  const catalogueBytes = options.catalogueBytes ?? DEFAULT_CATALOGUE_BYTES;
70
131
  if (!(Number.isFinite(catalogueBytes) && catalogueBytes >= 0)) {
71
132
  throw new ProtocolError('bad-route', 'catalogueBytes must be a non-negative number');
@@ -104,6 +165,11 @@ export function createSpool(options) {
104
165
  config.collectors = collectors;
105
166
  if (options.thirdParty)
106
167
  config.thirdParty = options.thirdParty;
168
+ if (writesOn)
169
+ config.writes = writes.map(serializeWrite);
170
+ const principalSets = [...datasets.values()].flatMap((d) => d.scope === 'principal' ? [{ name: d.name, maxStaleMs: d.maxStaleMs }] : []);
171
+ if (principalOn && principalSets.length > 0)
172
+ config.principalDatasets = principalSets;
107
173
  if (options.views === false)
108
174
  config.views = false;
109
175
  let script;
@@ -115,7 +181,8 @@ export function createSpool(options) {
115
181
  params,
116
182
  identity,
117
183
  record: collector.record,
118
- projection: collector.projection
184
+ projection: collector.projection,
185
+ row: collector.row
119
186
  });
120
187
  /** drops the records and projections the worker already holds at a version no change has passed */
121
188
  async function trim(artifact, have) {
@@ -154,9 +221,6 @@ export function createSpool(options) {
154
221
  const ctx = context(request, new URL(path, url), hit.params, identity, collector);
155
222
  const dependsOn = [...(await hit.route.depends(ctx))];
156
223
  const loaded = await hit.route.load(ctx);
157
- if (hit.route.query && !(await verifyProof(hit.route.query, loaded))) {
158
- throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof`);
159
- }
160
224
  const body = hit.route.strategy === 'assembly' ? loaded : { data: loaded };
161
225
  const collected = collector.collect(vector);
162
226
  const stored = new Set([...collected.records.map((r) => r.id)]);
@@ -172,6 +236,26 @@ export function createSpool(options) {
172
236
  throw new ProtocolError('bad-artifact', `${path} references projection ${id} it did not register`);
173
237
  }
174
238
  }
239
+ for (const id of refs.rows ?? []) {
240
+ const at = splitRowId(id);
241
+ if (!at || datasets.get(at.dataset)?.scope !== hit.route.scope) {
242
+ throw new ProtocolError('bad-artifact', `${path} references row ${id} of no dataset here`);
243
+ }
244
+ }
245
+ if (hit.route.query && hit.route.query.dataset === undefined) {
246
+ // the worker checks the proof after it resolves refs, so the server checks the same text
247
+ const recordsById = new Map(collected.records.map((r) => [r.id, r]));
248
+ const projectionsById = new Map(collected.projections.map((p) => [p.id, p]));
249
+ const lookup = {
250
+ record: (id) => recordsById.get(id),
251
+ projection: (id) => projectionsById.get(id)
252
+ };
253
+ const resolved = resolveRefs(loaded, lookup);
254
+ if (!resolved.ok ||
255
+ !(await verifyProof(hit.route.query, resolved.value, keyField(hit.route.query)))) {
256
+ throw new ProtocolError('bad-artifact', `${path}: a query route's load needs an array at "${hit.route.query.list}" and a matching proof over its resolved rows`);
257
+ }
258
+ }
175
259
  const artifact = parseArtifact({
176
260
  protocol: PROTOCOL_VERSION,
177
261
  id: path,
@@ -184,18 +268,24 @@ export function createSpool(options) {
184
268
  ...(collected.projections.length > 0 ? { projections: collected.projections } : {})
185
269
  });
186
270
  const trimmed = await trim(artifact, have);
187
- if (weighed && hit.route.scope === 'public') {
271
+ if (weighed) {
188
272
  const bytes = new TextEncoder().encode(JSON.stringify(trimmed)).length;
189
- const mine = { ...prior, ...hit.route.provision };
190
- const data = hit.route.strategy === 'data';
191
- const worth = benefit({
192
- id: path,
193
- kind: 'route',
194
- provision: { bytes, requests: 1 },
195
- avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
196
- }, mine);
197
- if (worth <= 0)
198
- return { declined: true };
273
+ // a query.dataset route's rows are bounded by the dataset's maxBytes at its seed
274
+ const budget = hit.route.query?.dataset === undefined ? hit.route.provision?.maxBytes : undefined;
275
+ if (budget !== undefined && bytes > budget)
276
+ return { declined: 'budget' };
277
+ if (hit.route.scope === 'public') {
278
+ const mine = { ...prior, ...hit.route.provision };
279
+ const data = hit.route.strategy === 'data';
280
+ const worth = benefit({
281
+ id: path,
282
+ kind: 'route',
283
+ provision: { bytes, requests: 1 },
284
+ avoided: { bytes: data ? bytes : prior.avoidedBytes, requests: 1 }
285
+ }, mine);
286
+ if (worth <= 0)
287
+ return { declined: 'benefit' };
288
+ }
199
289
  }
200
290
  const out = { artifact: trimmed };
201
291
  if (identity && hit.route.scope === 'principal') {
@@ -226,7 +316,11 @@ export function createSpool(options) {
226
316
  if ('declined' in built) {
227
317
  return new Response(null, {
228
318
  status: 204,
229
- headers: { 'cache-control': 'no-store', [HEADERS.provision]: 'declined' }
319
+ headers: {
320
+ 'cache-control': 'no-store',
321
+ [HEADERS.provision]: 'declined',
322
+ [HEADERS.provisionReason]: built.declined
323
+ }
230
324
  });
231
325
  }
232
326
  const headers = { [HEADERS.protocol]: String(PROTOCOL_VERSION) };
@@ -364,6 +458,20 @@ export function createSpool(options) {
364
458
  }
365
459
  if (url.pathname === PATHS.artifact && get)
366
460
  return await respondArtifact(request, url);
461
+ if (url.pathname === PATHS.dataset && get) {
462
+ return await respondDataset(request, url, datasets, { prior, stats: datasetStats }, async () => {
463
+ const identity = await who(request);
464
+ if (!identity || !options.identity)
465
+ return null;
466
+ return {
467
+ identity,
468
+ partition: await partitionId(options.identity.secret, identity.principal)
469
+ };
470
+ });
471
+ }
472
+ if (url.pathname === PATHS.write) {
473
+ return await writeServer.respond(request, writesOn ? await who(request) : null);
474
+ }
367
475
  if (url.pathname === PATHS.beacon) {
368
476
  return await respondBeacon(request, sink, provide(request), catalogue(request));
369
477
  }
@@ -0,0 +1,27 @@
1
+ import type { Identity } from '../protocol/route.js';
2
+ import { type Write } from '../protocol/write.js';
3
+ /** how many `(class, principal, key)` slots the server remembers an applied write id for */
4
+ export declare const DEFAULT_REMEMBER = 10000;
5
+ /** what {@link createWriteServer} takes */
6
+ export type WriteServerOptions = {
7
+ /** false answers every request with `writes-off` */
8
+ enabled: boolean;
9
+ writes: readonly Write[];
10
+ /** slots whose last applied id is remembered, oldest forgotten first; default {@link DEFAULT_REMEMBER} */
11
+ remember?: number;
12
+ };
13
+ /** the handler `createSpool` routes `PUT /.spool/write` to */
14
+ export type WriteServer = {
15
+ respond(request: Request, identity: Identity | null): Promise<Response>;
16
+ };
17
+ /**
18
+ * Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
19
+ * It checks the declaration, the principal, the key the site derives for that principal, the version
20
+ * header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
21
+ * a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
22
+ * answers 200 with the current version and applies nothing, and the same id with another body is
23
+ * refused. Memory is per server instance, so a deployment with several instances needs the site's
24
+ * `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
25
+ * reading as a conflict.
26
+ */
27
+ export declare function createWriteServer(options: WriteServerOptions): WriteServer;
@@ -0,0 +1,127 @@
1
+ import { shellHash } from '../protocol/hash.js';
2
+ import { isWriteId, isWriteKey, readIfMatch, WRITE_HEADERS, WRITE_STATUS } from '../protocol/write.js';
3
+ import { json } from './sync.js';
4
+ /** how many `(class, principal, key)` slots the server remembers an applied write id for */
5
+ export const DEFAULT_REMEMBER = 10_000;
6
+ const refuse = (code, error, headers = {}) => json({ code, error }, WRITE_STATUS[code], headers);
7
+ /** the body as text, or null when it is longer than `max` bytes; stops reading once it is */
8
+ async function readBounded(request, max) {
9
+ const declared = Number(request.headers.get('content-length') ?? Number.NaN);
10
+ if (declared > max)
11
+ return null;
12
+ if (!request.body)
13
+ return '';
14
+ const reader = request.body.getReader();
15
+ const chunks = [];
16
+ let total = 0;
17
+ for (;;) {
18
+ const { done, value } = await reader.read();
19
+ if (done)
20
+ break;
21
+ total += value.byteLength;
22
+ if (total > max) {
23
+ await reader.cancel();
24
+ return null;
25
+ }
26
+ chunks.push(value);
27
+ }
28
+ const whole = new Uint8Array(total);
29
+ let at = 0;
30
+ for (const chunk of chunks) {
31
+ whole.set(chunk, at);
32
+ at += chunk.byteLength;
33
+ }
34
+ return new TextDecoder().decode(whole);
35
+ }
36
+ /**
37
+ * Answers `PUT /.spool/write?name=<n>&key=<k>` with `If-Match: "<version>"` and a `Spool-Write-Id`.
38
+ * It checks the declaration, the principal, the key the site derives for that principal, the version
39
+ * header and the body size before `apply` runs, and runs one request per `(class, principal, key)` at
40
+ * a time. The last applied write id of each slot is kept in memory: a repeat of that id and body
41
+ * answers 200 with the current version and applies nothing, and the same id with another body is
42
+ * refused. Memory is per server instance, so a deployment with several instances needs the site's
43
+ * `apply` to store `ctx.writeId` with the value to keep a retry that lands on another instance from
44
+ * reading as a conflict.
45
+ */
46
+ export function createWriteServer(options) {
47
+ const byName = new Map(options.writes.map((write) => [write.name, write]));
48
+ const limit = options.remember ?? DEFAULT_REMEMBER;
49
+ const applied = new Map();
50
+ const locks = new Map();
51
+ const remember = (slot, entry) => {
52
+ applied.delete(slot);
53
+ applied.set(slot, entry);
54
+ while (applied.size > limit)
55
+ applied.delete(applied.keys().next().value);
56
+ };
57
+ /** runs `fn` after every earlier request for the same slot */
58
+ function exclusive(slot, fn) {
59
+ const run = (locks.get(slot) ?? Promise.resolve()).then(fn, fn);
60
+ const tail = run.catch(() => undefined);
61
+ locks.set(slot, tail);
62
+ void tail.then(() => {
63
+ if (locks.get(slot) === tail)
64
+ locks.delete(slot);
65
+ });
66
+ return run;
67
+ }
68
+ return {
69
+ async respond(request, identity) {
70
+ if (!options.enabled)
71
+ return refuse('writes-off', 'declared writes are off');
72
+ if (request.method !== 'PUT') {
73
+ return refuse('bad-method', 'use PUT', { allow: 'PUT' });
74
+ }
75
+ const url = new URL(request.url);
76
+ const write = byName.get(url.searchParams.get('name') ?? '');
77
+ if (!write)
78
+ return refuse('unknown-write', 'no such write class');
79
+ if (!identity)
80
+ return refuse('sign-in', 'sign in to write');
81
+ const key = url.searchParams.get('key');
82
+ if (!isWriteKey(key))
83
+ return refuse('bad-key', 'key is missing or malformed');
84
+ const id = request.headers.get(WRITE_HEADERS.id);
85
+ if (!isWriteId(id))
86
+ return refuse('bad-id', 'Spool-Write-Id is missing or malformed');
87
+ const header = request.headers.get('if-match');
88
+ if (header === null)
89
+ return refuse('missing-version', 'If-Match is required');
90
+ const version = readIfMatch(header);
91
+ if (version === null)
92
+ return refuse('bad-version', 'If-Match is not one version');
93
+ const ctx = { identity, writeId: id };
94
+ if ((await write.key(ctx, request)) !== key) {
95
+ return refuse('forbidden', 'that key is not yours');
96
+ }
97
+ const text = await readBounded(request, write.maxBytes);
98
+ if (text === null)
99
+ return refuse('too-large', `the body is over ${write.maxBytes} bytes`);
100
+ let value;
101
+ try {
102
+ const body = JSON.parse(text);
103
+ if (typeof body !== 'object' || body === null || !('value' in body))
104
+ throw new Error();
105
+ value = body.value;
106
+ }
107
+ catch {
108
+ return refuse('bad-body', 'the body must be JSON of the form {"value": ...}');
109
+ }
110
+ const slot = `${write.name}\u0000${identity.principal}\u0000${key}`;
111
+ return exclusive(slot, async () => {
112
+ const hash = await shellHash(text);
113
+ const last = applied.get(slot);
114
+ if (last?.id === id) {
115
+ if (last.hash !== hash)
116
+ return refuse('id-reused', 'that id was used for another body');
117
+ return json({ version: (await write.read(ctx, key)).version });
118
+ }
119
+ const result = await write.apply(ctx, key, value, version);
120
+ if ('conflict' in result)
121
+ return json({ code: 'conflict', conflict: result.conflict }, 412);
122
+ remember(slot, { id, hash });
123
+ return json({ version: result.version });
124
+ });
125
+ }
126
+ };
127
+ }
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
78
78
  * browser's. A request that fails, or a network that is down, loses the event for good.
79
79
  */
80
80
  sendSameOrigin(timeoutMs: number): Promise<void>;
81
+ /**
82
+ * Drops the held same-origin requests without sending them, for a response that names another
83
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
84
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
85
+ */
86
+ dropSameOrigin(): Promise<void>;
81
87
  }
@@ -199,6 +199,20 @@ export class BeaconQueue {
199
199
  }
200
200
  await this.#report(lost);
201
201
  }
202
+ /**
203
+ * Drops the held same-origin requests without sending them, for a response that names another
204
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
205
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
206
+ */
207
+ async dropSameOrigin() {
208
+ if (this.#flushing)
209
+ await this.#flushing.catch(() => false);
210
+ const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
211
+ if (mine.length === 0)
212
+ return;
213
+ await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
214
+ await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
215
+ }
202
216
  async #run() {
203
217
  const { storage, send, replay, want, catalogue } = this.#deps;
204
218
  await this.#expire();
@@ -0,0 +1,76 @@
1
+ import type { DatasetStore } from './dataset.js';
2
+ /** why a dataset was not held or not brought current */
3
+ export type DatasetRefusal =
4
+ /** the rows would use more than the storage the device has to give */
5
+ 'budget'
6
+ /** the proof failed again after a patch and one reseed */
7
+ | 'proof'
8
+ /** the server has no such dataset, or refused the request */
9
+ | 'unavailable'
10
+ /** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
11
+ | 'declined'
12
+ /** the server's answer was not a dataset response, or its paging did not advance */
13
+ | 'bad-response'
14
+ /** the replica was wiped while the apply ran */
15
+ | 'epoch'
16
+ /** a principal dataset's answer named another partition than the one held, or none */
17
+ | 'partition'
18
+ /** the server answered a principal dataset with a sign-in request: the session ended */
19
+ | 'ended'
20
+ /** the server forbids this principal the dataset: its held rows were dropped */
21
+ | 'revoked'
22
+ /** the storage transaction failed, for example on quota */
23
+ | 'storage';
24
+ /**
25
+ * What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
26
+ * changed them, `current` found nothing to change. `offline` means no answer came and the held
27
+ * rows, if any, are as they were; `refused` names why a dataset is not held.
28
+ */
29
+ export type DatasetOutcome = {
30
+ name: string;
31
+ requests: number;
32
+ } & ({
33
+ status: 'seeded' | 'reseeded' | 'patched' | 'current';
34
+ cursor: number;
35
+ rows: number;
36
+ } | {
37
+ status: 'offline';
38
+ } | {
39
+ status: 'refused';
40
+ reason: DatasetRefusal;
41
+ by?: string | null;
42
+ });
43
+ /** what a sync needs from the worker around it */
44
+ export type DatasetSyncDeps = {
45
+ store: DatasetStore;
46
+ fetch(url: string, init?: RequestInit): Promise<Response>;
47
+ origin: string;
48
+ /** the replica's wipe count */
49
+ epoch(): number;
50
+ /** the most bytes of rows one dataset may use; undefined when the device cannot tell */
51
+ budget(): Promise<number | undefined>;
52
+ /**
53
+ * Set for a principal dataset: reads what a response says about whose rows it carries before
54
+ * anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
55
+ */
56
+ guard?(response: Response): 'ok' | DatasetGuardVerdict;
57
+ };
58
+ /** why a guard stopped a response */
59
+ export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
60
+ /**
61
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
62
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
63
+ */
64
+ export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
65
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
66
+ export declare const DATASET_MAX_PAGES = 1000;
67
+ /**
68
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
69
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
70
+ * rows are applied to a staging area and written in one transaction after the collection proof
71
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
72
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
73
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
74
+ * an outcome.
75
+ */
76
+ export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;
@@ -0,0 +1,149 @@
1
+ import { PATHS } from '../protocol/config.js';
2
+ import { encodeDatasetRequest, parseDatasetResponse } from '../protocol/dataset.js';
3
+ import { HEADERS } from '../protocol/envelope.js';
4
+ import { ProtocolError } from '../protocol/errors.js';
5
+ /**
6
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
7
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
8
+ */
9
+ export function principalGuard(partition) {
10
+ return (response) => {
11
+ if (response.status === 401)
12
+ return 'ended';
13
+ if (response.status === 403)
14
+ return 'revoked';
15
+ if (response.ok && response.headers.get(HEADERS.partition) !== partition)
16
+ return 'partition';
17
+ return 'ok';
18
+ };
19
+ }
20
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
21
+ export const DATASET_MAX_PAGES = 1000;
22
+ class Refused extends Error {
23
+ reason;
24
+ by;
25
+ constructor(reason, by) {
26
+ super(reason);
27
+ this.reason = reason;
28
+ this.by = by;
29
+ }
30
+ }
31
+ /**
32
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
33
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
34
+ * rows are applied to a staging area and written in one transaction after the collection proof
35
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
36
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
37
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
38
+ * an outcome.
39
+ */
40
+ export async function syncDataset(deps, name) {
41
+ const { store, origin } = deps;
42
+ const epoch = deps.epoch();
43
+ let requests = 0;
44
+ const get = async (request) => {
45
+ requests += 1;
46
+ const response = await deps.fetch(`${origin}${PATHS.dataset}?${encodeDatasetRequest(request)}`, { cache: 'no-store', credentials: 'same-origin' });
47
+ const verdict = deps.guard?.(response) ?? 'ok';
48
+ if (verdict !== 'ok')
49
+ throw new Refused(verdict);
50
+ if (response.status === 409)
51
+ throw new Refused('declined', response.headers.get(HEADERS.provisionReason));
52
+ if (response.status === 400 || response.status === 404)
53
+ throw new Refused('unavailable');
54
+ if (!response.ok)
55
+ throw new TypeError(`dataset answered ${response.status}`);
56
+ const parsed = parseDatasetResponse(await response.json());
57
+ if (parsed.name !== name)
58
+ throw new Refused('bad-response');
59
+ return parsed;
60
+ };
61
+ const write = async (staging, at, status) => {
62
+ try {
63
+ if (deps.epoch() !== epoch)
64
+ throw new Refused('epoch');
65
+ if (!(await store.commit(staging, at, epoch)))
66
+ throw new Refused('epoch');
67
+ }
68
+ catch (error) {
69
+ if (error instanceof Refused)
70
+ throw error;
71
+ throw new Refused('storage');
72
+ }
73
+ return { name, requests, status, cursor: at.cursor, rows: staging.rows.size };
74
+ };
75
+ const seed = async (first, status, retry) => {
76
+ let page = first ?? (await get({ name }));
77
+ const { cursor, schema } = page;
78
+ const budget = await deps.budget();
79
+ const staging = store.stage(name);
80
+ let matches;
81
+ let after;
82
+ for (let pages = 1;; pages += 1) {
83
+ if (deps.epoch() !== epoch)
84
+ throw new Refused('epoch');
85
+ matches = await staging.add(page);
86
+ if (budget !== undefined && staging.bytes > budget)
87
+ throw new Refused('budget');
88
+ if (page.next === undefined)
89
+ break;
90
+ if (pages >= DATASET_MAX_PAGES || (after !== undefined && page.next <= after)) {
91
+ throw new Refused('bad-response');
92
+ }
93
+ after = page.next;
94
+ page = await get({ name, after });
95
+ }
96
+ if (matches === true)
97
+ return write(staging, { cursor, schema }, status);
98
+ // rows moved while the pages were read: the changes since the first page repair the copy
99
+ const patch = await get({ name, since: cursor, schema });
100
+ if (!patch.reset && (await staging.add(patch)) === true) {
101
+ return write(staging, { cursor: patch.cursor, schema }, status);
102
+ }
103
+ if (retry)
104
+ return seed(undefined, 'reseeded', false);
105
+ throw new Refused('proof');
106
+ };
107
+ try {
108
+ const held = await store.get(name);
109
+ if (!held)
110
+ return await seed(undefined, 'seeded', true);
111
+ const { cursor, schema, digest, count } = held.meta;
112
+ const response = await get({ name, since: cursor, schema });
113
+ if (response.reset)
114
+ return await seed(response, 'reseeded', true);
115
+ if (response.cursor < cursor)
116
+ return { name, requests, status: 'current', cursor, rows: count };
117
+ const proof = response.proof;
118
+ if (response.cursor === cursor) {
119
+ // nothing moved; a proof that disagrees with the held rows means the copy is damaged
120
+ if (proof && (proof.count !== count || proof.digest !== digest)) {
121
+ await store.drop(name);
122
+ return await seed(undefined, 'reseeded', false);
123
+ }
124
+ if (!(await store.touch(name, epoch)))
125
+ throw new Refused('epoch');
126
+ return { name, requests, status: 'current', cursor, rows: count };
127
+ }
128
+ const staging = store.stage(name, held);
129
+ if ((await staging.add(response)) === true) {
130
+ return await write(staging, { cursor: response.cursor, schema }, 'patched');
131
+ }
132
+ // the patched rows do not match the server's proof: nothing wrong is kept
133
+ await store.drop(name);
134
+ return await seed(undefined, 'reseeded', false);
135
+ }
136
+ catch (error) {
137
+ if (error instanceof Refused) {
138
+ // the server stopped admitting this principal to the dataset, so no row of it stays
139
+ if (error.reason === 'revoked' && deps.epoch() === epoch) {
140
+ await store.drop(name).catch(() => undefined);
141
+ }
142
+ return { name, requests, status: 'refused', reason: error.reason, by: error.by };
143
+ }
144
+ if (error instanceof ProtocolError) {
145
+ return { name, requests, status: 'refused', reason: 'bad-response' };
146
+ }
147
+ return { name, requests, status: 'offline' };
148
+ }
149
+ }