@drupflare/spool 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +499 -43
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +34 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/client/register.d.ts +23 -1
  11. package/dist/client/register.js +53 -7
  12. package/dist/media/embeds.d.ts +3 -2
  13. package/dist/media/embeds.js +16 -1
  14. package/dist/protocol/config.d.ts +16 -1
  15. package/dist/protocol/config.js +3 -1
  16. package/dist/protocol/dataset.d.ts +183 -0
  17. package/dist/protocol/dataset.js +206 -0
  18. package/dist/protocol/envelope.d.ts +3 -0
  19. package/dist/protocol/envelope.js +14 -3
  20. package/dist/protocol/errors.d.ts +1 -1
  21. package/dist/protocol/graph.d.ts +23 -4
  22. package/dist/protocol/graph.js +34 -3
  23. package/dist/protocol/hash.d.ts +2 -0
  24. package/dist/protocol/hash.js +8 -0
  25. package/dist/protocol/index.d.ts +7 -0
  26. package/dist/protocol/index.js +7 -0
  27. package/dist/protocol/inline.d.ts +14 -0
  28. package/dist/protocol/inline.js +43 -0
  29. package/dist/protocol/navigation.d.ts +81 -0
  30. package/dist/protocol/navigation.js +249 -0
  31. package/dist/protocol/normalise.d.ts +4 -3
  32. package/dist/protocol/normalise.js +6 -4
  33. package/dist/protocol/plan.d.ts +100 -0
  34. package/dist/protocol/plan.js +661 -0
  35. package/dist/protocol/provision.d.ts +52 -1
  36. package/dist/protocol/provision.js +53 -5
  37. package/dist/protocol/query.d.ts +141 -0
  38. package/dist/protocol/query.js +252 -0
  39. package/dist/protocol/route.d.ts +85 -5
  40. package/dist/protocol/route.js +91 -0
  41. package/dist/protocol/write.d.ts +146 -0
  42. package/dist/protocol/write.js +101 -0
  43. package/dist/proxy/bases.d.ts +53 -0
  44. package/dist/proxy/bases.js +76 -0
  45. package/dist/proxy/index.d.ts +1 -0
  46. package/dist/proxy/index.js +1 -0
  47. package/dist/proxy/proxy.js +8 -2
  48. package/dist/report/capacity.d.ts +59 -0
  49. package/dist/report/capacity.js +76 -0
  50. package/dist/report/channels.d.ts +60 -0
  51. package/dist/report/channels.js +65 -0
  52. package/dist/report/dataset.d.ts +48 -0
  53. package/dist/report/dataset.js +93 -0
  54. package/dist/report/derive.d.ts +285 -0
  55. package/dist/report/derive.js +676 -1
  56. package/dist/report/economics.d.ts +11 -1
  57. package/dist/report/economics.js +19 -1
  58. package/dist/report/index.d.ts +5 -0
  59. package/dist/report/index.js +5 -0
  60. package/dist/report/inputs.d.ts +97 -0
  61. package/dist/report/inputs.js +133 -0
  62. package/dist/report/institution.d.ts +284 -0
  63. package/dist/report/institution.js +225 -0
  64. package/dist/report/ledger.d.ts +28 -0
  65. package/dist/report/ledger.js +30 -0
  66. package/dist/report/loops.d.ts +73 -0
  67. package/dist/report/loops.js +101 -0
  68. package/dist/report/measurement.d.ts +1 -1
  69. package/dist/report/measurement.js +6 -1
  70. package/dist/report/model.d.ts +2 -0
  71. package/dist/report/model.js +4 -0
  72. package/dist/report/thirdparty.d.ts +57 -0
  73. package/dist/report/thirdparty.js +111 -0
  74. package/dist/report/traffic.d.ts +11 -2
  75. package/dist/report/traffic.js +13 -3
  76. package/dist/report/trim.d.ts +56 -0
  77. package/dist/report/trim.js +83 -1
  78. package/dist/server/dataset.d.ts +48 -0
  79. package/dist/server/dataset.js +308 -0
  80. package/dist/server/embedded.d.ts +3 -3
  81. package/dist/server/embedded.js +3 -3
  82. package/dist/server/index.d.ts +3 -0
  83. package/dist/server/index.js +3 -0
  84. package/dist/server/spool.d.ts +40 -0
  85. package/dist/server/spool.js +184 -23
  86. package/dist/server/write.d.ts +27 -0
  87. package/dist/server/write.js +127 -0
  88. package/dist/sw/analytics.d.ts +6 -0
  89. package/dist/sw/analytics.js +14 -0
  90. package/dist/sw/dataset-sync.d.ts +76 -0
  91. package/dist/sw/dataset-sync.js +149 -0
  92. package/dist/sw/dataset.d.ts +91 -0
  93. package/dist/sw/dataset.js +318 -0
  94. package/dist/sw/graph/index.d.ts +31 -1
  95. package/dist/sw/graph/index.js +57 -5
  96. package/dist/sw/graph/invalidate.d.ts +15 -3
  97. package/dist/sw/graph/invalidate.js +42 -4
  98. package/dist/sw/idb.js +78 -9
  99. package/dist/sw/partition.d.ts +6 -1
  100. package/dist/sw/partition.js +21 -2
  101. package/dist/sw/principal-dataset.d.ts +64 -0
  102. package/dist/sw/principal-dataset.js +254 -0
  103. package/dist/sw/render.d.ts +8 -1
  104. package/dist/sw/render.js +28 -1
  105. package/dist/sw/replica.d.ts +14 -3
  106. package/dist/sw/replica.js +46 -5
  107. package/dist/sw/router.js +2 -2
  108. package/dist/sw/runtime.d.ts +43 -2
  109. package/dist/sw/runtime.js +440 -29
  110. package/dist/sw/seal.d.ts +13 -0
  111. package/dist/sw/seal.js +13 -4
  112. package/dist/sw/static.d.ts +18 -0
  113. package/dist/sw/static.js +84 -1
  114. package/dist/sw/storage.d.ts +51 -1
  115. package/dist/sw/storage.js +33 -1
  116. package/dist/sw/write.d.ts +107 -0
  117. package/dist/sw/write.js +442 -0
  118. package/package.json +5 -2
@@ -3,12 +3,17 @@ import { MAX_WANT } from '../protocol/beacon.js';
3
3
  import { applyStamp } from '../protocol/collector.js';
4
4
  import { PATHS } from '../protocol/config.js';
5
5
  import { decodeEnvelope, decodeEnvelopeOrClear, HEADERS } from '../protocol/envelope.js';
6
+ import { shellHash } from '../protocol/hash.js';
7
+ import { readInline } from '../protocol/inline.js';
6
8
  import { manifestRoutes, parseManifest } from '../protocol/manifest.js';
9
+ import { PROOF_MISMATCH } from '../protocol/query.js';
7
10
  import { matchRoute } from '../protocol/route.js';
8
11
  import { encodeHeld, parseSyncResponse } from '../protocol/sync.js';
9
12
  import { engineOf, FROM_EXTERNAL, loadsToBeacon, REASON_TIER, viewToBeacon } from '../protocol/view.js';
13
+ import { ifMatch, WRITE_HEADERS } from '../protocol/write.js';
10
14
  import { BeaconQueue, LOGOUT_SEND_MS } from './analytics.js';
11
15
  import { collectorAnswer } from './collect.js';
16
+ import { principalGuard, syncDataset } from './dataset-sync.js';
12
17
  import { verdict } from './freshness.js';
13
18
  import { policyHeaders } from './policy.js';
14
19
  import { Ledger, plan, priorFrom } from './provision.js';
@@ -18,8 +23,9 @@ import { classify } from './router.js';
18
23
  import { installRoutes } from './routing.js';
19
24
  import { Scheduler } from './scheduler.js';
20
25
  import { differs, modeFor, runsShadow } from './shadow.js';
21
- import { matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
26
+ import { branch, matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
22
27
  import { THIRD_CACHE, THIRD_HEADERS, versionClass } from './thirdparty.js';
28
+ import { WriteQueue } from './write.js';
23
29
  /** the strategies a catalogue entry may have: data for a template or renderer the device holds */
24
30
  const ENTRY_STRATEGIES = ['template', 'projection', 'data'];
25
31
  const ASSET_CACHE = 'spool-assets-';
@@ -48,7 +54,7 @@ export function createRuntime(deps) {
48
54
  const ledger = new Ledger();
49
55
  const background = new Set();
50
56
  const provisioning = new Map();
51
- const declined = new Set();
57
+ const declined = new Map();
52
58
  const wants = new Set();
53
59
  const asked = new Set();
54
60
  let currentPath = null;
@@ -90,9 +96,11 @@ export function createRuntime(deps) {
90
96
  background.add(settled);
91
97
  void settled.then(() => background.delete(settled));
92
98
  };
93
- const settleBackground = async () => {
94
- while (background.size > 0)
95
- await Promise.allSettled([...background]);
99
+ // only work begun after `before` is waited for, not an earlier request's held body
100
+ const settleBackground = async (before) => {
101
+ const fresh = () => [...background].filter((p) => !before.has(p));
102
+ while (fresh().length > 0)
103
+ await Promise.allSettled(fresh());
96
104
  };
97
105
  /** runs `fn` after every earlier partition change, so two responses never swap the store at once */
98
106
  const serial = (fn) => {
@@ -254,23 +262,128 @@ export function createRuntime(deps) {
254
262
  replica,
255
263
  scheduler: makeScheduler(replica),
256
264
  deferred: new Set(),
257
- loaded: Promise.resolve()
265
+ loaded: Promise.resolve(),
266
+ dead: false
258
267
  };
259
268
  const scheduler = publicSide.scheduler;
269
+ // #region datasets
270
+ const datasetRuns = new Map();
271
+ const datasetCounts = {};
272
+ const datasetRefused = new Map();
273
+ const datasetsDeferred = new Set();
274
+ /** principal datasets whose held rows are past maxStaleMs; the next network moment syncs them */
275
+ const principalDeferred = new Set();
276
+ let persistAsked = false;
277
+ let persisted = null;
278
+ const estimate = async () => {
279
+ if (deps.estimate)
280
+ return deps.estimate();
281
+ return ((typeof navigator === 'undefined'
282
+ ? undefined
283
+ : await navigator.storage?.estimate?.()) ?? {});
284
+ };
285
+ /** the bytes of rows one dataset may take: the headroom times the declared share; unknown when the browser will not say */
286
+ const datasetBudget = async () => {
287
+ try {
288
+ const { quota, usage } = await estimate();
289
+ if (quota === undefined || usage === undefined)
290
+ return undefined;
291
+ return Math.max(0, (quota - usage) * (config.datasetShare ?? 1));
292
+ }
293
+ catch {
294
+ return undefined;
295
+ }
296
+ };
297
+ const tally = (outcome) => {
298
+ const key = outcome.status === 'refused'
299
+ ? `${outcome.name}:refused:${outcome.reason}`
300
+ : `${outcome.name}:${outcome.status}`;
301
+ datasetCounts[key] = (datasetCounts[key] ?? 0) + 1;
302
+ if (outcome.status === 'refused') {
303
+ datasetRefused.set(outcome.name, outcome.reason);
304
+ if (outcome.reason === 'declined')
305
+ declined.set(outcome.name, outcome.by ?? null);
306
+ }
307
+ else if (outcome.status !== 'offline')
308
+ datasetRefused.delete(outcome.name);
309
+ const held = outcome.status !== 'offline' && outcome.status !== 'refused';
310
+ if (held && !persistAsked) {
311
+ persistAsked = true;
312
+ deps.broadcast?.({ type: 'spool:persist' });
313
+ }
314
+ };
315
+ /** the principal datasets the server declared, with how long each may answer after a confirmed sync */
316
+ const principalLimits = () => new Map((config.principalDatasets ?? []).map((d) => [d.name, d.maxStaleMs]));
317
+ /**
318
+ * A principal dataset's request, read for who it answers. A late answer for a partition the
319
+ * worker has left is returned untouched, since the sync discards it; a session that ended wipes
320
+ * the store, and an answer for another partition swaps it, exactly as a navigation does.
321
+ */
322
+ async function principalFetch(side, url, init) {
323
+ const response = await deps.fetch(url, init);
324
+ if (side.dead || principal !== side)
325
+ return response;
326
+ if (response.status === 401)
327
+ await wipePartition('session-ended');
328
+ else if (response.ok) {
329
+ if (response.headers.has(HEADERS.partition)) {
330
+ await early(decodeEnvelopeOrClear(response.headers), true);
331
+ }
332
+ else
333
+ await wipePartition('session-ended');
334
+ }
335
+ return response;
336
+ }
337
+ /** brings one dataset current; a second caller joins the run in flight, for the same side */
338
+ function syncDatasetOnce(name, side = publicSide) {
339
+ const key = `${side.id ?? ''}\u0000${name}`;
340
+ const running = datasetRuns.get(key);
341
+ if (running)
342
+ return running;
343
+ const partition = side.id;
344
+ const run = syncDataset({
345
+ store: side.replica.datasets,
346
+ fetch: partition === null
347
+ ? (url, init) => deps.fetch(url, init)
348
+ : (url, init) => principalFetch(side, url, init),
349
+ origin,
350
+ epoch: () => (side.dead ? -1 : side.replica.epoch),
351
+ budget: datasetBudget,
352
+ ...(partition === null ? {} : { guard: principalGuard(partition) })
353
+ }, name)
354
+ .then((outcome) => {
355
+ tally(outcome);
356
+ return outcome;
357
+ })
358
+ .finally(() => datasetRuns.delete(key));
359
+ datasetRuns.set(key, run);
360
+ return run;
361
+ }
362
+ const datasetCurrent = (outcome) => outcome.status !== 'offline' && outcome.status !== 'refused';
363
+ /** seeds a dataset in the background unless this run already learned it will not be held */
364
+ function seedDataset(name, side = publicSide) {
365
+ if (side !== publicSide && !principalLimits().has(name))
366
+ return;
367
+ const key = `${side.id ?? ''}\u0000${name}`;
368
+ if (!datasetRefused.has(name) && !datasetRuns.has(key))
369
+ track(syncDatasetOnce(name, side));
370
+ }
371
+ // #endregion
260
372
  // #region partitions
261
373
  const retentionOf = (id) => matchRoute(config.routes, id)?.route.retention ?? 'session';
262
374
  const sharedRetention = () => config.routes.some((r) => r.scope === 'principal' && r.retention === 'persistent')
263
375
  ? 'persistent'
264
376
  : 'session';
265
377
  function makeSide(id) {
266
- const sealed = host.open(id, retentionOf, sharedRetention());
378
+ const sealed = host.open(id, retentionOf, sharedRetention(), config.principalDatasets !== undefined);
267
379
  const target = new Replica(sealed, config.version, now);
268
380
  return {
269
381
  id,
270
382
  replica: target,
271
383
  scheduler: makeScheduler(target),
272
384
  deferred: new Set(),
273
- loaded: target.load()
385
+ loaded: target.load(),
386
+ dead: false
274
387
  };
275
388
  }
276
389
  async function loadPartition() {
@@ -287,12 +400,22 @@ export function createRuntime(deps) {
287
400
  await side.loaded;
288
401
  return side;
289
402
  }
403
+ /** marks a side left for good, and forgets what the next principal must not inherit */
404
+ function retire(side) {
405
+ side.dead = true;
406
+ principalDeferred.clear();
407
+ for (const name of principalLimits().keys())
408
+ datasetRefused.delete(name);
409
+ }
290
410
  /** shuts the principal store, shreds its keys and deletes it; true when there was one */
291
411
  function wipePartition(reason) {
292
412
  return serial(async () => {
293
413
  await ready();
294
414
  const old = principal;
295
415
  principal = null;
416
+ old?.writes?.kill();
417
+ if (old)
418
+ retire(old);
296
419
  if (!old || !host)
297
420
  return false;
298
421
  await host.setCurrent(null);
@@ -309,6 +432,9 @@ export function createRuntime(deps) {
309
432
  if (old?.id === id)
310
433
  return;
311
434
  principal = makeSide(id);
435
+ old?.writes?.kill();
436
+ if (old)
437
+ retire(old);
312
438
  await host.setCurrent(id);
313
439
  if (old) {
314
440
  await host.destroy(old.id);
@@ -329,6 +455,7 @@ export function createRuntime(deps) {
329
455
  await ready();
330
456
  const clear = envelope.clear ?? [];
331
457
  if (clear.includes('*') || (principal && clear.includes(principal.id))) {
458
+ await analytics.dropSameOrigin();
332
459
  await wipePartition('clear');
333
460
  return;
334
461
  }
@@ -336,11 +463,66 @@ export function createRuntime(deps) {
336
463
  if (signedOut)
337
464
  return;
338
465
  if (envelope.partition !== undefined) {
339
- if (principal?.id !== envelope.partition)
466
+ if (principal?.id !== envelope.partition) {
467
+ await analytics.dropSameOrigin();
340
468
  await switchTo(envelope.partition);
469
+ }
341
470
  }
342
- else if (navigation && principal)
471
+ else if (navigation && principal) {
472
+ await analytics.dropSameOrigin();
343
473
  await wipePartition('session-ended');
474
+ }
475
+ }
476
+ // #endregion
477
+ // #region writes
478
+ // a worker in shadow mode only observes, so it queues nothing
479
+ const writesOn = () => config.writes !== undefined && config.mode !== 'shadow';
480
+ const writeRefusal = (code, message) => ({
481
+ ok: false,
482
+ code,
483
+ message
484
+ });
485
+ async function sendWrite(record) {
486
+ const query = `name=${encodeURIComponent(record.name)}&key=${encodeURIComponent(record.key)}`;
487
+ const response = await deps.fetch(`${origin}${PATHS.write}?${query}`, {
488
+ method: 'PUT',
489
+ headers: {
490
+ 'content-type': 'application/json',
491
+ 'if-match': ifMatch(record.base),
492
+ [WRITE_HEADERS.id]: record.id
493
+ },
494
+ body: JSON.stringify({ value: record.value }),
495
+ cache: 'no-store',
496
+ credentials: 'same-origin'
497
+ });
498
+ const body = await response.json().catch(() => undefined);
499
+ return { status: response.status, body };
500
+ }
501
+ function queueOf(side) {
502
+ if (!writesOn() || !host?.openWrites)
503
+ return null;
504
+ side.writes ??= new WriteQueue({
505
+ store: host.openWrites(side.id),
506
+ classes: () => config.writes ?? [],
507
+ now,
508
+ send: sendWrite,
509
+ onFlush: (flush) => deps.broadcast?.({ type: 'spool:written', ...flush })
510
+ });
511
+ return side.writes;
512
+ }
513
+ async function pageQueue() {
514
+ if (!writesOn())
515
+ return writeRefusal('writes-off', 'declared writes are off');
516
+ const side = await principalSide();
517
+ if (!side)
518
+ return writeRefusal('no-partition', 'no signed-in principal is held');
519
+ return queueOf(side) ?? writeRefusal('unavailable', 'this worker has no partition storage');
520
+ }
521
+ /** sends the signed-in partition's queue at a network moment that is happening anyway */
522
+ async function flushWrites() {
523
+ const queue = principal ? queueOf(principal) : null;
524
+ if (queue)
525
+ await queue.flush();
344
526
  }
345
527
  // #endregion
346
528
  async function provision(path, route, partition) {
@@ -351,7 +533,7 @@ export function createRuntime(deps) {
351
533
  const query = Object.keys(have).length > 0 ? `&have=${encodeURIComponent(encodeHeld(have))}` : '';
352
534
  const response = await deps.fetch(`${origin}${PATHS.artifact}?path=${encodeURIComponent(path)}${query}`, { cache: 'no-store', credentials: 'same-origin' });
353
535
  if (response.status === 204 && response.headers.get(HEADERS.provision) === 'declined') {
354
- declined.add(path);
536
+ declined.set(path, response.headers.get(HEADERS.provisionReason));
355
537
  return;
356
538
  }
357
539
  if (!response.ok) {
@@ -428,7 +610,18 @@ export function createRuntime(deps) {
428
610
  await drainDeferred(publicSide);
429
611
  if (principal)
430
612
  await drainDeferred(principal);
613
+ const names = [...datasetsDeferred];
614
+ datasetsDeferred.clear();
615
+ for (const name of names)
616
+ await syncDatasetOnce(name);
617
+ const owed = [...principalDeferred];
618
+ principalDeferred.clear();
619
+ const side = principal;
620
+ if (side)
621
+ for (const name of owed)
622
+ await syncDatasetOnce(name, side);
431
623
  await analytics.flushAtMoment();
624
+ await flushWrites();
432
625
  await revalidateThirdParty();
433
626
  }
434
627
  }
@@ -483,19 +676,22 @@ export function createRuntime(deps) {
483
676
  await early(envelope, navigation, after?.signedOut);
484
677
  // a cacheable GET may be answered by the HTTP cache with no radio wake, so only these count
485
678
  const moment = navigation || request.method !== 'GET';
486
- const copy = after?.clone ? response.clone() : null;
679
+ const clone = after?.clone;
680
+ const kept = typeof clone === 'function' ? clone(response, envelope) : clone;
681
+ const { visitor, copy } = kept ? branch(response) : { visitor: response, copy: null };
487
682
  track((async () => {
488
683
  try {
489
- await after?.run(copy);
684
+ await after?.run(copy, envelope);
490
685
  }
491
686
  catch {
492
687
  // a view that cannot be recorded must not stop the response being observed
493
688
  }
494
689
  await observe(response.headers, envelope, sent, navigation, moment, after?.provision ?? navigation, () => keepPolicy(request, response, envelope, navigation, after));
495
690
  })());
496
- return response;
691
+ return visitor;
497
692
  }
498
- async function decide(path, route, dry) {
693
+ /** `search` is the request's query string; the stored artifact is always the path's own */
694
+ async function decide(path, route, dry, search = '') {
499
695
  let side = publicSide;
500
696
  if (route.scope === 'principal') {
501
697
  if (!host)
@@ -522,9 +718,47 @@ export function createRuntime(deps) {
522
718
  await rep.drop([path]);
523
719
  return { reason: 'invalidated', staleSec: age };
524
720
  }
721
+ // a page answers from the rows the device holds, so their age bounds the answer too
722
+ const queried = route.query?.dataset;
723
+ const joined = (route.query?.plan?.join ?? []).map((j) => j.dataset);
724
+ const dsNames = [
725
+ ...new Set([
726
+ ...(queried ? [queried] : []),
727
+ ...joined,
728
+ ...(await rep.datasetsOf(stored.artifact))
729
+ ])
730
+ ];
731
+ // a principal side answers only from the principal datasets the server declared
732
+ const limits = side === publicSide ? undefined : principalLimits();
733
+ if (limits && dsNames.some((name) => !limits.has(name))) {
734
+ return { reason: 'refused', staleSec: staleSec() };
735
+ }
736
+ const heldAt = async () => Promise.all(dsNames.map(async (name) => (await rep.datasets.get(name))?.meta.appliedAt));
737
+ let appliedAt = await heldAt();
738
+ const missing = dsNames.filter((_, i) => appliedAt[i] === undefined);
739
+ if (missing.length > 0) {
740
+ if (!dry)
741
+ for (const name of missing)
742
+ seedDataset(name, side);
743
+ return { reason: 'refused', staleSec: staleSec() };
744
+ }
745
+ if (limits) {
746
+ // held principal rows answer only while the last confirmed sync is inside maxStaleMs
747
+ const stale = dsNames.filter((name, i) => {
748
+ const age = now() - appliedAt[i];
749
+ return !(age >= 0 && age < limits.get(name));
750
+ });
751
+ if (stale.length > 0) {
752
+ if (!dry)
753
+ for (const name of stale)
754
+ principalDeferred.add(name);
755
+ return { reason: 'refused', staleSec: staleSec() };
756
+ }
757
+ }
758
+ const known = (at) => at.filter((v) => v !== undefined);
525
759
  const fromCatalogue = stored.catalogue === true;
526
- const action = verdict(route.freshness, rep.validatedAt(stored), now());
527
- const age = staleSec();
760
+ const action = verdict(route.freshness, Math.min(rep.validatedAt(stored), ...known(appliedAt)), now());
761
+ const age = Math.max(staleSec(), ...known(appliedAt).map((at) => Math.floor((now() - at) / 1000)));
528
762
  const base = rep.needFor(stored.artifact, route.sync === 'deferred' || route.scope === 'principal');
529
763
  // a principal sync also names the catalogue entries no page has opened, so a revoked one is dropped
530
764
  const need = (route.scope === 'principal' && base ? await rep.sweep(base) : base) ?? undefined;
@@ -533,8 +767,14 @@ export function createRuntime(deps) {
533
767
  reason = 'hard-sync';
534
768
  if (!dry) {
535
769
  const synced = await sched.sync(need, age);
536
- if (!synced && route.offline === 'refuse')
770
+ const dsSynced = (await Promise.all(dsNames.map((name) => syncDatasetOnce(name, side)))).every(datasetCurrent);
771
+ if (!(synced && dsSynced) && route.offline === 'refuse')
537
772
  return { reason: 'hard-refused', staleSec: age };
773
+ if (dsNames.length > 0) {
774
+ appliedAt = await heldAt();
775
+ if (appliedAt.includes(undefined))
776
+ return { reason: 'refused', staleSec: age };
777
+ }
538
778
  if (synced) {
539
779
  if (need)
540
780
  await rep.validate(path);
@@ -543,7 +783,7 @@ export function createRuntime(deps) {
543
783
  return { reason: 'stale-after-sync', staleSec: age };
544
784
  }
545
785
  }
546
- else
786
+ if (!(synced && dsSynced))
547
787
  reason = 'offline-stale';
548
788
  }
549
789
  }
@@ -552,6 +792,11 @@ export function createRuntime(deps) {
552
792
  reason = 'soft-deferred';
553
793
  if (!dry && need && sched.defer(need, age))
554
794
  side.deferred.add(path);
795
+ if (!dry) {
796
+ for (const name of dsNames) {
797
+ (side === publicSide ? datasetsDeferred : principalDeferred).add(name);
798
+ }
799
+ }
555
800
  }
556
801
  else {
557
802
  reason = 'soft-sync';
@@ -560,6 +805,8 @@ export function createRuntime(deps) {
560
805
  if (ok && need)
561
806
  await rep.validate(path);
562
807
  }));
808
+ for (const name of dsNames)
809
+ track(syncDatasetOnce(name, side));
563
810
  }
564
811
  }
565
812
  }
@@ -577,13 +824,33 @@ export function createRuntime(deps) {
577
824
  return { reason: 'refused', staleSec: age };
578
825
  const plain = await rep.materialize(stored.artifact);
579
826
  if (!plain.ok) {
827
+ // rows of a dataset not held yet: the page is sound, so keep it and seed the dataset
828
+ if (plain.datasets) {
829
+ if (!dry)
830
+ for (const name of plain.datasets)
831
+ seedDataset(name, side);
832
+ return { reason: 'refused', staleSec: age };
833
+ }
580
834
  // a reference with nothing behind it: drop the page so the next network view fetches it whole
581
835
  await rep.drop([path]);
582
836
  return { reason: 'refused', staleSec: age };
583
837
  }
584
- const rendered = await renderLocal(config, route, plain.artifact, renderers);
585
- if (!rendered.ok)
838
+ const planned = route.query?.plan !== undefined;
839
+ const rows = queried === undefined || planned ? undefined : await rep.datasets.rows(queried);
840
+ if (queried !== undefined && !planned && !rows)
841
+ return { reason: 'refused', staleSec: age };
842
+ const tables = queried !== undefined && planned
843
+ ? await rep.datasets.tables(queried, joined)
844
+ : undefined;
845
+ if (queried !== undefined && planned && !tables)
846
+ return { reason: 'refused', staleSec: age };
847
+ const rendered = await renderLocal(config, route, plain.artifact, renderers, search, rows, tables);
848
+ if (!rendered.ok) {
849
+ // a collection that fails its proof is dropped, so the next network view fetches it whole
850
+ if (rendered.reason === PROOF_MISMATCH)
851
+ await rep.drop([path]);
586
852
  return { reason: 'refused', staleSec: age };
853
+ }
587
854
  if (fromCatalogue) {
588
855
  await rep.open(path);
589
856
  if (REASON_TIER[reason] === 'local')
@@ -612,7 +879,7 @@ export function createRuntime(deps) {
612
879
  const gapMs = lastViewAt === null ? null : at - lastViewAt;
613
880
  lastViewAt = at;
614
881
  currentPath = route.catalogue ? path : null;
615
- const decision = await decide(path, route, shadow);
882
+ const decision = await decide(path, route, shadow, new URL(request.url).search);
616
883
  const from = referrerRoute(request.referrer);
617
884
  const id = crypto.randomUUID();
618
885
  latestView.delete(path);
@@ -656,11 +923,13 @@ export function createRuntime(deps) {
656
923
  if (decision.html === undefined) {
657
924
  const keeps = route.strategy === 'static';
658
925
  const response = await passthrough(request, true, {
659
- clone: keeps,
926
+ clone: keeps || ((res, envelope) => derivable(path, route, res, envelope)),
660
927
  route,
661
- run: async (copy) => {
928
+ run: async (copy, envelope) => {
662
929
  if (keeps)
663
930
  await capture(path, route, copy);
931
+ else
932
+ await derive(path, route, copy, envelope);
664
933
  await record(null);
665
934
  }
666
935
  });
@@ -688,7 +957,7 @@ export function createRuntime(deps) {
688
957
  }
689
958
  /** answers a GET for a `data` route from the stored value; the page makes no request */
690
959
  async function dataFile(request, path, route) {
691
- const decision = await decide(path, route, false);
960
+ const decision = await decide(path, route, false, new URL(request.url).search);
692
961
  if (decision.html !== undefined) {
693
962
  ledger.serve(path, decision.reason === 'catalogue');
694
963
  const headers = new Headers(decision.headers);
@@ -716,8 +985,11 @@ export function createRuntime(deps) {
716
985
  * does not describe; the next view asks the network again.
717
986
  */
718
987
  async function capture(path, asked, copy) {
719
- if (!copy || copy.status !== 200 || copy.redirected)
988
+ if (!copy || copy.status !== 200 || copy.redirected) {
989
+ // an unread tee branch holds every chunk the visitor reads
990
+ void copy?.body?.cancel().catch(() => undefined);
720
991
  return;
992
+ }
721
993
  // a sync in the same view may have replaced the manifest, so the route is read again
722
994
  const route = config.manifest ? matchRoute(config.routes, pathOf(path))?.route : asked;
723
995
  if (route?.strategy !== 'static')
@@ -771,6 +1043,88 @@ export function createRuntime(deps) {
771
1043
  ledger.spend(path, { bytes: bytes.byteLength, requests: 0 });
772
1044
  }
773
1045
  }
1046
+ /** whether a cold navigation's body can stand in for the artifact its response advertises */
1047
+ function derivable(path, route, response, envelope) {
1048
+ const advertised = envelope?.artifact;
1049
+ return ((route.strategy === 'assembly' || route.strategy === 'template') &&
1050
+ route.scope === 'public' &&
1051
+ envelope?.protocol === PROTOCOL_VERSION &&
1052
+ advertised?.hash !== undefined &&
1053
+ advertised.id === path &&
1054
+ advertised.scope === 'public' &&
1055
+ modeFor(config, route) !== 'shadow' &&
1056
+ !declined.has(path) &&
1057
+ response.status === 200 &&
1058
+ !response.redirected &&
1059
+ /text\/html/i.test(response.headers.get('content-type') ?? ''));
1060
+ }
1061
+ /**
1062
+ * The artifact a navigation body carries, or null. An `assembly` page is the artifact's shell
1063
+ * once the client tag is out, and a `template` page holds the artifact's JSON in its inline
1064
+ * element. Either must hash to the value `Spool-Artifact` named.
1065
+ */
1066
+ async function carried(path, route, text, advertised) {
1067
+ if (route.strategy === 'template') {
1068
+ const json = readInline(text);
1069
+ if (json === null || (await shellHash(json)) !== advertised.hash)
1070
+ return null;
1071
+ const artifact = parseArtifact(JSON.parse(json));
1072
+ if (artifact.id !== path || artifact.scope !== 'public')
1073
+ return null;
1074
+ if (artifact.strategy !== 'template')
1075
+ return null;
1076
+ return { artifact, bytes: encoder.encode(json).length };
1077
+ }
1078
+ const at = text.indexOf(config.clientTag);
1079
+ const shells = at < 0 ? [text] : [text.slice(0, at) + text.slice(at + config.clientTag.length), text];
1080
+ for (const shell of shells) {
1081
+ if ((await shellHash(shell)) !== advertised.hash)
1082
+ continue;
1083
+ return {
1084
+ artifact: {
1085
+ protocol: PROTOCOL_VERSION,
1086
+ id: path,
1087
+ scope: 'public',
1088
+ strategy: 'assembly',
1089
+ deps: [...new Set(advertised.deps)],
1090
+ vector: advertised.version,
1091
+ body: { shell }
1092
+ },
1093
+ bytes: 0
1094
+ };
1095
+ }
1096
+ return null;
1097
+ }
1098
+ /**
1099
+ * Keeps the artifact a navigation's own body carries, when it matches the hash `Spool-Artifact`
1100
+ * named. Anything else leaves nothing stored, and the advertised artifact is then fetched as
1101
+ * before.
1102
+ */
1103
+ async function derive(path, route, copy, envelope) {
1104
+ const advertised = envelope?.artifact;
1105
+ if (!copy || !advertised?.hash || !derivable(path, route, copy, envelope))
1106
+ return;
1107
+ // a wipe while the body is read means the copy predates it, so it is not kept
1108
+ const epoch = replica.epoch;
1109
+ try {
1110
+ const bytes = await readCapped(copy, MAX_STATIC_BYTES);
1111
+ if (!bytes)
1112
+ return;
1113
+ const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
1114
+ const found = await carried(path, route, text, {
1115
+ ...advertised,
1116
+ hash: advertised.hash
1117
+ });
1118
+ if (!found || (await replica.get(path)) || replica.epoch !== epoch)
1119
+ return;
1120
+ if (await replica.put(found.artifact, route.persist)) {
1121
+ ledger.spend(path, { bytes: found.bytes, requests: 0 });
1122
+ }
1123
+ }
1124
+ catch {
1125
+ // an unreadable body leaves the network fetch as the only way to the artifact
1126
+ }
1127
+ }
774
1128
  async function staticFile(request, path, route) {
775
1129
  const decision = await decide(path, route, false);
776
1130
  if (decision.response) {
@@ -966,8 +1320,11 @@ export function createRuntime(deps) {
966
1320
  request.mode !== 'navigate') {
967
1321
  return undefined;
968
1322
  }
1323
+ const before = new Set(background);
969
1324
  const work = kind === 'asset' && !manifestPending() ? asset(request) : run(request);
970
- return work.catch(() => deps.fetch(request)).finally(() => wait(settleBackground()));
1325
+ return work
1326
+ .catch(() => deps.fetch(request))
1327
+ .finally(() => wait(settleBackground(before)));
971
1328
  },
972
1329
  async install(event) {
973
1330
  if (assetMode === 'static-routing')
@@ -1041,7 +1398,8 @@ export function createRuntime(deps) {
1041
1398
  partition: null,
1042
1399
  artifacts: [],
1043
1400
  records: [],
1044
- projections: []
1401
+ projections: [],
1402
+ datasets: []
1045
1403
  };
1046
1404
  }
1047
1405
  await side.replica.held();
@@ -1050,12 +1408,65 @@ export function createRuntime(deps) {
1050
1408
  partition: side.id,
1051
1409
  artifacts: await side.replica.ids(),
1052
1410
  records: side.replica.graph.records.ids(),
1053
- projections: side.replica.graph.projections.ids()
1411
+ projections: side.replica.graph.projections.ids(),
1412
+ datasets: await side.replica.datasets.list()
1054
1413
  };
1055
1414
  }
1056
1415
  if (message.type === 'spool:assets') {
1057
1416
  return { type: 'spool:assets', mode: assetMode, routed };
1058
1417
  }
1418
+ if (message.type === 'spool:datasets') {
1419
+ await ready();
1420
+ return {
1421
+ type: 'spool:datasets',
1422
+ datasets: await replica.datasets.list(),
1423
+ outcomes: { ...datasetCounts },
1424
+ refused: Object.fromEntries(datasetRefused),
1425
+ declined: Object.fromEntries(declined),
1426
+ evicted: replica.datasets.evicted,
1427
+ persisted
1428
+ };
1429
+ }
1430
+ if (message.type === 'spool:persisted') {
1431
+ persisted = message.granted === true;
1432
+ return undefined;
1433
+ }
1434
+ if (message.type === 'spool:write') {
1435
+ const { name, key, value, version } = message;
1436
+ const queue = await pageQueue();
1437
+ return {
1438
+ type: 'spool:write',
1439
+ result: queue instanceof WriteQueue
1440
+ ? await queue.write(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', value, version)
1441
+ : queue
1442
+ };
1443
+ }
1444
+ if (message.type === 'spool:flush') {
1445
+ const queue = await pageQueue();
1446
+ return {
1447
+ type: 'spool:flush',
1448
+ result: queue instanceof WriteQueue ? { ok: true, ...(await queue.flush()) } : queue
1449
+ };
1450
+ }
1451
+ if (message.type === 'spool:writes') {
1452
+ const queue = await pageQueue();
1453
+ return {
1454
+ type: 'spool:writes',
1455
+ result: queue instanceof WriteQueue
1456
+ ? { ok: true, writes: await queue.list() }
1457
+ : queue
1458
+ };
1459
+ }
1460
+ if (message.type === 'spool:resolve') {
1461
+ const { name, key, use } = message;
1462
+ const queue = await pageQueue();
1463
+ return {
1464
+ type: 'spool:resolve',
1465
+ result: queue instanceof WriteQueue
1466
+ ? await queue.resolve(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', use)
1467
+ : queue
1468
+ };
1469
+ }
1059
1470
  if (message.type === 'spool:held') {
1060
1471
  const cache = await cacheNamed(THIRD_CACHE);
1061
1472
  return {