@drupflare/spool 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +263 -41
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/entry.js +33 -1
  6. package/dist/client/register.d.ts +23 -1
  7. package/dist/client/register.js +53 -7
  8. package/dist/protocol/config.d.ts +16 -1
  9. package/dist/protocol/config.js +3 -1
  10. package/dist/protocol/dataset.d.ts +183 -0
  11. package/dist/protocol/dataset.js +206 -0
  12. package/dist/protocol/envelope.d.ts +1 -0
  13. package/dist/protocol/envelope.js +2 -1
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/graph.d.ts +23 -4
  16. package/dist/protocol/graph.js +34 -3
  17. package/dist/protocol/index.d.ts +3 -0
  18. package/dist/protocol/index.js +3 -0
  19. package/dist/protocol/plan.d.ts +100 -0
  20. package/dist/protocol/plan.js +661 -0
  21. package/dist/protocol/provision.d.ts +46 -0
  22. package/dist/protocol/provision.js +38 -0
  23. package/dist/protocol/query.d.ts +66 -5
  24. package/dist/protocol/query.js +103 -4
  25. package/dist/protocol/route.d.ts +6 -2
  26. package/dist/protocol/route.js +31 -3
  27. package/dist/protocol/write.d.ts +146 -0
  28. package/dist/protocol/write.js +101 -0
  29. package/dist/report/dataset.d.ts +48 -0
  30. package/dist/report/dataset.js +93 -0
  31. package/dist/report/derive.d.ts +285 -0
  32. package/dist/report/derive.js +676 -1
  33. package/dist/report/economics.d.ts +11 -1
  34. package/dist/report/economics.js +19 -1
  35. package/dist/report/index.d.ts +2 -0
  36. package/dist/report/index.js +2 -0
  37. package/dist/report/inputs.d.ts +97 -0
  38. package/dist/report/inputs.js +133 -0
  39. package/dist/report/institution.d.ts +188 -1
  40. package/dist/report/institution.js +146 -0
  41. package/dist/report/loops.d.ts +73 -0
  42. package/dist/report/loops.js +101 -0
  43. package/dist/report/measurement.d.ts +1 -1
  44. package/dist/report/measurement.js +3 -1
  45. package/dist/report/model.d.ts +2 -0
  46. package/dist/report/model.js +4 -0
  47. package/dist/report/thirdparty.d.ts +57 -0
  48. package/dist/report/thirdparty.js +111 -0
  49. package/dist/report/trim.d.ts +56 -0
  50. package/dist/report/trim.js +83 -1
  51. package/dist/server/dataset.d.ts +48 -0
  52. package/dist/server/dataset.js +308 -0
  53. package/dist/server/embedded.d.ts +3 -3
  54. package/dist/server/embedded.js +3 -3
  55. package/dist/server/index.d.ts +2 -0
  56. package/dist/server/index.js +2 -0
  57. package/dist/server/spool.d.ts +20 -0
  58. package/dist/server/spool.js +126 -18
  59. package/dist/server/write.d.ts +27 -0
  60. package/dist/server/write.js +127 -0
  61. package/dist/sw/analytics.d.ts +6 -0
  62. package/dist/sw/analytics.js +14 -0
  63. package/dist/sw/dataset-sync.d.ts +76 -0
  64. package/dist/sw/dataset-sync.js +149 -0
  65. package/dist/sw/dataset.d.ts +91 -0
  66. package/dist/sw/dataset.js +318 -0
  67. package/dist/sw/graph/index.d.ts +31 -1
  68. package/dist/sw/graph/index.js +57 -5
  69. package/dist/sw/graph/invalidate.d.ts +15 -3
  70. package/dist/sw/graph/invalidate.js +42 -4
  71. package/dist/sw/idb.js +78 -9
  72. package/dist/sw/partition.d.ts +6 -1
  73. package/dist/sw/partition.js +21 -2
  74. package/dist/sw/principal-dataset.d.ts +64 -0
  75. package/dist/sw/principal-dataset.js +254 -0
  76. package/dist/sw/render.d.ts +7 -2
  77. package/dist/sw/render.js +19 -5
  78. package/dist/sw/replica.d.ts +12 -3
  79. package/dist/sw/replica.js +40 -5
  80. package/dist/sw/runtime.d.ts +43 -2
  81. package/dist/sw/runtime.js +340 -24
  82. package/dist/sw/seal.d.ts +13 -0
  83. package/dist/sw/seal.js +13 -4
  84. package/dist/sw/static.d.ts +18 -0
  85. package/dist/sw/static.js +84 -1
  86. package/dist/sw/storage.d.ts +51 -1
  87. package/dist/sw/storage.js +33 -1
  88. package/dist/sw/write.d.ts +107 -0
  89. package/dist/sw/write.js +442 -0
  90. package/package.json +3 -2
@@ -10,8 +10,10 @@ import { PROOF_MISMATCH } from '../protocol/query.js';
10
10
  import { matchRoute } from '../protocol/route.js';
11
11
  import { encodeHeld, parseSyncResponse } from '../protocol/sync.js';
12
12
  import { engineOf, FROM_EXTERNAL, loadsToBeacon, REASON_TIER, viewToBeacon } from '../protocol/view.js';
13
+ import { ifMatch, WRITE_HEADERS } from '../protocol/write.js';
13
14
  import { BeaconQueue, LOGOUT_SEND_MS } from './analytics.js';
14
15
  import { collectorAnswer } from './collect.js';
16
+ import { principalGuard, syncDataset } from './dataset-sync.js';
15
17
  import { verdict } from './freshness.js';
16
18
  import { policyHeaders } from './policy.js';
17
19
  import { Ledger, plan, priorFrom } from './provision.js';
@@ -21,8 +23,9 @@ import { classify } from './router.js';
21
23
  import { installRoutes } from './routing.js';
22
24
  import { Scheduler } from './scheduler.js';
23
25
  import { differs, modeFor, runsShadow } from './shadow.js';
24
- import { matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
26
+ import { branch, matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
25
27
  import { THIRD_CACHE, THIRD_HEADERS, versionClass } from './thirdparty.js';
28
+ import { WriteQueue } from './write.js';
26
29
  /** the strategies a catalogue entry may have: data for a template or renderer the device holds */
27
30
  const ENTRY_STRATEGIES = ['template', 'projection', 'data'];
28
31
  const ASSET_CACHE = 'spool-assets-';
@@ -51,7 +54,7 @@ export function createRuntime(deps) {
51
54
  const ledger = new Ledger();
52
55
  const background = new Set();
53
56
  const provisioning = new Map();
54
- const declined = new Set();
57
+ const declined = new Map();
55
58
  const wants = new Set();
56
59
  const asked = new Set();
57
60
  let currentPath = null;
@@ -93,9 +96,11 @@ export function createRuntime(deps) {
93
96
  background.add(settled);
94
97
  void settled.then(() => background.delete(settled));
95
98
  };
96
- const settleBackground = async () => {
97
- while (background.size > 0)
98
- await Promise.allSettled([...background]);
99
+ // only work begun after `before` is waited for, not an earlier request's held body
100
+ const settleBackground = async (before) => {
101
+ const fresh = () => [...background].filter((p) => !before.has(p));
102
+ while (fresh().length > 0)
103
+ await Promise.allSettled(fresh());
99
104
  };
100
105
  /** runs `fn` after every earlier partition change, so two responses never swap the store at once */
101
106
  const serial = (fn) => {
@@ -257,23 +262,128 @@ export function createRuntime(deps) {
257
262
  replica,
258
263
  scheduler: makeScheduler(replica),
259
264
  deferred: new Set(),
260
- loaded: Promise.resolve()
265
+ loaded: Promise.resolve(),
266
+ dead: false
261
267
  };
262
268
  const scheduler = publicSide.scheduler;
269
+ // #region datasets
270
+ const datasetRuns = new Map();
271
+ const datasetCounts = {};
272
+ const datasetRefused = new Map();
273
+ const datasetsDeferred = new Set();
274
+ /** principal datasets whose held rows are past maxStaleMs; the next network moment syncs them */
275
+ const principalDeferred = new Set();
276
+ let persistAsked = false;
277
+ let persisted = null;
278
+ const estimate = async () => {
279
+ if (deps.estimate)
280
+ return deps.estimate();
281
+ return ((typeof navigator === 'undefined'
282
+ ? undefined
283
+ : await navigator.storage?.estimate?.()) ?? {});
284
+ };
285
+ /** the bytes of rows one dataset may take: the headroom times the declared share; unknown when the browser will not say */
286
+ const datasetBudget = async () => {
287
+ try {
288
+ const { quota, usage } = await estimate();
289
+ if (quota === undefined || usage === undefined)
290
+ return undefined;
291
+ return Math.max(0, (quota - usage) * (config.datasetShare ?? 1));
292
+ }
293
+ catch {
294
+ return undefined;
295
+ }
296
+ };
297
+ const tally = (outcome) => {
298
+ const key = outcome.status === 'refused'
299
+ ? `${outcome.name}:refused:${outcome.reason}`
300
+ : `${outcome.name}:${outcome.status}`;
301
+ datasetCounts[key] = (datasetCounts[key] ?? 0) + 1;
302
+ if (outcome.status === 'refused') {
303
+ datasetRefused.set(outcome.name, outcome.reason);
304
+ if (outcome.reason === 'declined')
305
+ declined.set(outcome.name, outcome.by ?? null);
306
+ }
307
+ else if (outcome.status !== 'offline')
308
+ datasetRefused.delete(outcome.name);
309
+ const held = outcome.status !== 'offline' && outcome.status !== 'refused';
310
+ if (held && !persistAsked) {
311
+ persistAsked = true;
312
+ deps.broadcast?.({ type: 'spool:persist' });
313
+ }
314
+ };
315
+ /** the principal datasets the server declared, with how long each may answer after a confirmed sync */
316
+ const principalLimits = () => new Map((config.principalDatasets ?? []).map((d) => [d.name, d.maxStaleMs]));
317
+ /**
318
+ * A principal dataset's request, read for who it answers. A late answer for a partition the
319
+ * worker has left is returned untouched, since the sync discards it; a session that ended wipes
320
+ * the store, and an answer for another partition swaps it, exactly as a navigation does.
321
+ */
322
+ async function principalFetch(side, url, init) {
323
+ const response = await deps.fetch(url, init);
324
+ if (side.dead || principal !== side)
325
+ return response;
326
+ if (response.status === 401)
327
+ await wipePartition('session-ended');
328
+ else if (response.ok) {
329
+ if (response.headers.has(HEADERS.partition)) {
330
+ await early(decodeEnvelopeOrClear(response.headers), true);
331
+ }
332
+ else
333
+ await wipePartition('session-ended');
334
+ }
335
+ return response;
336
+ }
337
+ /** brings one dataset current; a second caller joins the run in flight, for the same side */
338
+ function syncDatasetOnce(name, side = publicSide) {
339
+ const key = `${side.id ?? ''}\u0000${name}`;
340
+ const running = datasetRuns.get(key);
341
+ if (running)
342
+ return running;
343
+ const partition = side.id;
344
+ const run = syncDataset({
345
+ store: side.replica.datasets,
346
+ fetch: partition === null
347
+ ? (url, init) => deps.fetch(url, init)
348
+ : (url, init) => principalFetch(side, url, init),
349
+ origin,
350
+ epoch: () => (side.dead ? -1 : side.replica.epoch),
351
+ budget: datasetBudget,
352
+ ...(partition === null ? {} : { guard: principalGuard(partition) })
353
+ }, name)
354
+ .then((outcome) => {
355
+ tally(outcome);
356
+ return outcome;
357
+ })
358
+ .finally(() => datasetRuns.delete(key));
359
+ datasetRuns.set(key, run);
360
+ return run;
361
+ }
362
+ const datasetCurrent = (outcome) => outcome.status !== 'offline' && outcome.status !== 'refused';
363
+ /** seeds a dataset in the background unless this run already learned it will not be held */
364
+ function seedDataset(name, side = publicSide) {
365
+ if (side !== publicSide && !principalLimits().has(name))
366
+ return;
367
+ const key = `${side.id ?? ''}\u0000${name}`;
368
+ if (!datasetRefused.has(name) && !datasetRuns.has(key))
369
+ track(syncDatasetOnce(name, side));
370
+ }
371
+ // #endregion
263
372
  // #region partitions
264
373
  const retentionOf = (id) => matchRoute(config.routes, id)?.route.retention ?? 'session';
265
374
  const sharedRetention = () => config.routes.some((r) => r.scope === 'principal' && r.retention === 'persistent')
266
375
  ? 'persistent'
267
376
  : 'session';
268
377
  function makeSide(id) {
269
- const sealed = host.open(id, retentionOf, sharedRetention());
378
+ const sealed = host.open(id, retentionOf, sharedRetention(), config.principalDatasets !== undefined);
270
379
  const target = new Replica(sealed, config.version, now);
271
380
  return {
272
381
  id,
273
382
  replica: target,
274
383
  scheduler: makeScheduler(target),
275
384
  deferred: new Set(),
276
- loaded: target.load()
385
+ loaded: target.load(),
386
+ dead: false
277
387
  };
278
388
  }
279
389
  async function loadPartition() {
@@ -290,12 +400,22 @@ export function createRuntime(deps) {
290
400
  await side.loaded;
291
401
  return side;
292
402
  }
403
+ /** marks a side left for good, and forgets what the next principal must not inherit */
404
+ function retire(side) {
405
+ side.dead = true;
406
+ principalDeferred.clear();
407
+ for (const name of principalLimits().keys())
408
+ datasetRefused.delete(name);
409
+ }
293
410
  /** shuts the principal store, shreds its keys and deletes it; true when there was one */
294
411
  function wipePartition(reason) {
295
412
  return serial(async () => {
296
413
  await ready();
297
414
  const old = principal;
298
415
  principal = null;
416
+ old?.writes?.kill();
417
+ if (old)
418
+ retire(old);
299
419
  if (!old || !host)
300
420
  return false;
301
421
  await host.setCurrent(null);
@@ -312,6 +432,9 @@ export function createRuntime(deps) {
312
432
  if (old?.id === id)
313
433
  return;
314
434
  principal = makeSide(id);
435
+ old?.writes?.kill();
436
+ if (old)
437
+ retire(old);
315
438
  await host.setCurrent(id);
316
439
  if (old) {
317
440
  await host.destroy(old.id);
@@ -332,6 +455,7 @@ export function createRuntime(deps) {
332
455
  await ready();
333
456
  const clear = envelope.clear ?? [];
334
457
  if (clear.includes('*') || (principal && clear.includes(principal.id))) {
458
+ await analytics.dropSameOrigin();
335
459
  await wipePartition('clear');
336
460
  return;
337
461
  }
@@ -339,11 +463,66 @@ export function createRuntime(deps) {
339
463
  if (signedOut)
340
464
  return;
341
465
  if (envelope.partition !== undefined) {
342
- if (principal?.id !== envelope.partition)
466
+ if (principal?.id !== envelope.partition) {
467
+ await analytics.dropSameOrigin();
343
468
  await switchTo(envelope.partition);
469
+ }
344
470
  }
345
- else if (navigation && principal)
471
+ else if (navigation && principal) {
472
+ await analytics.dropSameOrigin();
346
473
  await wipePartition('session-ended');
474
+ }
475
+ }
476
+ // #endregion
477
+ // #region writes
478
+ // a worker in shadow mode only observes, so it queues nothing
479
+ const writesOn = () => config.writes !== undefined && config.mode !== 'shadow';
480
+ const writeRefusal = (code, message) => ({
481
+ ok: false,
482
+ code,
483
+ message
484
+ });
485
+ async function sendWrite(record) {
486
+ const query = `name=${encodeURIComponent(record.name)}&key=${encodeURIComponent(record.key)}`;
487
+ const response = await deps.fetch(`${origin}${PATHS.write}?${query}`, {
488
+ method: 'PUT',
489
+ headers: {
490
+ 'content-type': 'application/json',
491
+ 'if-match': ifMatch(record.base),
492
+ [WRITE_HEADERS.id]: record.id
493
+ },
494
+ body: JSON.stringify({ value: record.value }),
495
+ cache: 'no-store',
496
+ credentials: 'same-origin'
497
+ });
498
+ const body = await response.json().catch(() => undefined);
499
+ return { status: response.status, body };
500
+ }
501
+ function queueOf(side) {
502
+ if (!writesOn() || !host?.openWrites)
503
+ return null;
504
+ side.writes ??= new WriteQueue({
505
+ store: host.openWrites(side.id),
506
+ classes: () => config.writes ?? [],
507
+ now,
508
+ send: sendWrite,
509
+ onFlush: (flush) => deps.broadcast?.({ type: 'spool:written', ...flush })
510
+ });
511
+ return side.writes;
512
+ }
513
+ async function pageQueue() {
514
+ if (!writesOn())
515
+ return writeRefusal('writes-off', 'declared writes are off');
516
+ const side = await principalSide();
517
+ if (!side)
518
+ return writeRefusal('no-partition', 'no signed-in principal is held');
519
+ return queueOf(side) ?? writeRefusal('unavailable', 'this worker has no partition storage');
520
+ }
521
+ /** sends the signed-in partition's queue at a network moment that is happening anyway */
522
+ async function flushWrites() {
523
+ const queue = principal ? queueOf(principal) : null;
524
+ if (queue)
525
+ await queue.flush();
347
526
  }
348
527
  // #endregion
349
528
  async function provision(path, route, partition) {
@@ -354,7 +533,7 @@ export function createRuntime(deps) {
354
533
  const query = Object.keys(have).length > 0 ? `&have=${encodeURIComponent(encodeHeld(have))}` : '';
355
534
  const response = await deps.fetch(`${origin}${PATHS.artifact}?path=${encodeURIComponent(path)}${query}`, { cache: 'no-store', credentials: 'same-origin' });
356
535
  if (response.status === 204 && response.headers.get(HEADERS.provision) === 'declined') {
357
- declined.add(path);
536
+ declined.set(path, response.headers.get(HEADERS.provisionReason));
358
537
  return;
359
538
  }
360
539
  if (!response.ok) {
@@ -431,7 +610,18 @@ export function createRuntime(deps) {
431
610
  await drainDeferred(publicSide);
432
611
  if (principal)
433
612
  await drainDeferred(principal);
613
+ const names = [...datasetsDeferred];
614
+ datasetsDeferred.clear();
615
+ for (const name of names)
616
+ await syncDatasetOnce(name);
617
+ const owed = [...principalDeferred];
618
+ principalDeferred.clear();
619
+ const side = principal;
620
+ if (side)
621
+ for (const name of owed)
622
+ await syncDatasetOnce(name, side);
434
623
  await analytics.flushAtMoment();
624
+ await flushWrites();
435
625
  await revalidateThirdParty();
436
626
  }
437
627
  }
@@ -487,9 +677,8 @@ export function createRuntime(deps) {
487
677
  // a cacheable GET may be answered by the HTTP cache with no radio wake, so only these count
488
678
  const moment = navigation || request.method !== 'GET';
489
679
  const clone = after?.clone;
490
- const copy = (typeof clone === 'function' ? clone(response, envelope) : clone)
491
- ? response.clone()
492
- : null;
680
+ const kept = typeof clone === 'function' ? clone(response, envelope) : clone;
681
+ const { visitor, copy } = kept ? branch(response) : { visitor: response, copy: null };
493
682
  track((async () => {
494
683
  try {
495
684
  await after?.run(copy, envelope);
@@ -499,7 +688,7 @@ export function createRuntime(deps) {
499
688
  }
500
689
  await observe(response.headers, envelope, sent, navigation, moment, after?.provision ?? navigation, () => keepPolicy(request, response, envelope, navigation, after));
501
690
  })());
502
- return response;
691
+ return visitor;
503
692
  }
504
693
  /** `search` is the request's query string; the stored artifact is always the path's own */
505
694
  async function decide(path, route, dry, search = '') {
@@ -529,9 +718,47 @@ export function createRuntime(deps) {
529
718
  await rep.drop([path]);
530
719
  return { reason: 'invalidated', staleSec: age };
531
720
  }
721
+ // a page answers from the rows the device holds, so their age bounds the answer too
722
+ const queried = route.query?.dataset;
723
+ const joined = (route.query?.plan?.join ?? []).map((j) => j.dataset);
724
+ const dsNames = [
725
+ ...new Set([
726
+ ...(queried ? [queried] : []),
727
+ ...joined,
728
+ ...(await rep.datasetsOf(stored.artifact))
729
+ ])
730
+ ];
731
+ // a principal side answers only from the principal datasets the server declared
732
+ const limits = side === publicSide ? undefined : principalLimits();
733
+ if (limits && dsNames.some((name) => !limits.has(name))) {
734
+ return { reason: 'refused', staleSec: staleSec() };
735
+ }
736
+ const heldAt = async () => Promise.all(dsNames.map(async (name) => (await rep.datasets.get(name))?.meta.appliedAt));
737
+ let appliedAt = await heldAt();
738
+ const missing = dsNames.filter((_, i) => appliedAt[i] === undefined);
739
+ if (missing.length > 0) {
740
+ if (!dry)
741
+ for (const name of missing)
742
+ seedDataset(name, side);
743
+ return { reason: 'refused', staleSec: staleSec() };
744
+ }
745
+ if (limits) {
746
+ // held principal rows answer only while the last confirmed sync is inside maxStaleMs
747
+ const stale = dsNames.filter((name, i) => {
748
+ const age = now() - appliedAt[i];
749
+ return !(age >= 0 && age < limits.get(name));
750
+ });
751
+ if (stale.length > 0) {
752
+ if (!dry)
753
+ for (const name of stale)
754
+ principalDeferred.add(name);
755
+ return { reason: 'refused', staleSec: staleSec() };
756
+ }
757
+ }
758
+ const known = (at) => at.filter((v) => v !== undefined);
532
759
  const fromCatalogue = stored.catalogue === true;
533
- const action = verdict(route.freshness, rep.validatedAt(stored), now());
534
- const age = staleSec();
760
+ const action = verdict(route.freshness, Math.min(rep.validatedAt(stored), ...known(appliedAt)), now());
761
+ const age = Math.max(staleSec(), ...known(appliedAt).map((at) => Math.floor((now() - at) / 1000)));
535
762
  const base = rep.needFor(stored.artifact, route.sync === 'deferred' || route.scope === 'principal');
536
763
  // a principal sync also names the catalogue entries no page has opened, so a revoked one is dropped
537
764
  const need = (route.scope === 'principal' && base ? await rep.sweep(base) : base) ?? undefined;
@@ -540,8 +767,14 @@ export function createRuntime(deps) {
540
767
  reason = 'hard-sync';
541
768
  if (!dry) {
542
769
  const synced = await sched.sync(need, age);
543
- if (!synced && route.offline === 'refuse')
770
+ const dsSynced = (await Promise.all(dsNames.map((name) => syncDatasetOnce(name, side)))).every(datasetCurrent);
771
+ if (!(synced && dsSynced) && route.offline === 'refuse')
544
772
  return { reason: 'hard-refused', staleSec: age };
773
+ if (dsNames.length > 0) {
774
+ appliedAt = await heldAt();
775
+ if (appliedAt.includes(undefined))
776
+ return { reason: 'refused', staleSec: age };
777
+ }
545
778
  if (synced) {
546
779
  if (need)
547
780
  await rep.validate(path);
@@ -550,7 +783,7 @@ export function createRuntime(deps) {
550
783
  return { reason: 'stale-after-sync', staleSec: age };
551
784
  }
552
785
  }
553
- else
786
+ if (!(synced && dsSynced))
554
787
  reason = 'offline-stale';
555
788
  }
556
789
  }
@@ -559,6 +792,11 @@ export function createRuntime(deps) {
559
792
  reason = 'soft-deferred';
560
793
  if (!dry && need && sched.defer(need, age))
561
794
  side.deferred.add(path);
795
+ if (!dry) {
796
+ for (const name of dsNames) {
797
+ (side === publicSide ? datasetsDeferred : principalDeferred).add(name);
798
+ }
799
+ }
562
800
  }
563
801
  else {
564
802
  reason = 'soft-sync';
@@ -567,6 +805,8 @@ export function createRuntime(deps) {
567
805
  if (ok && need)
568
806
  await rep.validate(path);
569
807
  }));
808
+ for (const name of dsNames)
809
+ track(syncDatasetOnce(name, side));
570
810
  }
571
811
  }
572
812
  }
@@ -584,11 +824,27 @@ export function createRuntime(deps) {
584
824
  return { reason: 'refused', staleSec: age };
585
825
  const plain = await rep.materialize(stored.artifact);
586
826
  if (!plain.ok) {
827
+ // rows of a dataset not held yet: the page is sound, so keep it and seed the dataset
828
+ if (plain.datasets) {
829
+ if (!dry)
830
+ for (const name of plain.datasets)
831
+ seedDataset(name, side);
832
+ return { reason: 'refused', staleSec: age };
833
+ }
587
834
  // a reference with nothing behind it: drop the page so the next network view fetches it whole
588
835
  await rep.drop([path]);
589
836
  return { reason: 'refused', staleSec: age };
590
837
  }
591
- const rendered = await renderLocal(config, route, plain.artifact, renderers, search);
838
+ const planned = route.query?.plan !== undefined;
839
+ const rows = queried === undefined || planned ? undefined : await rep.datasets.rows(queried);
840
+ if (queried !== undefined && !planned && !rows)
841
+ return { reason: 'refused', staleSec: age };
842
+ const tables = queried !== undefined && planned
843
+ ? await rep.datasets.tables(queried, joined)
844
+ : undefined;
845
+ if (queried !== undefined && planned && !tables)
846
+ return { reason: 'refused', staleSec: age };
847
+ const rendered = await renderLocal(config, route, plain.artifact, renderers, search, rows, tables);
592
848
  if (!rendered.ok) {
593
849
  // a collection that fails its proof is dropped, so the next network view fetches it whole
594
850
  if (rendered.reason === PROOF_MISMATCH)
@@ -729,8 +985,11 @@ export function createRuntime(deps) {
729
985
  * does not describe; the next view asks the network again.
730
986
  */
731
987
  async function capture(path, asked, copy) {
732
- if (!copy || copy.status !== 200 || copy.redirected)
988
+ if (!copy || copy.status !== 200 || copy.redirected) {
989
+ // an unread tee branch holds every chunk the visitor reads
990
+ void copy?.body?.cancel().catch(() => undefined);
733
991
  return;
992
+ }
734
993
  // a sync in the same view may have replaced the manifest, so the route is read again
735
994
  const route = config.manifest ? matchRoute(config.routes, pathOf(path))?.route : asked;
736
995
  if (route?.strategy !== 'static')
@@ -1061,8 +1320,11 @@ export function createRuntime(deps) {
1061
1320
  request.mode !== 'navigate') {
1062
1321
  return undefined;
1063
1322
  }
1323
+ const before = new Set(background);
1064
1324
  const work = kind === 'asset' && !manifestPending() ? asset(request) : run(request);
1065
- return work.catch(() => deps.fetch(request)).finally(() => wait(settleBackground()));
1325
+ return work
1326
+ .catch(() => deps.fetch(request))
1327
+ .finally(() => wait(settleBackground(before)));
1066
1328
  },
1067
1329
  async install(event) {
1068
1330
  if (assetMode === 'static-routing')
@@ -1136,7 +1398,8 @@ export function createRuntime(deps) {
1136
1398
  partition: null,
1137
1399
  artifacts: [],
1138
1400
  records: [],
1139
- projections: []
1401
+ projections: [],
1402
+ datasets: []
1140
1403
  };
1141
1404
  }
1142
1405
  await side.replica.held();
@@ -1145,12 +1408,65 @@ export function createRuntime(deps) {
1145
1408
  partition: side.id,
1146
1409
  artifacts: await side.replica.ids(),
1147
1410
  records: side.replica.graph.records.ids(),
1148
- projections: side.replica.graph.projections.ids()
1411
+ projections: side.replica.graph.projections.ids(),
1412
+ datasets: await side.replica.datasets.list()
1149
1413
  };
1150
1414
  }
1151
1415
  if (message.type === 'spool:assets') {
1152
1416
  return { type: 'spool:assets', mode: assetMode, routed };
1153
1417
  }
1418
+ if (message.type === 'spool:datasets') {
1419
+ await ready();
1420
+ return {
1421
+ type: 'spool:datasets',
1422
+ datasets: await replica.datasets.list(),
1423
+ outcomes: { ...datasetCounts },
1424
+ refused: Object.fromEntries(datasetRefused),
1425
+ declined: Object.fromEntries(declined),
1426
+ evicted: replica.datasets.evicted,
1427
+ persisted
1428
+ };
1429
+ }
1430
+ if (message.type === 'spool:persisted') {
1431
+ persisted = message.granted === true;
1432
+ return undefined;
1433
+ }
1434
+ if (message.type === 'spool:write') {
1435
+ const { name, key, value, version } = message;
1436
+ const queue = await pageQueue();
1437
+ return {
1438
+ type: 'spool:write',
1439
+ result: queue instanceof WriteQueue
1440
+ ? await queue.write(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', value, version)
1441
+ : queue
1442
+ };
1443
+ }
1444
+ if (message.type === 'spool:flush') {
1445
+ const queue = await pageQueue();
1446
+ return {
1447
+ type: 'spool:flush',
1448
+ result: queue instanceof WriteQueue ? { ok: true, ...(await queue.flush()) } : queue
1449
+ };
1450
+ }
1451
+ if (message.type === 'spool:writes') {
1452
+ const queue = await pageQueue();
1453
+ return {
1454
+ type: 'spool:writes',
1455
+ result: queue instanceof WriteQueue
1456
+ ? { ok: true, writes: await queue.list() }
1457
+ : queue
1458
+ };
1459
+ }
1460
+ if (message.type === 'spool:resolve') {
1461
+ const { name, key, use } = message;
1462
+ const queue = await pageQueue();
1463
+ return {
1464
+ type: 'spool:resolve',
1465
+ result: queue instanceof WriteQueue
1466
+ ? await queue.resolve(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', use)
1467
+ : queue
1468
+ };
1469
+ }
1154
1470
  if (message.type === 'spool:held') {
1155
1471
  const cache = await cacheNamed(THIRD_CACHE);
1156
1472
  return {
package/dist/sw/seal.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import type { Retention } from '../protocol/route.js';
2
+ import type { PrincipalDatasets } from './principal-dataset.js';
2
3
  import type { ReplicaStorage } from './storage.js';
3
4
  /** where the keys that seal a partition live */
4
5
  export interface KeyVault {
@@ -9,6 +10,16 @@ export interface KeyVault {
9
10
  }
10
11
  /** keys held in memory only, both retentions; for tests and for a worker with no IndexedDB */
11
12
  export declare function memoryVault(): KeyVault;
13
+ /** a sealed value as it sits in storage, beside the id and version the store needs as keys */
14
+ export type Row = {
15
+ sealed: {
16
+ iv: Uint8Array;
17
+ data: ArrayBuffer;
18
+ at: Retention;
19
+ };
20
+ };
21
+ export declare function seal(key: CryptoKey, aad: string, value: unknown, at: Retention): Promise<Row>;
22
+ export declare function unseal<T>(vault: KeyVault, partition: string, aad: string, row: Row | undefined): Promise<T | undefined>;
12
23
  /** what {@link sealedStorage} needs to decide a key and to bind a value to its place */
13
24
  export type SealOptions = {
14
25
  partition: string;
@@ -17,6 +28,8 @@ export type SealOptions = {
17
28
  retentionOf(id: string): Retention;
18
29
  /** the retention for records, projections and the bookkeeping, which no one route owns */
19
30
  shared: Retention;
31
+ /** the sealed dataset store of this partition; absent while principal datasets are off */
32
+ datasets?: PrincipalDatasets;
20
33
  };
21
34
  /** a partition's storage that can be shut for good */
22
35
  export type SealedStorage = ReplicaStorage & {
package/dist/sw/seal.js CHANGED
@@ -27,12 +27,12 @@ export function memoryVault() {
27
27
  })
28
28
  };
29
29
  }
30
- async function seal(key, aad, value, at) {
30
+ export async function seal(key, aad, value, at) {
31
31
  const iv = crypto.getRandomValues(new Uint8Array(12));
32
32
  const data = await crypto.subtle.encrypt({ name: 'AES-GCM', iv, additionalData: encoder.encode(aad) }, key, encoder.encode(JSON.stringify(value)));
33
33
  return { sealed: { iv, data, at } };
34
34
  }
35
- async function unseal(vault, partition, aad, row) {
35
+ export async function unseal(vault, partition, aad, row) {
36
36
  const sealed = row?.sealed;
37
37
  if (!sealed)
38
38
  return undefined;
@@ -75,7 +75,10 @@ export function sealedStorage(inner, options) {
75
75
  }
76
76
  const open = (kind, id, row) => unseal(vault, partition, aad(kind, id), row);
77
77
  return {
78
- kill: () => void (dead = true),
78
+ kill: () => {
79
+ dead = true;
80
+ options.datasets?.kill();
81
+ },
79
82
  async getArtifact(id) {
80
83
  if (dead)
81
84
  return undefined;
@@ -178,6 +181,12 @@ export function sealedStorage(inner, options) {
178
181
  return out;
179
182
  },
180
183
  deleteProjections: (ids) => (dead ? Promise.resolve() : inner.deleteProjections(ids)),
181
- clearGraph: () => (dead ? Promise.resolve() : inner.clearGraph())
184
+ clearGraph: () => (dead ? Promise.resolve() : inner.clearGraph()),
185
+ // without the principal datasets flag a partition holds no dataset and refuses to be given one
186
+ getDatasets: options.datasets?.getDatasets ?? (async () => []),
187
+ readDataset: options.datasets?.readDataset ?? (async () => []),
188
+ commitDataset: options.datasets?.commitDataset ??
189
+ (() => Promise.reject(new Error('a principal partition holds no dataset'))),
190
+ clearDatasets: options.datasets?.clearDatasets ?? (async () => undefined)
182
191
  };
183
192
  }
@@ -9,6 +9,24 @@ export declare function replayHeaders(headers: Headers): Headers;
9
9
  * counting, gives null and the rest is not read.
10
10
  */
11
11
  export declare function readCapped(response: Response, cap: number): Promise<Uint8Array | null>;
12
+ /**
13
+ * Splits a body into the visitor's stream and a capture stream, both fed by one read of `body`. A
14
+ * tee drops the chunks a branch has not read yet when the source errors, so the visitor would lose
15
+ * bytes the network already sent. Here the error reaches the visitor only after those chunks.
16
+ */
17
+ export declare function splitBody(body: ReadableStream<Uint8Array>): {
18
+ visitor: ReadableStream<Uint8Array>;
19
+ capture: ReadableStream<Uint8Array>;
20
+ };
21
+ /**
22
+ * The response a visitor gets and the copy the runtime keeps from one network answer. Every body is
23
+ * split (see {@link splitBody}), whatever its status. A redirect and a body-less answer are cloned,
24
+ * because a split response would lose the redirect's url and a body-less one has no stream.
25
+ */
26
+ export declare function branch(response: Response): {
27
+ visitor: Response;
28
+ copy: Response;
29
+ };
12
30
  /**
13
31
  * Whether `bytes` are the file a manifest `version` names. A `sha256-`, `sha384-` or `sha512-`
14
32
  * version must equal the digest of the bytes, and an entity tag must equal the response's `ETag`.