@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/dist/sw/runtime.js
CHANGED
|
@@ -10,8 +10,10 @@ import { PROOF_MISMATCH } from '../protocol/query.js';
|
|
|
10
10
|
import { matchRoute } from '../protocol/route.js';
|
|
11
11
|
import { encodeHeld, parseSyncResponse } from '../protocol/sync.js';
|
|
12
12
|
import { engineOf, FROM_EXTERNAL, loadsToBeacon, REASON_TIER, viewToBeacon } from '../protocol/view.js';
|
|
13
|
+
import { ifMatch, WRITE_HEADERS } from '../protocol/write.js';
|
|
13
14
|
import { BeaconQueue, LOGOUT_SEND_MS } from './analytics.js';
|
|
14
15
|
import { collectorAnswer } from './collect.js';
|
|
16
|
+
import { principalGuard, syncDataset } from './dataset-sync.js';
|
|
15
17
|
import { verdict } from './freshness.js';
|
|
16
18
|
import { policyHeaders } from './policy.js';
|
|
17
19
|
import { Ledger, plan, priorFrom } from './provision.js';
|
|
@@ -21,8 +23,9 @@ import { classify } from './router.js';
|
|
|
21
23
|
import { installRoutes } from './routing.js';
|
|
22
24
|
import { Scheduler } from './scheduler.js';
|
|
23
25
|
import { differs, modeFor, runsShadow } from './shadow.js';
|
|
24
|
-
import { matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
|
|
26
|
+
import { branch, matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
|
|
25
27
|
import { THIRD_CACHE, THIRD_HEADERS, versionClass } from './thirdparty.js';
|
|
28
|
+
import { WriteQueue } from './write.js';
|
|
26
29
|
/** the strategies a catalogue entry may have: data for a template or renderer the device holds */
|
|
27
30
|
const ENTRY_STRATEGIES = ['template', 'projection', 'data'];
|
|
28
31
|
const ASSET_CACHE = 'spool-assets-';
|
|
@@ -51,7 +54,7 @@ export function createRuntime(deps) {
|
|
|
51
54
|
const ledger = new Ledger();
|
|
52
55
|
const background = new Set();
|
|
53
56
|
const provisioning = new Map();
|
|
54
|
-
const declined = new
|
|
57
|
+
const declined = new Map();
|
|
55
58
|
const wants = new Set();
|
|
56
59
|
const asked = new Set();
|
|
57
60
|
let currentPath = null;
|
|
@@ -93,9 +96,11 @@ export function createRuntime(deps) {
|
|
|
93
96
|
background.add(settled);
|
|
94
97
|
void settled.then(() => background.delete(settled));
|
|
95
98
|
};
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
// only work begun after `before` is waited for, not an earlier request's held body
|
|
100
|
+
const settleBackground = async (before) => {
|
|
101
|
+
const fresh = () => [...background].filter((p) => !before.has(p));
|
|
102
|
+
while (fresh().length > 0)
|
|
103
|
+
await Promise.allSettled(fresh());
|
|
99
104
|
};
|
|
100
105
|
/** runs `fn` after every earlier partition change, so two responses never swap the store at once */
|
|
101
106
|
const serial = (fn) => {
|
|
@@ -257,23 +262,128 @@ export function createRuntime(deps) {
|
|
|
257
262
|
replica,
|
|
258
263
|
scheduler: makeScheduler(replica),
|
|
259
264
|
deferred: new Set(),
|
|
260
|
-
loaded: Promise.resolve()
|
|
265
|
+
loaded: Promise.resolve(),
|
|
266
|
+
dead: false
|
|
261
267
|
};
|
|
262
268
|
const scheduler = publicSide.scheduler;
|
|
269
|
+
// #region datasets
|
|
270
|
+
const datasetRuns = new Map();
|
|
271
|
+
const datasetCounts = {};
|
|
272
|
+
const datasetRefused = new Map();
|
|
273
|
+
const datasetsDeferred = new Set();
|
|
274
|
+
/** principal datasets whose held rows are past maxStaleMs; the next network moment syncs them */
|
|
275
|
+
const principalDeferred = new Set();
|
|
276
|
+
let persistAsked = false;
|
|
277
|
+
let persisted = null;
|
|
278
|
+
const estimate = async () => {
|
|
279
|
+
if (deps.estimate)
|
|
280
|
+
return deps.estimate();
|
|
281
|
+
return ((typeof navigator === 'undefined'
|
|
282
|
+
? undefined
|
|
283
|
+
: await navigator.storage?.estimate?.()) ?? {});
|
|
284
|
+
};
|
|
285
|
+
/** the bytes of rows one dataset may take: the headroom times the declared share; unknown when the browser will not say */
|
|
286
|
+
const datasetBudget = async () => {
|
|
287
|
+
try {
|
|
288
|
+
const { quota, usage } = await estimate();
|
|
289
|
+
if (quota === undefined || usage === undefined)
|
|
290
|
+
return undefined;
|
|
291
|
+
return Math.max(0, (quota - usage) * (config.datasetShare ?? 1));
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
const tally = (outcome) => {
|
|
298
|
+
const key = outcome.status === 'refused'
|
|
299
|
+
? `${outcome.name}:refused:${outcome.reason}`
|
|
300
|
+
: `${outcome.name}:${outcome.status}`;
|
|
301
|
+
datasetCounts[key] = (datasetCounts[key] ?? 0) + 1;
|
|
302
|
+
if (outcome.status === 'refused') {
|
|
303
|
+
datasetRefused.set(outcome.name, outcome.reason);
|
|
304
|
+
if (outcome.reason === 'declined')
|
|
305
|
+
declined.set(outcome.name, outcome.by ?? null);
|
|
306
|
+
}
|
|
307
|
+
else if (outcome.status !== 'offline')
|
|
308
|
+
datasetRefused.delete(outcome.name);
|
|
309
|
+
const held = outcome.status !== 'offline' && outcome.status !== 'refused';
|
|
310
|
+
if (held && !persistAsked) {
|
|
311
|
+
persistAsked = true;
|
|
312
|
+
deps.broadcast?.({ type: 'spool:persist' });
|
|
313
|
+
}
|
|
314
|
+
};
|
|
315
|
+
/** the principal datasets the server declared, with how long each may answer after a confirmed sync */
|
|
316
|
+
const principalLimits = () => new Map((config.principalDatasets ?? []).map((d) => [d.name, d.maxStaleMs]));
|
|
317
|
+
/**
|
|
318
|
+
* A principal dataset's request, read for who it answers. A late answer for a partition the
|
|
319
|
+
* worker has left is returned untouched, since the sync discards it; a session that ended wipes
|
|
320
|
+
* the store, and an answer for another partition swaps it, exactly as a navigation does.
|
|
321
|
+
*/
|
|
322
|
+
async function principalFetch(side, url, init) {
|
|
323
|
+
const response = await deps.fetch(url, init);
|
|
324
|
+
if (side.dead || principal !== side)
|
|
325
|
+
return response;
|
|
326
|
+
if (response.status === 401)
|
|
327
|
+
await wipePartition('session-ended');
|
|
328
|
+
else if (response.ok) {
|
|
329
|
+
if (response.headers.has(HEADERS.partition)) {
|
|
330
|
+
await early(decodeEnvelopeOrClear(response.headers), true);
|
|
331
|
+
}
|
|
332
|
+
else
|
|
333
|
+
await wipePartition('session-ended');
|
|
334
|
+
}
|
|
335
|
+
return response;
|
|
336
|
+
}
|
|
337
|
+
/** brings one dataset current; a second caller joins the run in flight, for the same side */
|
|
338
|
+
function syncDatasetOnce(name, side = publicSide) {
|
|
339
|
+
const key = `${side.id ?? ''}\u0000${name}`;
|
|
340
|
+
const running = datasetRuns.get(key);
|
|
341
|
+
if (running)
|
|
342
|
+
return running;
|
|
343
|
+
const partition = side.id;
|
|
344
|
+
const run = syncDataset({
|
|
345
|
+
store: side.replica.datasets,
|
|
346
|
+
fetch: partition === null
|
|
347
|
+
? (url, init) => deps.fetch(url, init)
|
|
348
|
+
: (url, init) => principalFetch(side, url, init),
|
|
349
|
+
origin,
|
|
350
|
+
epoch: () => (side.dead ? -1 : side.replica.epoch),
|
|
351
|
+
budget: datasetBudget,
|
|
352
|
+
...(partition === null ? {} : { guard: principalGuard(partition) })
|
|
353
|
+
}, name)
|
|
354
|
+
.then((outcome) => {
|
|
355
|
+
tally(outcome);
|
|
356
|
+
return outcome;
|
|
357
|
+
})
|
|
358
|
+
.finally(() => datasetRuns.delete(key));
|
|
359
|
+
datasetRuns.set(key, run);
|
|
360
|
+
return run;
|
|
361
|
+
}
|
|
362
|
+
const datasetCurrent = (outcome) => outcome.status !== 'offline' && outcome.status !== 'refused';
|
|
363
|
+
/** seeds a dataset in the background unless this run already learned it will not be held */
|
|
364
|
+
function seedDataset(name, side = publicSide) {
|
|
365
|
+
if (side !== publicSide && !principalLimits().has(name))
|
|
366
|
+
return;
|
|
367
|
+
const key = `${side.id ?? ''}\u0000${name}`;
|
|
368
|
+
if (!datasetRefused.has(name) && !datasetRuns.has(key))
|
|
369
|
+
track(syncDatasetOnce(name, side));
|
|
370
|
+
}
|
|
371
|
+
// #endregion
|
|
263
372
|
// #region partitions
|
|
264
373
|
const retentionOf = (id) => matchRoute(config.routes, id)?.route.retention ?? 'session';
|
|
265
374
|
const sharedRetention = () => config.routes.some((r) => r.scope === 'principal' && r.retention === 'persistent')
|
|
266
375
|
? 'persistent'
|
|
267
376
|
: 'session';
|
|
268
377
|
function makeSide(id) {
|
|
269
|
-
const sealed = host.open(id, retentionOf, sharedRetention());
|
|
378
|
+
const sealed = host.open(id, retentionOf, sharedRetention(), config.principalDatasets !== undefined);
|
|
270
379
|
const target = new Replica(sealed, config.version, now);
|
|
271
380
|
return {
|
|
272
381
|
id,
|
|
273
382
|
replica: target,
|
|
274
383
|
scheduler: makeScheduler(target),
|
|
275
384
|
deferred: new Set(),
|
|
276
|
-
loaded: target.load()
|
|
385
|
+
loaded: target.load(),
|
|
386
|
+
dead: false
|
|
277
387
|
};
|
|
278
388
|
}
|
|
279
389
|
async function loadPartition() {
|
|
@@ -290,12 +400,22 @@ export function createRuntime(deps) {
|
|
|
290
400
|
await side.loaded;
|
|
291
401
|
return side;
|
|
292
402
|
}
|
|
403
|
+
/** marks a side left for good, and forgets what the next principal must not inherit */
|
|
404
|
+
function retire(side) {
|
|
405
|
+
side.dead = true;
|
|
406
|
+
principalDeferred.clear();
|
|
407
|
+
for (const name of principalLimits().keys())
|
|
408
|
+
datasetRefused.delete(name);
|
|
409
|
+
}
|
|
293
410
|
/** shuts the principal store, shreds its keys and deletes it; true when there was one */
|
|
294
411
|
function wipePartition(reason) {
|
|
295
412
|
return serial(async () => {
|
|
296
413
|
await ready();
|
|
297
414
|
const old = principal;
|
|
298
415
|
principal = null;
|
|
416
|
+
old?.writes?.kill();
|
|
417
|
+
if (old)
|
|
418
|
+
retire(old);
|
|
299
419
|
if (!old || !host)
|
|
300
420
|
return false;
|
|
301
421
|
await host.setCurrent(null);
|
|
@@ -312,6 +432,9 @@ export function createRuntime(deps) {
|
|
|
312
432
|
if (old?.id === id)
|
|
313
433
|
return;
|
|
314
434
|
principal = makeSide(id);
|
|
435
|
+
old?.writes?.kill();
|
|
436
|
+
if (old)
|
|
437
|
+
retire(old);
|
|
315
438
|
await host.setCurrent(id);
|
|
316
439
|
if (old) {
|
|
317
440
|
await host.destroy(old.id);
|
|
@@ -332,6 +455,7 @@ export function createRuntime(deps) {
|
|
|
332
455
|
await ready();
|
|
333
456
|
const clear = envelope.clear ?? [];
|
|
334
457
|
if (clear.includes('*') || (principal && clear.includes(principal.id))) {
|
|
458
|
+
await analytics.dropSameOrigin();
|
|
335
459
|
await wipePartition('clear');
|
|
336
460
|
return;
|
|
337
461
|
}
|
|
@@ -339,11 +463,66 @@ export function createRuntime(deps) {
|
|
|
339
463
|
if (signedOut)
|
|
340
464
|
return;
|
|
341
465
|
if (envelope.partition !== undefined) {
|
|
342
|
-
if (principal?.id !== envelope.partition)
|
|
466
|
+
if (principal?.id !== envelope.partition) {
|
|
467
|
+
await analytics.dropSameOrigin();
|
|
343
468
|
await switchTo(envelope.partition);
|
|
469
|
+
}
|
|
344
470
|
}
|
|
345
|
-
else if (navigation && principal)
|
|
471
|
+
else if (navigation && principal) {
|
|
472
|
+
await analytics.dropSameOrigin();
|
|
346
473
|
await wipePartition('session-ended');
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
// #endregion
|
|
477
|
+
// #region writes
|
|
478
|
+
// a worker in shadow mode only observes, so it queues nothing
|
|
479
|
+
const writesOn = () => config.writes !== undefined && config.mode !== 'shadow';
|
|
480
|
+
const writeRefusal = (code, message) => ({
|
|
481
|
+
ok: false,
|
|
482
|
+
code,
|
|
483
|
+
message
|
|
484
|
+
});
|
|
485
|
+
async function sendWrite(record) {
|
|
486
|
+
const query = `name=${encodeURIComponent(record.name)}&key=${encodeURIComponent(record.key)}`;
|
|
487
|
+
const response = await deps.fetch(`${origin}${PATHS.write}?${query}`, {
|
|
488
|
+
method: 'PUT',
|
|
489
|
+
headers: {
|
|
490
|
+
'content-type': 'application/json',
|
|
491
|
+
'if-match': ifMatch(record.base),
|
|
492
|
+
[WRITE_HEADERS.id]: record.id
|
|
493
|
+
},
|
|
494
|
+
body: JSON.stringify({ value: record.value }),
|
|
495
|
+
cache: 'no-store',
|
|
496
|
+
credentials: 'same-origin'
|
|
497
|
+
});
|
|
498
|
+
const body = await response.json().catch(() => undefined);
|
|
499
|
+
return { status: response.status, body };
|
|
500
|
+
}
|
|
501
|
+
function queueOf(side) {
|
|
502
|
+
if (!writesOn() || !host?.openWrites)
|
|
503
|
+
return null;
|
|
504
|
+
side.writes ??= new WriteQueue({
|
|
505
|
+
store: host.openWrites(side.id),
|
|
506
|
+
classes: () => config.writes ?? [],
|
|
507
|
+
now,
|
|
508
|
+
send: sendWrite,
|
|
509
|
+
onFlush: (flush) => deps.broadcast?.({ type: 'spool:written', ...flush })
|
|
510
|
+
});
|
|
511
|
+
return side.writes;
|
|
512
|
+
}
|
|
513
|
+
async function pageQueue() {
|
|
514
|
+
if (!writesOn())
|
|
515
|
+
return writeRefusal('writes-off', 'declared writes are off');
|
|
516
|
+
const side = await principalSide();
|
|
517
|
+
if (!side)
|
|
518
|
+
return writeRefusal('no-partition', 'no signed-in principal is held');
|
|
519
|
+
return queueOf(side) ?? writeRefusal('unavailable', 'this worker has no partition storage');
|
|
520
|
+
}
|
|
521
|
+
/** sends the signed-in partition's queue at a network moment that is happening anyway */
|
|
522
|
+
async function flushWrites() {
|
|
523
|
+
const queue = principal ? queueOf(principal) : null;
|
|
524
|
+
if (queue)
|
|
525
|
+
await queue.flush();
|
|
347
526
|
}
|
|
348
527
|
// #endregion
|
|
349
528
|
async function provision(path, route, partition) {
|
|
@@ -354,7 +533,7 @@ export function createRuntime(deps) {
|
|
|
354
533
|
const query = Object.keys(have).length > 0 ? `&have=${encodeURIComponent(encodeHeld(have))}` : '';
|
|
355
534
|
const response = await deps.fetch(`${origin}${PATHS.artifact}?path=${encodeURIComponent(path)}${query}`, { cache: 'no-store', credentials: 'same-origin' });
|
|
356
535
|
if (response.status === 204 && response.headers.get(HEADERS.provision) === 'declined') {
|
|
357
|
-
declined.
|
|
536
|
+
declined.set(path, response.headers.get(HEADERS.provisionReason));
|
|
358
537
|
return;
|
|
359
538
|
}
|
|
360
539
|
if (!response.ok) {
|
|
@@ -431,7 +610,18 @@ export function createRuntime(deps) {
|
|
|
431
610
|
await drainDeferred(publicSide);
|
|
432
611
|
if (principal)
|
|
433
612
|
await drainDeferred(principal);
|
|
613
|
+
const names = [...datasetsDeferred];
|
|
614
|
+
datasetsDeferred.clear();
|
|
615
|
+
for (const name of names)
|
|
616
|
+
await syncDatasetOnce(name);
|
|
617
|
+
const owed = [...principalDeferred];
|
|
618
|
+
principalDeferred.clear();
|
|
619
|
+
const side = principal;
|
|
620
|
+
if (side)
|
|
621
|
+
for (const name of owed)
|
|
622
|
+
await syncDatasetOnce(name, side);
|
|
434
623
|
await analytics.flushAtMoment();
|
|
624
|
+
await flushWrites();
|
|
435
625
|
await revalidateThirdParty();
|
|
436
626
|
}
|
|
437
627
|
}
|
|
@@ -487,9 +677,8 @@ export function createRuntime(deps) {
|
|
|
487
677
|
// a cacheable GET may be answered by the HTTP cache with no radio wake, so only these count
|
|
488
678
|
const moment = navigation || request.method !== 'GET';
|
|
489
679
|
const clone = after?.clone;
|
|
490
|
-
const
|
|
491
|
-
|
|
492
|
-
: null;
|
|
680
|
+
const kept = typeof clone === 'function' ? clone(response, envelope) : clone;
|
|
681
|
+
const { visitor, copy } = kept ? branch(response) : { visitor: response, copy: null };
|
|
493
682
|
track((async () => {
|
|
494
683
|
try {
|
|
495
684
|
await after?.run(copy, envelope);
|
|
@@ -499,7 +688,7 @@ export function createRuntime(deps) {
|
|
|
499
688
|
}
|
|
500
689
|
await observe(response.headers, envelope, sent, navigation, moment, after?.provision ?? navigation, () => keepPolicy(request, response, envelope, navigation, after));
|
|
501
690
|
})());
|
|
502
|
-
return
|
|
691
|
+
return visitor;
|
|
503
692
|
}
|
|
504
693
|
/** `search` is the request's query string; the stored artifact is always the path's own */
|
|
505
694
|
async function decide(path, route, dry, search = '') {
|
|
@@ -529,9 +718,47 @@ export function createRuntime(deps) {
|
|
|
529
718
|
await rep.drop([path]);
|
|
530
719
|
return { reason: 'invalidated', staleSec: age };
|
|
531
720
|
}
|
|
721
|
+
// a page answers from the rows the device holds, so their age bounds the answer too
|
|
722
|
+
const queried = route.query?.dataset;
|
|
723
|
+
const joined = (route.query?.plan?.join ?? []).map((j) => j.dataset);
|
|
724
|
+
const dsNames = [
|
|
725
|
+
...new Set([
|
|
726
|
+
...(queried ? [queried] : []),
|
|
727
|
+
...joined,
|
|
728
|
+
...(await rep.datasetsOf(stored.artifact))
|
|
729
|
+
])
|
|
730
|
+
];
|
|
731
|
+
// a principal side answers only from the principal datasets the server declared
|
|
732
|
+
const limits = side === publicSide ? undefined : principalLimits();
|
|
733
|
+
if (limits && dsNames.some((name) => !limits.has(name))) {
|
|
734
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
735
|
+
}
|
|
736
|
+
const heldAt = async () => Promise.all(dsNames.map(async (name) => (await rep.datasets.get(name))?.meta.appliedAt));
|
|
737
|
+
let appliedAt = await heldAt();
|
|
738
|
+
const missing = dsNames.filter((_, i) => appliedAt[i] === undefined);
|
|
739
|
+
if (missing.length > 0) {
|
|
740
|
+
if (!dry)
|
|
741
|
+
for (const name of missing)
|
|
742
|
+
seedDataset(name, side);
|
|
743
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
744
|
+
}
|
|
745
|
+
if (limits) {
|
|
746
|
+
// held principal rows answer only while the last confirmed sync is inside maxStaleMs
|
|
747
|
+
const stale = dsNames.filter((name, i) => {
|
|
748
|
+
const age = now() - appliedAt[i];
|
|
749
|
+
return !(age >= 0 && age < limits.get(name));
|
|
750
|
+
});
|
|
751
|
+
if (stale.length > 0) {
|
|
752
|
+
if (!dry)
|
|
753
|
+
for (const name of stale)
|
|
754
|
+
principalDeferred.add(name);
|
|
755
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
const known = (at) => at.filter((v) => v !== undefined);
|
|
532
759
|
const fromCatalogue = stored.catalogue === true;
|
|
533
|
-
const action = verdict(route.freshness, rep.validatedAt(stored), now());
|
|
534
|
-
const age = staleSec();
|
|
760
|
+
const action = verdict(route.freshness, Math.min(rep.validatedAt(stored), ...known(appliedAt)), now());
|
|
761
|
+
const age = Math.max(staleSec(), ...known(appliedAt).map((at) => Math.floor((now() - at) / 1000)));
|
|
535
762
|
const base = rep.needFor(stored.artifact, route.sync === 'deferred' || route.scope === 'principal');
|
|
536
763
|
// a principal sync also names the catalogue entries no page has opened, so a revoked one is dropped
|
|
537
764
|
const need = (route.scope === 'principal' && base ? await rep.sweep(base) : base) ?? undefined;
|
|
@@ -540,8 +767,14 @@ export function createRuntime(deps) {
|
|
|
540
767
|
reason = 'hard-sync';
|
|
541
768
|
if (!dry) {
|
|
542
769
|
const synced = await sched.sync(need, age);
|
|
543
|
-
|
|
770
|
+
const dsSynced = (await Promise.all(dsNames.map((name) => syncDatasetOnce(name, side)))).every(datasetCurrent);
|
|
771
|
+
if (!(synced && dsSynced) && route.offline === 'refuse')
|
|
544
772
|
return { reason: 'hard-refused', staleSec: age };
|
|
773
|
+
if (dsNames.length > 0) {
|
|
774
|
+
appliedAt = await heldAt();
|
|
775
|
+
if (appliedAt.includes(undefined))
|
|
776
|
+
return { reason: 'refused', staleSec: age };
|
|
777
|
+
}
|
|
545
778
|
if (synced) {
|
|
546
779
|
if (need)
|
|
547
780
|
await rep.validate(path);
|
|
@@ -550,7 +783,7 @@ export function createRuntime(deps) {
|
|
|
550
783
|
return { reason: 'stale-after-sync', staleSec: age };
|
|
551
784
|
}
|
|
552
785
|
}
|
|
553
|
-
|
|
786
|
+
if (!(synced && dsSynced))
|
|
554
787
|
reason = 'offline-stale';
|
|
555
788
|
}
|
|
556
789
|
}
|
|
@@ -559,6 +792,11 @@ export function createRuntime(deps) {
|
|
|
559
792
|
reason = 'soft-deferred';
|
|
560
793
|
if (!dry && need && sched.defer(need, age))
|
|
561
794
|
side.deferred.add(path);
|
|
795
|
+
if (!dry) {
|
|
796
|
+
for (const name of dsNames) {
|
|
797
|
+
(side === publicSide ? datasetsDeferred : principalDeferred).add(name);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
562
800
|
}
|
|
563
801
|
else {
|
|
564
802
|
reason = 'soft-sync';
|
|
@@ -567,6 +805,8 @@ export function createRuntime(deps) {
|
|
|
567
805
|
if (ok && need)
|
|
568
806
|
await rep.validate(path);
|
|
569
807
|
}));
|
|
808
|
+
for (const name of dsNames)
|
|
809
|
+
track(syncDatasetOnce(name, side));
|
|
570
810
|
}
|
|
571
811
|
}
|
|
572
812
|
}
|
|
@@ -584,11 +824,27 @@ export function createRuntime(deps) {
|
|
|
584
824
|
return { reason: 'refused', staleSec: age };
|
|
585
825
|
const plain = await rep.materialize(stored.artifact);
|
|
586
826
|
if (!plain.ok) {
|
|
827
|
+
// rows of a dataset not held yet: the page is sound, so keep it and seed the dataset
|
|
828
|
+
if (plain.datasets) {
|
|
829
|
+
if (!dry)
|
|
830
|
+
for (const name of plain.datasets)
|
|
831
|
+
seedDataset(name, side);
|
|
832
|
+
return { reason: 'refused', staleSec: age };
|
|
833
|
+
}
|
|
587
834
|
// a reference with nothing behind it: drop the page so the next network view fetches it whole
|
|
588
835
|
await rep.drop([path]);
|
|
589
836
|
return { reason: 'refused', staleSec: age };
|
|
590
837
|
}
|
|
591
|
-
const
|
|
838
|
+
const planned = route.query?.plan !== undefined;
|
|
839
|
+
const rows = queried === undefined || planned ? undefined : await rep.datasets.rows(queried);
|
|
840
|
+
if (queried !== undefined && !planned && !rows)
|
|
841
|
+
return { reason: 'refused', staleSec: age };
|
|
842
|
+
const tables = queried !== undefined && planned
|
|
843
|
+
? await rep.datasets.tables(queried, joined)
|
|
844
|
+
: undefined;
|
|
845
|
+
if (queried !== undefined && planned && !tables)
|
|
846
|
+
return { reason: 'refused', staleSec: age };
|
|
847
|
+
const rendered = await renderLocal(config, route, plain.artifact, renderers, search, rows, tables);
|
|
592
848
|
if (!rendered.ok) {
|
|
593
849
|
// a collection that fails its proof is dropped, so the next network view fetches it whole
|
|
594
850
|
if (rendered.reason === PROOF_MISMATCH)
|
|
@@ -729,8 +985,11 @@ export function createRuntime(deps) {
|
|
|
729
985
|
* does not describe; the next view asks the network again.
|
|
730
986
|
*/
|
|
731
987
|
async function capture(path, asked, copy) {
|
|
732
|
-
if (!copy || copy.status !== 200 || copy.redirected)
|
|
988
|
+
if (!copy || copy.status !== 200 || copy.redirected) {
|
|
989
|
+
// an unread tee branch holds every chunk the visitor reads
|
|
990
|
+
void copy?.body?.cancel().catch(() => undefined);
|
|
733
991
|
return;
|
|
992
|
+
}
|
|
734
993
|
// a sync in the same view may have replaced the manifest, so the route is read again
|
|
735
994
|
const route = config.manifest ? matchRoute(config.routes, pathOf(path))?.route : asked;
|
|
736
995
|
if (route?.strategy !== 'static')
|
|
@@ -1061,8 +1320,11 @@ export function createRuntime(deps) {
|
|
|
1061
1320
|
request.mode !== 'navigate') {
|
|
1062
1321
|
return undefined;
|
|
1063
1322
|
}
|
|
1323
|
+
const before = new Set(background);
|
|
1064
1324
|
const work = kind === 'asset' && !manifestPending() ? asset(request) : run(request);
|
|
1065
|
-
return work
|
|
1325
|
+
return work
|
|
1326
|
+
.catch(() => deps.fetch(request))
|
|
1327
|
+
.finally(() => wait(settleBackground(before)));
|
|
1066
1328
|
},
|
|
1067
1329
|
async install(event) {
|
|
1068
1330
|
if (assetMode === 'static-routing')
|
|
@@ -1136,7 +1398,8 @@ export function createRuntime(deps) {
|
|
|
1136
1398
|
partition: null,
|
|
1137
1399
|
artifacts: [],
|
|
1138
1400
|
records: [],
|
|
1139
|
-
projections: []
|
|
1401
|
+
projections: [],
|
|
1402
|
+
datasets: []
|
|
1140
1403
|
};
|
|
1141
1404
|
}
|
|
1142
1405
|
await side.replica.held();
|
|
@@ -1145,12 +1408,65 @@ export function createRuntime(deps) {
|
|
|
1145
1408
|
partition: side.id,
|
|
1146
1409
|
artifacts: await side.replica.ids(),
|
|
1147
1410
|
records: side.replica.graph.records.ids(),
|
|
1148
|
-
projections: side.replica.graph.projections.ids()
|
|
1411
|
+
projections: side.replica.graph.projections.ids(),
|
|
1412
|
+
datasets: await side.replica.datasets.list()
|
|
1149
1413
|
};
|
|
1150
1414
|
}
|
|
1151
1415
|
if (message.type === 'spool:assets') {
|
|
1152
1416
|
return { type: 'spool:assets', mode: assetMode, routed };
|
|
1153
1417
|
}
|
|
1418
|
+
if (message.type === 'spool:datasets') {
|
|
1419
|
+
await ready();
|
|
1420
|
+
return {
|
|
1421
|
+
type: 'spool:datasets',
|
|
1422
|
+
datasets: await replica.datasets.list(),
|
|
1423
|
+
outcomes: { ...datasetCounts },
|
|
1424
|
+
refused: Object.fromEntries(datasetRefused),
|
|
1425
|
+
declined: Object.fromEntries(declined),
|
|
1426
|
+
evicted: replica.datasets.evicted,
|
|
1427
|
+
persisted
|
|
1428
|
+
};
|
|
1429
|
+
}
|
|
1430
|
+
if (message.type === 'spool:persisted') {
|
|
1431
|
+
persisted = message.granted === true;
|
|
1432
|
+
return undefined;
|
|
1433
|
+
}
|
|
1434
|
+
if (message.type === 'spool:write') {
|
|
1435
|
+
const { name, key, value, version } = message;
|
|
1436
|
+
const queue = await pageQueue();
|
|
1437
|
+
return {
|
|
1438
|
+
type: 'spool:write',
|
|
1439
|
+
result: queue instanceof WriteQueue
|
|
1440
|
+
? await queue.write(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', value, version)
|
|
1441
|
+
: queue
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
if (message.type === 'spool:flush') {
|
|
1445
|
+
const queue = await pageQueue();
|
|
1446
|
+
return {
|
|
1447
|
+
type: 'spool:flush',
|
|
1448
|
+
result: queue instanceof WriteQueue ? { ok: true, ...(await queue.flush()) } : queue
|
|
1449
|
+
};
|
|
1450
|
+
}
|
|
1451
|
+
if (message.type === 'spool:writes') {
|
|
1452
|
+
const queue = await pageQueue();
|
|
1453
|
+
return {
|
|
1454
|
+
type: 'spool:writes',
|
|
1455
|
+
result: queue instanceof WriteQueue
|
|
1456
|
+
? { ok: true, writes: await queue.list() }
|
|
1457
|
+
: queue
|
|
1458
|
+
};
|
|
1459
|
+
}
|
|
1460
|
+
if (message.type === 'spool:resolve') {
|
|
1461
|
+
const { name, key, use } = message;
|
|
1462
|
+
const queue = await pageQueue();
|
|
1463
|
+
return {
|
|
1464
|
+
type: 'spool:resolve',
|
|
1465
|
+
result: queue instanceof WriteQueue
|
|
1466
|
+
? await queue.resolve(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', use)
|
|
1467
|
+
: queue
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1154
1470
|
if (message.type === 'spool:held') {
|
|
1155
1471
|
const cache = await cacheNamed(THIRD_CACHE);
|
|
1156
1472
|
return {
|
package/dist/sw/seal.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Retention } from '../protocol/route.js';
|
|
2
|
+
import type { PrincipalDatasets } from './principal-dataset.js';
|
|
2
3
|
import type { ReplicaStorage } from './storage.js';
|
|
3
4
|
/** where the keys that seal a partition live */
|
|
4
5
|
export interface KeyVault {
|
|
@@ -9,6 +10,16 @@ export interface KeyVault {
|
|
|
9
10
|
}
|
|
10
11
|
/** keys held in memory only, both retentions; for tests and for a worker with no IndexedDB */
|
|
11
12
|
export declare function memoryVault(): KeyVault;
|
|
13
|
+
/** a sealed value as it sits in storage, beside the id and version the store needs as keys */
|
|
14
|
+
export type Row = {
|
|
15
|
+
sealed: {
|
|
16
|
+
iv: Uint8Array;
|
|
17
|
+
data: ArrayBuffer;
|
|
18
|
+
at: Retention;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
export declare function seal(key: CryptoKey, aad: string, value: unknown, at: Retention): Promise<Row>;
|
|
22
|
+
export declare function unseal<T>(vault: KeyVault, partition: string, aad: string, row: Row | undefined): Promise<T | undefined>;
|
|
12
23
|
/** what {@link sealedStorage} needs to decide a key and to bind a value to its place */
|
|
13
24
|
export type SealOptions = {
|
|
14
25
|
partition: string;
|
|
@@ -17,6 +28,8 @@ export type SealOptions = {
|
|
|
17
28
|
retentionOf(id: string): Retention;
|
|
18
29
|
/** the retention for records, projections and the bookkeeping, which no one route owns */
|
|
19
30
|
shared: Retention;
|
|
31
|
+
/** the sealed dataset store of this partition; absent while principal datasets are off */
|
|
32
|
+
datasets?: PrincipalDatasets;
|
|
20
33
|
};
|
|
21
34
|
/** a partition's storage that can be shut for good */
|
|
22
35
|
export type SealedStorage = ReplicaStorage & {
|
package/dist/sw/seal.js
CHANGED
|
@@ -27,12 +27,12 @@ export function memoryVault() {
|
|
|
27
27
|
})
|
|
28
28
|
};
|
|
29
29
|
}
|
|
30
|
-
async function seal(key, aad, value, at) {
|
|
30
|
+
export async function seal(key, aad, value, at) {
|
|
31
31
|
const iv = crypto.getRandomValues(new Uint8Array(12));
|
|
32
32
|
const data = await crypto.subtle.encrypt({ name: 'AES-GCM', iv, additionalData: encoder.encode(aad) }, key, encoder.encode(JSON.stringify(value)));
|
|
33
33
|
return { sealed: { iv, data, at } };
|
|
34
34
|
}
|
|
35
|
-
async function unseal(vault, partition, aad, row) {
|
|
35
|
+
export async function unseal(vault, partition, aad, row) {
|
|
36
36
|
const sealed = row?.sealed;
|
|
37
37
|
if (!sealed)
|
|
38
38
|
return undefined;
|
|
@@ -75,7 +75,10 @@ export function sealedStorage(inner, options) {
|
|
|
75
75
|
}
|
|
76
76
|
const open = (kind, id, row) => unseal(vault, partition, aad(kind, id), row);
|
|
77
77
|
return {
|
|
78
|
-
kill: () =>
|
|
78
|
+
kill: () => {
|
|
79
|
+
dead = true;
|
|
80
|
+
options.datasets?.kill();
|
|
81
|
+
},
|
|
79
82
|
async getArtifact(id) {
|
|
80
83
|
if (dead)
|
|
81
84
|
return undefined;
|
|
@@ -178,6 +181,12 @@ export function sealedStorage(inner, options) {
|
|
|
178
181
|
return out;
|
|
179
182
|
},
|
|
180
183
|
deleteProjections: (ids) => (dead ? Promise.resolve() : inner.deleteProjections(ids)),
|
|
181
|
-
clearGraph: () => (dead ? Promise.resolve() : inner.clearGraph())
|
|
184
|
+
clearGraph: () => (dead ? Promise.resolve() : inner.clearGraph()),
|
|
185
|
+
// without the principal datasets flag a partition holds no dataset and refuses to be given one
|
|
186
|
+
getDatasets: options.datasets?.getDatasets ?? (async () => []),
|
|
187
|
+
readDataset: options.datasets?.readDataset ?? (async () => []),
|
|
188
|
+
commitDataset: options.datasets?.commitDataset ??
|
|
189
|
+
(() => Promise.reject(new Error('a principal partition holds no dataset'))),
|
|
190
|
+
clearDatasets: options.datasets?.clearDatasets ?? (async () => undefined)
|
|
182
191
|
};
|
|
183
192
|
}
|
package/dist/sw/static.d.ts
CHANGED
|
@@ -9,6 +9,24 @@ export declare function replayHeaders(headers: Headers): Headers;
|
|
|
9
9
|
* counting, gives null and the rest is not read.
|
|
10
10
|
*/
|
|
11
11
|
export declare function readCapped(response: Response, cap: number): Promise<Uint8Array | null>;
|
|
12
|
+
/**
|
|
13
|
+
* Splits a body into the visitor's stream and a capture stream, both fed by one read of `body`. A
|
|
14
|
+
* tee drops the chunks a branch has not read yet when the source errors, so the visitor would lose
|
|
15
|
+
* bytes the network already sent. Here the error reaches the visitor only after those chunks.
|
|
16
|
+
*/
|
|
17
|
+
export declare function splitBody(body: ReadableStream<Uint8Array>): {
|
|
18
|
+
visitor: ReadableStream<Uint8Array>;
|
|
19
|
+
capture: ReadableStream<Uint8Array>;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* The response a visitor gets and the copy the runtime keeps from one network answer. Every body is
|
|
23
|
+
* split (see {@link splitBody}), whatever its status. A redirect and a body-less answer are cloned,
|
|
24
|
+
* because a split response would lose the redirect's url and a body-less one has no stream.
|
|
25
|
+
*/
|
|
26
|
+
export declare function branch(response: Response): {
|
|
27
|
+
visitor: Response;
|
|
28
|
+
copy: Response;
|
|
29
|
+
};
|
|
12
30
|
/**
|
|
13
31
|
* Whether `bytes` are the file a manifest `version` names. A `sha256-`, `sha384-` or `sha512-`
|
|
14
32
|
* version must equal the digest of the bytes, and an entity tag must equal the response's `ETag`.
|