@drupflare/spool 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +499 -43
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +34 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/media/embeds.d.ts +3 -2
- package/dist/media/embeds.js +16 -1
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +3 -0
- package/dist/protocol/envelope.js +14 -3
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +7 -0
- package/dist/protocol/index.js +7 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +52 -1
- package/dist/protocol/provision.js +53 -5
- package/dist/protocol/query.d.ts +141 -0
- package/dist/protocol/query.js +252 -0
- package/dist/protocol/route.d.ts +85 -5
- package/dist/protocol/route.js +91 -0
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/index.d.ts +1 -0
- package/dist/proxy/index.js +1 -0
- package/dist/proxy/proxy.js +8 -2
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +5 -0
- package/dist/report/index.js +5 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +284 -0
- package/dist/report/institution.js +225 -0
- package/dist/report/ledger.d.ts +28 -0
- package/dist/report/ledger.js +30 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +6 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.js +3 -0
- package/dist/server/spool.d.ts +40 -0
- package/dist/server/spool.js +184 -23
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +8 -1
- package/dist/sw/render.js +28 -1
- package/dist/sw/replica.d.ts +14 -3
- package/dist/sw/replica.js +46 -5
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +440 -29
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +5 -2
package/dist/sw/runtime.js
CHANGED
|
@@ -3,12 +3,17 @@ import { MAX_WANT } from '../protocol/beacon.js';
|
|
|
3
3
|
import { applyStamp } from '../protocol/collector.js';
|
|
4
4
|
import { PATHS } from '../protocol/config.js';
|
|
5
5
|
import { decodeEnvelope, decodeEnvelopeOrClear, HEADERS } from '../protocol/envelope.js';
|
|
6
|
+
import { shellHash } from '../protocol/hash.js';
|
|
7
|
+
import { readInline } from '../protocol/inline.js';
|
|
6
8
|
import { manifestRoutes, parseManifest } from '../protocol/manifest.js';
|
|
9
|
+
import { PROOF_MISMATCH } from '../protocol/query.js';
|
|
7
10
|
import { matchRoute } from '../protocol/route.js';
|
|
8
11
|
import { encodeHeld, parseSyncResponse } from '../protocol/sync.js';
|
|
9
12
|
import { engineOf, FROM_EXTERNAL, loadsToBeacon, REASON_TIER, viewToBeacon } from '../protocol/view.js';
|
|
13
|
+
import { ifMatch, WRITE_HEADERS } from '../protocol/write.js';
|
|
10
14
|
import { BeaconQueue, LOGOUT_SEND_MS } from './analytics.js';
|
|
11
15
|
import { collectorAnswer } from './collect.js';
|
|
16
|
+
import { principalGuard, syncDataset } from './dataset-sync.js';
|
|
12
17
|
import { verdict } from './freshness.js';
|
|
13
18
|
import { policyHeaders } from './policy.js';
|
|
14
19
|
import { Ledger, plan, priorFrom } from './provision.js';
|
|
@@ -18,8 +23,9 @@ import { classify } from './router.js';
|
|
|
18
23
|
import { installRoutes } from './routing.js';
|
|
19
24
|
import { Scheduler } from './scheduler.js';
|
|
20
25
|
import { differs, modeFor, runsShadow } from './shadow.js';
|
|
21
|
-
import { matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
|
|
26
|
+
import { branch, matchesVersion, MAX_STATIC_BYTES, readCapped, replayHeaders, STATIC_CACHE } from './static.js';
|
|
22
27
|
import { THIRD_CACHE, THIRD_HEADERS, versionClass } from './thirdparty.js';
|
|
28
|
+
import { WriteQueue } from './write.js';
|
|
23
29
|
/** the strategies a catalogue entry may have: data for a template or renderer the device holds */
|
|
24
30
|
const ENTRY_STRATEGIES = ['template', 'projection', 'data'];
|
|
25
31
|
const ASSET_CACHE = 'spool-assets-';
|
|
@@ -48,7 +54,7 @@ export function createRuntime(deps) {
|
|
|
48
54
|
const ledger = new Ledger();
|
|
49
55
|
const background = new Set();
|
|
50
56
|
const provisioning = new Map();
|
|
51
|
-
const declined = new
|
|
57
|
+
const declined = new Map();
|
|
52
58
|
const wants = new Set();
|
|
53
59
|
const asked = new Set();
|
|
54
60
|
let currentPath = null;
|
|
@@ -90,9 +96,11 @@ export function createRuntime(deps) {
|
|
|
90
96
|
background.add(settled);
|
|
91
97
|
void settled.then(() => background.delete(settled));
|
|
92
98
|
};
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
// only work begun after `before` is waited for, not an earlier request's held body
|
|
100
|
+
const settleBackground = async (before) => {
|
|
101
|
+
const fresh = () => [...background].filter((p) => !before.has(p));
|
|
102
|
+
while (fresh().length > 0)
|
|
103
|
+
await Promise.allSettled(fresh());
|
|
96
104
|
};
|
|
97
105
|
/** runs `fn` after every earlier partition change, so two responses never swap the store at once */
|
|
98
106
|
const serial = (fn) => {
|
|
@@ -254,23 +262,128 @@ export function createRuntime(deps) {
|
|
|
254
262
|
replica,
|
|
255
263
|
scheduler: makeScheduler(replica),
|
|
256
264
|
deferred: new Set(),
|
|
257
|
-
loaded: Promise.resolve()
|
|
265
|
+
loaded: Promise.resolve(),
|
|
266
|
+
dead: false
|
|
258
267
|
};
|
|
259
268
|
const scheduler = publicSide.scheduler;
|
|
269
|
+
// #region datasets
|
|
270
|
+
const datasetRuns = new Map();
|
|
271
|
+
const datasetCounts = {};
|
|
272
|
+
const datasetRefused = new Map();
|
|
273
|
+
const datasetsDeferred = new Set();
|
|
274
|
+
/** principal datasets whose held rows are past maxStaleMs; the next network moment syncs them */
|
|
275
|
+
const principalDeferred = new Set();
|
|
276
|
+
let persistAsked = false;
|
|
277
|
+
let persisted = null;
|
|
278
|
+
const estimate = async () => {
|
|
279
|
+
if (deps.estimate)
|
|
280
|
+
return deps.estimate();
|
|
281
|
+
return ((typeof navigator === 'undefined'
|
|
282
|
+
? undefined
|
|
283
|
+
: await navigator.storage?.estimate?.()) ?? {});
|
|
284
|
+
};
|
|
285
|
+
/** the bytes of rows one dataset may take: the headroom times the declared share; unknown when the browser will not say */
|
|
286
|
+
const datasetBudget = async () => {
|
|
287
|
+
try {
|
|
288
|
+
const { quota, usage } = await estimate();
|
|
289
|
+
if (quota === undefined || usage === undefined)
|
|
290
|
+
return undefined;
|
|
291
|
+
return Math.max(0, (quota - usage) * (config.datasetShare ?? 1));
|
|
292
|
+
}
|
|
293
|
+
catch {
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
const tally = (outcome) => {
|
|
298
|
+
const key = outcome.status === 'refused'
|
|
299
|
+
? `${outcome.name}:refused:${outcome.reason}`
|
|
300
|
+
: `${outcome.name}:${outcome.status}`;
|
|
301
|
+
datasetCounts[key] = (datasetCounts[key] ?? 0) + 1;
|
|
302
|
+
if (outcome.status === 'refused') {
|
|
303
|
+
datasetRefused.set(outcome.name, outcome.reason);
|
|
304
|
+
if (outcome.reason === 'declined')
|
|
305
|
+
declined.set(outcome.name, outcome.by ?? null);
|
|
306
|
+
}
|
|
307
|
+
else if (outcome.status !== 'offline')
|
|
308
|
+
datasetRefused.delete(outcome.name);
|
|
309
|
+
const held = outcome.status !== 'offline' && outcome.status !== 'refused';
|
|
310
|
+
if (held && !persistAsked) {
|
|
311
|
+
persistAsked = true;
|
|
312
|
+
deps.broadcast?.({ type: 'spool:persist' });
|
|
313
|
+
}
|
|
314
|
+
};
|
|
315
|
+
/** the principal datasets the server declared, with how long each may answer after a confirmed sync */
|
|
316
|
+
const principalLimits = () => new Map((config.principalDatasets ?? []).map((d) => [d.name, d.maxStaleMs]));
|
|
317
|
+
/**
|
|
318
|
+
* A principal dataset's request, read for who it answers. A late answer for a partition the
|
|
319
|
+
* worker has left is returned untouched, since the sync discards it; a session that ended wipes
|
|
320
|
+
* the store, and an answer for another partition swaps it, exactly as a navigation does.
|
|
321
|
+
*/
|
|
322
|
+
async function principalFetch(side, url, init) {
|
|
323
|
+
const response = await deps.fetch(url, init);
|
|
324
|
+
if (side.dead || principal !== side)
|
|
325
|
+
return response;
|
|
326
|
+
if (response.status === 401)
|
|
327
|
+
await wipePartition('session-ended');
|
|
328
|
+
else if (response.ok) {
|
|
329
|
+
if (response.headers.has(HEADERS.partition)) {
|
|
330
|
+
await early(decodeEnvelopeOrClear(response.headers), true);
|
|
331
|
+
}
|
|
332
|
+
else
|
|
333
|
+
await wipePartition('session-ended');
|
|
334
|
+
}
|
|
335
|
+
return response;
|
|
336
|
+
}
|
|
337
|
+
/** brings one dataset current; a second caller joins the run in flight, for the same side */
|
|
338
|
+
function syncDatasetOnce(name, side = publicSide) {
|
|
339
|
+
const key = `${side.id ?? ''}\u0000${name}`;
|
|
340
|
+
const running = datasetRuns.get(key);
|
|
341
|
+
if (running)
|
|
342
|
+
return running;
|
|
343
|
+
const partition = side.id;
|
|
344
|
+
const run = syncDataset({
|
|
345
|
+
store: side.replica.datasets,
|
|
346
|
+
fetch: partition === null
|
|
347
|
+
? (url, init) => deps.fetch(url, init)
|
|
348
|
+
: (url, init) => principalFetch(side, url, init),
|
|
349
|
+
origin,
|
|
350
|
+
epoch: () => (side.dead ? -1 : side.replica.epoch),
|
|
351
|
+
budget: datasetBudget,
|
|
352
|
+
...(partition === null ? {} : { guard: principalGuard(partition) })
|
|
353
|
+
}, name)
|
|
354
|
+
.then((outcome) => {
|
|
355
|
+
tally(outcome);
|
|
356
|
+
return outcome;
|
|
357
|
+
})
|
|
358
|
+
.finally(() => datasetRuns.delete(key));
|
|
359
|
+
datasetRuns.set(key, run);
|
|
360
|
+
return run;
|
|
361
|
+
}
|
|
362
|
+
const datasetCurrent = (outcome) => outcome.status !== 'offline' && outcome.status !== 'refused';
|
|
363
|
+
/** seeds a dataset in the background unless this run already learned it will not be held */
|
|
364
|
+
function seedDataset(name, side = publicSide) {
|
|
365
|
+
if (side !== publicSide && !principalLimits().has(name))
|
|
366
|
+
return;
|
|
367
|
+
const key = `${side.id ?? ''}\u0000${name}`;
|
|
368
|
+
if (!datasetRefused.has(name) && !datasetRuns.has(key))
|
|
369
|
+
track(syncDatasetOnce(name, side));
|
|
370
|
+
}
|
|
371
|
+
// #endregion
|
|
260
372
|
// #region partitions
|
|
261
373
|
const retentionOf = (id) => matchRoute(config.routes, id)?.route.retention ?? 'session';
|
|
262
374
|
const sharedRetention = () => config.routes.some((r) => r.scope === 'principal' && r.retention === 'persistent')
|
|
263
375
|
? 'persistent'
|
|
264
376
|
: 'session';
|
|
265
377
|
function makeSide(id) {
|
|
266
|
-
const sealed = host.open(id, retentionOf, sharedRetention());
|
|
378
|
+
const sealed = host.open(id, retentionOf, sharedRetention(), config.principalDatasets !== undefined);
|
|
267
379
|
const target = new Replica(sealed, config.version, now);
|
|
268
380
|
return {
|
|
269
381
|
id,
|
|
270
382
|
replica: target,
|
|
271
383
|
scheduler: makeScheduler(target),
|
|
272
384
|
deferred: new Set(),
|
|
273
|
-
loaded: target.load()
|
|
385
|
+
loaded: target.load(),
|
|
386
|
+
dead: false
|
|
274
387
|
};
|
|
275
388
|
}
|
|
276
389
|
async function loadPartition() {
|
|
@@ -287,12 +400,22 @@ export function createRuntime(deps) {
|
|
|
287
400
|
await side.loaded;
|
|
288
401
|
return side;
|
|
289
402
|
}
|
|
403
|
+
/** marks a side left for good, and forgets what the next principal must not inherit */
|
|
404
|
+
function retire(side) {
|
|
405
|
+
side.dead = true;
|
|
406
|
+
principalDeferred.clear();
|
|
407
|
+
for (const name of principalLimits().keys())
|
|
408
|
+
datasetRefused.delete(name);
|
|
409
|
+
}
|
|
290
410
|
/** shuts the principal store, shreds its keys and deletes it; true when there was one */
|
|
291
411
|
function wipePartition(reason) {
|
|
292
412
|
return serial(async () => {
|
|
293
413
|
await ready();
|
|
294
414
|
const old = principal;
|
|
295
415
|
principal = null;
|
|
416
|
+
old?.writes?.kill();
|
|
417
|
+
if (old)
|
|
418
|
+
retire(old);
|
|
296
419
|
if (!old || !host)
|
|
297
420
|
return false;
|
|
298
421
|
await host.setCurrent(null);
|
|
@@ -309,6 +432,9 @@ export function createRuntime(deps) {
|
|
|
309
432
|
if (old?.id === id)
|
|
310
433
|
return;
|
|
311
434
|
principal = makeSide(id);
|
|
435
|
+
old?.writes?.kill();
|
|
436
|
+
if (old)
|
|
437
|
+
retire(old);
|
|
312
438
|
await host.setCurrent(id);
|
|
313
439
|
if (old) {
|
|
314
440
|
await host.destroy(old.id);
|
|
@@ -329,6 +455,7 @@ export function createRuntime(deps) {
|
|
|
329
455
|
await ready();
|
|
330
456
|
const clear = envelope.clear ?? [];
|
|
331
457
|
if (clear.includes('*') || (principal && clear.includes(principal.id))) {
|
|
458
|
+
await analytics.dropSameOrigin();
|
|
332
459
|
await wipePartition('clear');
|
|
333
460
|
return;
|
|
334
461
|
}
|
|
@@ -336,11 +463,66 @@ export function createRuntime(deps) {
|
|
|
336
463
|
if (signedOut)
|
|
337
464
|
return;
|
|
338
465
|
if (envelope.partition !== undefined) {
|
|
339
|
-
if (principal?.id !== envelope.partition)
|
|
466
|
+
if (principal?.id !== envelope.partition) {
|
|
467
|
+
await analytics.dropSameOrigin();
|
|
340
468
|
await switchTo(envelope.partition);
|
|
469
|
+
}
|
|
341
470
|
}
|
|
342
|
-
else if (navigation && principal)
|
|
471
|
+
else if (navigation && principal) {
|
|
472
|
+
await analytics.dropSameOrigin();
|
|
343
473
|
await wipePartition('session-ended');
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
// #endregion
|
|
477
|
+
// #region writes
|
|
478
|
+
// a worker in shadow mode only observes, so it queues nothing
|
|
479
|
+
const writesOn = () => config.writes !== undefined && config.mode !== 'shadow';
|
|
480
|
+
const writeRefusal = (code, message) => ({
|
|
481
|
+
ok: false,
|
|
482
|
+
code,
|
|
483
|
+
message
|
|
484
|
+
});
|
|
485
|
+
async function sendWrite(record) {
|
|
486
|
+
const query = `name=${encodeURIComponent(record.name)}&key=${encodeURIComponent(record.key)}`;
|
|
487
|
+
const response = await deps.fetch(`${origin}${PATHS.write}?${query}`, {
|
|
488
|
+
method: 'PUT',
|
|
489
|
+
headers: {
|
|
490
|
+
'content-type': 'application/json',
|
|
491
|
+
'if-match': ifMatch(record.base),
|
|
492
|
+
[WRITE_HEADERS.id]: record.id
|
|
493
|
+
},
|
|
494
|
+
body: JSON.stringify({ value: record.value }),
|
|
495
|
+
cache: 'no-store',
|
|
496
|
+
credentials: 'same-origin'
|
|
497
|
+
});
|
|
498
|
+
const body = await response.json().catch(() => undefined);
|
|
499
|
+
return { status: response.status, body };
|
|
500
|
+
}
|
|
501
|
+
function queueOf(side) {
|
|
502
|
+
if (!writesOn() || !host?.openWrites)
|
|
503
|
+
return null;
|
|
504
|
+
side.writes ??= new WriteQueue({
|
|
505
|
+
store: host.openWrites(side.id),
|
|
506
|
+
classes: () => config.writes ?? [],
|
|
507
|
+
now,
|
|
508
|
+
send: sendWrite,
|
|
509
|
+
onFlush: (flush) => deps.broadcast?.({ type: 'spool:written', ...flush })
|
|
510
|
+
});
|
|
511
|
+
return side.writes;
|
|
512
|
+
}
|
|
513
|
+
async function pageQueue() {
|
|
514
|
+
if (!writesOn())
|
|
515
|
+
return writeRefusal('writes-off', 'declared writes are off');
|
|
516
|
+
const side = await principalSide();
|
|
517
|
+
if (!side)
|
|
518
|
+
return writeRefusal('no-partition', 'no signed-in principal is held');
|
|
519
|
+
return queueOf(side) ?? writeRefusal('unavailable', 'this worker has no partition storage');
|
|
520
|
+
}
|
|
521
|
+
/** sends the signed-in partition's queue at a network moment that is happening anyway */
|
|
522
|
+
async function flushWrites() {
|
|
523
|
+
const queue = principal ? queueOf(principal) : null;
|
|
524
|
+
if (queue)
|
|
525
|
+
await queue.flush();
|
|
344
526
|
}
|
|
345
527
|
// #endregion
|
|
346
528
|
async function provision(path, route, partition) {
|
|
@@ -351,7 +533,7 @@ export function createRuntime(deps) {
|
|
|
351
533
|
const query = Object.keys(have).length > 0 ? `&have=${encodeURIComponent(encodeHeld(have))}` : '';
|
|
352
534
|
const response = await deps.fetch(`${origin}${PATHS.artifact}?path=${encodeURIComponent(path)}${query}`, { cache: 'no-store', credentials: 'same-origin' });
|
|
353
535
|
if (response.status === 204 && response.headers.get(HEADERS.provision) === 'declined') {
|
|
354
|
-
declined.
|
|
536
|
+
declined.set(path, response.headers.get(HEADERS.provisionReason));
|
|
355
537
|
return;
|
|
356
538
|
}
|
|
357
539
|
if (!response.ok) {
|
|
@@ -428,7 +610,18 @@ export function createRuntime(deps) {
|
|
|
428
610
|
await drainDeferred(publicSide);
|
|
429
611
|
if (principal)
|
|
430
612
|
await drainDeferred(principal);
|
|
613
|
+
const names = [...datasetsDeferred];
|
|
614
|
+
datasetsDeferred.clear();
|
|
615
|
+
for (const name of names)
|
|
616
|
+
await syncDatasetOnce(name);
|
|
617
|
+
const owed = [...principalDeferred];
|
|
618
|
+
principalDeferred.clear();
|
|
619
|
+
const side = principal;
|
|
620
|
+
if (side)
|
|
621
|
+
for (const name of owed)
|
|
622
|
+
await syncDatasetOnce(name, side);
|
|
431
623
|
await analytics.flushAtMoment();
|
|
624
|
+
await flushWrites();
|
|
432
625
|
await revalidateThirdParty();
|
|
433
626
|
}
|
|
434
627
|
}
|
|
@@ -483,19 +676,22 @@ export function createRuntime(deps) {
|
|
|
483
676
|
await early(envelope, navigation, after?.signedOut);
|
|
484
677
|
// a cacheable GET may be answered by the HTTP cache with no radio wake, so only these count
|
|
485
678
|
const moment = navigation || request.method !== 'GET';
|
|
486
|
-
const
|
|
679
|
+
const clone = after?.clone;
|
|
680
|
+
const kept = typeof clone === 'function' ? clone(response, envelope) : clone;
|
|
681
|
+
const { visitor, copy } = kept ? branch(response) : { visitor: response, copy: null };
|
|
487
682
|
track((async () => {
|
|
488
683
|
try {
|
|
489
|
-
await after?.run(copy);
|
|
684
|
+
await after?.run(copy, envelope);
|
|
490
685
|
}
|
|
491
686
|
catch {
|
|
492
687
|
// a view that cannot be recorded must not stop the response being observed
|
|
493
688
|
}
|
|
494
689
|
await observe(response.headers, envelope, sent, navigation, moment, after?.provision ?? navigation, () => keepPolicy(request, response, envelope, navigation, after));
|
|
495
690
|
})());
|
|
496
|
-
return
|
|
691
|
+
return visitor;
|
|
497
692
|
}
|
|
498
|
-
|
|
693
|
+
/** `search` is the request's query string; the stored artifact is always the path's own */
|
|
694
|
+
async function decide(path, route, dry, search = '') {
|
|
499
695
|
let side = publicSide;
|
|
500
696
|
if (route.scope === 'principal') {
|
|
501
697
|
if (!host)
|
|
@@ -522,9 +718,47 @@ export function createRuntime(deps) {
|
|
|
522
718
|
await rep.drop([path]);
|
|
523
719
|
return { reason: 'invalidated', staleSec: age };
|
|
524
720
|
}
|
|
721
|
+
// a page answers from the rows the device holds, so their age bounds the answer too
|
|
722
|
+
const queried = route.query?.dataset;
|
|
723
|
+
const joined = (route.query?.plan?.join ?? []).map((j) => j.dataset);
|
|
724
|
+
const dsNames = [
|
|
725
|
+
...new Set([
|
|
726
|
+
...(queried ? [queried] : []),
|
|
727
|
+
...joined,
|
|
728
|
+
...(await rep.datasetsOf(stored.artifact))
|
|
729
|
+
])
|
|
730
|
+
];
|
|
731
|
+
// a principal side answers only from the principal datasets the server declared
|
|
732
|
+
const limits = side === publicSide ? undefined : principalLimits();
|
|
733
|
+
if (limits && dsNames.some((name) => !limits.has(name))) {
|
|
734
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
735
|
+
}
|
|
736
|
+
const heldAt = async () => Promise.all(dsNames.map(async (name) => (await rep.datasets.get(name))?.meta.appliedAt));
|
|
737
|
+
let appliedAt = await heldAt();
|
|
738
|
+
const missing = dsNames.filter((_, i) => appliedAt[i] === undefined);
|
|
739
|
+
if (missing.length > 0) {
|
|
740
|
+
if (!dry)
|
|
741
|
+
for (const name of missing)
|
|
742
|
+
seedDataset(name, side);
|
|
743
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
744
|
+
}
|
|
745
|
+
if (limits) {
|
|
746
|
+
// held principal rows answer only while the last confirmed sync is inside maxStaleMs
|
|
747
|
+
const stale = dsNames.filter((name, i) => {
|
|
748
|
+
const age = now() - appliedAt[i];
|
|
749
|
+
return !(age >= 0 && age < limits.get(name));
|
|
750
|
+
});
|
|
751
|
+
if (stale.length > 0) {
|
|
752
|
+
if (!dry)
|
|
753
|
+
for (const name of stale)
|
|
754
|
+
principalDeferred.add(name);
|
|
755
|
+
return { reason: 'refused', staleSec: staleSec() };
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
const known = (at) => at.filter((v) => v !== undefined);
|
|
525
759
|
const fromCatalogue = stored.catalogue === true;
|
|
526
|
-
const action = verdict(route.freshness, rep.validatedAt(stored), now());
|
|
527
|
-
const age = staleSec();
|
|
760
|
+
const action = verdict(route.freshness, Math.min(rep.validatedAt(stored), ...known(appliedAt)), now());
|
|
761
|
+
const age = Math.max(staleSec(), ...known(appliedAt).map((at) => Math.floor((now() - at) / 1000)));
|
|
528
762
|
const base = rep.needFor(stored.artifact, route.sync === 'deferred' || route.scope === 'principal');
|
|
529
763
|
// a principal sync also names the catalogue entries no page has opened, so a revoked one is dropped
|
|
530
764
|
const need = (route.scope === 'principal' && base ? await rep.sweep(base) : base) ?? undefined;
|
|
@@ -533,8 +767,14 @@ export function createRuntime(deps) {
|
|
|
533
767
|
reason = 'hard-sync';
|
|
534
768
|
if (!dry) {
|
|
535
769
|
const synced = await sched.sync(need, age);
|
|
536
|
-
|
|
770
|
+
const dsSynced = (await Promise.all(dsNames.map((name) => syncDatasetOnce(name, side)))).every(datasetCurrent);
|
|
771
|
+
if (!(synced && dsSynced) && route.offline === 'refuse')
|
|
537
772
|
return { reason: 'hard-refused', staleSec: age };
|
|
773
|
+
if (dsNames.length > 0) {
|
|
774
|
+
appliedAt = await heldAt();
|
|
775
|
+
if (appliedAt.includes(undefined))
|
|
776
|
+
return { reason: 'refused', staleSec: age };
|
|
777
|
+
}
|
|
538
778
|
if (synced) {
|
|
539
779
|
if (need)
|
|
540
780
|
await rep.validate(path);
|
|
@@ -543,7 +783,7 @@ export function createRuntime(deps) {
|
|
|
543
783
|
return { reason: 'stale-after-sync', staleSec: age };
|
|
544
784
|
}
|
|
545
785
|
}
|
|
546
|
-
|
|
786
|
+
if (!(synced && dsSynced))
|
|
547
787
|
reason = 'offline-stale';
|
|
548
788
|
}
|
|
549
789
|
}
|
|
@@ -552,6 +792,11 @@ export function createRuntime(deps) {
|
|
|
552
792
|
reason = 'soft-deferred';
|
|
553
793
|
if (!dry && need && sched.defer(need, age))
|
|
554
794
|
side.deferred.add(path);
|
|
795
|
+
if (!dry) {
|
|
796
|
+
for (const name of dsNames) {
|
|
797
|
+
(side === publicSide ? datasetsDeferred : principalDeferred).add(name);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
555
800
|
}
|
|
556
801
|
else {
|
|
557
802
|
reason = 'soft-sync';
|
|
@@ -560,6 +805,8 @@ export function createRuntime(deps) {
|
|
|
560
805
|
if (ok && need)
|
|
561
806
|
await rep.validate(path);
|
|
562
807
|
}));
|
|
808
|
+
for (const name of dsNames)
|
|
809
|
+
track(syncDatasetOnce(name, side));
|
|
563
810
|
}
|
|
564
811
|
}
|
|
565
812
|
}
|
|
@@ -577,13 +824,33 @@ export function createRuntime(deps) {
|
|
|
577
824
|
return { reason: 'refused', staleSec: age };
|
|
578
825
|
const plain = await rep.materialize(stored.artifact);
|
|
579
826
|
if (!plain.ok) {
|
|
827
|
+
// rows of a dataset not held yet: the page is sound, so keep it and seed the dataset
|
|
828
|
+
if (plain.datasets) {
|
|
829
|
+
if (!dry)
|
|
830
|
+
for (const name of plain.datasets)
|
|
831
|
+
seedDataset(name, side);
|
|
832
|
+
return { reason: 'refused', staleSec: age };
|
|
833
|
+
}
|
|
580
834
|
// a reference with nothing behind it: drop the page so the next network view fetches it whole
|
|
581
835
|
await rep.drop([path]);
|
|
582
836
|
return { reason: 'refused', staleSec: age };
|
|
583
837
|
}
|
|
584
|
-
const
|
|
585
|
-
|
|
838
|
+
const planned = route.query?.plan !== undefined;
|
|
839
|
+
const rows = queried === undefined || planned ? undefined : await rep.datasets.rows(queried);
|
|
840
|
+
if (queried !== undefined && !planned && !rows)
|
|
841
|
+
return { reason: 'refused', staleSec: age };
|
|
842
|
+
const tables = queried !== undefined && planned
|
|
843
|
+
? await rep.datasets.tables(queried, joined)
|
|
844
|
+
: undefined;
|
|
845
|
+
if (queried !== undefined && planned && !tables)
|
|
846
|
+
return { reason: 'refused', staleSec: age };
|
|
847
|
+
const rendered = await renderLocal(config, route, plain.artifact, renderers, search, rows, tables);
|
|
848
|
+
if (!rendered.ok) {
|
|
849
|
+
// a collection that fails its proof is dropped, so the next network view fetches it whole
|
|
850
|
+
if (rendered.reason === PROOF_MISMATCH)
|
|
851
|
+
await rep.drop([path]);
|
|
586
852
|
return { reason: 'refused', staleSec: age };
|
|
853
|
+
}
|
|
587
854
|
if (fromCatalogue) {
|
|
588
855
|
await rep.open(path);
|
|
589
856
|
if (REASON_TIER[reason] === 'local')
|
|
@@ -612,7 +879,7 @@ export function createRuntime(deps) {
|
|
|
612
879
|
const gapMs = lastViewAt === null ? null : at - lastViewAt;
|
|
613
880
|
lastViewAt = at;
|
|
614
881
|
currentPath = route.catalogue ? path : null;
|
|
615
|
-
const decision = await decide(path, route, shadow);
|
|
882
|
+
const decision = await decide(path, route, shadow, new URL(request.url).search);
|
|
616
883
|
const from = referrerRoute(request.referrer);
|
|
617
884
|
const id = crypto.randomUUID();
|
|
618
885
|
latestView.delete(path);
|
|
@@ -656,11 +923,13 @@ export function createRuntime(deps) {
|
|
|
656
923
|
if (decision.html === undefined) {
|
|
657
924
|
const keeps = route.strategy === 'static';
|
|
658
925
|
const response = await passthrough(request, true, {
|
|
659
|
-
clone: keeps,
|
|
926
|
+
clone: keeps || ((res, envelope) => derivable(path, route, res, envelope)),
|
|
660
927
|
route,
|
|
661
|
-
run: async (copy) => {
|
|
928
|
+
run: async (copy, envelope) => {
|
|
662
929
|
if (keeps)
|
|
663
930
|
await capture(path, route, copy);
|
|
931
|
+
else
|
|
932
|
+
await derive(path, route, copy, envelope);
|
|
664
933
|
await record(null);
|
|
665
934
|
}
|
|
666
935
|
});
|
|
@@ -688,7 +957,7 @@ export function createRuntime(deps) {
|
|
|
688
957
|
}
|
|
689
958
|
/** answers a GET for a `data` route from the stored value; the page makes no request */
|
|
690
959
|
async function dataFile(request, path, route) {
|
|
691
|
-
const decision = await decide(path, route, false);
|
|
960
|
+
const decision = await decide(path, route, false, new URL(request.url).search);
|
|
692
961
|
if (decision.html !== undefined) {
|
|
693
962
|
ledger.serve(path, decision.reason === 'catalogue');
|
|
694
963
|
const headers = new Headers(decision.headers);
|
|
@@ -716,8 +985,11 @@ export function createRuntime(deps) {
|
|
|
716
985
|
* does not describe; the next view asks the network again.
|
|
717
986
|
*/
|
|
718
987
|
async function capture(path, asked, copy) {
|
|
719
|
-
if (!copy || copy.status !== 200 || copy.redirected)
|
|
988
|
+
if (!copy || copy.status !== 200 || copy.redirected) {
|
|
989
|
+
// an unread tee branch holds every chunk the visitor reads
|
|
990
|
+
void copy?.body?.cancel().catch(() => undefined);
|
|
720
991
|
return;
|
|
992
|
+
}
|
|
721
993
|
// a sync in the same view may have replaced the manifest, so the route is read again
|
|
722
994
|
const route = config.manifest ? matchRoute(config.routes, pathOf(path))?.route : asked;
|
|
723
995
|
if (route?.strategy !== 'static')
|
|
@@ -771,6 +1043,88 @@ export function createRuntime(deps) {
|
|
|
771
1043
|
ledger.spend(path, { bytes: bytes.byteLength, requests: 0 });
|
|
772
1044
|
}
|
|
773
1045
|
}
|
|
1046
|
+
/** whether a cold navigation's body can stand in for the artifact its response advertises */
|
|
1047
|
+
function derivable(path, route, response, envelope) {
|
|
1048
|
+
const advertised = envelope?.artifact;
|
|
1049
|
+
return ((route.strategy === 'assembly' || route.strategy === 'template') &&
|
|
1050
|
+
route.scope === 'public' &&
|
|
1051
|
+
envelope?.protocol === PROTOCOL_VERSION &&
|
|
1052
|
+
advertised?.hash !== undefined &&
|
|
1053
|
+
advertised.id === path &&
|
|
1054
|
+
advertised.scope === 'public' &&
|
|
1055
|
+
modeFor(config, route) !== 'shadow' &&
|
|
1056
|
+
!declined.has(path) &&
|
|
1057
|
+
response.status === 200 &&
|
|
1058
|
+
!response.redirected &&
|
|
1059
|
+
/text\/html/i.test(response.headers.get('content-type') ?? ''));
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* The artifact a navigation body carries, or null. An `assembly` page is the artifact's shell
|
|
1063
|
+
* once the client tag is out, and a `template` page holds the artifact's JSON in its inline
|
|
1064
|
+
* element. Either must hash to the value `Spool-Artifact` named.
|
|
1065
|
+
*/
|
|
1066
|
+
async function carried(path, route, text, advertised) {
|
|
1067
|
+
if (route.strategy === 'template') {
|
|
1068
|
+
const json = readInline(text);
|
|
1069
|
+
if (json === null || (await shellHash(json)) !== advertised.hash)
|
|
1070
|
+
return null;
|
|
1071
|
+
const artifact = parseArtifact(JSON.parse(json));
|
|
1072
|
+
if (artifact.id !== path || artifact.scope !== 'public')
|
|
1073
|
+
return null;
|
|
1074
|
+
if (artifact.strategy !== 'template')
|
|
1075
|
+
return null;
|
|
1076
|
+
return { artifact, bytes: encoder.encode(json).length };
|
|
1077
|
+
}
|
|
1078
|
+
const at = text.indexOf(config.clientTag);
|
|
1079
|
+
const shells = at < 0 ? [text] : [text.slice(0, at) + text.slice(at + config.clientTag.length), text];
|
|
1080
|
+
for (const shell of shells) {
|
|
1081
|
+
if ((await shellHash(shell)) !== advertised.hash)
|
|
1082
|
+
continue;
|
|
1083
|
+
return {
|
|
1084
|
+
artifact: {
|
|
1085
|
+
protocol: PROTOCOL_VERSION,
|
|
1086
|
+
id: path,
|
|
1087
|
+
scope: 'public',
|
|
1088
|
+
strategy: 'assembly',
|
|
1089
|
+
deps: [...new Set(advertised.deps)],
|
|
1090
|
+
vector: advertised.version,
|
|
1091
|
+
body: { shell }
|
|
1092
|
+
},
|
|
1093
|
+
bytes: 0
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
return null;
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Keeps the artifact a navigation's own body carries, when it matches the hash `Spool-Artifact`
|
|
1100
|
+
* named. Anything else leaves nothing stored, and the advertised artifact is then fetched as
|
|
1101
|
+
* before.
|
|
1102
|
+
*/
|
|
1103
|
+
async function derive(path, route, copy, envelope) {
|
|
1104
|
+
const advertised = envelope?.artifact;
|
|
1105
|
+
if (!copy || !advertised?.hash || !derivable(path, route, copy, envelope))
|
|
1106
|
+
return;
|
|
1107
|
+
// a wipe while the body is read means the copy predates it, so it is not kept
|
|
1108
|
+
const epoch = replica.epoch;
|
|
1109
|
+
try {
|
|
1110
|
+
const bytes = await readCapped(copy, MAX_STATIC_BYTES);
|
|
1111
|
+
if (!bytes)
|
|
1112
|
+
return;
|
|
1113
|
+
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
1114
|
+
const found = await carried(path, route, text, {
|
|
1115
|
+
...advertised,
|
|
1116
|
+
hash: advertised.hash
|
|
1117
|
+
});
|
|
1118
|
+
if (!found || (await replica.get(path)) || replica.epoch !== epoch)
|
|
1119
|
+
return;
|
|
1120
|
+
if (await replica.put(found.artifact, route.persist)) {
|
|
1121
|
+
ledger.spend(path, { bytes: found.bytes, requests: 0 });
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
catch {
|
|
1125
|
+
// an unreadable body leaves the network fetch as the only way to the artifact
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
774
1128
|
async function staticFile(request, path, route) {
|
|
775
1129
|
const decision = await decide(path, route, false);
|
|
776
1130
|
if (decision.response) {
|
|
@@ -966,8 +1320,11 @@ export function createRuntime(deps) {
|
|
|
966
1320
|
request.mode !== 'navigate') {
|
|
967
1321
|
return undefined;
|
|
968
1322
|
}
|
|
1323
|
+
const before = new Set(background);
|
|
969
1324
|
const work = kind === 'asset' && !manifestPending() ? asset(request) : run(request);
|
|
970
|
-
return work
|
|
1325
|
+
return work
|
|
1326
|
+
.catch(() => deps.fetch(request))
|
|
1327
|
+
.finally(() => wait(settleBackground(before)));
|
|
971
1328
|
},
|
|
972
1329
|
async install(event) {
|
|
973
1330
|
if (assetMode === 'static-routing')
|
|
@@ -1041,7 +1398,8 @@ export function createRuntime(deps) {
|
|
|
1041
1398
|
partition: null,
|
|
1042
1399
|
artifacts: [],
|
|
1043
1400
|
records: [],
|
|
1044
|
-
projections: []
|
|
1401
|
+
projections: [],
|
|
1402
|
+
datasets: []
|
|
1045
1403
|
};
|
|
1046
1404
|
}
|
|
1047
1405
|
await side.replica.held();
|
|
@@ -1050,12 +1408,65 @@ export function createRuntime(deps) {
|
|
|
1050
1408
|
partition: side.id,
|
|
1051
1409
|
artifacts: await side.replica.ids(),
|
|
1052
1410
|
records: side.replica.graph.records.ids(),
|
|
1053
|
-
projections: side.replica.graph.projections.ids()
|
|
1411
|
+
projections: side.replica.graph.projections.ids(),
|
|
1412
|
+
datasets: await side.replica.datasets.list()
|
|
1054
1413
|
};
|
|
1055
1414
|
}
|
|
1056
1415
|
if (message.type === 'spool:assets') {
|
|
1057
1416
|
return { type: 'spool:assets', mode: assetMode, routed };
|
|
1058
1417
|
}
|
|
1418
|
+
if (message.type === 'spool:datasets') {
|
|
1419
|
+
await ready();
|
|
1420
|
+
return {
|
|
1421
|
+
type: 'spool:datasets',
|
|
1422
|
+
datasets: await replica.datasets.list(),
|
|
1423
|
+
outcomes: { ...datasetCounts },
|
|
1424
|
+
refused: Object.fromEntries(datasetRefused),
|
|
1425
|
+
declined: Object.fromEntries(declined),
|
|
1426
|
+
evicted: replica.datasets.evicted,
|
|
1427
|
+
persisted
|
|
1428
|
+
};
|
|
1429
|
+
}
|
|
1430
|
+
if (message.type === 'spool:persisted') {
|
|
1431
|
+
persisted = message.granted === true;
|
|
1432
|
+
return undefined;
|
|
1433
|
+
}
|
|
1434
|
+
if (message.type === 'spool:write') {
|
|
1435
|
+
const { name, key, value, version } = message;
|
|
1436
|
+
const queue = await pageQueue();
|
|
1437
|
+
return {
|
|
1438
|
+
type: 'spool:write',
|
|
1439
|
+
result: queue instanceof WriteQueue
|
|
1440
|
+
? await queue.write(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', value, version)
|
|
1441
|
+
: queue
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
if (message.type === 'spool:flush') {
|
|
1445
|
+
const queue = await pageQueue();
|
|
1446
|
+
return {
|
|
1447
|
+
type: 'spool:flush',
|
|
1448
|
+
result: queue instanceof WriteQueue ? { ok: true, ...(await queue.flush()) } : queue
|
|
1449
|
+
};
|
|
1450
|
+
}
|
|
1451
|
+
if (message.type === 'spool:writes') {
|
|
1452
|
+
const queue = await pageQueue();
|
|
1453
|
+
return {
|
|
1454
|
+
type: 'spool:writes',
|
|
1455
|
+
result: queue instanceof WriteQueue
|
|
1456
|
+
? { ok: true, writes: await queue.list() }
|
|
1457
|
+
: queue
|
|
1458
|
+
};
|
|
1459
|
+
}
|
|
1460
|
+
if (message.type === 'spool:resolve') {
|
|
1461
|
+
const { name, key, use } = message;
|
|
1462
|
+
const queue = await pageQueue();
|
|
1463
|
+
return {
|
|
1464
|
+
type: 'spool:resolve',
|
|
1465
|
+
result: queue instanceof WriteQueue
|
|
1466
|
+
? await queue.resolve(typeof name === 'string' ? name : '', typeof key === 'string' ? key : '', use)
|
|
1467
|
+
: queue
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1059
1470
|
if (message.type === 'spool:held') {
|
|
1060
1471
|
const cache = await cacheNamed(THIRD_CACHE);
|
|
1061
1472
|
return {
|