@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/dist/report/trim.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { quantile } from './derive.js';
|
|
1
|
+
import { median, quantile } from './derive.js';
|
|
2
2
|
import { attributionalJoules } from './model.js';
|
|
3
3
|
/** {@link Quartiles} of a list, or null when it is empty */
|
|
4
4
|
export function quartiles(values) {
|
|
@@ -90,6 +90,88 @@ function shares(pages, kind, field) {
|
|
|
90
90
|
return total > 0 ? [same.reduce((t, f) => t + f[field], 0) / total] : [];
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
|
+
/** whether a URL is a Drupal aggregate: a CSS or JS file directly under a `files/css/` or `files/js/` folder */
|
|
94
|
+
export function isAggregate(url) {
|
|
95
|
+
return /\/files\/(css|js)\/[^/]+$/.test(new URL(url).pathname);
|
|
96
|
+
}
|
|
97
|
+
/** the aggregates of each site of a trim reading, one entry per path across its pages */
|
|
98
|
+
export function aggregatesOf(rows) {
|
|
99
|
+
const out = new Map();
|
|
100
|
+
for (const row of rows) {
|
|
101
|
+
const byPath = new Map();
|
|
102
|
+
for (const page of row.pages ?? []) {
|
|
103
|
+
for (const file of page.files ?? []) {
|
|
104
|
+
if (file.origin !== 'same' || !isAggregate(file.url))
|
|
105
|
+
continue;
|
|
106
|
+
const path = new URL(file.url).pathname;
|
|
107
|
+
const entry = byPath.get(path) ?? {
|
|
108
|
+
path,
|
|
109
|
+
kind: file.kind,
|
|
110
|
+
raw: file.raw,
|
|
111
|
+
transfer: null,
|
|
112
|
+
base: null
|
|
113
|
+
};
|
|
114
|
+
if (file.transfer !== null &&
|
|
115
|
+
(entry.transfer === null || file.transfer > entry.transfer))
|
|
116
|
+
entry.transfer = file.transfer;
|
|
117
|
+
entry.base ??= file.base ?? null;
|
|
118
|
+
byPath.set(path, entry);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
out.set(row.site, [...byPath.values()]);
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
|
125
|
+
const summary = (xs) => ({
|
|
126
|
+
median: xs.length === 0 ? null : median(xs),
|
|
127
|
+
mean: xs.length === 0 ? null : xs.reduce((n, x) => n + x, 0) / xs.length,
|
|
128
|
+
sites: xs.length
|
|
129
|
+
});
|
|
130
|
+
/**
|
|
131
|
+
* Churn between an earlier and a later trim reading. Names are compared by path, so a changed
|
|
132
|
+
* name is an aggregate the later reading holds and the earlier one did not. The dictionary ratio
|
|
133
|
+
* uses the base sizes the later reading recorded; the earlier reading carries none.
|
|
134
|
+
*/
|
|
135
|
+
export function churnFigures(prior, current) {
|
|
136
|
+
const before = aggregatesOf(prior);
|
|
137
|
+
const now = aggregatesOf(current);
|
|
138
|
+
const sites = [];
|
|
139
|
+
for (const [site, files] of now) {
|
|
140
|
+
const earlier = before.get(site);
|
|
141
|
+
if (!earlier)
|
|
142
|
+
continue;
|
|
143
|
+
const old = new Set(earlier.map((a) => a.path));
|
|
144
|
+
const sized = files.filter((a) => a.transfer !== null);
|
|
145
|
+
const changed = files.filter((a) => !old.has(a.path));
|
|
146
|
+
const changedSized = changed.filter((a) => a.transfer !== null);
|
|
147
|
+
const total = sized.reduce((n, a) => n + (a.transfer ?? 0), 0);
|
|
148
|
+
const changedBytes = changedSized.reduce((n, a) => n + (a.transfer ?? 0), 0);
|
|
149
|
+
const pairs = changed.flatMap((a) => {
|
|
150
|
+
const b = a.base;
|
|
151
|
+
return b && b.zstdAlone !== null && b.zstdDict !== null
|
|
152
|
+
? [{ alone: b.zstdAlone, dict: b.zstdDict }]
|
|
153
|
+
: [];
|
|
154
|
+
});
|
|
155
|
+
const alone = pairs.reduce((n, p) => n + p.alone, 0);
|
|
156
|
+
sites.push({
|
|
157
|
+
site,
|
|
158
|
+
aggregates: files.length,
|
|
159
|
+
changed: changed.length,
|
|
160
|
+
changedShare: total > 0 ? changedBytes / total : null,
|
|
161
|
+
compared: pairs.length,
|
|
162
|
+
dictRatio: pairs.length > 0 ? pairs.reduce((n, p) => n + p.dict, 0) / alone : null,
|
|
163
|
+
unchanged: files.length > 0 &&
|
|
164
|
+
files.length === earlier.length &&
|
|
165
|
+
files.every((a) => old.has(a.path))
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return {
|
|
169
|
+
sites,
|
|
170
|
+
changedShare: summary(sites.flatMap((s) => (s.changedShare === null ? [] : [s.changedShare]))),
|
|
171
|
+
dictRatio: summary(sites.flatMap((s) => (s.dictRatio === null ? [] : [s.dictRatio]))),
|
|
172
|
+
unchangedSites: sites.filter((s) => s.unchanged).length
|
|
173
|
+
};
|
|
174
|
+
}
|
|
93
175
|
/** coverage shares of the coverage reading (`measurements/trim/<date>.json`), pooled over its pages */
|
|
94
176
|
export function coverageFigures(rows) {
|
|
95
177
|
const pages = rows
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { type Dataset, type DatasetContext, type DatasetRequest, type DatasetResponse } from '../protocol/dataset.js';
|
|
2
|
+
import { type DatasetInputs, type ProvisionPrior } from '../protocol/provision.js';
|
|
3
|
+
import type { Identity } from '../protocol/route.js';
|
|
4
|
+
/**
|
|
5
|
+
* The body for a request. With `since` and the current schema it is a patch of the rows that
|
|
6
|
+
* changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
|
|
7
|
+
* `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
|
|
8
|
+
* seed page, with the proof on the last.
|
|
9
|
+
*/
|
|
10
|
+
export declare function answerDataset(dataset: Dataset, ask: DatasetRequest, ctx: DatasetContext): Promise<DatasetResponse>;
|
|
11
|
+
/** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
|
|
12
|
+
export declare const NATIVE_LOOK_DAYS = 1;
|
|
13
|
+
/** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
|
|
14
|
+
export declare const NOT_MODIFIED_BYTES = 220;
|
|
15
|
+
/** what the server has seen of one `provision: 'auto'` dataset, in memory */
|
|
16
|
+
export type DatasetStats = {
|
|
17
|
+
/** compressed bytes of a full seed, once one has been built */
|
|
18
|
+
seedBytes?: number;
|
|
19
|
+
/** compressed bytes of a patch: the server's own one-update patch first, then an exponentially weighted mean of the patches answered */
|
|
20
|
+
patchBytes?: number;
|
|
21
|
+
/** the first and latest cursor the server answered, with the time */
|
|
22
|
+
first?: {
|
|
23
|
+
cursor: number;
|
|
24
|
+
at: number;
|
|
25
|
+
};
|
|
26
|
+
last?: {
|
|
27
|
+
cursor: number;
|
|
28
|
+
at: number;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
/** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
|
|
32
|
+
export declare function datasetInputs(seed: number, stats: DatasetStats, prior: ProvisionPrior): DatasetInputs;
|
|
33
|
+
/** the admission inputs `respondDataset` needs from the server: its prior and what it has seen, by dataset name */
|
|
34
|
+
export type Admission = {
|
|
35
|
+
prior: ProvisionPrior;
|
|
36
|
+
stats: Map<string, DatasetStats>;
|
|
37
|
+
};
|
|
38
|
+
/** who is asking a principal dataset, and the `Spool-Partition` value that names them */
|
|
39
|
+
export type DatasetPrincipal = {
|
|
40
|
+
identity: Identity;
|
|
41
|
+
partition: string;
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
|
|
45
|
+
* first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
|
|
46
|
+
* with the control that bound.
|
|
47
|
+
*/
|
|
48
|
+
export declare function respondDataset(request: Request, url: URL, datasets: ReadonlyMap<string, Dataset>, admission: Admission, who?: () => Promise<DatasetPrincipal | null>): Promise<Response>;
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
import { DATASET_PAGE_ROWS, parseDatasetRequest } from '../protocol/dataset.js';
|
|
2
|
+
import { HEADERS } from '../protocol/envelope.js';
|
|
3
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
4
|
+
import { datasetBenefit } from '../protocol/provision.js';
|
|
5
|
+
import { isSetProof, setProof } from '../protocol/query.js';
|
|
6
|
+
import { json } from './sync.js';
|
|
7
|
+
const encoder = new TextEncoder();
|
|
8
|
+
function bad(dataset, message) {
|
|
9
|
+
throw new ProtocolError('bad-dataset', `${dataset.name}: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
function limit(dataset, message) {
|
|
12
|
+
throw new ProtocolError('dataset-limit', `${dataset.name}: ${message}`);
|
|
13
|
+
}
|
|
14
|
+
function checkCursor(dataset, value, what) {
|
|
15
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
|
16
|
+
return bad(dataset, `${what} must be a non-negative integer`);
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
function checkProof(dataset, proof) {
|
|
21
|
+
if (proof === undefined)
|
|
22
|
+
return undefined;
|
|
23
|
+
if (!isSetProof(proof))
|
|
24
|
+
return bad(dataset, 'a proof must be a set proof');
|
|
25
|
+
return proof;
|
|
26
|
+
}
|
|
27
|
+
/** the key of a row, checked: a plain object with a non-empty string key */
|
|
28
|
+
function keyed(dataset, row) {
|
|
29
|
+
if (typeof row !== 'object' || row === null || Array.isArray(row)) {
|
|
30
|
+
return bad(dataset, 'a row must be an object');
|
|
31
|
+
}
|
|
32
|
+
const key = dataset.key(row);
|
|
33
|
+
if (typeof key !== 'string' || key === '')
|
|
34
|
+
return bad(dataset, 'key(row) must be a non-empty string');
|
|
35
|
+
return [key, row];
|
|
36
|
+
}
|
|
37
|
+
async function* rowsOf(dataset, ctx, after) {
|
|
38
|
+
const source = await dataset.seed(after === undefined ? ctx : { ...ctx, after });
|
|
39
|
+
if (Array.isArray(source)) {
|
|
40
|
+
for (const row of source)
|
|
41
|
+
yield row;
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
for await (const page of source) {
|
|
45
|
+
if (!Array.isArray(page))
|
|
46
|
+
return bad(dataset, 'seed must give rows or pages of rows');
|
|
47
|
+
for (const row of page)
|
|
48
|
+
yield row;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* One pass over the seed in key order. Keeps the first `size` rows after `after`, notes whether
|
|
53
|
+
* another follows, and with `admit` keeps going to count rows and bytes against the dataset's limits.
|
|
54
|
+
*/
|
|
55
|
+
async function scan(dataset, ctx, size, after, admit) {
|
|
56
|
+
const page = new Map();
|
|
57
|
+
let more = false;
|
|
58
|
+
let count = 0;
|
|
59
|
+
let bytes = 0;
|
|
60
|
+
let previous;
|
|
61
|
+
for await (const raw of rowsOf(dataset, ctx, after)) {
|
|
62
|
+
const [key, row] = keyed(dataset, raw);
|
|
63
|
+
if (after !== undefined && key <= after)
|
|
64
|
+
continue;
|
|
65
|
+
if (previous !== undefined && key <= previous) {
|
|
66
|
+
return bad(dataset, 'seed rows must come in strictly ascending key order');
|
|
67
|
+
}
|
|
68
|
+
previous = key;
|
|
69
|
+
count += 1;
|
|
70
|
+
if (page.size < size)
|
|
71
|
+
page.set(key, row);
|
|
72
|
+
else {
|
|
73
|
+
more = true;
|
|
74
|
+
if (!admit)
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
if (admit) {
|
|
78
|
+
bytes += encoder.encode(JSON.stringify(row)).length;
|
|
79
|
+
if (count > dataset.maxRows)
|
|
80
|
+
limit(dataset, `a seed over ${dataset.maxRows} rows`);
|
|
81
|
+
if (bytes > dataset.maxBytes)
|
|
82
|
+
limit(dataset, `a seed over ${dataset.maxBytes} bytes`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return { page, more, count };
|
|
86
|
+
}
|
|
87
|
+
/** the proof from reading every row; the cost a backend avoids by returning one */
|
|
88
|
+
async function readProof(dataset, ctx) {
|
|
89
|
+
const all = new Map();
|
|
90
|
+
let previous;
|
|
91
|
+
for await (const raw of rowsOf(dataset, ctx)) {
|
|
92
|
+
const [key, row] = keyed(dataset, raw);
|
|
93
|
+
if (previous !== undefined && key <= previous) {
|
|
94
|
+
return bad(dataset, 'seed rows must come in strictly ascending key order');
|
|
95
|
+
}
|
|
96
|
+
previous = key;
|
|
97
|
+
all.set(key, row);
|
|
98
|
+
}
|
|
99
|
+
return setProof(all);
|
|
100
|
+
}
|
|
101
|
+
function response(dataset, cursor, body) {
|
|
102
|
+
return { protocol: 1, name: dataset.name, schema: dataset.schema, cursor, ...body };
|
|
103
|
+
}
|
|
104
|
+
async function seedOf(dataset, ask, ctx) {
|
|
105
|
+
const head = await dataset.head(ctx);
|
|
106
|
+
const cursor = checkCursor(dataset, head?.cursor, 'head().cursor');
|
|
107
|
+
const known = checkProof(dataset, head.proof);
|
|
108
|
+
if (known && known.count > dataset.maxRows) {
|
|
109
|
+
limit(dataset, `a seed over ${dataset.maxRows} rows`);
|
|
110
|
+
}
|
|
111
|
+
const size = ask.limit ?? DATASET_PAGE_ROWS;
|
|
112
|
+
const { page, more } = await scan(dataset, ctx, size, ask.after, ask.after === undefined);
|
|
113
|
+
const puts = [...page].map(([key, row]) => ({ key, row }));
|
|
114
|
+
const paging = ask.after === undefined ? {} : { after: ask.after };
|
|
115
|
+
if (more) {
|
|
116
|
+
const next = puts[puts.length - 1]?.key;
|
|
117
|
+
return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, next });
|
|
118
|
+
}
|
|
119
|
+
const proof = known ?? (ask.after === undefined ? await setProof(page) : await readProof(dataset, ctx));
|
|
120
|
+
return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, proof });
|
|
121
|
+
}
|
|
122
|
+
async function patchOf(dataset, since, ctx) {
|
|
123
|
+
const oldest = checkCursor(dataset, await dataset.horizon(ctx), 'horizon()');
|
|
124
|
+
if (since < oldest)
|
|
125
|
+
return null;
|
|
126
|
+
const found = await dataset.changes(since, ctx);
|
|
127
|
+
if (found === 'reset')
|
|
128
|
+
return null;
|
|
129
|
+
const cursor = checkCursor(dataset, found?.cursor, 'changes().cursor');
|
|
130
|
+
if (since > cursor)
|
|
131
|
+
return null;
|
|
132
|
+
const wanted = [...new Set(found.puts)].sort();
|
|
133
|
+
if (wanted.some((key) => typeof key !== 'string' || key === '')) {
|
|
134
|
+
return bad(dataset, 'changes().puts must be non-empty strings');
|
|
135
|
+
}
|
|
136
|
+
const gone = new Map();
|
|
137
|
+
for (const del of found.deletes) {
|
|
138
|
+
if (typeof del?.key !== 'string' || del.key === '') {
|
|
139
|
+
return bad(dataset, 'changes().deletes need a key');
|
|
140
|
+
}
|
|
141
|
+
gone.set(del.key, {
|
|
142
|
+
key: del.key,
|
|
143
|
+
cursor: checkCursor(dataset, del.cursor, 'a delete cursor')
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const asked = new Set(wanted);
|
|
147
|
+
for (const key of gone.keys()) {
|
|
148
|
+
if (asked.has(key))
|
|
149
|
+
return bad(dataset, `${key} is both a put and a delete`);
|
|
150
|
+
}
|
|
151
|
+
const read = new Map();
|
|
152
|
+
if (wanted.length > 0) {
|
|
153
|
+
for (const raw of await dataset.read(wanted, ctx)) {
|
|
154
|
+
const [key, row] = keyed(dataset, raw);
|
|
155
|
+
if (!asked.has(key))
|
|
156
|
+
return bad(dataset, `read returned ${key}, which was not asked for`);
|
|
157
|
+
read.set(key, row);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
for (const key of wanted) {
|
|
161
|
+
if (!read.has(key))
|
|
162
|
+
gone.set(key, { key, cursor });
|
|
163
|
+
}
|
|
164
|
+
const proof = checkProof(dataset, found.proof) ?? (await readProof(dataset, ctx));
|
|
165
|
+
return response(dataset, cursor, {
|
|
166
|
+
reset: false,
|
|
167
|
+
puts: wanted.flatMap((key) => read.has(key) ? [{ key, row: read.get(key) }] : []),
|
|
168
|
+
deletes: [...gone.values()].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)),
|
|
169
|
+
proof
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The body for a request. With `since` and the current schema it is a patch of the rows that
|
|
174
|
+
* changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
|
|
175
|
+
* `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
|
|
176
|
+
* seed page, with the proof on the last.
|
|
177
|
+
*/
|
|
178
|
+
export async function answerDataset(dataset, ask, ctx) {
|
|
179
|
+
if (ask.since !== undefined && ask.schema === dataset.schema) {
|
|
180
|
+
const patch = await patchOf(dataset, ask.since, ctx);
|
|
181
|
+
if (patch)
|
|
182
|
+
return patch;
|
|
183
|
+
}
|
|
184
|
+
return seedOf(dataset, ask, ctx);
|
|
185
|
+
}
|
|
186
|
+
/** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
|
|
187
|
+
export const NATIVE_LOOK_DAYS = 1;
|
|
188
|
+
/** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
|
|
189
|
+
export const NOT_MODIFIED_BYTES = 220;
|
|
190
|
+
const DAY_MS = 86_400_000;
|
|
191
|
+
const PATCH_WEIGHT = 0.2;
|
|
192
|
+
/** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
|
|
193
|
+
export function datasetInputs(seed, stats, prior) {
|
|
194
|
+
const perDay = updatesPerDay(stats) ?? prior.changesPerDay;
|
|
195
|
+
const changed = 1 - Math.exp(-perDay * NATIVE_LOOK_DAYS);
|
|
196
|
+
return {
|
|
197
|
+
seedBytes: seed,
|
|
198
|
+
patchBytes: stats.patchBytes ?? seed,
|
|
199
|
+
updatesPerDay: perDay,
|
|
200
|
+
v03: { seedBytes: seed, refetchBytes: seed },
|
|
201
|
+
nativeBytesPerView: changed * seed + (1 - changed) * NOT_MODIFIED_BYTES
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
function updatesPerDay(stats) {
|
|
205
|
+
const { first, last } = stats;
|
|
206
|
+
if (!first || !last || last.at <= first.at || last.cursor < first.cursor)
|
|
207
|
+
return undefined;
|
|
208
|
+
return ((last.cursor - first.cursor) * DAY_MS) / (last.at - first.at);
|
|
209
|
+
}
|
|
210
|
+
async function gzipBytes(text) {
|
|
211
|
+
const packed = new Blob([text]).stream().pipeThrough(new CompressionStream('gzip'));
|
|
212
|
+
return (await new Response(packed).arrayBuffer()).byteLength;
|
|
213
|
+
}
|
|
214
|
+
// walks every page of a seed without sending it, so the server knows what a device would fetch
|
|
215
|
+
async function measureSeed(dataset, stats, ctx) {
|
|
216
|
+
let total = 0;
|
|
217
|
+
let after;
|
|
218
|
+
for (;;) {
|
|
219
|
+
const ask = after === undefined ? { name: dataset.name } : { name: dataset.name, after };
|
|
220
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
221
|
+
total += await gzipBytes(JSON.stringify(body));
|
|
222
|
+
if (body.next === undefined)
|
|
223
|
+
break;
|
|
224
|
+
after = body.next;
|
|
225
|
+
}
|
|
226
|
+
stats.seedBytes = total;
|
|
227
|
+
return total;
|
|
228
|
+
}
|
|
229
|
+
// built before any device holds the dataset, so the first decision has a patch to weigh
|
|
230
|
+
async function measurePatch(dataset, stats, ctx) {
|
|
231
|
+
const { cursor } = await dataset.head(ctx);
|
|
232
|
+
if (cursor < 1)
|
|
233
|
+
return;
|
|
234
|
+
const body = await answerDataset(dataset, { name: dataset.name, schema: dataset.schema, since: cursor - 1 }, ctx);
|
|
235
|
+
if (!body.reset)
|
|
236
|
+
stats.patchBytes = await gzipBytes(JSON.stringify(body));
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Answers a principal dataset. Everything the dataset's functions see comes from the identity the
|
|
240
|
+
* server resolved, never from the URL, and a request with no identity gets `401`. The answer is
|
|
241
|
+
* never shared-cacheable and names the partition it belongs to, which the device checks before it
|
|
242
|
+
* applies a row.
|
|
243
|
+
*/
|
|
244
|
+
async function respondPrincipalDataset(request, url, dataset, ask, who) {
|
|
245
|
+
const caller = who ? await who() : null;
|
|
246
|
+
if (!caller)
|
|
247
|
+
return json({ error: 'sign in' }, 401, { 'cache-control': 'private, no-store' });
|
|
248
|
+
let body;
|
|
249
|
+
try {
|
|
250
|
+
body = await answerDataset(dataset, ask, { request, url, principal: caller.identity });
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
if (error instanceof ProtocolError && error.code === 'dataset-forbidden') {
|
|
254
|
+
return json({ error: 'forbidden' }, 403, { 'cache-control': 'private, no-store' });
|
|
255
|
+
}
|
|
256
|
+
throw error;
|
|
257
|
+
}
|
|
258
|
+
return json(body, 200, {
|
|
259
|
+
[HEADERS.protocol]: '1',
|
|
260
|
+
[HEADERS.partition]: caller.partition,
|
|
261
|
+
'cache-control': 'private, no-store'
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
|
|
266
|
+
* first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
|
|
267
|
+
* with the control that bound.
|
|
268
|
+
*/
|
|
269
|
+
export async function respondDataset(request, url, datasets, admission, who) {
|
|
270
|
+
const ask = parseDatasetRequest(url.searchParams);
|
|
271
|
+
const dataset = datasets.get(ask.name);
|
|
272
|
+
if (!dataset)
|
|
273
|
+
return json({ error: 'no dataset' }, 404);
|
|
274
|
+
if (dataset.scope === 'principal')
|
|
275
|
+
return respondPrincipalDataset(request, url, dataset, ask, who);
|
|
276
|
+
const ctx = { request, url };
|
|
277
|
+
if (dataset.provision !== 'auto') {
|
|
278
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
279
|
+
return json(body, 200, { [HEADERS.protocol]: '1' });
|
|
280
|
+
}
|
|
281
|
+
const stats = admission.stats.get(dataset.name) ?? {};
|
|
282
|
+
admission.stats.set(dataset.name, stats);
|
|
283
|
+
if (ask.since === undefined && ask.after === undefined) {
|
|
284
|
+
const seedBytes = stats.seedBytes ?? (await measureSeed(dataset, stats, ctx));
|
|
285
|
+
if (stats.patchBytes === undefined)
|
|
286
|
+
await measurePatch(dataset, stats, ctx);
|
|
287
|
+
const inputs = datasetInputs(seedBytes, stats, admission.prior);
|
|
288
|
+
const decision = datasetBenefit(inputs, admission.prior);
|
|
289
|
+
if (!decision.ship) {
|
|
290
|
+
return json({ error: 'declined', controls: decision.controls }, 409, {
|
|
291
|
+
[HEADERS.provision]: 'declined',
|
|
292
|
+
[HEADERS.provisionReason]: decision.controls
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const body = await answerDataset(dataset, ask, ctx);
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
stats.first ??= { cursor: body.cursor, at: now };
|
|
299
|
+
stats.last = { cursor: body.cursor, at: now };
|
|
300
|
+
if (!body.reset) {
|
|
301
|
+
const bytes = await gzipBytes(JSON.stringify(body));
|
|
302
|
+
stats.patchBytes =
|
|
303
|
+
stats.patchBytes === undefined
|
|
304
|
+
? bytes
|
|
305
|
+
: (1 - PATCH_WEIGHT) * stats.patchBytes + PATCH_WEIGHT * bytes;
|
|
306
|
+
}
|
|
307
|
+
return json(body, 200, { [HEADERS.protocol]: '1' });
|
|
308
|
+
}
|