@drupflare/spool 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +499 -43
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +34 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/client/register.d.ts +23 -1
  11. package/dist/client/register.js +53 -7
  12. package/dist/media/embeds.d.ts +3 -2
  13. package/dist/media/embeds.js +16 -1
  14. package/dist/protocol/config.d.ts +16 -1
  15. package/dist/protocol/config.js +3 -1
  16. package/dist/protocol/dataset.d.ts +183 -0
  17. package/dist/protocol/dataset.js +206 -0
  18. package/dist/protocol/envelope.d.ts +3 -0
  19. package/dist/protocol/envelope.js +14 -3
  20. package/dist/protocol/errors.d.ts +1 -1
  21. package/dist/protocol/graph.d.ts +23 -4
  22. package/dist/protocol/graph.js +34 -3
  23. package/dist/protocol/hash.d.ts +2 -0
  24. package/dist/protocol/hash.js +8 -0
  25. package/dist/protocol/index.d.ts +7 -0
  26. package/dist/protocol/index.js +7 -0
  27. package/dist/protocol/inline.d.ts +14 -0
  28. package/dist/protocol/inline.js +43 -0
  29. package/dist/protocol/navigation.d.ts +81 -0
  30. package/dist/protocol/navigation.js +249 -0
  31. package/dist/protocol/normalise.d.ts +4 -3
  32. package/dist/protocol/normalise.js +6 -4
  33. package/dist/protocol/plan.d.ts +100 -0
  34. package/dist/protocol/plan.js +661 -0
  35. package/dist/protocol/provision.d.ts +52 -1
  36. package/dist/protocol/provision.js +53 -5
  37. package/dist/protocol/query.d.ts +141 -0
  38. package/dist/protocol/query.js +252 -0
  39. package/dist/protocol/route.d.ts +85 -5
  40. package/dist/protocol/route.js +91 -0
  41. package/dist/protocol/write.d.ts +146 -0
  42. package/dist/protocol/write.js +101 -0
  43. package/dist/proxy/bases.d.ts +53 -0
  44. package/dist/proxy/bases.js +76 -0
  45. package/dist/proxy/index.d.ts +1 -0
  46. package/dist/proxy/index.js +1 -0
  47. package/dist/proxy/proxy.js +8 -2
  48. package/dist/report/capacity.d.ts +59 -0
  49. package/dist/report/capacity.js +76 -0
  50. package/dist/report/channels.d.ts +60 -0
  51. package/dist/report/channels.js +65 -0
  52. package/dist/report/dataset.d.ts +48 -0
  53. package/dist/report/dataset.js +93 -0
  54. package/dist/report/derive.d.ts +285 -0
  55. package/dist/report/derive.js +676 -1
  56. package/dist/report/economics.d.ts +11 -1
  57. package/dist/report/economics.js +19 -1
  58. package/dist/report/index.d.ts +5 -0
  59. package/dist/report/index.js +5 -0
  60. package/dist/report/inputs.d.ts +97 -0
  61. package/dist/report/inputs.js +133 -0
  62. package/dist/report/institution.d.ts +284 -0
  63. package/dist/report/institution.js +225 -0
  64. package/dist/report/ledger.d.ts +28 -0
  65. package/dist/report/ledger.js +30 -0
  66. package/dist/report/loops.d.ts +73 -0
  67. package/dist/report/loops.js +101 -0
  68. package/dist/report/measurement.d.ts +1 -1
  69. package/dist/report/measurement.js +6 -1
  70. package/dist/report/model.d.ts +2 -0
  71. package/dist/report/model.js +4 -0
  72. package/dist/report/thirdparty.d.ts +57 -0
  73. package/dist/report/thirdparty.js +111 -0
  74. package/dist/report/traffic.d.ts +11 -2
  75. package/dist/report/traffic.js +13 -3
  76. package/dist/report/trim.d.ts +56 -0
  77. package/dist/report/trim.js +83 -1
  78. package/dist/server/dataset.d.ts +48 -0
  79. package/dist/server/dataset.js +308 -0
  80. package/dist/server/embedded.d.ts +3 -3
  81. package/dist/server/embedded.js +3 -3
  82. package/dist/server/index.d.ts +3 -0
  83. package/dist/server/index.js +3 -0
  84. package/dist/server/spool.d.ts +40 -0
  85. package/dist/server/spool.js +184 -23
  86. package/dist/server/write.d.ts +27 -0
  87. package/dist/server/write.js +127 -0
  88. package/dist/sw/analytics.d.ts +6 -0
  89. package/dist/sw/analytics.js +14 -0
  90. package/dist/sw/dataset-sync.d.ts +76 -0
  91. package/dist/sw/dataset-sync.js +149 -0
  92. package/dist/sw/dataset.d.ts +91 -0
  93. package/dist/sw/dataset.js +318 -0
  94. package/dist/sw/graph/index.d.ts +31 -1
  95. package/dist/sw/graph/index.js +57 -5
  96. package/dist/sw/graph/invalidate.d.ts +15 -3
  97. package/dist/sw/graph/invalidate.js +42 -4
  98. package/dist/sw/idb.js +78 -9
  99. package/dist/sw/partition.d.ts +6 -1
  100. package/dist/sw/partition.js +21 -2
  101. package/dist/sw/principal-dataset.d.ts +64 -0
  102. package/dist/sw/principal-dataset.js +254 -0
  103. package/dist/sw/render.d.ts +8 -1
  104. package/dist/sw/render.js +28 -1
  105. package/dist/sw/replica.d.ts +14 -3
  106. package/dist/sw/replica.js +46 -5
  107. package/dist/sw/router.js +2 -2
  108. package/dist/sw/runtime.d.ts +43 -2
  109. package/dist/sw/runtime.js +440 -29
  110. package/dist/sw/seal.d.ts +13 -0
  111. package/dist/sw/seal.js +13 -4
  112. package/dist/sw/static.d.ts +18 -0
  113. package/dist/sw/static.js +84 -1
  114. package/dist/sw/storage.d.ts +51 -1
  115. package/dist/sw/storage.js +33 -1
  116. package/dist/sw/write.d.ts +107 -0
  117. package/dist/sw/write.js +442 -0
  118. package/package.json +5 -2
@@ -1,3 +1,4 @@
1
+ import { loaded, median } from './derive.js';
1
2
  /**
2
3
  * A short table of common third-party hosts. It is a starting point for a site to extend, not a
3
4
  * claim about any vendor's behaviour: the class and remedy come from reading vendor documentation
@@ -48,6 +49,116 @@ export function classifyOrigin(origin, table = KNOWN_ORIGINS) {
48
49
  ? { class: best.class, remedy: best.remedy }
49
50
  : { class: 'unknown', remedy: 'review' };
50
51
  }
52
+ const DAY_S = 86400;
53
+ const YEAR_S = 365 * DAY_S;
54
+ const CLASSES = ['immutable', 'long', 'short', 'uncacheable'];
55
+ /**
56
+ * The class of one recorded response from its cache headers. `immutable` is an `immutable`
57
+ * directive or a max-age of a year; `long` a max-age of a day up to a year; `short` a max-age under
58
+ * a day, or a validator with no freshness; `uncacheable` a `no-store`, or neither freshness nor a
59
+ * validator. The reading keeps `expires` as a flag only, so it is not read.
60
+ */
61
+ export function cacheability(response) {
62
+ const directives = new Map();
63
+ for (const part of String(response.cacheControl ?? '').split(',')) {
64
+ const [name = '', value = ''] = part.trim().toLowerCase().split('=');
65
+ if (name)
66
+ directives.set(name, value.replace(/"/g, ''));
67
+ }
68
+ if (directives.has('no-store'))
69
+ return 'uncacheable';
70
+ const raw = directives.get('max-age');
71
+ const maxAge = raw === undefined ? NaN : Number(raw);
72
+ if (directives.has('immutable') || maxAge >= YEAR_S)
73
+ return 'immutable';
74
+ if (maxAge >= DAY_S)
75
+ return 'long';
76
+ if (Number.isFinite(maxAge))
77
+ return 'short';
78
+ return response.etag === true || response.lastModified === true ? 'short' : 'uncacheable';
79
+ }
80
+ /**
81
+ * The third-party requests of one repeat view that reached the network, split by cacheability.
82
+ * First-party requests and requests the browser cache answered are left out, as in `splitRequestBytes`.
83
+ */
84
+ export function thirdPartyCacheSplit(requests) {
85
+ const zeros = () => ({
86
+ immutable: 0,
87
+ long: 0,
88
+ short: 0,
89
+ uncacheable: 0
90
+ });
91
+ const split = {
92
+ bytes: zeros(),
93
+ requests: zeros(),
94
+ total: 0,
95
+ answerable: 0,
96
+ byOrigin: {}
97
+ };
98
+ for (const request of requests) {
99
+ if (request.fromCache === true || request.sameOrigin === true)
100
+ continue;
101
+ const bytes = typeof request.bytes === 'number' ? request.bytes : 0;
102
+ const kind = cacheability(request);
103
+ const { class: origin, remedy } = classifyOrigin(String(request.url ?? ''));
104
+ const longLived = kind === 'immutable' || kind === 'long' ? bytes : 0;
105
+ split.bytes[kind] += bytes;
106
+ split.requests[kind] += 1;
107
+ split.total += bytes;
108
+ if (remedy === 'cache')
109
+ split.answerable += longLived;
110
+ const key = `${origin}|${remedy}`;
111
+ const cell = split.byOrigin[key] ?? { bytes: 0, longLived: 0 };
112
+ cell.bytes += bytes;
113
+ cell.longLived += longLived;
114
+ split.byOrigin[key] = cell;
115
+ }
116
+ return split;
117
+ }
118
+ /** {@link thirdPartyCacheSplit} for each site's loaded home-page repeat view, picked as `repeatSplitBySite` picks it */
119
+ export function thirdPartyCacheBySite(rows) {
120
+ const out = new Map();
121
+ for (const row of rows) {
122
+ for (const nav of row.navs ?? []) {
123
+ if (loaded(nav) && nav.kind === 'warm-home-repeat' && Array.isArray(nav.requestList)) {
124
+ out.set(String(row.site ?? ''), thirdPartyCacheSplit(nav.requestList));
125
+ }
126
+ }
127
+ }
128
+ return out;
129
+ }
130
+ const ratio = (part, whole) => (whole === 0 ? 0 : part / whole);
131
+ const mean = (xs) => xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length;
132
+ const middle = (xs) => (xs.length === 0 ? 0 : median(xs));
133
+ /** the median, mean and pooled figures of the splits, one per class and for the answerable bytes */
134
+ export function cacheabilityFigures(splits) {
135
+ const timed = splits.filter((s) => s.total > 0);
136
+ const pooledTotal = splits.reduce((n, s) => n + s.total, 0);
137
+ const shares = {};
138
+ for (const kind of CLASSES) {
139
+ const xs = timed.map((s) => s.bytes[kind] / s.total);
140
+ shares[kind] = {
141
+ median: middle(xs),
142
+ mean: mean(xs),
143
+ pooled: ratio(splits.reduce((n, s) => n + s.bytes[kind], 0), pooledTotal)
144
+ };
145
+ }
146
+ const answerable = timed.map((s) => s.answerable / s.total);
147
+ return {
148
+ sites: splits.length,
149
+ withThirdParty: timed.length,
150
+ shares,
151
+ answerableBytes: {
152
+ median: middle(splits.map((s) => s.answerable)),
153
+ mean: mean(splits.map((s) => s.answerable))
154
+ },
155
+ answerableShare: {
156
+ median: middle(answerable),
157
+ mean: mean(answerable),
158
+ pooled: ratio(splits.reduce((n, s) => n + s.answerable, 0), pooledTotal)
159
+ }
160
+ };
161
+ }
51
162
  /**
52
163
  * What blocks zero-network on this site: for each third-party origin the views that loaded from it,
53
164
  * ranked by how many views it alone blocks. Pass the views from `viewsOf`, which joins each page's
@@ -136,5 +136,14 @@ export type Transitions = {
136
136
  };
137
137
  /** counts per `(from, route)` pair and per `from`; a view with no `from` counts in neither */
138
138
  export declare function transitions(views: readonly TimedView[]): Transitions;
139
- /** the share of views from `from` that went to each route, in route order; empty when none came from it */
140
- export declare function nextShare(views: readonly TimedView[], from: string): Record<string, number>;
139
+ /**
140
+ * The share of views from `from` that went to each route, in route order; empty when none came from
141
+ * it. With `first`, only a first view of the route counts toward its share, over every view from
142
+ * `from`. A catalogue entry saves only a first view, so that share is the one to rank entries by:
143
+ * the share over all views counts returning visits and, in a replay of 989,818 public visitors,
144
+ * ran 1.5 to 7 times the observed chance of a first view in the bins that decide an entry
145
+ * (`measurements/nav-replay`).
146
+ */
147
+ export declare function nextShare(views: readonly TimedView[], from: string, options?: {
148
+ first?: boolean;
149
+ }): Record<string, number>;
@@ -214,9 +214,19 @@ export function transitions(views) {
214
214
  const sorted = [...pairs.values()].sort((a, b) => a.from === b.from ? (a.route < b.route ? -1 : 1) : a.from < b.from ? -1 : 1);
215
215
  return { pairs: sorted, byFrom };
216
216
  }
217
- /** the share of views from `from` that went to each route, in route order; empty when none came from it */
218
- export function nextShare(views, from) {
219
- const { pairs, byFrom } = transitions(views);
217
+ /** a view of a page the device did not hold: a network view of a cold route, or a catalogue entry's first open */
218
+ const isFirstView = (view) => view.reason === 'cold' || view.reason === 'catalogue';
219
+ /**
220
+ * The share of views from `from` that went to each route, in route order; empty when none came from
221
+ * it. With `first`, only a first view of the route counts toward its share, over every view from
222
+ * `from`. A catalogue entry saves only a first view, so that share is the one to rank entries by:
223
+ * the share over all views counts returning visits and, in a replay of 989,818 public visitors,
224
+ * ran 1.5 to 7 times the observed chance of a first view in the bins that decide an entry
225
+ * (`measurements/nav-replay`).
226
+ */
227
+ export function nextShare(views, from, options = {}) {
228
+ const { byFrom } = transitions(views);
229
+ const { pairs } = options.first ? transitions(views.filter(isFirstView)) : transitions(views);
220
230
  const total = byFrom[from];
221
231
  const out = {};
222
232
  if (!total)
@@ -75,6 +75,62 @@ export type CoverageFigures = {
75
75
  jsLoad: Quartiles | null;
76
76
  jsScroll: Quartiles | null;
77
77
  };
78
+ /** a changed aggregate's comparison with an earlier file of its kind; the zstd sizes are null when the base did not load */
79
+ export type Base = {
80
+ url: string | null;
81
+ status: number | null;
82
+ zstdAlone: number | null;
83
+ zstdDict: number | null;
84
+ };
85
+ /** one same-origin CSS or JS aggregate of a site, keyed by its path with the query dropped */
86
+ export type Aggregate = {
87
+ path: string;
88
+ kind: 'css' | 'js';
89
+ raw: number;
90
+ /** the largest transfer a page recorded for it, or null when no page did */
91
+ transfer: number | null;
92
+ /** only on a reading taken with an earlier reading to compare against, for a name that changed */
93
+ base: Base | null;
94
+ };
95
+ /** whether a URL is a Drupal aggregate: a CSS or JS file directly under a `files/css/` or `files/js/` folder */
96
+ export declare function isAggregate(url: string): boolean;
97
+ /** the aggregates of each site of a trim reading, one entry per path across its pages */
98
+ export declare function aggregatesOf(rows: readonly Row[]): Map<string, Aggregate[]>;
99
+ /** one site's aggregates on the second reading against the first */
100
+ export type ChurnSite = {
101
+ site: string;
102
+ /** aggregates on the second reading */
103
+ aggregates: number;
104
+ /** of which the name is new since the first reading */
105
+ changed: number;
106
+ /** the changed names' share of the second reading's aggregate transfer bytes; null without any transfer */
107
+ changedShare: number | null;
108
+ /** changed aggregates whose base loaded, and the zstd dictionary size over standalone size across them */
109
+ compared: number;
110
+ dictRatio: number | null;
111
+ /** the same non-empty set of names on both readings */
112
+ unchanged: boolean;
113
+ };
114
+ /** the median and mean over the sites that have a value, and how many did */
115
+ export type Summary = {
116
+ median: number | null;
117
+ mean: number | null;
118
+ sites: number;
119
+ };
120
+ /** the churn between two readings of the same sites, over the sites both hold */
121
+ export type ChurnFigures = {
122
+ sites: ChurnSite[];
123
+ changedShare: Summary;
124
+ dictRatio: Summary;
125
+ /** sites whose aggregate names are identical on both readings */
126
+ unchangedSites: number;
127
+ };
128
+ /**
129
+ * Churn between an earlier and a later trim reading. Names are compared by path, so a changed
130
+ * name is an aggregate the later reading holds and the earlier one did not. The dictionary ratio
131
+ * uses the base sizes the later reading recorded; the earlier reading carries none.
132
+ */
133
+ export declare function churnFigures(prior: readonly Row[], current: readonly Row[]): ChurnFigures;
78
134
  /** coverage shares of the coverage reading (`measurements/trim/<date>.json`), pooled over its pages */
79
135
  export declare function coverageFigures(rows: readonly Row[]): CoverageFigures & {
80
136
  pages: number;
@@ -1,4 +1,4 @@
1
- import { quantile } from './derive.js';
1
+ import { median, quantile } from './derive.js';
2
2
  import { attributionalJoules } from './model.js';
3
3
  /** {@link Quartiles} of a list, or null when it is empty */
4
4
  export function quartiles(values) {
@@ -90,6 +90,88 @@ function shares(pages, kind, field) {
90
90
  return total > 0 ? [same.reduce((t, f) => t + f[field], 0) / total] : [];
91
91
  });
92
92
  }
93
+ /** whether a URL is a Drupal aggregate: a CSS or JS file directly under a `files/css/` or `files/js/` folder */
94
+ export function isAggregate(url) {
95
+ return /\/files\/(css|js)\/[^/]+$/.test(new URL(url).pathname);
96
+ }
97
+ /** the aggregates of each site of a trim reading, one entry per path across its pages */
98
+ export function aggregatesOf(rows) {
99
+ const out = new Map();
100
+ for (const row of rows) {
101
+ const byPath = new Map();
102
+ for (const page of row.pages ?? []) {
103
+ for (const file of page.files ?? []) {
104
+ if (file.origin !== 'same' || !isAggregate(file.url))
105
+ continue;
106
+ const path = new URL(file.url).pathname;
107
+ const entry = byPath.get(path) ?? {
108
+ path,
109
+ kind: file.kind,
110
+ raw: file.raw,
111
+ transfer: null,
112
+ base: null
113
+ };
114
+ if (file.transfer !== null &&
115
+ (entry.transfer === null || file.transfer > entry.transfer))
116
+ entry.transfer = file.transfer;
117
+ entry.base ??= file.base ?? null;
118
+ byPath.set(path, entry);
119
+ }
120
+ }
121
+ out.set(row.site, [...byPath.values()]);
122
+ }
123
+ return out;
124
+ }
125
+ const summary = (xs) => ({
126
+ median: xs.length === 0 ? null : median(xs),
127
+ mean: xs.length === 0 ? null : xs.reduce((n, x) => n + x, 0) / xs.length,
128
+ sites: xs.length
129
+ });
130
+ /**
131
+ * Churn between an earlier and a later trim reading. Names are compared by path, so a changed
132
+ * name is an aggregate the later reading holds and the earlier one did not. The dictionary ratio
133
+ * uses the base sizes the later reading recorded; the earlier reading carries none.
134
+ */
135
+ export function churnFigures(prior, current) {
136
+ const before = aggregatesOf(prior);
137
+ const now = aggregatesOf(current);
138
+ const sites = [];
139
+ for (const [site, files] of now) {
140
+ const earlier = before.get(site);
141
+ if (!earlier)
142
+ continue;
143
+ const old = new Set(earlier.map((a) => a.path));
144
+ const sized = files.filter((a) => a.transfer !== null);
145
+ const changed = files.filter((a) => !old.has(a.path));
146
+ const changedSized = changed.filter((a) => a.transfer !== null);
147
+ const total = sized.reduce((n, a) => n + (a.transfer ?? 0), 0);
148
+ const changedBytes = changedSized.reduce((n, a) => n + (a.transfer ?? 0), 0);
149
+ const pairs = changed.flatMap((a) => {
150
+ const b = a.base;
151
+ return b && b.zstdAlone !== null && b.zstdDict !== null
152
+ ? [{ alone: b.zstdAlone, dict: b.zstdDict }]
153
+ : [];
154
+ });
155
+ const alone = pairs.reduce((n, p) => n + p.alone, 0);
156
+ sites.push({
157
+ site,
158
+ aggregates: files.length,
159
+ changed: changed.length,
160
+ changedShare: total > 0 ? changedBytes / total : null,
161
+ compared: pairs.length,
162
+ dictRatio: pairs.length > 0 ? pairs.reduce((n, p) => n + p.dict, 0) / alone : null,
163
+ unchanged: files.length > 0 &&
164
+ files.length === earlier.length &&
165
+ files.every((a) => old.has(a.path))
166
+ });
167
+ }
168
+ return {
169
+ sites,
170
+ changedShare: summary(sites.flatMap((s) => (s.changedShare === null ? [] : [s.changedShare]))),
171
+ dictRatio: summary(sites.flatMap((s) => (s.dictRatio === null ? [] : [s.dictRatio]))),
172
+ unchangedSites: sites.filter((s) => s.unchanged).length
173
+ };
174
+ }
93
175
  /** coverage shares of the coverage reading (`measurements/trim/<date>.json`), pooled over its pages */
94
176
  export function coverageFigures(rows) {
95
177
  const pages = rows
@@ -0,0 +1,48 @@
1
+ import { type Dataset, type DatasetContext, type DatasetRequest, type DatasetResponse } from '../protocol/dataset.js';
2
+ import { type DatasetInputs, type ProvisionPrior } from '../protocol/provision.js';
3
+ import type { Identity } from '../protocol/route.js';
4
+ /**
5
+ * The body for a request. With `since` and the current schema it is a patch of the rows that
6
+ * changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
7
+ * `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
8
+ * seed page, with the proof on the last.
9
+ */
10
+ export declare function answerDataset(dataset: Dataset, ask: DatasetRequest, ctx: DatasetContext): Promise<DatasetResponse>;
11
+ /** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
12
+ export declare const NATIVE_LOOK_DAYS = 1;
13
+ /** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
14
+ export declare const NOT_MODIFIED_BYTES = 220;
15
+ /** what the server has seen of one `provision: 'auto'` dataset, in memory */
16
+ export type DatasetStats = {
17
+ /** compressed bytes of a full seed, once one has been built */
18
+ seedBytes?: number;
19
+ /** compressed bytes of a patch: the server's own one-update patch first, then an exponentially weighted mean of the patches answered */
20
+ patchBytes?: number;
21
+ /** the first and latest cursor the server answered, with the time */
22
+ first?: {
23
+ cursor: number;
24
+ at: number;
25
+ };
26
+ last?: {
27
+ cursor: number;
28
+ at: number;
29
+ };
30
+ };
31
+ /** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
32
+ export declare function datasetInputs(seed: number, stats: DatasetStats, prior: ProvisionPrior): DatasetInputs;
33
+ /** the admission inputs `respondDataset` needs from the server: its prior and what it has seen, by dataset name */
34
+ export type Admission = {
35
+ prior: ProvisionPrior;
36
+ stats: Map<string, DatasetStats>;
37
+ };
38
+ /** who is asking a principal dataset, and the `Spool-Partition` value that names them */
39
+ export type DatasetPrincipal = {
40
+ identity: Identity;
41
+ partition: string;
42
+ };
43
+ /**
44
+ * answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
45
+ * first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
46
+ * with the control that bound.
47
+ */
48
+ export declare function respondDataset(request: Request, url: URL, datasets: ReadonlyMap<string, Dataset>, admission: Admission, who?: () => Promise<DatasetPrincipal | null>): Promise<Response>;
@@ -0,0 +1,308 @@
1
+ import { DATASET_PAGE_ROWS, parseDatasetRequest } from '../protocol/dataset.js';
2
+ import { HEADERS } from '../protocol/envelope.js';
3
+ import { ProtocolError } from '../protocol/errors.js';
4
+ import { datasetBenefit } from '../protocol/provision.js';
5
+ import { isSetProof, setProof } from '../protocol/query.js';
6
+ import { json } from './sync.js';
7
+ const encoder = new TextEncoder();
8
+ function bad(dataset, message) {
9
+ throw new ProtocolError('bad-dataset', `${dataset.name}: ${message}`);
10
+ }
11
+ function limit(dataset, message) {
12
+ throw new ProtocolError('dataset-limit', `${dataset.name}: ${message}`);
13
+ }
14
+ function checkCursor(dataset, value, what) {
15
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
16
+ return bad(dataset, `${what} must be a non-negative integer`);
17
+ }
18
+ return value;
19
+ }
20
+ function checkProof(dataset, proof) {
21
+ if (proof === undefined)
22
+ return undefined;
23
+ if (!isSetProof(proof))
24
+ return bad(dataset, 'a proof must be a set proof');
25
+ return proof;
26
+ }
27
+ /** the key of a row, checked: a plain object with a non-empty string key */
28
+ function keyed(dataset, row) {
29
+ if (typeof row !== 'object' || row === null || Array.isArray(row)) {
30
+ return bad(dataset, 'a row must be an object');
31
+ }
32
+ const key = dataset.key(row);
33
+ if (typeof key !== 'string' || key === '')
34
+ return bad(dataset, 'key(row) must be a non-empty string');
35
+ return [key, row];
36
+ }
37
+ async function* rowsOf(dataset, ctx, after) {
38
+ const source = await dataset.seed(after === undefined ? ctx : { ...ctx, after });
39
+ if (Array.isArray(source)) {
40
+ for (const row of source)
41
+ yield row;
42
+ return;
43
+ }
44
+ for await (const page of source) {
45
+ if (!Array.isArray(page))
46
+ return bad(dataset, 'seed must give rows or pages of rows');
47
+ for (const row of page)
48
+ yield row;
49
+ }
50
+ }
51
+ /**
52
+ * One pass over the seed in key order. Keeps the first `size` rows after `after`, notes whether
53
+ * another follows, and with `admit` keeps going to count rows and bytes against the dataset's limits.
54
+ */
55
+ async function scan(dataset, ctx, size, after, admit) {
56
+ const page = new Map();
57
+ let more = false;
58
+ let count = 0;
59
+ let bytes = 0;
60
+ let previous;
61
+ for await (const raw of rowsOf(dataset, ctx, after)) {
62
+ const [key, row] = keyed(dataset, raw);
63
+ if (after !== undefined && key <= after)
64
+ continue;
65
+ if (previous !== undefined && key <= previous) {
66
+ return bad(dataset, 'seed rows must come in strictly ascending key order');
67
+ }
68
+ previous = key;
69
+ count += 1;
70
+ if (page.size < size)
71
+ page.set(key, row);
72
+ else {
73
+ more = true;
74
+ if (!admit)
75
+ break;
76
+ }
77
+ if (admit) {
78
+ bytes += encoder.encode(JSON.stringify(row)).length;
79
+ if (count > dataset.maxRows)
80
+ limit(dataset, `a seed over ${dataset.maxRows} rows`);
81
+ if (bytes > dataset.maxBytes)
82
+ limit(dataset, `a seed over ${dataset.maxBytes} bytes`);
83
+ }
84
+ }
85
+ return { page, more, count };
86
+ }
87
+ /** the proof from reading every row; the cost a backend avoids by returning one */
88
+ async function readProof(dataset, ctx) {
89
+ const all = new Map();
90
+ let previous;
91
+ for await (const raw of rowsOf(dataset, ctx)) {
92
+ const [key, row] = keyed(dataset, raw);
93
+ if (previous !== undefined && key <= previous) {
94
+ return bad(dataset, 'seed rows must come in strictly ascending key order');
95
+ }
96
+ previous = key;
97
+ all.set(key, row);
98
+ }
99
+ return setProof(all);
100
+ }
101
+ function response(dataset, cursor, body) {
102
+ return { protocol: 1, name: dataset.name, schema: dataset.schema, cursor, ...body };
103
+ }
104
+ async function seedOf(dataset, ask, ctx) {
105
+ const head = await dataset.head(ctx);
106
+ const cursor = checkCursor(dataset, head?.cursor, 'head().cursor');
107
+ const known = checkProof(dataset, head.proof);
108
+ if (known && known.count > dataset.maxRows) {
109
+ limit(dataset, `a seed over ${dataset.maxRows} rows`);
110
+ }
111
+ const size = ask.limit ?? DATASET_PAGE_ROWS;
112
+ const { page, more } = await scan(dataset, ctx, size, ask.after, ask.after === undefined);
113
+ const puts = [...page].map(([key, row]) => ({ key, row }));
114
+ const paging = ask.after === undefined ? {} : { after: ask.after };
115
+ if (more) {
116
+ const next = puts[puts.length - 1]?.key;
117
+ return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, next });
118
+ }
119
+ const proof = known ?? (ask.after === undefined ? await setProof(page) : await readProof(dataset, ctx));
120
+ return response(dataset, cursor, { reset: true, puts, deletes: [], ...paging, proof });
121
+ }
122
+ async function patchOf(dataset, since, ctx) {
123
+ const oldest = checkCursor(dataset, await dataset.horizon(ctx), 'horizon()');
124
+ if (since < oldest)
125
+ return null;
126
+ const found = await dataset.changes(since, ctx);
127
+ if (found === 'reset')
128
+ return null;
129
+ const cursor = checkCursor(dataset, found?.cursor, 'changes().cursor');
130
+ if (since > cursor)
131
+ return null;
132
+ const wanted = [...new Set(found.puts)].sort();
133
+ if (wanted.some((key) => typeof key !== 'string' || key === '')) {
134
+ return bad(dataset, 'changes().puts must be non-empty strings');
135
+ }
136
+ const gone = new Map();
137
+ for (const del of found.deletes) {
138
+ if (typeof del?.key !== 'string' || del.key === '') {
139
+ return bad(dataset, 'changes().deletes need a key');
140
+ }
141
+ gone.set(del.key, {
142
+ key: del.key,
143
+ cursor: checkCursor(dataset, del.cursor, 'a delete cursor')
144
+ });
145
+ }
146
+ const asked = new Set(wanted);
147
+ for (const key of gone.keys()) {
148
+ if (asked.has(key))
149
+ return bad(dataset, `${key} is both a put and a delete`);
150
+ }
151
+ const read = new Map();
152
+ if (wanted.length > 0) {
153
+ for (const raw of await dataset.read(wanted, ctx)) {
154
+ const [key, row] = keyed(dataset, raw);
155
+ if (!asked.has(key))
156
+ return bad(dataset, `read returned ${key}, which was not asked for`);
157
+ read.set(key, row);
158
+ }
159
+ }
160
+ for (const key of wanted) {
161
+ if (!read.has(key))
162
+ gone.set(key, { key, cursor });
163
+ }
164
+ const proof = checkProof(dataset, found.proof) ?? (await readProof(dataset, ctx));
165
+ return response(dataset, cursor, {
166
+ reset: false,
167
+ puts: wanted.flatMap((key) => read.has(key) ? [{ key, row: read.get(key) }] : []),
168
+ deletes: [...gone.values()].sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)),
169
+ proof
170
+ });
171
+ }
172
+ /**
173
+ * The body for a request. With `since` and the current schema it is a patch of the rows that
174
+ * changed, read by key. A `since` before the horizon, a schema that differs, a backend that says
175
+ * `reset` or a device ahead of the backend gets the first seed page instead. Without `since` it is a
176
+ * seed page, with the proof on the last.
177
+ */
178
+ export async function answerDataset(dataset, ask, ctx) {
179
+ if (ask.since !== undefined && ask.schema === dataset.schema) {
180
+ const patch = await patchOf(dataset, ask.since, ctx);
181
+ if (patch)
182
+ return patch;
183
+ }
184
+ return seedOf(dataset, ask, ctx);
185
+ }
186
+ /** days between a device's looks at a page, for the chance a native copy sees a change (L3) */
187
+ export const NATIVE_LOOK_DAYS = 1;
188
+ /** bytes of the 304 a plain HTTP cache gets back when nothing changed (L1, `measurements/dataset-baseline/2026-11-15.json`) */
189
+ export const NOT_MODIFIED_BYTES = 220;
190
+ const DAY_MS = 86_400_000;
191
+ const PATCH_WEIGHT = 0.2;
192
+ /** the decision's inputs for one dataset; a size not seen yet takes its L3 stand-in */
193
+ export function datasetInputs(seed, stats, prior) {
194
+ const perDay = updatesPerDay(stats) ?? prior.changesPerDay;
195
+ const changed = 1 - Math.exp(-perDay * NATIVE_LOOK_DAYS);
196
+ return {
197
+ seedBytes: seed,
198
+ patchBytes: stats.patchBytes ?? seed,
199
+ updatesPerDay: perDay,
200
+ v03: { seedBytes: seed, refetchBytes: seed },
201
+ nativeBytesPerView: changed * seed + (1 - changed) * NOT_MODIFIED_BYTES
202
+ };
203
+ }
204
+ function updatesPerDay(stats) {
205
+ const { first, last } = stats;
206
+ if (!first || !last || last.at <= first.at || last.cursor < first.cursor)
207
+ return undefined;
208
+ return ((last.cursor - first.cursor) * DAY_MS) / (last.at - first.at);
209
+ }
210
+ async function gzipBytes(text) {
211
+ const packed = new Blob([text]).stream().pipeThrough(new CompressionStream('gzip'));
212
+ return (await new Response(packed).arrayBuffer()).byteLength;
213
+ }
214
+ // walks every page of a seed without sending it, so the server knows what a device would fetch
215
+ async function measureSeed(dataset, stats, ctx) {
216
+ let total = 0;
217
+ let after;
218
+ for (;;) {
219
+ const ask = after === undefined ? { name: dataset.name } : { name: dataset.name, after };
220
+ const body = await answerDataset(dataset, ask, ctx);
221
+ total += await gzipBytes(JSON.stringify(body));
222
+ if (body.next === undefined)
223
+ break;
224
+ after = body.next;
225
+ }
226
+ stats.seedBytes = total;
227
+ return total;
228
+ }
229
+ // built before any device holds the dataset, so the first decision has a patch to weigh
230
+ async function measurePatch(dataset, stats, ctx) {
231
+ const { cursor } = await dataset.head(ctx);
232
+ if (cursor < 1)
233
+ return;
234
+ const body = await answerDataset(dataset, { name: dataset.name, schema: dataset.schema, since: cursor - 1 }, ctx);
235
+ if (!body.reset)
236
+ stats.patchBytes = await gzipBytes(JSON.stringify(body));
237
+ }
238
+ /**
239
+ * Answers a principal dataset. Everything the dataset's functions see comes from the identity the
240
+ * server resolved, never from the URL, and a request with no identity gets `401`. The answer is
241
+ * never shared-cacheable and names the partition it belongs to, which the device checks before it
242
+ * applies a row.
243
+ */
244
+ async function respondPrincipalDataset(request, url, dataset, ask, who) {
245
+ const caller = who ? await who() : null;
246
+ if (!caller)
247
+ return json({ error: 'sign in' }, 401, { 'cache-control': 'private, no-store' });
248
+ let body;
249
+ try {
250
+ body = await answerDataset(dataset, ask, { request, url, principal: caller.identity });
251
+ }
252
+ catch (error) {
253
+ if (error instanceof ProtocolError && error.code === 'dataset-forbidden') {
254
+ return json({ error: 'forbidden' }, 403, { 'cache-control': 'private, no-store' });
255
+ }
256
+ throw error;
257
+ }
258
+ return json(body, 200, {
259
+ [HEADERS.protocol]: '1',
260
+ [HEADERS.partition]: caller.partition,
261
+ 'cache-control': 'private, no-store'
262
+ });
263
+ }
264
+ /**
265
+ * answers `/.spool/dataset`; `datasets` is keyed by name. A `provision: 'auto'` dataset answers its
266
+ * first seed only while `datasetBenefit` says it beats both controls, and otherwise answers `409`
267
+ * with the control that bound.
268
+ */
269
+ export async function respondDataset(request, url, datasets, admission, who) {
270
+ const ask = parseDatasetRequest(url.searchParams);
271
+ const dataset = datasets.get(ask.name);
272
+ if (!dataset)
273
+ return json({ error: 'no dataset' }, 404);
274
+ if (dataset.scope === 'principal')
275
+ return respondPrincipalDataset(request, url, dataset, ask, who);
276
+ const ctx = { request, url };
277
+ if (dataset.provision !== 'auto') {
278
+ const body = await answerDataset(dataset, ask, ctx);
279
+ return json(body, 200, { [HEADERS.protocol]: '1' });
280
+ }
281
+ const stats = admission.stats.get(dataset.name) ?? {};
282
+ admission.stats.set(dataset.name, stats);
283
+ if (ask.since === undefined && ask.after === undefined) {
284
+ const seedBytes = stats.seedBytes ?? (await measureSeed(dataset, stats, ctx));
285
+ if (stats.patchBytes === undefined)
286
+ await measurePatch(dataset, stats, ctx);
287
+ const inputs = datasetInputs(seedBytes, stats, admission.prior);
288
+ const decision = datasetBenefit(inputs, admission.prior);
289
+ if (!decision.ship) {
290
+ return json({ error: 'declined', controls: decision.controls }, 409, {
291
+ [HEADERS.provision]: 'declined',
292
+ [HEADERS.provisionReason]: decision.controls
293
+ });
294
+ }
295
+ }
296
+ const body = await answerDataset(dataset, ask, ctx);
297
+ const now = Date.now();
298
+ stats.first ??= { cursor: body.cursor, at: now };
299
+ stats.last = { cursor: body.cursor, at: now };
300
+ if (!body.reset) {
301
+ const bytes = await gzipBytes(JSON.stringify(body));
302
+ stats.patchBytes =
303
+ stats.patchBytes === undefined
304
+ ? bytes
305
+ : (1 - PATCH_WEIGHT) * stats.patchBytes + PATCH_WEIGHT * bytes;
306
+ }
307
+ return json(body, 200, { [HEADERS.protocol]: '1' });
308
+ }