@drupflare/spool 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +256 -22
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +2 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/media/embeds.d.ts +3 -2
  11. package/dist/media/embeds.js +16 -1
  12. package/dist/protocol/envelope.d.ts +2 -0
  13. package/dist/protocol/envelope.js +12 -2
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/hash.d.ts +2 -0
  16. package/dist/protocol/hash.js +8 -0
  17. package/dist/protocol/index.d.ts +4 -0
  18. package/dist/protocol/index.js +4 -0
  19. package/dist/protocol/inline.d.ts +14 -0
  20. package/dist/protocol/inline.js +43 -0
  21. package/dist/protocol/navigation.d.ts +81 -0
  22. package/dist/protocol/navigation.js +249 -0
  23. package/dist/protocol/normalise.d.ts +4 -3
  24. package/dist/protocol/normalise.js +6 -4
  25. package/dist/protocol/provision.d.ts +6 -1
  26. package/dist/protocol/provision.js +15 -5
  27. package/dist/protocol/query.d.ts +80 -0
  28. package/dist/protocol/query.js +153 -0
  29. package/dist/protocol/route.d.ts +80 -4
  30. package/dist/protocol/route.js +63 -0
  31. package/dist/proxy/bases.d.ts +53 -0
  32. package/dist/proxy/bases.js +76 -0
  33. package/dist/proxy/index.d.ts +1 -0
  34. package/dist/proxy/index.js +1 -0
  35. package/dist/proxy/proxy.js +8 -2
  36. package/dist/report/capacity.d.ts +59 -0
  37. package/dist/report/capacity.js +76 -0
  38. package/dist/report/channels.d.ts +60 -0
  39. package/dist/report/channels.js +65 -0
  40. package/dist/report/index.d.ts +3 -0
  41. package/dist/report/index.js +3 -0
  42. package/dist/report/institution.d.ts +97 -0
  43. package/dist/report/institution.js +79 -0
  44. package/dist/report/ledger.d.ts +28 -0
  45. package/dist/report/ledger.js +30 -0
  46. package/dist/report/measurement.d.ts +1 -1
  47. package/dist/report/measurement.js +4 -1
  48. package/dist/report/traffic.d.ts +11 -2
  49. package/dist/report/traffic.js +13 -3
  50. package/dist/server/embedded.d.ts +3 -3
  51. package/dist/server/embedded.js +3 -3
  52. package/dist/server/index.d.ts +1 -0
  53. package/dist/server/index.js +1 -0
  54. package/dist/server/spool.d.ts +20 -0
  55. package/dist/server/spool.js +62 -9
  56. package/dist/sw/render.d.ts +3 -1
  57. package/dist/sw/render.js +14 -1
  58. package/dist/sw/replica.d.ts +2 -0
  59. package/dist/sw/replica.js +6 -0
  60. package/dist/sw/router.js +2 -2
  61. package/dist/sw/runtime.js +104 -9
  62. package/package.json +4 -2
@@ -0,0 +1,153 @@
1
+ import { shellHash } from './hash.js';
2
+ /** the refusal reason when the stored items no longer match their proof */
3
+ export const PROOF_MISMATCH = 'dataset proof does not match its items';
4
+ const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
5
+ const own = (value, key) => Object.hasOwn(value, key) ? value[key] : undefined;
6
+ function canonical(value) {
7
+ const convert = value?.toJSON;
8
+ if (typeof value === 'object' && typeof convert === 'function') {
9
+ return canonical(convert.call(value));
10
+ }
11
+ if (Array.isArray(value))
12
+ return `[${value.map((v) => canonical(v) ?? 'null').join(',')}]`;
13
+ if (isPlain(value)) {
14
+ const parts = [];
15
+ for (const key of Object.keys(value).sort()) {
16
+ const part = canonical(value[key]);
17
+ if (part !== undefined)
18
+ parts.push(`${JSON.stringify(key)}:${part}`);
19
+ }
20
+ return `{${parts.join(',')}}`;
21
+ }
22
+ return JSON.stringify(value);
23
+ }
24
+ /** JSON with object keys sorted and no whitespace, so equal values give equal text on any engine */
25
+ export function canonicalJson(value) {
26
+ return canonical(value) ?? 'null';
27
+ }
28
+ /** the proof a query route's `load` result carries for its collection */
29
+ export async function datasetProof(items) {
30
+ return { count: items.length, digest: await shellHash(canonicalJson(items)) };
31
+ }
32
+ /** whether `data` holds the collection at `schema.list` and a `proof` that matches it */
33
+ export async function verifyProof(schema, data) {
34
+ if (!isPlain(data))
35
+ return false;
36
+ const items = own(data, schema.list);
37
+ const proof = own(data, 'proof');
38
+ if (!Array.isArray(items) || !isPlain(proof))
39
+ return false;
40
+ if (proof['count'] !== items.length || typeof proof['digest'] !== 'string')
41
+ return false;
42
+ return (await shellHash(canonicalJson(items))) === proof['digest'];
43
+ }
44
+ const numberOf = (text) => text.trim() !== '' && Number.isFinite(Number(text)) ? Number(text) : null;
45
+ const rank = (value) => typeof value === 'number' ? 0 : typeof value === 'string' ? 1 : 2;
46
+ /** numbers before strings before anything else; numbers by value, strings by code unit */
47
+ function compare(a, b) {
48
+ const [ra, rb] = [rank(a), rank(b)];
49
+ if (ra !== rb)
50
+ return ra - rb;
51
+ if (ra === 2)
52
+ return 0;
53
+ return a < b
54
+ ? -1
55
+ : a > b
56
+ ? 1
57
+ : 0;
58
+ }
59
+ /**
60
+ * Applies a query string to the collection in `data`: filters by equality, sorts stably (ties keep
61
+ * the stored order), and pages. Returns `null` for anything the schema does not declare, so the
62
+ * caller goes to the network: an unknown parameter, a number filter that is not a finite number, a
63
+ * `sort` that is repeated or not a listed field, a `page` that is not a positive integer or lies
64
+ * past the last, or an item that is not an object. An empty query string selects everything.
65
+ */
66
+ export function evaluateQuery(schema, data, search) {
67
+ const list = isPlain(data) ? own(data, schema.list) : undefined;
68
+ if (!Array.isArray(list) || !list.every(isPlain))
69
+ return null;
70
+ const params = new URLSearchParams(search);
71
+ const filter = schema.filter ?? {};
72
+ for (const key of new Set(params.keys())) {
73
+ if (key !== 'sort' && key !== 'page' && !Object.hasOwn(filter, key))
74
+ return null;
75
+ }
76
+ let items = list;
77
+ for (const [field, type] of Object.entries(filter)) {
78
+ const wanted = params.getAll(field);
79
+ if (wanted.length === 0)
80
+ continue;
81
+ if (type === 'number') {
82
+ const numbers = wanted.map(numberOf);
83
+ if (numbers.includes(null))
84
+ return null;
85
+ items = items.filter((item) => {
86
+ const value = own(item, field);
87
+ return typeof value === 'number' && numbers.includes(value);
88
+ });
89
+ }
90
+ else {
91
+ items = items.filter((item) => {
92
+ const value = own(item, field);
93
+ const text = typeof value === 'string' ||
94
+ typeof value === 'number' ||
95
+ typeof value === 'boolean'
96
+ ? String(value)
97
+ : undefined;
98
+ return text !== undefined && wanted.includes(text);
99
+ });
100
+ }
101
+ }
102
+ const sorts = params.getAll('sort');
103
+ if (sorts.length > 1)
104
+ return null;
105
+ items = [...items];
106
+ if (sorts.length === 1) {
107
+ const raw = sorts[0];
108
+ const listed = schema.sort ?? [];
109
+ const descending = !listed.includes(raw) && raw.startsWith('-');
110
+ const field = descending ? raw.slice(1) : raw;
111
+ if (!listed.includes(field))
112
+ return null;
113
+ items.sort((a, b) => {
114
+ const order = compare(own(a, field), own(b, field));
115
+ return descending ? -order : order;
116
+ });
117
+ }
118
+ const pageParams = params.getAll('page');
119
+ if (pageParams.length > 1)
120
+ return null;
121
+ let page = 1;
122
+ if (pageParams.length === 1) {
123
+ if (!/^[1-9]\d{0,14}$/.test(pageParams[0]))
124
+ return null;
125
+ page = Number(pageParams[0]);
126
+ }
127
+ const total = items.length;
128
+ const size = schema.pageSize;
129
+ const pages = size === undefined ? 1 : Math.max(1, Math.ceil(total / size));
130
+ if (page > pages)
131
+ return null;
132
+ return {
133
+ items: size === undefined ? items : items.slice((page - 1) * size, page * size),
134
+ total,
135
+ page,
136
+ pages
137
+ };
138
+ }
139
+ /**
140
+ * The data a template or renderer gets for a query URL: the stored data with `schema.list` replaced
141
+ * by the page's items and a `query` key added. Refuses when the proof fails or the query is not one
142
+ * the schema declares.
143
+ */
144
+ export async function shapeQuery(schema, data, search) {
145
+ if (!(await verifyProof(schema, data)))
146
+ return { ok: false, reason: PROOF_MISMATCH };
147
+ const result = evaluateQuery(schema, data, search);
148
+ if (!result)
149
+ return { ok: false, reason: 'query is not one the route declares' };
150
+ const { items, total, page, pages } = result;
151
+ const view = { total, page, pages };
152
+ return { ok: true, data: { ...data, [schema.list]: items, query: view } };
153
+ }
@@ -1,6 +1,7 @@
1
1
  import type { AssemblyBody, Scope } from './artifact.js';
2
2
  import type { Collector } from './graph.js';
3
3
  import { type Params } from './pattern.js';
4
+ import type { QuerySchema } from './query.js';
4
5
  import { type CompiledTemplate, type TemplateSource } from './template/compile.js';
5
6
  import type { Mode } from './view.js';
6
7
  /**
@@ -17,14 +18,18 @@ export type Retention = 'persistent' | 'session';
17
18
  export type FreshnessMode = 'hard' | 'soft';
18
19
  /** the staleness contract of a route; `maxStale` is in seconds */
19
20
  export type Freshness = {
21
+ /** seconds a copy may age before the mode applies */
20
22
  maxStale: number;
23
+ /** what happens past maxStale: block on a sync, or show first and sync later */
21
24
  mode: FreshnessMode;
22
25
  };
23
26
  /** whether the worker may show a snapshot it cannot validate because the network is down */
24
27
  export type OfflineRule = 'serve' | 'refuse';
25
28
  /** who is asking, as the app's `identity.resolve` reports it */
26
29
  export type Identity = {
30
+ /** the principal's id, as identity.resolve reports it */
27
31
  principal: string;
32
+ /** role names the principal holds; absent when none */
28
33
  roles?: readonly string[];
29
34
  };
30
35
  /** when a soft route past its bound refreshes: right after the view, or at the next network moment */
@@ -34,24 +39,38 @@ export type SyncRule = 'background' | 'deferred';
34
39
  * and return the reference to embed in the page's data; see {@link Collector}.
35
40
  */
36
41
  export type RouteContext = {
42
+ /** the request being answered */
37
43
  request: Request;
44
+ /** the request URL, parsed */
38
45
  url: URL;
46
+ /** the values the route pattern captured */
39
47
  params: Params;
48
+ /** the principal the app resolved for this request; null when none */
40
49
  identity: Identity | null;
41
50
  } & Pick<Collector, 'record' | 'projection'>;
42
51
  /** public routes default to 15 minutes soft, principal routes to 5 minutes hard */
43
52
  export declare const DEFAULT_FRESHNESS: Record<Scope, Freshness>;
44
- /** a page a route's catalogue suggests, with the share of devices expected to open it next */
53
+ /**
54
+ * a page a route's catalogue suggests, with the share of devices expected to open it next and,
55
+ * when the share is an estimate, `n`, the observations behind it
56
+ */
45
57
  export type CatalogueGuess = {
58
+ /** the page path a device may open next */
46
59
  path: string;
60
+ /** share of devices expected to open the page next, from 0 to 1 */
47
61
  p: number;
62
+ /** observations behind p; absent when p is a fixed prior */
63
+ n?: number;
48
64
  };
49
65
  /** the declaration fields every strategy shares */
50
66
  export type RouteCommon = {
67
+ /** public or principal; public when absent */
51
68
  scope?: Scope;
69
+ /** overrides the scope's default maxStale and mode */
52
70
  freshness?: Partial<Freshness>;
53
71
  /** keep the artifact across worker restarts; default true */
54
72
  persist?: boolean;
73
+ /** serve shows a snapshot offline, refuse does not; default follows the scope */
55
74
  offline?: OfflineRule;
56
75
  /** `shadow` renders locally and compares but never serves; the server's mode when absent */
57
76
  mode?: Mode;
@@ -63,7 +82,9 @@ export type RouteCommon = {
63
82
  retention?: Retention;
64
83
  /** overrides the provisioning priors for this route when the server decides whether to ship it */
65
84
  provision?: {
85
+ /** chance a provisioned artifact serves a later view; the server's prior when absent */
66
86
  reuse?: number;
87
+ /** later views one provisioned artifact is expected to serve; the server's prior when absent */
67
88
  futureViews?: number;
68
89
  };
69
90
  /**
@@ -78,13 +99,26 @@ export type RouteCommon = {
78
99
  };
79
100
  /** a route answered by filling placeholders in a stored shell */
80
101
  export type AssemblyRoute = RouteCommon & {
102
+ /** fills placeholders in a stored shell */
81
103
  strategy: 'assembly';
104
+ /** returns the body that fills the shell's placeholders */
82
105
  load(ctx: RouteContext): AssemblyBody | Promise<AssemblyBody>;
83
106
  };
107
+ /** the declaration of a public route that answers query strings from a collection it holds */
108
+ export type QueryRoute = {
109
+ /**
110
+ * The query strings the worker answers from the stored collection. `load` then returns an
111
+ * object whose `list` key holds the array and which carries `proof: await datasetProof(items)`.
112
+ */
113
+ query?: QuerySchema;
114
+ };
84
115
  /** a route answered by rendering stored JSON through a compiled template */
85
- export type TemplateRoute = RouteCommon & {
116
+ export type TemplateRoute = RouteCommon & QueryRoute & {
117
+ /** renders stored JSON through the template */
86
118
  strategy: 'template';
119
+ /** the template source, compiled once when the route is defined */
87
120
  template: TemplateSource;
121
+ /** returns the JSON the template renders */
88
122
  load(ctx: RouteContext): unknown;
89
123
  };
90
124
  /**
@@ -94,10 +128,15 @@ export type TemplateRoute = RouteCommon & {
94
128
  * is answered in `serve` mode only.
95
129
  */
96
130
  export type StaticRoute = Omit<RouteCommon, 'scope' | 'persist' | 'mode' | 'retention'> & {
131
+ /** the file's response, stored and shown with no request */
97
132
  strategy: 'static';
133
+ /** always absent; a static route is public */
98
134
  scope?: undefined;
135
+ /** always absent; a static route is always kept */
99
136
  persist?: undefined;
137
+ /** always absent; a static route has no mode */
100
138
  mode?: undefined;
139
+ /** always absent; a static route has no load */
101
140
  load?: undefined;
102
141
  };
103
142
  /**
@@ -105,16 +144,20 @@ export type StaticRoute = Omit<RouteCommon, 'scope' | 'persist' | 'mode' | 'rete
105
144
  * `@drupflare/spool/sw` takes a renderer keyed by this pattern. Without one the worker refuses the
106
145
  * local view and the network answers.
107
146
  */
108
- export type ProjectionRoute = RouteCommon & {
147
+ export type ProjectionRoute = RouteCommon & QueryRoute & {
148
+ /** renders stored JSON with the renderer the bundled worker holds */
109
149
  strategy: 'projection';
150
+ /** returns the JSON the renderer receives */
110
151
  load(ctx: RouteContext): unknown;
111
152
  };
112
153
  /**
113
154
  * A route for a JSON read. The worker answers a GET (not a navigation) from the stored value with
114
155
  * `JSON.stringify`, so the application's own handler has to return the same JSON `load` does.
115
156
  */
116
- export type DataRoute = RouteCommon & {
157
+ export type DataRoute = RouteCommon & QueryRoute & {
158
+ /** answers a GET with the stored JSON */
117
159
  strategy: 'data';
160
+ /** returns the JSON the local answer serializes */
118
161
  load(ctx: RouteContext): unknown;
119
162
  /** the content type of the local answer; default `application/json` */
120
163
  contentType?: string;
@@ -123,35 +166,63 @@ export type DataRoute = RouteCommon & {
123
166
  export type RouteDeclaration = AssemblyRoute | TemplateRoute | StaticRoute | ProjectionRoute | DataRoute;
124
167
  /** a validated route with every default applied */
125
168
  export type Route = {
169
+ /** the path pattern the route matches */
126
170
  pattern: string;
171
+ /** how the route is answered locally */
127
172
  strategy: Strategy;
173
+ /** public or principal, after the default */
128
174
  scope: Scope;
175
+ /** the staleness contract with every default applied */
129
176
  freshness: Freshness;
177
+ /** whether the artifact survives a worker restart; true by default */
130
178
  persist: boolean;
179
+ /** whether a snapshot may be shown while offline, after the default */
131
180
  offline: OfflineRule;
181
+ /** serve or shadow; absent means the server decides */
132
182
  mode?: Mode;
183
+ /** when a soft route past its bound refreshes */
133
184
  sync: SyncRule;
185
+ /** at-rest key lifetime; set only on principal routes */
134
186
  retention?: Retention;
187
+ /** content type of a data route's local answer */
135
188
  contentType?: string;
189
+ /** provisioning priors that override the server's for this route */
136
190
  provision?: {
191
+ /** chance a provisioned artifact serves a later view; the server's prior when absent */
137
192
  reuse?: number;
193
+ /** later views one provisioned artifact is expected to serve; the server's prior when absent */
138
194
  futureViews?: number;
139
195
  };
196
+ /** the compiled template of a template route */
140
197
  template?: CompiledTemplate;
198
+ /** the route's catalogue function, when it declares one */
141
199
  catalogue?: RouteCommon['catalogue'];
200
+ /** compressed bytes of the document the route renders */
142
201
  documentBytes?: number;
202
+ /** query strings the worker answers from the stored collection */
203
+ query?: QuerySchema;
204
+ /** tags the artifact is built from; a change to one invalidates it */
143
205
  depends(ctx: RouteContext): readonly string[] | Promise<readonly string[]>;
206
+ /** returns the data the artifact is built from */
144
207
  load(ctx: RouteContext): unknown;
145
208
  };
146
209
  /** the part of a route the service worker needs; functions stay on the server */
147
210
  export type SerializedRoute = {
211
+ /** the path pattern the route matches */
148
212
  pattern: string;
213
+ /** how the route is answered locally */
149
214
  strategy: Strategy;
215
+ /** public or principal */
150
216
  scope: Scope;
217
+ /** the staleness contract, with maxStale in seconds */
151
218
  freshness: Freshness;
219
+ /** whether the artifact survives a worker restart */
152
220
  persist: boolean;
221
+ /** whether a snapshot may be shown while offline */
153
222
  offline: OfflineRule;
223
+ /** serve or shadow; absent means the server decides */
154
224
  mode?: Mode;
225
+ /** when a soft route past its bound refreshes */
155
226
  sync: SyncRule;
156
227
  /** a principal route's at-rest key; absent on a public route */
157
228
  retention?: Retention;
@@ -161,6 +232,8 @@ export type SerializedRoute = {
161
232
  version?: string;
162
233
  /** the route declares a `catalogue`, so the worker asks for one when a flush goes out on it */
163
234
  catalogue?: true;
235
+ /** the query strings the worker answers from the stored collection of a public route */
236
+ query?: QuerySchema;
164
237
  };
165
238
  /**
166
239
  * Declares a route. Validates the pattern and the freshness contract, compiles a template, and
@@ -170,9 +243,12 @@ export type SerializedRoute = {
170
243
  export declare function defineRoute(pattern: string, declaration: RouteDeclaration): Route;
171
244
  /** the first route whose pattern fits `path`, with the params it captured */
172
245
  export declare function matchRoute<T extends {
246
+ /** the pattern a route is matched by */
173
247
  pattern: string;
174
248
  }>(routes: readonly T[], path: string): {
249
+ /** the first route whose pattern matches the path */
175
250
  route: T;
251
+ /** the values the pattern captured from the path */
176
252
  params: Params;
177
253
  } | null;
178
254
  /** strips a route to what the service worker config carries */
@@ -9,6 +9,58 @@ export const DEFAULT_FRESHNESS = {
9
9
  function bad(pattern, message) {
10
10
  throw new ProtocolError('bad-route', `${pattern}: ${message}`);
11
11
  }
12
+ const QUERY_KEYS = ['list', 'filter', 'sort', 'pageSize'];
13
+ const RESERVED = ['sort', 'page'];
14
+ /** the validated copy of a route's `query`; throws `bad-route` for a schema the worker could not apply */
15
+ function checkQuery(pattern, value) {
16
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
17
+ return bad(pattern, 'query must be an object');
18
+ }
19
+ const given = value;
20
+ for (const key of Object.keys(given)) {
21
+ if (!QUERY_KEYS.includes(key))
22
+ return bad(pattern, `unknown query option "${key}"`);
23
+ }
24
+ const { list, filter, sort, pageSize } = given;
25
+ if (typeof list !== 'string' || list === '')
26
+ return bad(pattern, 'query.list must be a name');
27
+ if (list === 'proof' || list === 'query') {
28
+ return bad(pattern, `query.list cannot be "${list}", which the data already uses`);
29
+ }
30
+ const out = { list };
31
+ if (filter !== undefined) {
32
+ if (typeof filter !== 'object' || filter === null || Array.isArray(filter)) {
33
+ return bad(pattern, 'query.filter must be an object');
34
+ }
35
+ out.filter = {};
36
+ for (const [name, type] of Object.entries(filter)) {
37
+ if (name === '' || RESERVED.includes(name)) {
38
+ return bad(pattern, `query.filter cannot name "${name}"`);
39
+ }
40
+ if (type !== 'string' && type !== 'number') {
41
+ return bad(pattern, `query.filter.${name} must be "string" or "number"`);
42
+ }
43
+ out.filter[name] = type;
44
+ }
45
+ }
46
+ if (sort !== undefined) {
47
+ if (!Array.isArray(sort))
48
+ return bad(pattern, 'query.sort must be a list of field names');
49
+ for (const field of sort) {
50
+ if (typeof field !== 'string' || field === '' || RESERVED.includes(field)) {
51
+ return bad(pattern, `query.sort cannot name ${JSON.stringify(field)}`);
52
+ }
53
+ }
54
+ out.sort = [...sort];
55
+ }
56
+ if (pageSize !== undefined) {
57
+ if (typeof pageSize !== 'number' || !Number.isSafeInteger(pageSize) || pageSize < 1) {
58
+ return bad(pattern, 'query.pageSize must be a positive integer');
59
+ }
60
+ out.pageSize = pageSize;
61
+ }
62
+ return out;
63
+ }
12
64
  /**
13
65
  * Declares a route. Validates the pattern and the freshness contract, compiles a template, and
14
66
  * fills the defaults: freshness by scope, and `offline` as `serve` for public, `refuse` for
@@ -95,6 +147,15 @@ export function defineRoute(pattern, declaration) {
95
147
  if (declaration.strategy === 'data' && declaration.contentType) {
96
148
  route.contentType = declaration.contentType;
97
149
  }
150
+ const query = declaration.query;
151
+ if (query !== undefined) {
152
+ if (scope !== 'public')
153
+ return bad(pattern, 'query applies to a public route');
154
+ if (!['template', 'projection', 'data'].includes(declaration.strategy)) {
155
+ return bad(pattern, `a ${declaration.strategy} route cannot declare query`);
156
+ }
157
+ route.query = checkQuery(pattern, query);
158
+ }
98
159
  if (declaration.strategy === 'template') {
99
160
  if (!declaration.template)
100
161
  return bad(pattern, 'a template route needs a template');
@@ -131,5 +192,7 @@ export function serializeRoute(route) {
131
192
  out.type = route.contentType;
132
193
  if (route.catalogue)
133
194
  out.catalogue = true;
195
+ if (route.query)
196
+ out.query = structuredClone(route.query);
134
197
  return out;
135
198
  }
@@ -0,0 +1,53 @@
1
+ /** one file of a crawl: its URL, its kind and its plain compressed size */
2
+ export type BaseCandidate = {
3
+ /** the URL the file was crawled from */
4
+ url: string;
5
+ /** whether the file is a stylesheet or a script */
6
+ kind: 'style' | 'script';
7
+ /** compressed size of the file alone, with no dictionary, in bytes */
8
+ plain: number;
9
+ };
10
+ /** compressed size of `file` against `dictionary`, both indexes into the candidate list */
11
+ export type DeltaSize = (file: number, dictionary: number) => number | Promise<number>;
12
+ /**
13
+ * What a candidate base is scored on. `held` counts what the other files cost against it, for a
14
+ * visitor who already holds the base. `first-visit` adds the base's own plain size, for a visitor
15
+ * who loads every file of the group and has not held it. Neither weighs how often each file is
16
+ * requested or how long a browser keeps the base, so neither is a proven minimum of total transfer.
17
+ */
18
+ export type BasePolicy = 'held' | 'first-visit';
19
+ export type BaseChoice = {
20
+ /** whether the base is a stylesheet or a script */
21
+ kind: 'style' | 'script';
22
+ /** the path up to the last slash that every member shares */
23
+ directory: string;
24
+ /** the URL of the chosen base file */
25
+ url: string;
26
+ /** how many files of the crawl share the directory and kind, the base included */
27
+ files: number;
28
+ /** mean bytes each other file costs against this base, capped at its plain size */
29
+ meanDelta: number;
30
+ };
31
+ /** the path of a URL up to and including its last slash */
32
+ export declare function dirOf(url: string): string;
33
+ /**
34
+ * The member of `members` that minimizes the sum, over the other members, of the smaller of their
35
+ * delta against it and their plain size, plus the member's own plain size under `first-visit`.
36
+ * Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
37
+ * base's own size.
38
+ */
39
+ export declare function pickBase(members: readonly number[], delta: (file: number, dictionary: number) => number, plain: (file: number) => number, policy?: BasePolicy): {
40
+ best: number;
41
+ score: number;
42
+ others: number;
43
+ };
44
+ /**
45
+ * One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
46
+ * before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
47
+ * one file has no base.
48
+ */
49
+ export declare function chooseBases(files: readonly BaseCandidate[], delta: DeltaSize, options?: {
50
+ policy?: BasePolicy;
51
+ }): Promise<BaseChoice[]>;
52
+ /** true for the exact pathnames of the chosen base URLs; a query string does not count */
53
+ export declare function baseMatcher(choices: readonly BaseChoice[]): (pathname: string) => boolean;
@@ -0,0 +1,76 @@
1
+ /** the path of a URL up to and including its last slash */
2
+ export function dirOf(url) {
3
+ const path = new URL(url).pathname;
4
+ return path.slice(0, path.lastIndexOf('/') + 1);
5
+ }
6
+ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
7
+ /**
8
+ * The member of `members` that minimizes the sum, over the other members, of the smaller of their
9
+ * delta against it and their plain size, plus the member's own plain size under `first-visit`.
10
+ * Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
11
+ * base's own size.
12
+ */
13
+ export function pickBase(members, delta, plain, policy = 'held') {
14
+ let best = -1;
15
+ let bestScore = Infinity;
16
+ let bestOthers = Infinity;
17
+ for (const c of members) {
18
+ let others = 0;
19
+ for (const f of members) {
20
+ if (f !== c)
21
+ others += Math.min(delta(f, c), plain(f));
22
+ }
23
+ const score = policy === 'first-visit' ? others + plain(c) : others;
24
+ if (score < bestScore) {
25
+ best = c;
26
+ bestScore = score;
27
+ bestOthers = others;
28
+ }
29
+ }
30
+ return { best, score: bestScore, others: bestOthers };
31
+ }
32
+ /**
33
+ * One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
34
+ * before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
35
+ * one file has no base.
36
+ */
37
+ export async function chooseBases(files, delta, options = {}) {
38
+ const at = (i) => files[i];
39
+ const groups = new Map();
40
+ for (const [i, f] of files.entries()) {
41
+ const key = JSON.stringify([f.kind, dirOf(f.url)]);
42
+ const members = groups.get(key);
43
+ if (members)
44
+ members.push(i);
45
+ else
46
+ groups.set(key, [i]);
47
+ }
48
+ const choices = [];
49
+ for (const members of groups.values()) {
50
+ if (members.length < 2)
51
+ continue;
52
+ const sorted = [...members].sort((a, b) => compare(at(a).url, at(b).url));
53
+ const sizes = new Map();
54
+ for (const f of sorted) {
55
+ for (const c of sorted) {
56
+ if (f !== c)
57
+ sizes.set(`${f}|${c}`, await delta(f, c));
58
+ }
59
+ }
60
+ const { best, others } = pickBase(sorted, (f, c) => sizes.get(`${f}|${c}`), (f) => at(f).plain, options.policy);
61
+ const { kind, url } = at(best);
62
+ choices.push({
63
+ kind,
64
+ directory: dirOf(url),
65
+ url,
66
+ files: sorted.length,
67
+ meanDelta: others / (sorted.length - 1)
68
+ });
69
+ }
70
+ return choices.sort((a, b) => compare(a.directory, b.directory) || compare(a.kind, b.kind));
71
+ }
72
+ /** true for the exact pathnames of the chosen base URLs; a query string does not count */
73
+ export function baseMatcher(choices) {
74
+ const paths = new Set(choices.map((choice) => new URL(choice.url).pathname));
75
+ return (pathname) => paths.has(pathname);
76
+ }
@@ -1,3 +1,4 @@
1
+ export { baseMatcher, chooseBases, type BaseCandidate, type BaseChoice, type BasePolicy, type DeltaSize } from './bases.js';
1
2
  export * from './dictionary.js';
2
3
  export * from './eligible.js';
3
4
  export * from './proxy.js';
@@ -1,3 +1,4 @@
1
+ export { baseMatcher, chooseBases } from './bases.js';
1
2
  export * from './dictionary.js';
2
3
  export * from './eligible.js';
3
4
  export * from './proxy.js';
@@ -211,9 +211,11 @@ export function spoolProxy(options) {
211
211
  ...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
212
212
  });
213
213
  }
214
- /** the headers the browser gets: bodies arrive decoded, so the encoding and its length go */
214
+ /** the headers the browser gets: bodies arrive decoded, so the encoding and its length go, and so does what only described the hop to the proxy */
215
215
  function tidy(source, url) {
216
216
  const headers = new Headers(source);
217
+ for (const name of HOP_BY_HOP)
218
+ headers.delete(name);
217
219
  if (headers.has('content-encoding')) {
218
220
  headers.delete('content-encoding');
219
221
  headers.delete('content-length');
@@ -371,6 +373,7 @@ export function spoolProxy(options) {
371
373
  if (url.pathname === PATHS.worker && request.method === 'GET')
372
374
  return spool.stub();
373
375
  const converting = images !== undefined && imageRequest(images, request);
376
+ let visitorRewrite = false;
374
377
  let upstream;
375
378
  try {
376
379
  upstream = await call(forward(request, url, converting));
@@ -427,6 +430,7 @@ export function spoolProxy(options) {
427
430
  videoStats.rewritten += result.rewritten;
428
431
  // a page rewritten for one visitor's connection is not another's copy
429
432
  if (video.when !== 'always') {
433
+ visitorRewrite = true;
430
434
  makePrivate(headers);
431
435
  mergeList(headers, 'vary', ['Save-Data', 'ECT']);
432
436
  }
@@ -488,7 +492,9 @@ export function spoolProxy(options) {
488
492
  path: url.pathname,
489
493
  page: Promise.resolve({ text, tags: effective })
490
494
  });
491
- return spool.decorate(request, new Response(text, init));
495
+ // a visitor-signal rewrite is not in the copy /.spool/artifact builds, so the worker asks for it
496
+ const same = !visitorRewrite && !data && !principal;
497
+ return spool.decorate(request, new Response(text, init), same ? { assembled: text } : {});
492
498
  }
493
499
  return Object.assign(handle, {
494
500
  spool,