@drupflare/spool 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +256 -22
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +2 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/media/embeds.d.ts +3 -2
- package/dist/media/embeds.js +16 -1
- package/dist/protocol/envelope.d.ts +2 -0
- package/dist/protocol/envelope.js +12 -2
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +4 -0
- package/dist/protocol/index.js +4 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/provision.d.ts +6 -1
- package/dist/protocol/provision.js +15 -5
- package/dist/protocol/query.d.ts +80 -0
- package/dist/protocol/query.js +153 -0
- package/dist/protocol/route.d.ts +80 -4
- package/dist/protocol/route.js +63 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/index.d.ts +1 -0
- package/dist/proxy/index.js +1 -0
- package/dist/proxy/proxy.js +8 -2
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/index.d.ts +3 -0
- package/dist/report/index.js +3 -0
- package/dist/report/institution.d.ts +97 -0
- package/dist/report/institution.js +79 -0
- package/dist/report/ledger.d.ts +28 -0
- package/dist/report/ledger.js +30 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +4 -1
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +62 -9
- package/dist/sw/render.d.ts +3 -1
- package/dist/sw/render.js +14 -1
- package/dist/sw/replica.d.ts +2 -0
- package/dist/sw/replica.js +6 -0
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.js +104 -9
- package/package.json +4 -2
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { shellHash } from './hash.js';
|
|
2
|
+
/** the refusal reason when the stored items no longer match their proof */
|
|
3
|
+
export const PROOF_MISMATCH = 'dataset proof does not match its items';
|
|
4
|
+
const isPlain = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
5
|
+
const own = (value, key) => Object.hasOwn(value, key) ? value[key] : undefined;
|
|
6
|
+
function canonical(value) {
|
|
7
|
+
const convert = value?.toJSON;
|
|
8
|
+
if (typeof value === 'object' && typeof convert === 'function') {
|
|
9
|
+
return canonical(convert.call(value));
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(value))
|
|
12
|
+
return `[${value.map((v) => canonical(v) ?? 'null').join(',')}]`;
|
|
13
|
+
if (isPlain(value)) {
|
|
14
|
+
const parts = [];
|
|
15
|
+
for (const key of Object.keys(value).sort()) {
|
|
16
|
+
const part = canonical(value[key]);
|
|
17
|
+
if (part !== undefined)
|
|
18
|
+
parts.push(`${JSON.stringify(key)}:${part}`);
|
|
19
|
+
}
|
|
20
|
+
return `{${parts.join(',')}}`;
|
|
21
|
+
}
|
|
22
|
+
return JSON.stringify(value);
|
|
23
|
+
}
|
|
24
|
+
/** JSON with object keys sorted and no whitespace, so equal values give equal text on any engine */
|
|
25
|
+
export function canonicalJson(value) {
|
|
26
|
+
return canonical(value) ?? 'null';
|
|
27
|
+
}
|
|
28
|
+
/** the proof a query route's `load` result carries for its collection */
|
|
29
|
+
export async function datasetProof(items) {
|
|
30
|
+
return { count: items.length, digest: await shellHash(canonicalJson(items)) };
|
|
31
|
+
}
|
|
32
|
+
/** whether `data` holds the collection at `schema.list` and a `proof` that matches it */
|
|
33
|
+
export async function verifyProof(schema, data) {
|
|
34
|
+
if (!isPlain(data))
|
|
35
|
+
return false;
|
|
36
|
+
const items = own(data, schema.list);
|
|
37
|
+
const proof = own(data, 'proof');
|
|
38
|
+
if (!Array.isArray(items) || !isPlain(proof))
|
|
39
|
+
return false;
|
|
40
|
+
if (proof['count'] !== items.length || typeof proof['digest'] !== 'string')
|
|
41
|
+
return false;
|
|
42
|
+
return (await shellHash(canonicalJson(items))) === proof['digest'];
|
|
43
|
+
}
|
|
44
|
+
const numberOf = (text) => text.trim() !== '' && Number.isFinite(Number(text)) ? Number(text) : null;
|
|
45
|
+
const rank = (value) => typeof value === 'number' ? 0 : typeof value === 'string' ? 1 : 2;
|
|
46
|
+
/** numbers before strings before anything else; numbers by value, strings by code unit */
|
|
47
|
+
function compare(a, b) {
|
|
48
|
+
const [ra, rb] = [rank(a), rank(b)];
|
|
49
|
+
if (ra !== rb)
|
|
50
|
+
return ra - rb;
|
|
51
|
+
if (ra === 2)
|
|
52
|
+
return 0;
|
|
53
|
+
return a < b
|
|
54
|
+
? -1
|
|
55
|
+
: a > b
|
|
56
|
+
? 1
|
|
57
|
+
: 0;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Applies a query string to the collection in `data`: filters by equality, sorts stably (ties keep
|
|
61
|
+
* the stored order), and pages. Returns `null` for anything the schema does not declare, so the
|
|
62
|
+
* caller goes to the network: an unknown parameter, a number filter that is not a finite number, a
|
|
63
|
+
* `sort` that is repeated or not a listed field, a `page` that is not a positive integer or lies
|
|
64
|
+
* past the last, or an item that is not an object. An empty query string selects everything.
|
|
65
|
+
*/
|
|
66
|
+
export function evaluateQuery(schema, data, search) {
|
|
67
|
+
const list = isPlain(data) ? own(data, schema.list) : undefined;
|
|
68
|
+
if (!Array.isArray(list) || !list.every(isPlain))
|
|
69
|
+
return null;
|
|
70
|
+
const params = new URLSearchParams(search);
|
|
71
|
+
const filter = schema.filter ?? {};
|
|
72
|
+
for (const key of new Set(params.keys())) {
|
|
73
|
+
if (key !== 'sort' && key !== 'page' && !Object.hasOwn(filter, key))
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
let items = list;
|
|
77
|
+
for (const [field, type] of Object.entries(filter)) {
|
|
78
|
+
const wanted = params.getAll(field);
|
|
79
|
+
if (wanted.length === 0)
|
|
80
|
+
continue;
|
|
81
|
+
if (type === 'number') {
|
|
82
|
+
const numbers = wanted.map(numberOf);
|
|
83
|
+
if (numbers.includes(null))
|
|
84
|
+
return null;
|
|
85
|
+
items = items.filter((item) => {
|
|
86
|
+
const value = own(item, field);
|
|
87
|
+
return typeof value === 'number' && numbers.includes(value);
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
items = items.filter((item) => {
|
|
92
|
+
const value = own(item, field);
|
|
93
|
+
const text = typeof value === 'string' ||
|
|
94
|
+
typeof value === 'number' ||
|
|
95
|
+
typeof value === 'boolean'
|
|
96
|
+
? String(value)
|
|
97
|
+
: undefined;
|
|
98
|
+
return text !== undefined && wanted.includes(text);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const sorts = params.getAll('sort');
|
|
103
|
+
if (sorts.length > 1)
|
|
104
|
+
return null;
|
|
105
|
+
items = [...items];
|
|
106
|
+
if (sorts.length === 1) {
|
|
107
|
+
const raw = sorts[0];
|
|
108
|
+
const listed = schema.sort ?? [];
|
|
109
|
+
const descending = !listed.includes(raw) && raw.startsWith('-');
|
|
110
|
+
const field = descending ? raw.slice(1) : raw;
|
|
111
|
+
if (!listed.includes(field))
|
|
112
|
+
return null;
|
|
113
|
+
items.sort((a, b) => {
|
|
114
|
+
const order = compare(own(a, field), own(b, field));
|
|
115
|
+
return descending ? -order : order;
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
const pageParams = params.getAll('page');
|
|
119
|
+
if (pageParams.length > 1)
|
|
120
|
+
return null;
|
|
121
|
+
let page = 1;
|
|
122
|
+
if (pageParams.length === 1) {
|
|
123
|
+
if (!/^[1-9]\d{0,14}$/.test(pageParams[0]))
|
|
124
|
+
return null;
|
|
125
|
+
page = Number(pageParams[0]);
|
|
126
|
+
}
|
|
127
|
+
const total = items.length;
|
|
128
|
+
const size = schema.pageSize;
|
|
129
|
+
const pages = size === undefined ? 1 : Math.max(1, Math.ceil(total / size));
|
|
130
|
+
if (page > pages)
|
|
131
|
+
return null;
|
|
132
|
+
return {
|
|
133
|
+
items: size === undefined ? items : items.slice((page - 1) * size, page * size),
|
|
134
|
+
total,
|
|
135
|
+
page,
|
|
136
|
+
pages
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* The data a template or renderer gets for a query URL: the stored data with `schema.list` replaced
|
|
141
|
+
* by the page's items and a `query` key added. Refuses when the proof fails or the query is not one
|
|
142
|
+
* the schema declares.
|
|
143
|
+
*/
|
|
144
|
+
export async function shapeQuery(schema, data, search) {
|
|
145
|
+
if (!(await verifyProof(schema, data)))
|
|
146
|
+
return { ok: false, reason: PROOF_MISMATCH };
|
|
147
|
+
const result = evaluateQuery(schema, data, search);
|
|
148
|
+
if (!result)
|
|
149
|
+
return { ok: false, reason: 'query is not one the route declares' };
|
|
150
|
+
const { items, total, page, pages } = result;
|
|
151
|
+
const view = { total, page, pages };
|
|
152
|
+
return { ok: true, data: { ...data, [schema.list]: items, query: view } };
|
|
153
|
+
}
|
package/dist/protocol/route.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AssemblyBody, Scope } from './artifact.js';
|
|
2
2
|
import type { Collector } from './graph.js';
|
|
3
3
|
import { type Params } from './pattern.js';
|
|
4
|
+
import type { QuerySchema } from './query.js';
|
|
4
5
|
import { type CompiledTemplate, type TemplateSource } from './template/compile.js';
|
|
5
6
|
import type { Mode } from './view.js';
|
|
6
7
|
/**
|
|
@@ -17,14 +18,18 @@ export type Retention = 'persistent' | 'session';
|
|
|
17
18
|
export type FreshnessMode = 'hard' | 'soft';
|
|
18
19
|
/** the staleness contract of a route; `maxStale` is in seconds */
|
|
19
20
|
export type Freshness = {
|
|
21
|
+
/** seconds a copy may age before the mode applies */
|
|
20
22
|
maxStale: number;
|
|
23
|
+
/** what happens past maxStale: block on a sync, or show first and sync later */
|
|
21
24
|
mode: FreshnessMode;
|
|
22
25
|
};
|
|
23
26
|
/** whether the worker may show a snapshot it cannot validate because the network is down */
|
|
24
27
|
export type OfflineRule = 'serve' | 'refuse';
|
|
25
28
|
/** who is asking, as the app's `identity.resolve` reports it */
|
|
26
29
|
export type Identity = {
|
|
30
|
+
/** the principal's id, as identity.resolve reports it */
|
|
27
31
|
principal: string;
|
|
32
|
+
/** role names the principal holds; absent when none */
|
|
28
33
|
roles?: readonly string[];
|
|
29
34
|
};
|
|
30
35
|
/** when a soft route past its bound refreshes: right after the view, or at the next network moment */
|
|
@@ -34,24 +39,38 @@ export type SyncRule = 'background' | 'deferred';
|
|
|
34
39
|
* and return the reference to embed in the page's data; see {@link Collector}.
|
|
35
40
|
*/
|
|
36
41
|
export type RouteContext = {
|
|
42
|
+
/** the request being answered */
|
|
37
43
|
request: Request;
|
|
44
|
+
/** the request URL, parsed */
|
|
38
45
|
url: URL;
|
|
46
|
+
/** the values the route pattern captured */
|
|
39
47
|
params: Params;
|
|
48
|
+
/** the principal the app resolved for this request; null when none */
|
|
40
49
|
identity: Identity | null;
|
|
41
50
|
} & Pick<Collector, 'record' | 'projection'>;
|
|
42
51
|
/** public routes default to 15 minutes soft, principal routes to 5 minutes hard */
|
|
43
52
|
export declare const DEFAULT_FRESHNESS: Record<Scope, Freshness>;
|
|
44
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* a page a route's catalogue suggests, with the share of devices expected to open it next and,
|
|
55
|
+
* when the share is an estimate, `n`, the observations behind it
|
|
56
|
+
*/
|
|
45
57
|
export type CatalogueGuess = {
|
|
58
|
+
/** the page path a device may open next */
|
|
46
59
|
path: string;
|
|
60
|
+
/** share of devices expected to open the page next, from 0 to 1 */
|
|
47
61
|
p: number;
|
|
62
|
+
/** observations behind p; absent when p is a fixed prior */
|
|
63
|
+
n?: number;
|
|
48
64
|
};
|
|
49
65
|
/** the declaration fields every strategy shares */
|
|
50
66
|
export type RouteCommon = {
|
|
67
|
+
/** public or principal; public when absent */
|
|
51
68
|
scope?: Scope;
|
|
69
|
+
/** overrides the scope's default maxStale and mode */
|
|
52
70
|
freshness?: Partial<Freshness>;
|
|
53
71
|
/** keep the artifact across worker restarts; default true */
|
|
54
72
|
persist?: boolean;
|
|
73
|
+
/** serve shows a snapshot offline, refuse does not; default follows the scope */
|
|
55
74
|
offline?: OfflineRule;
|
|
56
75
|
/** `shadow` renders locally and compares but never serves; the server's mode when absent */
|
|
57
76
|
mode?: Mode;
|
|
@@ -63,7 +82,9 @@ export type RouteCommon = {
|
|
|
63
82
|
retention?: Retention;
|
|
64
83
|
/** overrides the provisioning priors for this route when the server decides whether to ship it */
|
|
65
84
|
provision?: {
|
|
85
|
+
/** chance a provisioned artifact serves a later view; the server's prior when absent */
|
|
66
86
|
reuse?: number;
|
|
87
|
+
/** later views one provisioned artifact is expected to serve; the server's prior when absent */
|
|
67
88
|
futureViews?: number;
|
|
68
89
|
};
|
|
69
90
|
/**
|
|
@@ -78,13 +99,26 @@ export type RouteCommon = {
|
|
|
78
99
|
};
|
|
79
100
|
/** a route answered by filling placeholders in a stored shell */
|
|
80
101
|
export type AssemblyRoute = RouteCommon & {
|
|
102
|
+
/** fills placeholders in a stored shell */
|
|
81
103
|
strategy: 'assembly';
|
|
104
|
+
/** returns the body that fills the shell's placeholders */
|
|
82
105
|
load(ctx: RouteContext): AssemblyBody | Promise<AssemblyBody>;
|
|
83
106
|
};
|
|
107
|
+
/** the declaration of a public route that answers query strings from a collection it holds */
|
|
108
|
+
export type QueryRoute = {
|
|
109
|
+
/**
|
|
110
|
+
* The query strings the worker answers from the stored collection. `load` then returns an
|
|
111
|
+
* object whose `list` key holds the array and which carries `proof: await datasetProof(items)`.
|
|
112
|
+
*/
|
|
113
|
+
query?: QuerySchema;
|
|
114
|
+
};
|
|
84
115
|
/** a route answered by rendering stored JSON through a compiled template */
|
|
85
|
-
export type TemplateRoute = RouteCommon & {
|
|
116
|
+
export type TemplateRoute = RouteCommon & QueryRoute & {
|
|
117
|
+
/** renders stored JSON through the template */
|
|
86
118
|
strategy: 'template';
|
|
119
|
+
/** the template source, compiled once when the route is defined */
|
|
87
120
|
template: TemplateSource;
|
|
121
|
+
/** returns the JSON the template renders */
|
|
88
122
|
load(ctx: RouteContext): unknown;
|
|
89
123
|
};
|
|
90
124
|
/**
|
|
@@ -94,10 +128,15 @@ export type TemplateRoute = RouteCommon & {
|
|
|
94
128
|
* is answered in `serve` mode only.
|
|
95
129
|
*/
|
|
96
130
|
export type StaticRoute = Omit<RouteCommon, 'scope' | 'persist' | 'mode' | 'retention'> & {
|
|
131
|
+
/** the file's response, stored and shown with no request */
|
|
97
132
|
strategy: 'static';
|
|
133
|
+
/** always absent; a static route is public */
|
|
98
134
|
scope?: undefined;
|
|
135
|
+
/** always absent; a static route is always kept */
|
|
99
136
|
persist?: undefined;
|
|
137
|
+
/** always absent; a static route has no mode */
|
|
100
138
|
mode?: undefined;
|
|
139
|
+
/** always absent; a static route has no load */
|
|
101
140
|
load?: undefined;
|
|
102
141
|
};
|
|
103
142
|
/**
|
|
@@ -105,16 +144,20 @@ export type StaticRoute = Omit<RouteCommon, 'scope' | 'persist' | 'mode' | 'rete
|
|
|
105
144
|
* `@drupflare/spool/sw` takes a renderer keyed by this pattern. Without one the worker refuses the
|
|
106
145
|
* local view and the network answers.
|
|
107
146
|
*/
|
|
108
|
-
export type ProjectionRoute = RouteCommon & {
|
|
147
|
+
export type ProjectionRoute = RouteCommon & QueryRoute & {
|
|
148
|
+
/** renders stored JSON with the renderer the bundled worker holds */
|
|
109
149
|
strategy: 'projection';
|
|
150
|
+
/** returns the JSON the renderer receives */
|
|
110
151
|
load(ctx: RouteContext): unknown;
|
|
111
152
|
};
|
|
112
153
|
/**
|
|
113
154
|
* A route for a JSON read. The worker answers a GET (not a navigation) from the stored value with
|
|
114
155
|
* `JSON.stringify`, so the application's own handler has to return the same JSON `load` does.
|
|
115
156
|
*/
|
|
116
|
-
export type DataRoute = RouteCommon & {
|
|
157
|
+
export type DataRoute = RouteCommon & QueryRoute & {
|
|
158
|
+
/** answers a GET with the stored JSON */
|
|
117
159
|
strategy: 'data';
|
|
160
|
+
/** returns the JSON the local answer serializes */
|
|
118
161
|
load(ctx: RouteContext): unknown;
|
|
119
162
|
/** the content type of the local answer; default `application/json` */
|
|
120
163
|
contentType?: string;
|
|
@@ -123,35 +166,63 @@ export type DataRoute = RouteCommon & {
|
|
|
123
166
|
export type RouteDeclaration = AssemblyRoute | TemplateRoute | StaticRoute | ProjectionRoute | DataRoute;
|
|
124
167
|
/** a validated route with every default applied */
|
|
125
168
|
export type Route = {
|
|
169
|
+
/** the path pattern the route matches */
|
|
126
170
|
pattern: string;
|
|
171
|
+
/** how the route is answered locally */
|
|
127
172
|
strategy: Strategy;
|
|
173
|
+
/** public or principal, after the default */
|
|
128
174
|
scope: Scope;
|
|
175
|
+
/** the staleness contract with every default applied */
|
|
129
176
|
freshness: Freshness;
|
|
177
|
+
/** whether the artifact survives a worker restart; true by default */
|
|
130
178
|
persist: boolean;
|
|
179
|
+
/** whether a snapshot may be shown while offline, after the default */
|
|
131
180
|
offline: OfflineRule;
|
|
181
|
+
/** serve or shadow; absent means the server decides */
|
|
132
182
|
mode?: Mode;
|
|
183
|
+
/** when a soft route past its bound refreshes */
|
|
133
184
|
sync: SyncRule;
|
|
185
|
+
/** at-rest key lifetime; set only on principal routes */
|
|
134
186
|
retention?: Retention;
|
|
187
|
+
/** content type of a data route's local answer */
|
|
135
188
|
contentType?: string;
|
|
189
|
+
/** provisioning priors that override the server's for this route */
|
|
136
190
|
provision?: {
|
|
191
|
+
/** chance a provisioned artifact serves a later view; the server's prior when absent */
|
|
137
192
|
reuse?: number;
|
|
193
|
+
/** later views one provisioned artifact is expected to serve; the server's prior when absent */
|
|
138
194
|
futureViews?: number;
|
|
139
195
|
};
|
|
196
|
+
/** the compiled template of a template route */
|
|
140
197
|
template?: CompiledTemplate;
|
|
198
|
+
/** the route's catalogue function, when it declares one */
|
|
141
199
|
catalogue?: RouteCommon['catalogue'];
|
|
200
|
+
/** compressed bytes of the document the route renders */
|
|
142
201
|
documentBytes?: number;
|
|
202
|
+
/** query strings the worker answers from the stored collection */
|
|
203
|
+
query?: QuerySchema;
|
|
204
|
+
/** tags the artifact is built from; a change to one invalidates it */
|
|
143
205
|
depends(ctx: RouteContext): readonly string[] | Promise<readonly string[]>;
|
|
206
|
+
/** returns the data the artifact is built from */
|
|
144
207
|
load(ctx: RouteContext): unknown;
|
|
145
208
|
};
|
|
146
209
|
/** the part of a route the service worker needs; functions stay on the server */
|
|
147
210
|
export type SerializedRoute = {
|
|
211
|
+
/** the path pattern the route matches */
|
|
148
212
|
pattern: string;
|
|
213
|
+
/** how the route is answered locally */
|
|
149
214
|
strategy: Strategy;
|
|
215
|
+
/** public or principal */
|
|
150
216
|
scope: Scope;
|
|
217
|
+
/** the staleness contract, with maxStale in seconds */
|
|
151
218
|
freshness: Freshness;
|
|
219
|
+
/** whether the artifact survives a worker restart */
|
|
152
220
|
persist: boolean;
|
|
221
|
+
/** whether a snapshot may be shown while offline */
|
|
153
222
|
offline: OfflineRule;
|
|
223
|
+
/** serve or shadow; absent means the server decides */
|
|
154
224
|
mode?: Mode;
|
|
225
|
+
/** when a soft route past its bound refreshes */
|
|
155
226
|
sync: SyncRule;
|
|
156
227
|
/** a principal route's at-rest key; absent on a public route */
|
|
157
228
|
retention?: Retention;
|
|
@@ -161,6 +232,8 @@ export type SerializedRoute = {
|
|
|
161
232
|
version?: string;
|
|
162
233
|
/** the route declares a `catalogue`, so the worker asks for one when a flush goes out on it */
|
|
163
234
|
catalogue?: true;
|
|
235
|
+
/** the query strings the worker answers from the stored collection of a public route */
|
|
236
|
+
query?: QuerySchema;
|
|
164
237
|
};
|
|
165
238
|
/**
|
|
166
239
|
* Declares a route. Validates the pattern and the freshness contract, compiles a template, and
|
|
@@ -170,9 +243,12 @@ export type SerializedRoute = {
|
|
|
170
243
|
export declare function defineRoute(pattern: string, declaration: RouteDeclaration): Route;
|
|
171
244
|
/** the first route whose pattern fits `path`, with the params it captured */
|
|
172
245
|
export declare function matchRoute<T extends {
|
|
246
|
+
/** the pattern a route is matched by */
|
|
173
247
|
pattern: string;
|
|
174
248
|
}>(routes: readonly T[], path: string): {
|
|
249
|
+
/** the first route whose pattern matches the path */
|
|
175
250
|
route: T;
|
|
251
|
+
/** the values the pattern captured from the path */
|
|
176
252
|
params: Params;
|
|
177
253
|
} | null;
|
|
178
254
|
/** strips a route to what the service worker config carries */
|
package/dist/protocol/route.js
CHANGED
|
@@ -9,6 +9,58 @@ export const DEFAULT_FRESHNESS = {
|
|
|
9
9
|
function bad(pattern, message) {
|
|
10
10
|
throw new ProtocolError('bad-route', `${pattern}: ${message}`);
|
|
11
11
|
}
|
|
12
|
+
const QUERY_KEYS = ['list', 'filter', 'sort', 'pageSize'];
|
|
13
|
+
const RESERVED = ['sort', 'page'];
|
|
14
|
+
/** the validated copy of a route's `query`; throws `bad-route` for a schema the worker could not apply */
|
|
15
|
+
function checkQuery(pattern, value) {
|
|
16
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
17
|
+
return bad(pattern, 'query must be an object');
|
|
18
|
+
}
|
|
19
|
+
const given = value;
|
|
20
|
+
for (const key of Object.keys(given)) {
|
|
21
|
+
if (!QUERY_KEYS.includes(key))
|
|
22
|
+
return bad(pattern, `unknown query option "${key}"`);
|
|
23
|
+
}
|
|
24
|
+
const { list, filter, sort, pageSize } = given;
|
|
25
|
+
if (typeof list !== 'string' || list === '')
|
|
26
|
+
return bad(pattern, 'query.list must be a name');
|
|
27
|
+
if (list === 'proof' || list === 'query') {
|
|
28
|
+
return bad(pattern, `query.list cannot be "${list}", which the data already uses`);
|
|
29
|
+
}
|
|
30
|
+
const out = { list };
|
|
31
|
+
if (filter !== undefined) {
|
|
32
|
+
if (typeof filter !== 'object' || filter === null || Array.isArray(filter)) {
|
|
33
|
+
return bad(pattern, 'query.filter must be an object');
|
|
34
|
+
}
|
|
35
|
+
out.filter = {};
|
|
36
|
+
for (const [name, type] of Object.entries(filter)) {
|
|
37
|
+
if (name === '' || RESERVED.includes(name)) {
|
|
38
|
+
return bad(pattern, `query.filter cannot name "${name}"`);
|
|
39
|
+
}
|
|
40
|
+
if (type !== 'string' && type !== 'number') {
|
|
41
|
+
return bad(pattern, `query.filter.${name} must be "string" or "number"`);
|
|
42
|
+
}
|
|
43
|
+
out.filter[name] = type;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (sort !== undefined) {
|
|
47
|
+
if (!Array.isArray(sort))
|
|
48
|
+
return bad(pattern, 'query.sort must be a list of field names');
|
|
49
|
+
for (const field of sort) {
|
|
50
|
+
if (typeof field !== 'string' || field === '' || RESERVED.includes(field)) {
|
|
51
|
+
return bad(pattern, `query.sort cannot name ${JSON.stringify(field)}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
out.sort = [...sort];
|
|
55
|
+
}
|
|
56
|
+
if (pageSize !== undefined) {
|
|
57
|
+
if (typeof pageSize !== 'number' || !Number.isSafeInteger(pageSize) || pageSize < 1) {
|
|
58
|
+
return bad(pattern, 'query.pageSize must be a positive integer');
|
|
59
|
+
}
|
|
60
|
+
out.pageSize = pageSize;
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
12
64
|
/**
|
|
13
65
|
* Declares a route. Validates the pattern and the freshness contract, compiles a template, and
|
|
14
66
|
* fills the defaults: freshness by scope, and `offline` as `serve` for public, `refuse` for
|
|
@@ -95,6 +147,15 @@ export function defineRoute(pattern, declaration) {
|
|
|
95
147
|
if (declaration.strategy === 'data' && declaration.contentType) {
|
|
96
148
|
route.contentType = declaration.contentType;
|
|
97
149
|
}
|
|
150
|
+
const query = declaration.query;
|
|
151
|
+
if (query !== undefined) {
|
|
152
|
+
if (scope !== 'public')
|
|
153
|
+
return bad(pattern, 'query applies to a public route');
|
|
154
|
+
if (!['template', 'projection', 'data'].includes(declaration.strategy)) {
|
|
155
|
+
return bad(pattern, `a ${declaration.strategy} route cannot declare query`);
|
|
156
|
+
}
|
|
157
|
+
route.query = checkQuery(pattern, query);
|
|
158
|
+
}
|
|
98
159
|
if (declaration.strategy === 'template') {
|
|
99
160
|
if (!declaration.template)
|
|
100
161
|
return bad(pattern, 'a template route needs a template');
|
|
@@ -131,5 +192,7 @@ export function serializeRoute(route) {
|
|
|
131
192
|
out.type = route.contentType;
|
|
132
193
|
if (route.catalogue)
|
|
133
194
|
out.catalogue = true;
|
|
195
|
+
if (route.query)
|
|
196
|
+
out.query = structuredClone(route.query);
|
|
134
197
|
return out;
|
|
135
198
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** one file of a crawl: its URL, its kind and its plain compressed size */
|
|
2
|
+
export type BaseCandidate = {
|
|
3
|
+
/** the URL the file was crawled from */
|
|
4
|
+
url: string;
|
|
5
|
+
/** whether the file is a stylesheet or a script */
|
|
6
|
+
kind: 'style' | 'script';
|
|
7
|
+
/** compressed size of the file alone, with no dictionary, in bytes */
|
|
8
|
+
plain: number;
|
|
9
|
+
};
|
|
10
|
+
/** compressed size of `file` against `dictionary`, both indexes into the candidate list */
|
|
11
|
+
export type DeltaSize = (file: number, dictionary: number) => number | Promise<number>;
|
|
12
|
+
/**
|
|
13
|
+
* What a candidate base is scored on. `held` counts what the other files cost against it, for a
|
|
14
|
+
* visitor who already holds the base. `first-visit` adds the base's own plain size, for a visitor
|
|
15
|
+
* who loads every file of the group and has not held it. Neither weighs how often each file is
|
|
16
|
+
* requested or how long a browser keeps the base, so neither is a proven minimum of total transfer.
|
|
17
|
+
*/
|
|
18
|
+
export type BasePolicy = 'held' | 'first-visit';
|
|
19
|
+
export type BaseChoice = {
|
|
20
|
+
/** whether the base is a stylesheet or a script */
|
|
21
|
+
kind: 'style' | 'script';
|
|
22
|
+
/** the path up to the last slash that every member shares */
|
|
23
|
+
directory: string;
|
|
24
|
+
/** the URL of the chosen base file */
|
|
25
|
+
url: string;
|
|
26
|
+
/** how many files of the crawl share the directory and kind, the base included */
|
|
27
|
+
files: number;
|
|
28
|
+
/** mean bytes each other file costs against this base, capped at its plain size */
|
|
29
|
+
meanDelta: number;
|
|
30
|
+
};
|
|
31
|
+
/** the path of a URL up to and including its last slash */
|
|
32
|
+
export declare function dirOf(url: string): string;
|
|
33
|
+
/**
|
|
34
|
+
* The member of `members` that minimizes the sum, over the other members, of the smaller of their
|
|
35
|
+
* delta against it and their plain size, plus the member's own plain size under `first-visit`.
|
|
36
|
+
* Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
|
|
37
|
+
* base's own size.
|
|
38
|
+
*/
|
|
39
|
+
export declare function pickBase(members: readonly number[], delta: (file: number, dictionary: number) => number, plain: (file: number) => number, policy?: BasePolicy): {
|
|
40
|
+
best: number;
|
|
41
|
+
score: number;
|
|
42
|
+
others: number;
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
|
|
46
|
+
* before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
|
|
47
|
+
* one file has no base.
|
|
48
|
+
*/
|
|
49
|
+
export declare function chooseBases(files: readonly BaseCandidate[], delta: DeltaSize, options?: {
|
|
50
|
+
policy?: BasePolicy;
|
|
51
|
+
}): Promise<BaseChoice[]>;
|
|
52
|
+
/** true for the exact pathnames of the chosen base URLs; a query string does not count */
|
|
53
|
+
export declare function baseMatcher(choices: readonly BaseChoice[]): (pathname: string) => boolean;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** the path of a URL up to and including its last slash */
|
|
2
|
+
export function dirOf(url) {
|
|
3
|
+
const path = new URL(url).pathname;
|
|
4
|
+
return path.slice(0, path.lastIndexOf('/') + 1);
|
|
5
|
+
}
|
|
6
|
+
const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
7
|
+
/**
|
|
8
|
+
* The member of `members` that minimizes the sum, over the other members, of the smaller of their
|
|
9
|
+
* delta against it and their plain size, plus the member's own plain size under `first-visit`.
|
|
10
|
+
* Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
|
|
11
|
+
* base's own size.
|
|
12
|
+
*/
|
|
13
|
+
export function pickBase(members, delta, plain, policy = 'held') {
|
|
14
|
+
let best = -1;
|
|
15
|
+
let bestScore = Infinity;
|
|
16
|
+
let bestOthers = Infinity;
|
|
17
|
+
for (const c of members) {
|
|
18
|
+
let others = 0;
|
|
19
|
+
for (const f of members) {
|
|
20
|
+
if (f !== c)
|
|
21
|
+
others += Math.min(delta(f, c), plain(f));
|
|
22
|
+
}
|
|
23
|
+
const score = policy === 'first-visit' ? others + plain(c) : others;
|
|
24
|
+
if (score < bestScore) {
|
|
25
|
+
best = c;
|
|
26
|
+
bestScore = score;
|
|
27
|
+
bestOthers = others;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return { best, score: bestScore, others: bestOthers };
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
|
|
34
|
+
* before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
|
|
35
|
+
* one file has no base.
|
|
36
|
+
*/
|
|
37
|
+
export async function chooseBases(files, delta, options = {}) {
|
|
38
|
+
const at = (i) => files[i];
|
|
39
|
+
const groups = new Map();
|
|
40
|
+
for (const [i, f] of files.entries()) {
|
|
41
|
+
const key = JSON.stringify([f.kind, dirOf(f.url)]);
|
|
42
|
+
const members = groups.get(key);
|
|
43
|
+
if (members)
|
|
44
|
+
members.push(i);
|
|
45
|
+
else
|
|
46
|
+
groups.set(key, [i]);
|
|
47
|
+
}
|
|
48
|
+
const choices = [];
|
|
49
|
+
for (const members of groups.values()) {
|
|
50
|
+
if (members.length < 2)
|
|
51
|
+
continue;
|
|
52
|
+
const sorted = [...members].sort((a, b) => compare(at(a).url, at(b).url));
|
|
53
|
+
const sizes = new Map();
|
|
54
|
+
for (const f of sorted) {
|
|
55
|
+
for (const c of sorted) {
|
|
56
|
+
if (f !== c)
|
|
57
|
+
sizes.set(`${f}|${c}`, await delta(f, c));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
const { best, others } = pickBase(sorted, (f, c) => sizes.get(`${f}|${c}`), (f) => at(f).plain, options.policy);
|
|
61
|
+
const { kind, url } = at(best);
|
|
62
|
+
choices.push({
|
|
63
|
+
kind,
|
|
64
|
+
directory: dirOf(url),
|
|
65
|
+
url,
|
|
66
|
+
files: sorted.length,
|
|
67
|
+
meanDelta: others / (sorted.length - 1)
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return choices.sort((a, b) => compare(a.directory, b.directory) || compare(a.kind, b.kind));
|
|
71
|
+
}
|
|
72
|
+
/** true for the exact pathnames of the chosen base URLs; a query string does not count */
|
|
73
|
+
export function baseMatcher(choices) {
|
|
74
|
+
const paths = new Set(choices.map((choice) => new URL(choice.url).pathname));
|
|
75
|
+
return (pathname) => paths.has(pathname);
|
|
76
|
+
}
|
package/dist/proxy/index.d.ts
CHANGED
package/dist/proxy/index.js
CHANGED
package/dist/proxy/proxy.js
CHANGED
|
@@ -211,9 +211,11 @@ export function spoolProxy(options) {
|
|
|
211
211
|
...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
|
|
212
212
|
});
|
|
213
213
|
}
|
|
214
|
-
/** the headers the browser gets: bodies arrive decoded, so the encoding and its length go */
|
|
214
|
+
/** the headers the browser gets: bodies arrive decoded, so the encoding and its length go, and so does what only described the hop to the proxy */
|
|
215
215
|
function tidy(source, url) {
|
|
216
216
|
const headers = new Headers(source);
|
|
217
|
+
for (const name of HOP_BY_HOP)
|
|
218
|
+
headers.delete(name);
|
|
217
219
|
if (headers.has('content-encoding')) {
|
|
218
220
|
headers.delete('content-encoding');
|
|
219
221
|
headers.delete('content-length');
|
|
@@ -371,6 +373,7 @@ export function spoolProxy(options) {
|
|
|
371
373
|
if (url.pathname === PATHS.worker && request.method === 'GET')
|
|
372
374
|
return spool.stub();
|
|
373
375
|
const converting = images !== undefined && imageRequest(images, request);
|
|
376
|
+
let visitorRewrite = false;
|
|
374
377
|
let upstream;
|
|
375
378
|
try {
|
|
376
379
|
upstream = await call(forward(request, url, converting));
|
|
@@ -427,6 +430,7 @@ export function spoolProxy(options) {
|
|
|
427
430
|
videoStats.rewritten += result.rewritten;
|
|
428
431
|
// a page rewritten for one visitor's connection is not another's copy
|
|
429
432
|
if (video.when !== 'always') {
|
|
433
|
+
visitorRewrite = true;
|
|
430
434
|
makePrivate(headers);
|
|
431
435
|
mergeList(headers, 'vary', ['Save-Data', 'ECT']);
|
|
432
436
|
}
|
|
@@ -488,7 +492,9 @@ export function spoolProxy(options) {
|
|
|
488
492
|
path: url.pathname,
|
|
489
493
|
page: Promise.resolve({ text, tags: effective })
|
|
490
494
|
});
|
|
491
|
-
|
|
495
|
+
// a visitor-signal rewrite is not in the copy /.spool/artifact builds, so the worker asks for it
|
|
496
|
+
const same = !visitorRewrite && !data && !principal;
|
|
497
|
+
return spool.decorate(request, new Response(text, init), same ? { assembled: text } : {});
|
|
492
498
|
}
|
|
493
499
|
return Object.assign(handle, {
|
|
494
500
|
spool,
|