@drupflare/spool 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +444 -18
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/activity.d.ts +37 -0
- package/dist/client/activity.js +114 -0
- package/dist/client/entry.js +2 -1
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/media/embeds.d.ts +60 -0
- package/dist/media/embeds.js +135 -0
- package/dist/media/headers.d.ts +13 -0
- package/dist/media/headers.js +44 -0
- package/dist/media/html.d.ts +27 -0
- package/dist/media/html.js +93 -0
- package/dist/media/image.d.ts +222 -0
- package/dist/media/image.js +413 -0
- package/dist/media/index.d.ts +6 -0
- package/dist/media/index.js +6 -0
- package/dist/media/lazy.d.ts +26 -0
- package/dist/media/lazy.js +74 -0
- package/dist/media/responsive.d.ts +45 -0
- package/dist/media/responsive.js +98 -0
- package/dist/media/video.d.ts +30 -0
- package/dist/media/video.js +75 -0
- package/dist/protocol/envelope.d.ts +2 -0
- package/dist/protocol/envelope.js +12 -2
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/hash.d.ts +2 -0
- package/dist/protocol/hash.js +8 -0
- package/dist/protocol/index.d.ts +4 -0
- package/dist/protocol/index.js +4 -0
- package/dist/protocol/inline.d.ts +14 -0
- package/dist/protocol/inline.js +43 -0
- package/dist/protocol/navigation.d.ts +81 -0
- package/dist/protocol/navigation.js +249 -0
- package/dist/protocol/normalise.d.ts +4 -3
- package/dist/protocol/normalise.js +6 -4
- package/dist/protocol/provision.d.ts +6 -1
- package/dist/protocol/provision.js +15 -5
- package/dist/protocol/query.d.ts +80 -0
- package/dist/protocol/query.js +153 -0
- package/dist/protocol/route.d.ts +80 -4
- package/dist/protocol/route.js +63 -0
- package/dist/proxy/bases.d.ts +53 -0
- package/dist/proxy/bases.js +76 -0
- package/dist/proxy/dictionary.d.ts +58 -0
- package/dist/proxy/dictionary.js +197 -0
- package/dist/proxy/index.d.ts +2 -0
- package/dist/proxy/index.js +2 -0
- package/dist/proxy/proxy.d.ts +47 -0
- package/dist/proxy/proxy.js +140 -15
- package/dist/report/capacity.d.ts +59 -0
- package/dist/report/capacity.js +76 -0
- package/dist/report/channels.d.ts +60 -0
- package/dist/report/channels.js +65 -0
- package/dist/report/derive.d.ts +2 -0
- package/dist/report/derive.js +6 -2
- package/dist/report/index.d.ts +4 -0
- package/dist/report/index.js +4 -0
- package/dist/report/institution.d.ts +97 -0
- package/dist/report/institution.js +79 -0
- package/dist/report/ledger.d.ts +197 -0
- package/dist/report/ledger.js +308 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +7 -1
- package/dist/report/traffic.d.ts +11 -2
- package/dist/report/traffic.js +13 -3
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +62 -9
- package/dist/sw/render.d.ts +3 -1
- package/dist/sw/render.js +14 -1
- package/dist/sw/replica.d.ts +2 -0
- package/dist/sw/replica.js +6 -0
- package/dist/sw/router.js +2 -2
- package/dist/sw/runtime.js +104 -9
- package/dist/trim/tokens.d.ts +4 -0
- package/dist/trim/tokens.js +1 -1
- package/package.json +10 -4
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** the path of a URL up to and including its last slash */
|
|
2
|
+
export function dirOf(url) {
|
|
3
|
+
const path = new URL(url).pathname;
|
|
4
|
+
return path.slice(0, path.lastIndexOf('/') + 1);
|
|
5
|
+
}
|
|
6
|
+
const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
7
|
+
/**
|
|
8
|
+
* The member of `members` that minimizes the sum, over the other members, of the smaller of their
|
|
9
|
+
* delta against it and their plain size, plus the member's own plain size under `first-visit`.
|
|
10
|
+
* Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
|
|
11
|
+
* base's own size.
|
|
12
|
+
*/
|
|
13
|
+
export function pickBase(members, delta, plain, policy = 'held') {
|
|
14
|
+
let best = -1;
|
|
15
|
+
let bestScore = Infinity;
|
|
16
|
+
let bestOthers = Infinity;
|
|
17
|
+
for (const c of members) {
|
|
18
|
+
let others = 0;
|
|
19
|
+
for (const f of members) {
|
|
20
|
+
if (f !== c)
|
|
21
|
+
others += Math.min(delta(f, c), plain(f));
|
|
22
|
+
}
|
|
23
|
+
const score = policy === 'first-visit' ? others + plain(c) : others;
|
|
24
|
+
if (score < bestScore) {
|
|
25
|
+
best = c;
|
|
26
|
+
bestScore = score;
|
|
27
|
+
bestOthers = others;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return { best, score: bestScore, others: bestOthers };
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
|
|
34
|
+
* before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
|
|
35
|
+
* one file has no base.
|
|
36
|
+
*/
|
|
37
|
+
export async function chooseBases(files, delta, options = {}) {
|
|
38
|
+
const at = (i) => files[i];
|
|
39
|
+
const groups = new Map();
|
|
40
|
+
for (const [i, f] of files.entries()) {
|
|
41
|
+
const key = JSON.stringify([f.kind, dirOf(f.url)]);
|
|
42
|
+
const members = groups.get(key);
|
|
43
|
+
if (members)
|
|
44
|
+
members.push(i);
|
|
45
|
+
else
|
|
46
|
+
groups.set(key, [i]);
|
|
47
|
+
}
|
|
48
|
+
const choices = [];
|
|
49
|
+
for (const members of groups.values()) {
|
|
50
|
+
if (members.length < 2)
|
|
51
|
+
continue;
|
|
52
|
+
const sorted = [...members].sort((a, b) => compare(at(a).url, at(b).url));
|
|
53
|
+
const sizes = new Map();
|
|
54
|
+
for (const f of sorted) {
|
|
55
|
+
for (const c of sorted) {
|
|
56
|
+
if (f !== c)
|
|
57
|
+
sizes.set(`${f}|${c}`, await delta(f, c));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
const { best, others } = pickBase(sorted, (f, c) => sizes.get(`${f}|${c}`), (f) => at(f).plain, options.policy);
|
|
61
|
+
const { kind, url } = at(best);
|
|
62
|
+
choices.push({
|
|
63
|
+
kind,
|
|
64
|
+
directory: dirOf(url),
|
|
65
|
+
url,
|
|
66
|
+
files: sorted.length,
|
|
67
|
+
meanDelta: others / (sorted.length - 1)
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return choices.sort((a, b) => compare(a.directory, b.directory) || compare(a.kind, b.kind));
|
|
71
|
+
}
|
|
72
|
+
/** true for the exact pathnames of the chosen base URLs; a query string does not count */
|
|
73
|
+
export function baseMatcher(choices) {
|
|
74
|
+
const paths = new Set(choices.map((choice) => new URL(choice.url).pathname));
|
|
75
|
+
return (pathname) => paths.has(pathname);
|
|
76
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type Gate } from './eligible.js';
|
|
2
|
+
/** the content codings of RFC 9842: Brotli (`dcb`) and Zstandard (`dcz`) against a dictionary */
|
|
3
|
+
export type DictionaryFormat = 'dcb' | 'dcz';
|
|
4
|
+
/**
|
|
5
|
+
* Compresses `input` against `dictionary` and returns the compressed stream only. spool adds the
|
|
6
|
+
* coding's header (the magic bytes and the SHA-256 of the dictionary). For `dcz` the dictionary is a
|
|
7
|
+
* raw content dictionary and the window must stay at or under 8 MiB or 1.25 times the dictionary,
|
|
8
|
+
* whichever is larger. For `dcb` it is a Brotli prefix dictionary. It throws for a format it cannot
|
|
9
|
+
* produce, and the proxy then answers with the plain body.
|
|
10
|
+
*/
|
|
11
|
+
export type DictionaryEncoder = (dictionary: Uint8Array, input: Uint8Array, format: DictionaryFormat) => Promise<Uint8Array>;
|
|
12
|
+
/** what `spoolProxy({ dictionary })` takes */
|
|
13
|
+
export type DictionaryOptions = {
|
|
14
|
+
/** produces the compressed stream; see {@link DictionaryEncoder} */
|
|
15
|
+
encoder: DictionaryEncoder;
|
|
16
|
+
/** the paths that become and use dictionaries, tested against the pathname; default `/\.(?:css|m?js)$/i` */
|
|
17
|
+
match?: RegExp;
|
|
18
|
+
/**
|
|
19
|
+
* Which of those responses register as dictionaries, as a function of the pathname, a regular
|
|
20
|
+
* expression or a list of them; default every one `match` accepts. The others get no
|
|
21
|
+
* `Use-As-Dictionary`, so the browser holds only the bases you name, and they are still encoded
|
|
22
|
+
* against a base the request names.
|
|
23
|
+
*/
|
|
24
|
+
base?: RegExp | readonly RegExp[] | ((pathname: string) => boolean);
|
|
25
|
+
/** largest body kept as a dictionary or encoded, in bytes; default 2 MiB */
|
|
26
|
+
maxBytes?: number;
|
|
27
|
+
/** the codings offered, in order of preference, when the request accepts them; default `['dcz']` */
|
|
28
|
+
formats?: readonly DictionaryFormat[];
|
|
29
|
+
/** bytes of dictionaries kept in memory, the least recently used going first; default 16 MiB */
|
|
30
|
+
cacheBytes?: number;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* What the proxy counted since it started: `registered` is the dictionaries stored (new content
|
|
34
|
+
* hashes), `encoded` the responses sent as `dcb` or `dcz`, `bytesIn` and `bytesOut` the plain and
|
|
35
|
+
* encoded sizes of those responses, and `skipped` the eligible responses left plain by reason:
|
|
36
|
+
* `unavailable` (the request named no dictionary the proxy holds, or accepted no coding offered),
|
|
37
|
+
* `size`, `larger` (the encoded body was not smaller), `failed` (the encoder threw), `range` and
|
|
38
|
+
* `pattern` (the path cannot be written as a `match` pattern).
|
|
39
|
+
*/
|
|
40
|
+
export type DictionaryStats = {
|
|
41
|
+
registered: number;
|
|
42
|
+
encoded: number;
|
|
43
|
+
bytesIn: number;
|
|
44
|
+
bytesOut: number;
|
|
45
|
+
skipped: Record<string, number>;
|
|
46
|
+
};
|
|
47
|
+
/** the body of a `dcb` or `dcz` response: the coding's magic bytes, the dictionary's SHA-256, then the stream */
|
|
48
|
+
export declare function frameDictionaryBody(format: DictionaryFormat, hash: Uint8Array, stream: Uint8Array): Uint8Array;
|
|
49
|
+
/** the state `spoolProxy` keeps for `dictionary`: checked options, the dictionaries and the counters */
|
|
50
|
+
export type Dictionaries = {
|
|
51
|
+
readonly stats: DictionaryStats;
|
|
52
|
+
/** whether a response could take part, by its content type and the request's method and path */
|
|
53
|
+
applies(request: Request, response: Response): boolean;
|
|
54
|
+
/** answers one origin response for `request`, registering it and encoding it when the request names a stored dictionary */
|
|
55
|
+
apply(request: Request, response: Response): Promise<Response>;
|
|
56
|
+
};
|
|
57
|
+
/** applies the defaults and checks the values; `ProtocolError('bad-route')` for one out of range */
|
|
58
|
+
export declare function createDictionaries(options: DictionaryOptions, gate: Gate): Dictionaries;
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { mergeList } from '../media/headers.js';
|
|
2
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
3
|
+
import { decline } from './eligible.js';
|
|
4
|
+
const MAGIC = {
|
|
5
|
+
dcb: [0xff, 0x44, 0x43, 0x42],
|
|
6
|
+
dcz: [0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00]
|
|
7
|
+
};
|
|
8
|
+
const DEFAULT_MATCH = /\.(?:css|m?js)$/i;
|
|
9
|
+
const DEFAULT_MAX_BYTES = 2 * 1024 * 1024;
|
|
10
|
+
const DEFAULT_CACHE_BYTES = 16 * 1024 * 1024;
|
|
11
|
+
const SAFE_DIRECTORY = /^[A-Za-z0-9/_.~%-]*$/;
|
|
12
|
+
const AVAILABLE = /^:([A-Za-z0-9+/]{43}=):$/;
|
|
13
|
+
/** the body of a `dcb` or `dcz` response: the coding's magic bytes, the dictionary's SHA-256, then the stream */
|
|
14
|
+
export function frameDictionaryBody(format, hash, stream) {
|
|
15
|
+
const magic = MAGIC[format];
|
|
16
|
+
const out = new Uint8Array(magic.length + hash.length + stream.length);
|
|
17
|
+
out.set(magic, 0);
|
|
18
|
+
out.set(hash, magic.length);
|
|
19
|
+
out.set(stream, magic.length + hash.length);
|
|
20
|
+
return out;
|
|
21
|
+
}
|
|
22
|
+
const toBase64 = (bytes) => btoa(String.fromCharCode(...bytes));
|
|
23
|
+
async function sha256(bytes) {
|
|
24
|
+
return new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
|
|
25
|
+
}
|
|
26
|
+
/** the codings `Accept-Encoding` lists with a quality above zero, lowercase */
|
|
27
|
+
function accepted(header) {
|
|
28
|
+
const out = new Set();
|
|
29
|
+
for (const part of (header ?? '').split(',')) {
|
|
30
|
+
const [token, ...params] = part.trim().toLowerCase().split(';');
|
|
31
|
+
const q = params.map((p) => p.trim()).find((p) => p.startsWith('q='));
|
|
32
|
+
if (token && !(q !== undefined && Number(q.slice(2)) === 0))
|
|
33
|
+
out.add(token.trim());
|
|
34
|
+
}
|
|
35
|
+
return out;
|
|
36
|
+
}
|
|
37
|
+
function kindOf(response) {
|
|
38
|
+
const type = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
|
|
39
|
+
if (type === 'text/css')
|
|
40
|
+
return 'style';
|
|
41
|
+
if (type === 'text/javascript' || type === 'application/javascript')
|
|
42
|
+
return 'script';
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
/** applies the defaults and checks the values; `ProtocolError('bad-route')` for one out of range */
|
|
46
|
+
export function createDictionaries(options, gate) {
|
|
47
|
+
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
48
|
+
const cacheBytes = options.cacheBytes ?? DEFAULT_CACHE_BYTES;
|
|
49
|
+
const formats = options.formats ?? ['dcz'];
|
|
50
|
+
const match = options.match ?? DEFAULT_MATCH;
|
|
51
|
+
const { base } = options;
|
|
52
|
+
const isBase = base === undefined
|
|
53
|
+
? () => true
|
|
54
|
+
: typeof base === 'function'
|
|
55
|
+
? base
|
|
56
|
+
: (pathname) => [].concat(base).some((pattern) => {
|
|
57
|
+
pattern.lastIndex = 0;
|
|
58
|
+
return pattern.test(pathname);
|
|
59
|
+
});
|
|
60
|
+
if (!Number.isInteger(maxBytes) || maxBytes < 1) {
|
|
61
|
+
throw new ProtocolError('bad-route', 'dictionary maxBytes must be a positive integer');
|
|
62
|
+
}
|
|
63
|
+
if (!Number.isInteger(cacheBytes) || cacheBytes < 1) {
|
|
64
|
+
throw new ProtocolError('bad-route', 'dictionary cacheBytes must be a positive integer');
|
|
65
|
+
}
|
|
66
|
+
if (formats.length === 0 || formats.some((f) => f !== 'dcb' && f !== 'dcz')) {
|
|
67
|
+
throw new ProtocolError('bad-route', `dictionary formats must list 'dcb' or 'dcz'`);
|
|
68
|
+
}
|
|
69
|
+
const stats = {
|
|
70
|
+
registered: 0,
|
|
71
|
+
encoded: 0,
|
|
72
|
+
bytesIn: 0,
|
|
73
|
+
bytesOut: 0,
|
|
74
|
+
skipped: {}
|
|
75
|
+
};
|
|
76
|
+
const held = new Map();
|
|
77
|
+
let size = 0;
|
|
78
|
+
const skip = (why) => void (stats.skipped[why] = (stats.skipped[why] ?? 0) + 1);
|
|
79
|
+
function touch(key, body) {
|
|
80
|
+
held.delete(key);
|
|
81
|
+
held.set(key, body);
|
|
82
|
+
}
|
|
83
|
+
function remember(key, bytes) {
|
|
84
|
+
const body = held.get(key);
|
|
85
|
+
if (body)
|
|
86
|
+
return touch(key, body);
|
|
87
|
+
held.set(key, bytes);
|
|
88
|
+
size += bytes.length;
|
|
89
|
+
stats.registered++;
|
|
90
|
+
for (const [oldest, body] of held) {
|
|
91
|
+
if (size <= cacheBytes || held.size === 1)
|
|
92
|
+
break;
|
|
93
|
+
held.delete(oldest);
|
|
94
|
+
size -= body.length;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function applies(request, response) {
|
|
98
|
+
if (request.method !== 'GET' || response.status !== 200)
|
|
99
|
+
return false;
|
|
100
|
+
match.lastIndex = 0;
|
|
101
|
+
if (!match.test(new URL(request.url).pathname))
|
|
102
|
+
return false;
|
|
103
|
+
return kindOf(response) !== null;
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
stats,
|
|
107
|
+
applies,
|
|
108
|
+
async apply(request, response) {
|
|
109
|
+
const url = new URL(request.url);
|
|
110
|
+
const auth = new Headers();
|
|
111
|
+
if (request.headers.has('authorization'))
|
|
112
|
+
auth.set('authorization', 'x');
|
|
113
|
+
const why = decline({
|
|
114
|
+
method: 'GET',
|
|
115
|
+
search: '',
|
|
116
|
+
requestHeaders: auth,
|
|
117
|
+
response,
|
|
118
|
+
tags: [],
|
|
119
|
+
data: true,
|
|
120
|
+
principal: false
|
|
121
|
+
}, gate);
|
|
122
|
+
if (why !== null)
|
|
123
|
+
return response;
|
|
124
|
+
const declared = Number(response.headers.get('content-length'));
|
|
125
|
+
if (request.headers.has('range')) {
|
|
126
|
+
skip('range');
|
|
127
|
+
return response;
|
|
128
|
+
}
|
|
129
|
+
if (Number.isFinite(declared) && declared > maxBytes) {
|
|
130
|
+
skip('size');
|
|
131
|
+
return response;
|
|
132
|
+
}
|
|
133
|
+
const directory = url.pathname.slice(0, url.pathname.lastIndexOf('/') + 1);
|
|
134
|
+
if (!SAFE_DIRECTORY.test(directory)) {
|
|
135
|
+
skip('pattern');
|
|
136
|
+
return response;
|
|
137
|
+
}
|
|
138
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
139
|
+
const headers = new Headers(response.headers);
|
|
140
|
+
const plain = () => new Response(bytes, {
|
|
141
|
+
status: 200,
|
|
142
|
+
statusText: response.statusText,
|
|
143
|
+
headers
|
|
144
|
+
});
|
|
145
|
+
if (bytes.length === 0 || bytes.length > maxBytes) {
|
|
146
|
+
skip('size');
|
|
147
|
+
return plain();
|
|
148
|
+
}
|
|
149
|
+
const hash = await sha256(bytes);
|
|
150
|
+
const key = AVAILABLE.exec(request.headers.get('available-dictionary') ?? '')?.[1];
|
|
151
|
+
const dictionary = key === undefined ? undefined : held.get(key);
|
|
152
|
+
if (key !== undefined && dictionary)
|
|
153
|
+
touch(key, dictionary);
|
|
154
|
+
mergeList(headers, 'vary', ['Accept-Encoding', 'Available-Dictionary']);
|
|
155
|
+
if (isBase(url.pathname)) {
|
|
156
|
+
headers.set('use-as-dictionary', `match="${directory}*", match-dest=("${kindOf(response)}")`);
|
|
157
|
+
remember(toBase64(hash), bytes);
|
|
158
|
+
}
|
|
159
|
+
if (!dictionary) {
|
|
160
|
+
skip('unavailable');
|
|
161
|
+
return plain();
|
|
162
|
+
}
|
|
163
|
+
const wanted = accepted(request.headers.get('accept-encoding'));
|
|
164
|
+
const format = formats.find((f) => wanted.has(f));
|
|
165
|
+
if (!format) {
|
|
166
|
+
skip('unavailable');
|
|
167
|
+
return plain();
|
|
168
|
+
}
|
|
169
|
+
let framed;
|
|
170
|
+
try {
|
|
171
|
+
const stream = await options.encoder(dictionary, bytes, format);
|
|
172
|
+
framed = frameDictionaryBody(format, await sha256(dictionary), stream);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
skip('failed');
|
|
176
|
+
return plain();
|
|
177
|
+
}
|
|
178
|
+
if (framed.length >= bytes.length) {
|
|
179
|
+
skip('larger');
|
|
180
|
+
return plain();
|
|
181
|
+
}
|
|
182
|
+
headers.set('content-encoding', format);
|
|
183
|
+
headers.delete('content-length');
|
|
184
|
+
const etag = headers.get('etag');
|
|
185
|
+
if (etag && !etag.startsWith('W/'))
|
|
186
|
+
headers.set('etag', `W/${etag}`);
|
|
187
|
+
stats.encoded++;
|
|
188
|
+
stats.bytesIn += bytes.length;
|
|
189
|
+
stats.bytesOut += framed.length;
|
|
190
|
+
return new Response(framed, {
|
|
191
|
+
status: 200,
|
|
192
|
+
statusText: response.statusText,
|
|
193
|
+
headers
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
}
|
package/dist/proxy/index.d.ts
CHANGED
package/dist/proxy/index.js
CHANGED
package/dist/proxy/proxy.d.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
import { type EmbedOptions, type EmbedStats } from '../media/embeds.js';
|
|
2
|
+
import { type ImageOptions, type ImageStats } from '../media/image.js';
|
|
3
|
+
import { type VideoOptions } from '../media/video.js';
|
|
1
4
|
import { type AssemblyRoute, type DataRoute } from '../protocol/route.js';
|
|
2
5
|
import { type Spool, type SpoolOptions } from '../server/spool.js';
|
|
3
6
|
import { type TrimManifest } from '../trim/index.js';
|
|
7
|
+
import { type DictionaryOptions, type DictionaryStats } from './dictionary.js';
|
|
4
8
|
/**
|
|
5
9
|
* A route the proxy answers locally, declared like a `createSpool` route minus `load`: the proxy
|
|
6
10
|
* loads the page from the origin. A `data` route parses the origin's JSON. `depends` adds tags to
|
|
@@ -49,6 +53,31 @@ export type ProxyOptions = {
|
|
|
49
53
|
* and every HTML page is checked against the guard first. Without it nothing is trimmed.
|
|
50
54
|
*/
|
|
51
55
|
trim?: TrimManifest;
|
|
56
|
+
/**
|
|
57
|
+
* Converts JPEG and PNG responses to WebP for a browser that accepts it, at the quality and
|
|
58
|
+
* width the request's headers call for, up to a width ceiling of 2560 pixels when the request
|
|
59
|
+
* names none. Identical simultaneous requests share one encoder call, and an encoder that runs
|
|
60
|
+
* past 10 seconds is abandoned for that request. See `ImageOptions`. Without it no image is
|
|
61
|
+
* touched.
|
|
62
|
+
*/
|
|
63
|
+
images?: ImageOptions;
|
|
64
|
+
/**
|
|
65
|
+
* Rewrites `<video>` tags in HTML responses, such as `preload="none"`; see `VideoOptions`.
|
|
66
|
+
* Without it no page is touched for video.
|
|
67
|
+
*/
|
|
68
|
+
video?: VideoOptions;
|
|
69
|
+
/**
|
|
70
|
+
* Replaces server-rendered YouTube and Vimeo `<iframe>` embeds with a link that loads the
|
|
71
|
+
* player on click; see `EmbedOptions`. A page that sends a `Content-Security-Policy` is left
|
|
72
|
+
* alone. Without it no embed is touched.
|
|
73
|
+
*/
|
|
74
|
+
embeds?: EmbedOptions;
|
|
75
|
+
/**
|
|
76
|
+
* Serves a stylesheet or script as `dcz` or `dcb` against an earlier public file of the same
|
|
77
|
+
* directory the visitor already holds (RFC 9842, Chromium 130 and later); see
|
|
78
|
+
* `DictionaryOptions`. Without it no response is marked or encoded.
|
|
79
|
+
*/
|
|
80
|
+
dictionary?: DictionaryOptions;
|
|
52
81
|
};
|
|
53
82
|
/** what the proxy counted since it started: trimmed answers, hash mismatches and guard firings */
|
|
54
83
|
export type TrimStats = {
|
|
@@ -56,6 +85,11 @@ export type TrimStats = {
|
|
|
56
85
|
mismatched: number;
|
|
57
86
|
guarded: number;
|
|
58
87
|
};
|
|
88
|
+
/** what the proxy counted for `video`: pages with at least one rewritten tag, and the tags */
|
|
89
|
+
export type VideoStats = {
|
|
90
|
+
pages: number;
|
|
91
|
+
rewritten: number;
|
|
92
|
+
};
|
|
59
93
|
/** where the proxy writes a guard firing or a hash mismatch to the sink, as JSON `{ kind, ... }` */
|
|
60
94
|
export declare const TRIM_PATH = "/.spool/trim";
|
|
61
95
|
/** the Fetch handler `spoolProxy` returns, with the spool it built */
|
|
@@ -64,6 +98,14 @@ export type SpoolProxy = {
|
|
|
64
98
|
readonly spool: Spool;
|
|
65
99
|
/** live counters; all zero when `trim` is not set */
|
|
66
100
|
readonly trim: Readonly<TrimStats>;
|
|
101
|
+
/** live counters; all zero when `images` is not set */
|
|
102
|
+
readonly images: Readonly<ImageStats>;
|
|
103
|
+
/** live counters; all zero when `video` is not set */
|
|
104
|
+
readonly video: Readonly<VideoStats>;
|
|
105
|
+
/** live counters; all zero when `embeds` is not set */
|
|
106
|
+
readonly embeds: Readonly<EmbedStats>;
|
|
107
|
+
/** live counters; all zero when `dictionary` is not set */
|
|
108
|
+
readonly dictionary: Readonly<DictionaryStats>;
|
|
67
109
|
};
|
|
68
110
|
/**
|
|
69
111
|
* A Fetch handler that sits in front of an origin that cannot load the library, such as PHP or a
|
|
@@ -76,6 +118,11 @@ export type SpoolProxy = {
|
|
|
76
118
|
* The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
|
|
77
119
|
* become the page's dependencies, and a page with none depends on its own path.
|
|
78
120
|
*
|
|
121
|
+
* With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
|
|
122
|
+
* converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
|
|
123
|
+
* `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
|
|
124
|
+
* kept, so a held page carries the rewritten tags.
|
|
125
|
+
*
|
|
79
126
|
* Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
|
|
80
127
|
* records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
|
|
81
128
|
* when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
|
package/dist/proxy/proxy.js
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { resolveEmbedOptions, rewriteEmbeds } from '../media/embeds.js';
|
|
2
|
+
import { makePrivate, mergeList, signalsOf } from '../media/headers.js';
|
|
3
|
+
import { convertImage, createImageConverter, IMAGE_HINTS, imageRequest, originSearch } from '../media/image.js';
|
|
4
|
+
import { rewriteLazy } from '../media/lazy.js';
|
|
5
|
+
import { rewriteResponsive } from '../media/responsive.js';
|
|
6
|
+
import { resolveVideoOptions, rewriteVideo } from '../media/video.js';
|
|
1
7
|
import { PATHS } from '../protocol/config.js';
|
|
2
8
|
import { writeEnvelope } from '../protocol/envelope.js';
|
|
3
9
|
import { ProtocolError } from '../protocol/errors.js';
|
|
@@ -6,6 +12,7 @@ import { defineRoute, matchRoute } from '../protocol/route.js';
|
|
|
6
12
|
import { artifactTag } from '../protocol/sync.js';
|
|
7
13
|
import { createSpool } from '../server/spool.js';
|
|
8
14
|
import { guardPage, sourceHash, stripTrimParam, TRIM_PARAM, trimKey } from '../trim/index.js';
|
|
15
|
+
import { createDictionaries } from './dictionary.js';
|
|
9
16
|
import { decline, htmlCharset, isUtf8, setsPrivateCookie } from './eligible.js';
|
|
10
17
|
import { PURGE_PATH, respondPurge } from './purge.js';
|
|
11
18
|
import { pathTag, readTags, stripTags } from './tags.js';
|
|
@@ -43,6 +50,11 @@ const toBytes = (text) => Uint8Array.from(text, (char) => char.charCodeAt(0));
|
|
|
43
50
|
* The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
|
|
44
51
|
* become the page's dependencies, and a page with none depends on its own path.
|
|
45
52
|
*
|
|
53
|
+
* With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
|
|
54
|
+
* converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
|
|
55
|
+
* `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
|
|
56
|
+
* kept, so a held page carries the rewritten tags.
|
|
57
|
+
*
|
|
46
58
|
* Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
|
|
47
59
|
* records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
|
|
48
60
|
* when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
|
|
@@ -74,6 +86,62 @@ export function spoolProxy(options) {
|
|
|
74
86
|
const { store } = options.spool;
|
|
75
87
|
const { trim } = options;
|
|
76
88
|
const stats = { served: 0, mismatched: 0, guarded: 0 };
|
|
89
|
+
const images = options.images ? createImageConverter(options.images) : undefined;
|
|
90
|
+
const video = options.video ? resolveVideoOptions(options.video) : undefined;
|
|
91
|
+
const videoStats = { pages: 0, rewritten: 0 };
|
|
92
|
+
const imageStats = images?.stats ?? {
|
|
93
|
+
converted: 0,
|
|
94
|
+
bytesIn: 0,
|
|
95
|
+
bytesOut: 0,
|
|
96
|
+
cacheHits: 0,
|
|
97
|
+
coalesced: 0,
|
|
98
|
+
lazied: 0,
|
|
99
|
+
responsive: 0,
|
|
100
|
+
skipped: {}
|
|
101
|
+
};
|
|
102
|
+
const dictionaries = options.dictionary
|
|
103
|
+
? createDictionaries(options.dictionary, gate)
|
|
104
|
+
: undefined;
|
|
105
|
+
const embeds = options.embeds ? resolveEmbedOptions(options.embeds) : undefined;
|
|
106
|
+
const embedStats = { pages: 0, rewritten: 0, skipped: {} };
|
|
107
|
+
const lazy = images?.options.lazy || undefined;
|
|
108
|
+
const responsive = images?.options.responsive || undefined;
|
|
109
|
+
const readsHtml = Boolean(trim || video || embeds || lazy || responsive || images?.options.hints);
|
|
110
|
+
/** the embed, responsive and lazy rewrites, which do not depend on the visitor; only the live path counts them */
|
|
111
|
+
function rewriteMarkup(text, headers, pageUrl, count) {
|
|
112
|
+
let out = text;
|
|
113
|
+
if (embeds) {
|
|
114
|
+
const result = rewriteEmbeds(out, embeds, {
|
|
115
|
+
csp: headers.has('content-security-policy')
|
|
116
|
+
});
|
|
117
|
+
out = result.html;
|
|
118
|
+
if (count) {
|
|
119
|
+
if (result.rewritten > 0) {
|
|
120
|
+
embedStats.pages++;
|
|
121
|
+
embedStats.rewritten += result.rewritten;
|
|
122
|
+
}
|
|
123
|
+
for (const [why, n] of Object.entries(result.skipped)) {
|
|
124
|
+
embedStats.skipped[why] = (embedStats.skipped[why] ?? 0) + n;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (responsive && images) {
|
|
129
|
+
const result = rewriteResponsive(out, responsive, {
|
|
130
|
+
pageUrl,
|
|
131
|
+
paths: images.options.paths
|
|
132
|
+
});
|
|
133
|
+
out = result.html;
|
|
134
|
+
if (count)
|
|
135
|
+
imageStats.responsive += result.rewritten;
|
|
136
|
+
}
|
|
137
|
+
if (lazy) {
|
|
138
|
+
const result = rewriteLazy(out, lazy);
|
|
139
|
+
out = result.html;
|
|
140
|
+
if (count)
|
|
141
|
+
imageStats.lazied += result.rewritten;
|
|
142
|
+
}
|
|
143
|
+
return out;
|
|
144
|
+
}
|
|
77
145
|
function note(kind, detail) {
|
|
78
146
|
if (!options.spool.sink)
|
|
79
147
|
return;
|
|
@@ -126,19 +194,28 @@ export function spoolProxy(options) {
|
|
|
126
194
|
headers.set('x-forwarded-proto', publicUrl.protocol.slice(0, -1));
|
|
127
195
|
return headers;
|
|
128
196
|
}
|
|
129
|
-
function forward(request, url) {
|
|
197
|
+
function forward(request, url, whole) {
|
|
130
198
|
const read = request.method === 'GET' || request.method === 'HEAD';
|
|
131
|
-
|
|
199
|
+
const headers = originHeaders(request.headers, url);
|
|
200
|
+
// a conversion needs the whole file, so the origin gets no conditional request
|
|
201
|
+
if (whole) {
|
|
202
|
+
headers.delete('if-none-match');
|
|
203
|
+
headers.delete('if-modified-since');
|
|
204
|
+
}
|
|
205
|
+
const search = images ? originSearch(images, url) : url.search;
|
|
206
|
+
return new Request(new URL(url.pathname + stripTrimParam(search), base), {
|
|
132
207
|
method: request.method,
|
|
133
|
-
headers
|
|
208
|
+
headers,
|
|
134
209
|
redirect: 'manual',
|
|
135
210
|
signal: request.signal,
|
|
136
211
|
...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
|
|
137
212
|
});
|
|
138
213
|
}
|
|
139
|
-
/** the headers the browser gets: bodies arrive decoded, so the encoding and its length go */
|
|
214
|
+
/** the headers the browser gets: bodies arrive decoded, so the encoding and its length go, and so does what only described the hop to the proxy */
|
|
140
215
|
function tidy(source, url) {
|
|
141
216
|
const headers = new Headers(source);
|
|
217
|
+
for (const name of HOP_BY_HOP)
|
|
218
|
+
headers.delete(name);
|
|
142
219
|
if (headers.has('content-encoding')) {
|
|
143
220
|
headers.delete('content-encoding');
|
|
144
221
|
headers.delete('content-length');
|
|
@@ -204,7 +281,13 @@ export function spoolProxy(options) {
|
|
|
204
281
|
if (!data && !isUtf8(htmlCharset(response.headers, bytes))) {
|
|
205
282
|
throw new ProtocolError('bad-artifact', `${path} is not kept (charset)`);
|
|
206
283
|
}
|
|
207
|
-
|
|
284
|
+
let text = guarded(new TextDecoder().decode(bytes), ctx.url.href);
|
|
285
|
+
// a signal-triggered rewrite is never part of a held copy
|
|
286
|
+
if (!data && video?.when === 'always') {
|
|
287
|
+
text = rewriteVideo(text, video, { saveData: false, slow: false }).html;
|
|
288
|
+
}
|
|
289
|
+
if (!data)
|
|
290
|
+
text = rewriteMarkup(text, response.headers, ctx.url.href, false);
|
|
208
291
|
if (text.length > gate.maxBytes) {
|
|
209
292
|
throw new ProtocolError('bad-artifact', `${path} is not kept (size)`);
|
|
210
293
|
}
|
|
@@ -289,9 +372,11 @@ export function spoolProxy(options) {
|
|
|
289
372
|
return answered;
|
|
290
373
|
if (url.pathname === PATHS.worker && request.method === 'GET')
|
|
291
374
|
return spool.stub();
|
|
375
|
+
const converting = images !== undefined && imageRequest(images, request);
|
|
376
|
+
let visitorRewrite = false;
|
|
292
377
|
let upstream;
|
|
293
378
|
try {
|
|
294
|
-
upstream = await call(forward(request, url));
|
|
379
|
+
upstream = await call(forward(request, url, converting));
|
|
295
380
|
}
|
|
296
381
|
catch {
|
|
297
382
|
return new Response('bad gateway', {
|
|
@@ -299,10 +384,13 @@ export function spoolProxy(options) {
|
|
|
299
384
|
headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' }
|
|
300
385
|
});
|
|
301
386
|
}
|
|
302
|
-
if (
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
387
|
+
if (images && converting)
|
|
388
|
+
upstream = await convertImage(images, request, upstream);
|
|
389
|
+
if (request.method === 'GET') {
|
|
390
|
+
const key = trim ? trimKey(url.href) : '';
|
|
391
|
+
const entry = trim?.styles[key];
|
|
392
|
+
if (trim &&
|
|
393
|
+
entry &&
|
|
306
394
|
url.searchParams.get(TRIM_PARAM) !== '0' &&
|
|
307
395
|
upstream.status === 200 &&
|
|
308
396
|
(upstream.headers.get('content-type') ?? '').toLowerCase().includes('text/css')) {
|
|
@@ -324,14 +412,33 @@ export function spoolProxy(options) {
|
|
|
324
412
|
headers
|
|
325
413
|
});
|
|
326
414
|
}
|
|
327
|
-
else if (
|
|
415
|
+
else if (readsHtml &&
|
|
416
|
+
isHtml(upstream) &&
|
|
417
|
+
![204, 205, 304].includes(upstream.status)) {
|
|
328
418
|
const bytes = new Uint8Array(await upstream.arrayBuffer());
|
|
329
|
-
// another charset is rewritten byte for byte; the
|
|
419
|
+
// another charset is rewritten byte for byte; the edits insert ASCII only
|
|
330
420
|
const utf8 = isUtf8(htmlCharset(upstream.headers, bytes));
|
|
331
421
|
const text = utf8 ? new TextDecoder().decode(bytes) : fromBytes(bytes);
|
|
332
|
-
|
|
422
|
+
let verdict = guarded(text, url.href);
|
|
333
423
|
const headers = new Headers(upstream.headers);
|
|
334
424
|
headers.delete('content-length');
|
|
425
|
+
if (video) {
|
|
426
|
+
const result = rewriteVideo(verdict, video, signalsOf(request.headers));
|
|
427
|
+
if (result.rewritten > 0) {
|
|
428
|
+
verdict = result.html;
|
|
429
|
+
videoStats.pages++;
|
|
430
|
+
videoStats.rewritten += result.rewritten;
|
|
431
|
+
// a page rewritten for one visitor's connection is not another's copy
|
|
432
|
+
if (video.when !== 'always') {
|
|
433
|
+
visitorRewrite = true;
|
|
434
|
+
makePrivate(headers);
|
|
435
|
+
mergeList(headers, 'vary', ['Save-Data', 'ECT']);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
verdict = rewriteMarkup(verdict, upstream.headers, url.href, true);
|
|
440
|
+
if (images?.options.hints)
|
|
441
|
+
mergeList(headers, 'accept-ch', IMAGE_HINTS);
|
|
335
442
|
if (verdict !== text)
|
|
336
443
|
weaken(headers);
|
|
337
444
|
const body = verdict === text ? bytes : utf8 ? verdict : toBytes(verdict);
|
|
@@ -351,6 +458,9 @@ export function spoolProxy(options) {
|
|
|
351
458
|
if (request.method !== 'GET' && request.method !== 'HEAD') {
|
|
352
459
|
return new Response(upstream.body, init);
|
|
353
460
|
}
|
|
461
|
+
if (dictionaries?.applies(request, upstream)) {
|
|
462
|
+
return dictionaries.apply(request, new Response(upstream.body, init));
|
|
463
|
+
}
|
|
354
464
|
const hit = matchRoute(routes, url.pathname);
|
|
355
465
|
const data = hit.route.strategy === 'data';
|
|
356
466
|
const principal = hit.route.scope === 'principal';
|
|
@@ -382,7 +492,22 @@ export function spoolProxy(options) {
|
|
|
382
492
|
path: url.pathname,
|
|
383
493
|
page: Promise.resolve({ text, tags: effective })
|
|
384
494
|
});
|
|
385
|
-
|
|
495
|
+
// a visitor-signal rewrite is not in the copy /.spool/artifact builds, so the worker asks for it
|
|
496
|
+
const same = !visitorRewrite && !data && !principal;
|
|
497
|
+
return spool.decorate(request, new Response(text, init), same ? { assembled: text } : {});
|
|
386
498
|
}
|
|
387
|
-
return Object.assign(handle, {
|
|
499
|
+
return Object.assign(handle, {
|
|
500
|
+
spool,
|
|
501
|
+
trim: stats,
|
|
502
|
+
images: imageStats,
|
|
503
|
+
video: videoStats,
|
|
504
|
+
embeds: embedStats,
|
|
505
|
+
dictionary: dictionaries?.stats ?? {
|
|
506
|
+
registered: 0,
|
|
507
|
+
encoded: 0,
|
|
508
|
+
bytesIn: 0,
|
|
509
|
+
bytesOut: 0,
|
|
510
|
+
skipped: {}
|
|
511
|
+
}
|
|
512
|
+
});
|
|
388
513
|
}
|