@drupflare/spool 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +444 -18
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +2 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/media/embeds.d.ts +60 -0
  11. package/dist/media/embeds.js +135 -0
  12. package/dist/media/headers.d.ts +13 -0
  13. package/dist/media/headers.js +44 -0
  14. package/dist/media/html.d.ts +27 -0
  15. package/dist/media/html.js +93 -0
  16. package/dist/media/image.d.ts +222 -0
  17. package/dist/media/image.js +413 -0
  18. package/dist/media/index.d.ts +6 -0
  19. package/dist/media/index.js +6 -0
  20. package/dist/media/lazy.d.ts +26 -0
  21. package/dist/media/lazy.js +74 -0
  22. package/dist/media/responsive.d.ts +45 -0
  23. package/dist/media/responsive.js +98 -0
  24. package/dist/media/video.d.ts +30 -0
  25. package/dist/media/video.js +75 -0
  26. package/dist/protocol/envelope.d.ts +2 -0
  27. package/dist/protocol/envelope.js +12 -2
  28. package/dist/protocol/errors.d.ts +1 -1
  29. package/dist/protocol/hash.d.ts +2 -0
  30. package/dist/protocol/hash.js +8 -0
  31. package/dist/protocol/index.d.ts +4 -0
  32. package/dist/protocol/index.js +4 -0
  33. package/dist/protocol/inline.d.ts +14 -0
  34. package/dist/protocol/inline.js +43 -0
  35. package/dist/protocol/navigation.d.ts +81 -0
  36. package/dist/protocol/navigation.js +249 -0
  37. package/dist/protocol/normalise.d.ts +4 -3
  38. package/dist/protocol/normalise.js +6 -4
  39. package/dist/protocol/provision.d.ts +6 -1
  40. package/dist/protocol/provision.js +15 -5
  41. package/dist/protocol/query.d.ts +80 -0
  42. package/dist/protocol/query.js +153 -0
  43. package/dist/protocol/route.d.ts +80 -4
  44. package/dist/protocol/route.js +63 -0
  45. package/dist/proxy/bases.d.ts +53 -0
  46. package/dist/proxy/bases.js +76 -0
  47. package/dist/proxy/dictionary.d.ts +58 -0
  48. package/dist/proxy/dictionary.js +197 -0
  49. package/dist/proxy/index.d.ts +2 -0
  50. package/dist/proxy/index.js +2 -0
  51. package/dist/proxy/proxy.d.ts +47 -0
  52. package/dist/proxy/proxy.js +140 -15
  53. package/dist/report/capacity.d.ts +59 -0
  54. package/dist/report/capacity.js +76 -0
  55. package/dist/report/channels.d.ts +60 -0
  56. package/dist/report/channels.js +65 -0
  57. package/dist/report/derive.d.ts +2 -0
  58. package/dist/report/derive.js +6 -2
  59. package/dist/report/index.d.ts +4 -0
  60. package/dist/report/index.js +4 -0
  61. package/dist/report/institution.d.ts +97 -0
  62. package/dist/report/institution.js +79 -0
  63. package/dist/report/ledger.d.ts +197 -0
  64. package/dist/report/ledger.js +308 -0
  65. package/dist/report/measurement.d.ts +1 -1
  66. package/dist/report/measurement.js +7 -1
  67. package/dist/report/traffic.d.ts +11 -2
  68. package/dist/report/traffic.js +13 -3
  69. package/dist/server/embedded.d.ts +3 -3
  70. package/dist/server/embedded.js +3 -3
  71. package/dist/server/index.d.ts +1 -0
  72. package/dist/server/index.js +1 -0
  73. package/dist/server/spool.d.ts +20 -0
  74. package/dist/server/spool.js +62 -9
  75. package/dist/sw/render.d.ts +3 -1
  76. package/dist/sw/render.js +14 -1
  77. package/dist/sw/replica.d.ts +2 -0
  78. package/dist/sw/replica.js +6 -0
  79. package/dist/sw/router.js +2 -2
  80. package/dist/sw/runtime.js +104 -9
  81. package/dist/trim/tokens.d.ts +4 -0
  82. package/dist/trim/tokens.js +1 -1
  83. package/package.json +10 -4
@@ -0,0 +1,76 @@
1
+ /** the path of a URL up to and including its last slash */
2
+ export function dirOf(url) {
3
+ const path = new URL(url).pathname;
4
+ return path.slice(0, path.lastIndexOf('/') + 1);
5
+ }
6
+ const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
7
+ /**
8
+ * The member of `members` that minimizes the sum, over the other members, of the smaller of their
9
+ * delta against it and their plain size, plus the member's own plain size under `first-visit`.
10
+ * Ties keep the member listed first. `score` is the minimized sum and `others` its part without the
11
+ * base's own size.
12
+ */
13
+ export function pickBase(members, delta, plain, policy = 'held') {
14
+ let best = -1;
15
+ let bestScore = Infinity;
16
+ let bestOthers = Infinity;
17
+ for (const c of members) {
18
+ let others = 0;
19
+ for (const f of members) {
20
+ if (f !== c)
21
+ others += Math.min(delta(f, c), plain(f));
22
+ }
23
+ const score = policy === 'first-visit' ? others + plain(c) : others;
24
+ if (score < bestScore) {
25
+ best = c;
26
+ bestScore = score;
27
+ bestOthers = others;
28
+ }
29
+ }
30
+ return { best, score: bestScore, others: bestOthers };
31
+ }
32
+ /**
33
+ * One base per directory and kind, scored by `policy` (default `held`). Members are ordered by URL
34
+ * before scoring, so a tie goes to the URL that sorts first whatever the input order. A group of
35
+ * one file has no base.
36
+ */
37
+ export async function chooseBases(files, delta, options = {}) {
38
+ const at = (i) => files[i];
39
+ const groups = new Map();
40
+ for (const [i, f] of files.entries()) {
41
+ const key = JSON.stringify([f.kind, dirOf(f.url)]);
42
+ const members = groups.get(key);
43
+ if (members)
44
+ members.push(i);
45
+ else
46
+ groups.set(key, [i]);
47
+ }
48
+ const choices = [];
49
+ for (const members of groups.values()) {
50
+ if (members.length < 2)
51
+ continue;
52
+ const sorted = [...members].sort((a, b) => compare(at(a).url, at(b).url));
53
+ const sizes = new Map();
54
+ for (const f of sorted) {
55
+ for (const c of sorted) {
56
+ if (f !== c)
57
+ sizes.set(`${f}|${c}`, await delta(f, c));
58
+ }
59
+ }
60
+ const { best, others } = pickBase(sorted, (f, c) => sizes.get(`${f}|${c}`), (f) => at(f).plain, options.policy);
61
+ const { kind, url } = at(best);
62
+ choices.push({
63
+ kind,
64
+ directory: dirOf(url),
65
+ url,
66
+ files: sorted.length,
67
+ meanDelta: others / (sorted.length - 1)
68
+ });
69
+ }
70
+ return choices.sort((a, b) => compare(a.directory, b.directory) || compare(a.kind, b.kind));
71
+ }
72
+ /** true for the exact pathnames of the chosen base URLs; a query string does not count */
73
+ export function baseMatcher(choices) {
74
+ const paths = new Set(choices.map((choice) => new URL(choice.url).pathname));
75
+ return (pathname) => paths.has(pathname);
76
+ }
@@ -0,0 +1,58 @@
1
+ import { type Gate } from './eligible.js';
2
+ /** the content codings of RFC 9842: Brotli (`dcb`) and Zstandard (`dcz`) against a dictionary */
3
+ export type DictionaryFormat = 'dcb' | 'dcz';
4
+ /**
5
+ * Compresses `input` against `dictionary` and returns the compressed stream only. spool adds the
6
+ * coding's header (the magic bytes and the SHA-256 of the dictionary). For `dcz` the dictionary is a
7
+ * raw content dictionary and the window must stay at or under 8 MiB or 1.25 times the dictionary,
8
+ * whichever is larger. For `dcb` it is a Brotli prefix dictionary. It throws for a format it cannot
9
+ * produce, and the proxy then answers with the plain body.
10
+ */
11
+ export type DictionaryEncoder = (dictionary: Uint8Array, input: Uint8Array, format: DictionaryFormat) => Promise<Uint8Array>;
12
+ /** what `spoolProxy({ dictionary })` takes */
13
+ export type DictionaryOptions = {
14
+ /** produces the compressed stream; see {@link DictionaryEncoder} */
15
+ encoder: DictionaryEncoder;
16
+ /** the paths that become and use dictionaries, tested against the pathname; default `/\.(?:css|m?js)$/i` */
17
+ match?: RegExp;
18
+ /**
19
+ * Which of those responses register as dictionaries, as a function of the pathname, a regular
20
+ * expression or a list of them; default every one `match` accepts. The others get no
21
+ * `Use-As-Dictionary`, so the browser holds only the bases you name, and they are still encoded
22
+ * against a base the request names.
23
+ */
24
+ base?: RegExp | readonly RegExp[] | ((pathname: string) => boolean);
25
+ /** largest body kept as a dictionary or encoded, in bytes; default 2 MiB */
26
+ maxBytes?: number;
27
+ /** the codings offered, in order of preference, when the request accepts them; default `['dcz']` */
28
+ formats?: readonly DictionaryFormat[];
29
+ /** bytes of dictionaries kept in memory, the least recently used going first; default 16 MiB */
30
+ cacheBytes?: number;
31
+ };
32
+ /**
33
+ * What the proxy counted since it started: `registered` is the dictionaries stored (new content
34
+ * hashes), `encoded` the responses sent as `dcb` or `dcz`, `bytesIn` and `bytesOut` the plain and
35
+ * encoded sizes of those responses, and `skipped` the eligible responses left plain by reason:
36
+ * `unavailable` (the request named no dictionary the proxy holds, or accepted no coding offered),
37
+ * `size`, `larger` (the encoded body was not smaller), `failed` (the encoder threw), `range` and
38
+ * `pattern` (the path cannot be written as a `match` pattern).
39
+ */
40
+ export type DictionaryStats = {
41
+ registered: number;
42
+ encoded: number;
43
+ bytesIn: number;
44
+ bytesOut: number;
45
+ skipped: Record<string, number>;
46
+ };
47
+ /** the body of a `dcb` or `dcz` response: the coding's magic bytes, the dictionary's SHA-256, then the stream */
48
+ export declare function frameDictionaryBody(format: DictionaryFormat, hash: Uint8Array, stream: Uint8Array): Uint8Array;
49
+ /** the state `spoolProxy` keeps for `dictionary`: checked options, the dictionaries and the counters */
50
+ export type Dictionaries = {
51
+ readonly stats: DictionaryStats;
52
+ /** whether a response could take part, by its content type and the request's method and path */
53
+ applies(request: Request, response: Response): boolean;
54
+ /** answers one origin response for `request`, registering it and encoding it when the request names a stored dictionary */
55
+ apply(request: Request, response: Response): Promise<Response>;
56
+ };
57
+ /** applies the defaults and checks the values; `ProtocolError('bad-route')` for one out of range */
58
+ export declare function createDictionaries(options: DictionaryOptions, gate: Gate): Dictionaries;
@@ -0,0 +1,197 @@
1
+ import { mergeList } from '../media/headers.js';
2
+ import { ProtocolError } from '../protocol/errors.js';
3
+ import { decline } from './eligible.js';
4
+ const MAGIC = {
5
+ dcb: [0xff, 0x44, 0x43, 0x42],
6
+ dcz: [0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00]
7
+ };
8
+ const DEFAULT_MATCH = /\.(?:css|m?js)$/i;
9
+ const DEFAULT_MAX_BYTES = 2 * 1024 * 1024;
10
+ const DEFAULT_CACHE_BYTES = 16 * 1024 * 1024;
11
+ const SAFE_DIRECTORY = /^[A-Za-z0-9/_.~%-]*$/;
12
+ const AVAILABLE = /^:([A-Za-z0-9+/]{43}=):$/;
13
+ /** the body of a `dcb` or `dcz` response: the coding's magic bytes, the dictionary's SHA-256, then the stream */
14
+ export function frameDictionaryBody(format, hash, stream) {
15
+ const magic = MAGIC[format];
16
+ const out = new Uint8Array(magic.length + hash.length + stream.length);
17
+ out.set(magic, 0);
18
+ out.set(hash, magic.length);
19
+ out.set(stream, magic.length + hash.length);
20
+ return out;
21
+ }
22
+ const toBase64 = (bytes) => btoa(String.fromCharCode(...bytes));
23
+ async function sha256(bytes) {
24
+ return new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
25
+ }
26
+ /** the codings `Accept-Encoding` lists with a quality above zero, lowercase */
27
+ function accepted(header) {
28
+ const out = new Set();
29
+ for (const part of (header ?? '').split(',')) {
30
+ const [token, ...params] = part.trim().toLowerCase().split(';');
31
+ const q = params.map((p) => p.trim()).find((p) => p.startsWith('q='));
32
+ if (token && !(q !== undefined && Number(q.slice(2)) === 0))
33
+ out.add(token.trim());
34
+ }
35
+ return out;
36
+ }
37
+ function kindOf(response) {
38
+ const type = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase();
39
+ if (type === 'text/css')
40
+ return 'style';
41
+ if (type === 'text/javascript' || type === 'application/javascript')
42
+ return 'script';
43
+ return null;
44
+ }
45
+ /** applies the defaults and checks the values; `ProtocolError('bad-route')` for one out of range */
46
+ export function createDictionaries(options, gate) {
47
+ const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
48
+ const cacheBytes = options.cacheBytes ?? DEFAULT_CACHE_BYTES;
49
+ const formats = options.formats ?? ['dcz'];
50
+ const match = options.match ?? DEFAULT_MATCH;
51
+ const { base } = options;
52
+ const isBase = base === undefined
53
+ ? () => true
54
+ : typeof base === 'function'
55
+ ? base
56
+ : (pathname) => [].concat(base).some((pattern) => {
57
+ pattern.lastIndex = 0;
58
+ return pattern.test(pathname);
59
+ });
60
+ if (!Number.isInteger(maxBytes) || maxBytes < 1) {
61
+ throw new ProtocolError('bad-route', 'dictionary maxBytes must be a positive integer');
62
+ }
63
+ if (!Number.isInteger(cacheBytes) || cacheBytes < 1) {
64
+ throw new ProtocolError('bad-route', 'dictionary cacheBytes must be a positive integer');
65
+ }
66
+ if (formats.length === 0 || formats.some((f) => f !== 'dcb' && f !== 'dcz')) {
67
+ throw new ProtocolError('bad-route', `dictionary formats must list 'dcb' or 'dcz'`);
68
+ }
69
+ const stats = {
70
+ registered: 0,
71
+ encoded: 0,
72
+ bytesIn: 0,
73
+ bytesOut: 0,
74
+ skipped: {}
75
+ };
76
+ const held = new Map();
77
+ let size = 0;
78
+ const skip = (why) => void (stats.skipped[why] = (stats.skipped[why] ?? 0) + 1);
79
+ function touch(key, body) {
80
+ held.delete(key);
81
+ held.set(key, body);
82
+ }
83
+ function remember(key, bytes) {
84
+ const body = held.get(key);
85
+ if (body)
86
+ return touch(key, body);
87
+ held.set(key, bytes);
88
+ size += bytes.length;
89
+ stats.registered++;
90
+ for (const [oldest, body] of held) {
91
+ if (size <= cacheBytes || held.size === 1)
92
+ break;
93
+ held.delete(oldest);
94
+ size -= body.length;
95
+ }
96
+ }
97
+ function applies(request, response) {
98
+ if (request.method !== 'GET' || response.status !== 200)
99
+ return false;
100
+ match.lastIndex = 0;
101
+ if (!match.test(new URL(request.url).pathname))
102
+ return false;
103
+ return kindOf(response) !== null;
104
+ }
105
+ return {
106
+ stats,
107
+ applies,
108
+ async apply(request, response) {
109
+ const url = new URL(request.url);
110
+ const auth = new Headers();
111
+ if (request.headers.has('authorization'))
112
+ auth.set('authorization', 'x');
113
+ const why = decline({
114
+ method: 'GET',
115
+ search: '',
116
+ requestHeaders: auth,
117
+ response,
118
+ tags: [],
119
+ data: true,
120
+ principal: false
121
+ }, gate);
122
+ if (why !== null)
123
+ return response;
124
+ const declared = Number(response.headers.get('content-length'));
125
+ if (request.headers.has('range')) {
126
+ skip('range');
127
+ return response;
128
+ }
129
+ if (Number.isFinite(declared) && declared > maxBytes) {
130
+ skip('size');
131
+ return response;
132
+ }
133
+ const directory = url.pathname.slice(0, url.pathname.lastIndexOf('/') + 1);
134
+ if (!SAFE_DIRECTORY.test(directory)) {
135
+ skip('pattern');
136
+ return response;
137
+ }
138
+ const bytes = new Uint8Array(await response.arrayBuffer());
139
+ const headers = new Headers(response.headers);
140
+ const plain = () => new Response(bytes, {
141
+ status: 200,
142
+ statusText: response.statusText,
143
+ headers
144
+ });
145
+ if (bytes.length === 0 || bytes.length > maxBytes) {
146
+ skip('size');
147
+ return plain();
148
+ }
149
+ const hash = await sha256(bytes);
150
+ const key = AVAILABLE.exec(request.headers.get('available-dictionary') ?? '')?.[1];
151
+ const dictionary = key === undefined ? undefined : held.get(key);
152
+ if (key !== undefined && dictionary)
153
+ touch(key, dictionary);
154
+ mergeList(headers, 'vary', ['Accept-Encoding', 'Available-Dictionary']);
155
+ if (isBase(url.pathname)) {
156
+ headers.set('use-as-dictionary', `match="${directory}*", match-dest=("${kindOf(response)}")`);
157
+ remember(toBase64(hash), bytes);
158
+ }
159
+ if (!dictionary) {
160
+ skip('unavailable');
161
+ return plain();
162
+ }
163
+ const wanted = accepted(request.headers.get('accept-encoding'));
164
+ const format = formats.find((f) => wanted.has(f));
165
+ if (!format) {
166
+ skip('unavailable');
167
+ return plain();
168
+ }
169
+ let framed;
170
+ try {
171
+ const stream = await options.encoder(dictionary, bytes, format);
172
+ framed = frameDictionaryBody(format, await sha256(dictionary), stream);
173
+ }
174
+ catch {
175
+ skip('failed');
176
+ return plain();
177
+ }
178
+ if (framed.length >= bytes.length) {
179
+ skip('larger');
180
+ return plain();
181
+ }
182
+ headers.set('content-encoding', format);
183
+ headers.delete('content-length');
184
+ const etag = headers.get('etag');
185
+ if (etag && !etag.startsWith('W/'))
186
+ headers.set('etag', `W/${etag}`);
187
+ stats.encoded++;
188
+ stats.bytesIn += bytes.length;
189
+ stats.bytesOut += framed.length;
190
+ return new Response(framed, {
191
+ status: 200,
192
+ statusText: response.statusText,
193
+ headers
194
+ });
195
+ }
196
+ };
197
+ }
@@ -1,3 +1,5 @@
1
+ export { baseMatcher, chooseBases, type BaseCandidate, type BaseChoice, type BasePolicy, type DeltaSize } from './bases.js';
2
+ export * from './dictionary.js';
1
3
  export * from './eligible.js';
2
4
  export * from './proxy.js';
3
5
  export * from './purge.js';
@@ -1,3 +1,5 @@
1
+ export { baseMatcher, chooseBases } from './bases.js';
2
+ export * from './dictionary.js';
1
3
  export * from './eligible.js';
2
4
  export * from './proxy.js';
3
5
  export * from './purge.js';
@@ -1,6 +1,10 @@
1
+ import { type EmbedOptions, type EmbedStats } from '../media/embeds.js';
2
+ import { type ImageOptions, type ImageStats } from '../media/image.js';
3
+ import { type VideoOptions } from '../media/video.js';
1
4
  import { type AssemblyRoute, type DataRoute } from '../protocol/route.js';
2
5
  import { type Spool, type SpoolOptions } from '../server/spool.js';
3
6
  import { type TrimManifest } from '../trim/index.js';
7
+ import { type DictionaryOptions, type DictionaryStats } from './dictionary.js';
4
8
  /**
5
9
  * A route the proxy answers locally, declared like a `createSpool` route minus `load`: the proxy
6
10
  * loads the page from the origin. A `data` route parses the origin's JSON. `depends` adds tags to
@@ -49,6 +53,31 @@ export type ProxyOptions = {
49
53
  * and every HTML page is checked against the guard first. Without it nothing is trimmed.
50
54
  */
51
55
  trim?: TrimManifest;
56
+ /**
57
+ * Converts JPEG and PNG responses to WebP for a browser that accepts it, at the quality and
58
+ * width the request's headers call for, up to a width ceiling of 2560 pixels when the request
59
+ * names none. Identical simultaneous requests share one encoder call, and an encoder that runs
60
+ * past 10 seconds is abandoned for that request. See `ImageOptions`. Without it no image is
61
+ * touched.
62
+ */
63
+ images?: ImageOptions;
64
+ /**
65
+ * Rewrites `<video>` tags in HTML responses, such as `preload="none"`; see `VideoOptions`.
66
+ * Without it no page is touched for video.
67
+ */
68
+ video?: VideoOptions;
69
+ /**
70
+ * Replaces server-rendered YouTube and Vimeo `<iframe>` embeds with a link that loads the
71
+ * player on click; see `EmbedOptions`. A page that sends a `Content-Security-Policy` is left
72
+ * alone. Without it no embed is touched.
73
+ */
74
+ embeds?: EmbedOptions;
75
+ /**
76
+ * Serves a stylesheet or script as `dcz` or `dcb` against an earlier public file of the same
77
+ * directory the visitor already holds (RFC 9842, Chromium 130 and later); see
78
+ * `DictionaryOptions`. Without it no response is marked or encoded.
79
+ */
80
+ dictionary?: DictionaryOptions;
52
81
  };
53
82
  /** what the proxy counted since it started: trimmed answers, hash mismatches and guard firings */
54
83
  export type TrimStats = {
@@ -56,6 +85,11 @@ export type TrimStats = {
56
85
  mismatched: number;
57
86
  guarded: number;
58
87
  };
88
+ /** what the proxy counted for `video`: pages with at least one rewritten tag, and the tags */
89
+ export type VideoStats = {
90
+ pages: number;
91
+ rewritten: number;
92
+ };
59
93
  /** where the proxy writes a guard firing or a hash mismatch to the sink, as JSON `{ kind, ... }` */
60
94
  export declare const TRIM_PATH = "/.spool/trim";
61
95
  /** the Fetch handler `spoolProxy` returns, with the spool it built */
@@ -64,6 +98,14 @@ export type SpoolProxy = {
64
98
  readonly spool: Spool;
65
99
  /** live counters; all zero when `trim` is not set */
66
100
  readonly trim: Readonly<TrimStats>;
101
+ /** live counters; all zero when `images` is not set */
102
+ readonly images: Readonly<ImageStats>;
103
+ /** live counters; all zero when `video` is not set */
104
+ readonly video: Readonly<VideoStats>;
105
+ /** live counters; all zero when `embeds` is not set */
106
+ readonly embeds: Readonly<EmbedStats>;
107
+ /** live counters; all zero when `dictionary` is not set */
108
+ readonly dictionary: Readonly<DictionaryStats>;
67
109
  };
68
110
  /**
69
111
  * A Fetch handler that sits in front of an origin that cannot load the library, such as PHP or a
@@ -76,6 +118,11 @@ export type SpoolProxy = {
76
118
  * The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
77
119
  * become the page's dependencies, and a page with none depends on its own path.
78
120
  *
121
+ * With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
122
+ * converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
123
+ * `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
124
+ * kept, so a held page carries the rewritten tags.
125
+ *
79
126
  * Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
80
127
  * records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
81
128
  * when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
@@ -1,3 +1,9 @@
1
+ import { resolveEmbedOptions, rewriteEmbeds } from '../media/embeds.js';
2
+ import { makePrivate, mergeList, signalsOf } from '../media/headers.js';
3
+ import { convertImage, createImageConverter, IMAGE_HINTS, imageRequest, originSearch } from '../media/image.js';
4
+ import { rewriteLazy } from '../media/lazy.js';
5
+ import { rewriteResponsive } from '../media/responsive.js';
6
+ import { resolveVideoOptions, rewriteVideo } from '../media/video.js';
1
7
  import { PATHS } from '../protocol/config.js';
2
8
  import { writeEnvelope } from '../protocol/envelope.js';
3
9
  import { ProtocolError } from '../protocol/errors.js';
@@ -6,6 +12,7 @@ import { defineRoute, matchRoute } from '../protocol/route.js';
6
12
  import { artifactTag } from '../protocol/sync.js';
7
13
  import { createSpool } from '../server/spool.js';
8
14
  import { guardPage, sourceHash, stripTrimParam, TRIM_PARAM, trimKey } from '../trim/index.js';
15
+ import { createDictionaries } from './dictionary.js';
9
16
  import { decline, htmlCharset, isUtf8, setsPrivateCookie } from './eligible.js';
10
17
  import { PURGE_PATH, respondPurge } from './purge.js';
11
18
  import { pathTag, readTags, stripTags } from './tags.js';
@@ -43,6 +50,11 @@ const toBytes = (text) => Uint8Array.from(text, (char) => char.charCodeAt(0));
43
50
  * The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
44
51
  * become the page's dependencies, and a page with none depends on its own path.
45
52
  *
53
+ * With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
54
+ * converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
55
+ * `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
56
+ * kept, so a held page carries the rewritten tags.
57
+ *
46
58
  * Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
47
59
  * records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
48
60
  * when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
@@ -74,6 +86,62 @@ export function spoolProxy(options) {
74
86
  const { store } = options.spool;
75
87
  const { trim } = options;
76
88
  const stats = { served: 0, mismatched: 0, guarded: 0 };
89
+ const images = options.images ? createImageConverter(options.images) : undefined;
90
+ const video = options.video ? resolveVideoOptions(options.video) : undefined;
91
+ const videoStats = { pages: 0, rewritten: 0 };
92
+ const imageStats = images?.stats ?? {
93
+ converted: 0,
94
+ bytesIn: 0,
95
+ bytesOut: 0,
96
+ cacheHits: 0,
97
+ coalesced: 0,
98
+ lazied: 0,
99
+ responsive: 0,
100
+ skipped: {}
101
+ };
102
+ const dictionaries = options.dictionary
103
+ ? createDictionaries(options.dictionary, gate)
104
+ : undefined;
105
+ const embeds = options.embeds ? resolveEmbedOptions(options.embeds) : undefined;
106
+ const embedStats = { pages: 0, rewritten: 0, skipped: {} };
107
+ const lazy = images?.options.lazy || undefined;
108
+ const responsive = images?.options.responsive || undefined;
109
+ const readsHtml = Boolean(trim || video || embeds || lazy || responsive || images?.options.hints);
110
+ /** the embed, responsive and lazy rewrites, which do not depend on the visitor; only the live path counts them */
111
+ function rewriteMarkup(text, headers, pageUrl, count) {
112
+ let out = text;
113
+ if (embeds) {
114
+ const result = rewriteEmbeds(out, embeds, {
115
+ csp: headers.has('content-security-policy')
116
+ });
117
+ out = result.html;
118
+ if (count) {
119
+ if (result.rewritten > 0) {
120
+ embedStats.pages++;
121
+ embedStats.rewritten += result.rewritten;
122
+ }
123
+ for (const [why, n] of Object.entries(result.skipped)) {
124
+ embedStats.skipped[why] = (embedStats.skipped[why] ?? 0) + n;
125
+ }
126
+ }
127
+ }
128
+ if (responsive && images) {
129
+ const result = rewriteResponsive(out, responsive, {
130
+ pageUrl,
131
+ paths: images.options.paths
132
+ });
133
+ out = result.html;
134
+ if (count)
135
+ imageStats.responsive += result.rewritten;
136
+ }
137
+ if (lazy) {
138
+ const result = rewriteLazy(out, lazy);
139
+ out = result.html;
140
+ if (count)
141
+ imageStats.lazied += result.rewritten;
142
+ }
143
+ return out;
144
+ }
77
145
  function note(kind, detail) {
78
146
  if (!options.spool.sink)
79
147
  return;
@@ -126,19 +194,28 @@ export function spoolProxy(options) {
126
194
  headers.set('x-forwarded-proto', publicUrl.protocol.slice(0, -1));
127
195
  return headers;
128
196
  }
129
- function forward(request, url) {
197
+ function forward(request, url, whole) {
130
198
  const read = request.method === 'GET' || request.method === 'HEAD';
131
- return new Request(new URL(url.pathname + stripTrimParam(url.search), base), {
199
+ const headers = originHeaders(request.headers, url);
200
+ // a conversion needs the whole file, so the origin gets no conditional request
201
+ if (whole) {
202
+ headers.delete('if-none-match');
203
+ headers.delete('if-modified-since');
204
+ }
205
+ const search = images ? originSearch(images, url) : url.search;
206
+ return new Request(new URL(url.pathname + stripTrimParam(search), base), {
132
207
  method: request.method,
133
- headers: originHeaders(request.headers, url),
208
+ headers,
134
209
  redirect: 'manual',
135
210
  signal: request.signal,
136
211
  ...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
137
212
  });
138
213
  }
139
- /** the headers the browser gets: bodies arrive decoded, so the encoding and its length go */
214
+ /** the headers the browser gets: bodies arrive decoded, so the encoding and its length go, and so does what only described the hop to the proxy */
140
215
  function tidy(source, url) {
141
216
  const headers = new Headers(source);
217
+ for (const name of HOP_BY_HOP)
218
+ headers.delete(name);
142
219
  if (headers.has('content-encoding')) {
143
220
  headers.delete('content-encoding');
144
221
  headers.delete('content-length');
@@ -204,7 +281,13 @@ export function spoolProxy(options) {
204
281
  if (!data && !isUtf8(htmlCharset(response.headers, bytes))) {
205
282
  throw new ProtocolError('bad-artifact', `${path} is not kept (charset)`);
206
283
  }
207
- const text = guarded(new TextDecoder().decode(bytes), ctx.url.href);
284
+ let text = guarded(new TextDecoder().decode(bytes), ctx.url.href);
285
+ // a signal-triggered rewrite is never part of a held copy
286
+ if (!data && video?.when === 'always') {
287
+ text = rewriteVideo(text, video, { saveData: false, slow: false }).html;
288
+ }
289
+ if (!data)
290
+ text = rewriteMarkup(text, response.headers, ctx.url.href, false);
208
291
  if (text.length > gate.maxBytes) {
209
292
  throw new ProtocolError('bad-artifact', `${path} is not kept (size)`);
210
293
  }
@@ -289,9 +372,11 @@ export function spoolProxy(options) {
289
372
  return answered;
290
373
  if (url.pathname === PATHS.worker && request.method === 'GET')
291
374
  return spool.stub();
375
+ const converting = images !== undefined && imageRequest(images, request);
376
+ let visitorRewrite = false;
292
377
  let upstream;
293
378
  try {
294
- upstream = await call(forward(request, url));
379
+ upstream = await call(forward(request, url, converting));
295
380
  }
296
381
  catch {
297
382
  return new Response('bad gateway', {
@@ -299,10 +384,13 @@ export function spoolProxy(options) {
299
384
  headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' }
300
385
  });
301
386
  }
302
- if (trim && request.method === 'GET') {
303
- const key = trimKey(url.href);
304
- const entry = trim.styles[key];
305
- if (entry &&
387
+ if (images && converting)
388
+ upstream = await convertImage(images, request, upstream);
389
+ if (request.method === 'GET') {
390
+ const key = trim ? trimKey(url.href) : '';
391
+ const entry = trim?.styles[key];
392
+ if (trim &&
393
+ entry &&
306
394
  url.searchParams.get(TRIM_PARAM) !== '0' &&
307
395
  upstream.status === 200 &&
308
396
  (upstream.headers.get('content-type') ?? '').toLowerCase().includes('text/css')) {
@@ -324,14 +412,33 @@ export function spoolProxy(options) {
324
412
  headers
325
413
  });
326
414
  }
327
- else if (isHtml(upstream) && ![204, 205, 304].includes(upstream.status)) {
415
+ else if (readsHtml &&
416
+ isHtml(upstream) &&
417
+ ![204, 205, 304].includes(upstream.status)) {
328
418
  const bytes = new Uint8Array(await upstream.arrayBuffer());
329
- // another charset is rewritten byte for byte; the guard inserts ASCII only
419
+ // another charset is rewritten byte for byte; the edits insert ASCII only
330
420
  const utf8 = isUtf8(htmlCharset(upstream.headers, bytes));
331
421
  const text = utf8 ? new TextDecoder().decode(bytes) : fromBytes(bytes);
332
- const verdict = guarded(text, url.href);
422
+ let verdict = guarded(text, url.href);
333
423
  const headers = new Headers(upstream.headers);
334
424
  headers.delete('content-length');
425
+ if (video) {
426
+ const result = rewriteVideo(verdict, video, signalsOf(request.headers));
427
+ if (result.rewritten > 0) {
428
+ verdict = result.html;
429
+ videoStats.pages++;
430
+ videoStats.rewritten += result.rewritten;
431
+ // a page rewritten for one visitor's connection is not another's copy
432
+ if (video.when !== 'always') {
433
+ visitorRewrite = true;
434
+ makePrivate(headers);
435
+ mergeList(headers, 'vary', ['Save-Data', 'ECT']);
436
+ }
437
+ }
438
+ }
439
+ verdict = rewriteMarkup(verdict, upstream.headers, url.href, true);
440
+ if (images?.options.hints)
441
+ mergeList(headers, 'accept-ch', IMAGE_HINTS);
335
442
  if (verdict !== text)
336
443
  weaken(headers);
337
444
  const body = verdict === text ? bytes : utf8 ? verdict : toBytes(verdict);
@@ -351,6 +458,9 @@ export function spoolProxy(options) {
351
458
  if (request.method !== 'GET' && request.method !== 'HEAD') {
352
459
  return new Response(upstream.body, init);
353
460
  }
461
+ if (dictionaries?.applies(request, upstream)) {
462
+ return dictionaries.apply(request, new Response(upstream.body, init));
463
+ }
354
464
  const hit = matchRoute(routes, url.pathname);
355
465
  const data = hit.route.strategy === 'data';
356
466
  const principal = hit.route.scope === 'principal';
@@ -382,7 +492,22 @@ export function spoolProxy(options) {
382
492
  path: url.pathname,
383
493
  page: Promise.resolve({ text, tags: effective })
384
494
  });
385
- return spool.decorate(request, new Response(text, init));
495
+ // a visitor-signal rewrite is not in the copy /.spool/artifact builds, so the worker asks for it
496
+ const same = !visitorRewrite && !data && !principal;
497
+ return spool.decorate(request, new Response(text, init), same ? { assembled: text } : {});
386
498
  }
387
- return Object.assign(handle, { spool, trim: stats });
499
+ return Object.assign(handle, {
500
+ spool,
501
+ trim: stats,
502
+ images: imageStats,
503
+ video: videoStats,
504
+ embeds: embedStats,
505
+ dictionary: dictionaries?.stats ?? {
506
+ registered: 0,
507
+ encoded: 0,
508
+ bytesIn: 0,
509
+ bytesOut: 0,
510
+ skipped: {}
511
+ }
512
+ });
388
513
  }