@drupflare/spool 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +1749 -2
  3. package/dist/cdn/client.js +1 -0
  4. package/dist/cdn/location.d.ts +11 -0
  5. package/dist/cdn/location.js +15 -0
  6. package/dist/cdn/manifest.json +16 -0
  7. package/dist/cdn/spool-sw.js +1 -0
  8. package/dist/client/entry.d.ts +1 -0
  9. package/dist/client/entry.js +25 -0
  10. package/dist/client/index.d.ts +3 -0
  11. package/dist/client/index.js +2 -0
  12. package/dist/client/lifecycle.d.ts +10 -0
  13. package/dist/client/lifecycle.js +10 -0
  14. package/dist/client/loads.d.ts +16 -0
  15. package/dist/client/loads.js +41 -0
  16. package/dist/client/register.d.ts +34 -0
  17. package/dist/client/register.js +58 -0
  18. package/dist/index.d.ts +2 -0
  19. package/dist/index.js +2 -0
  20. package/dist/media/embeds.d.ts +59 -0
  21. package/dist/media/embeds.js +120 -0
  22. package/dist/media/headers.d.ts +13 -0
  23. package/dist/media/headers.js +44 -0
  24. package/dist/media/html.d.ts +27 -0
  25. package/dist/media/html.js +93 -0
  26. package/dist/media/image.d.ts +222 -0
  27. package/dist/media/image.js +413 -0
  28. package/dist/media/index.d.ts +6 -0
  29. package/dist/media/index.js +6 -0
  30. package/dist/media/lazy.d.ts +26 -0
  31. package/dist/media/lazy.js +74 -0
  32. package/dist/media/responsive.d.ts +45 -0
  33. package/dist/media/responsive.js +98 -0
  34. package/dist/media/video.d.ts +30 -0
  35. package/dist/media/video.js +75 -0
  36. package/dist/node/listener.d.ts +19 -0
  37. package/dist/node/listener.js +76 -0
  38. package/dist/protocol/artifact.d.ts +54 -0
  39. package/dist/protocol/artifact.js +79 -0
  40. package/dist/protocol/assembly.d.ts +34 -0
  41. package/dist/protocol/assembly.js +72 -0
  42. package/dist/protocol/beacon.d.ts +59 -0
  43. package/dist/protocol/beacon.js +86 -0
  44. package/dist/protocol/collector.d.ts +41 -0
  45. package/dist/protocol/collector.js +84 -0
  46. package/dist/protocol/config.d.ts +89 -0
  47. package/dist/protocol/config.js +13 -0
  48. package/dist/protocol/envelope.d.ts +47 -0
  49. package/dist/protocol/envelope.js +162 -0
  50. package/dist/protocol/errors.d.ts +7 -0
  51. package/dist/protocol/errors.js +9 -0
  52. package/dist/protocol/graph.d.ts +81 -0
  53. package/dist/protocol/graph.js +201 -0
  54. package/dist/protocol/index.d.ts +19 -0
  55. package/dist/protocol/index.js +19 -0
  56. package/dist/protocol/manifest.d.ts +29 -0
  57. package/dist/protocol/manifest.js +94 -0
  58. package/dist/protocol/normalise.d.ts +20 -0
  59. package/dist/protocol/normalise.js +29 -0
  60. package/dist/protocol/pattern.d.ts +12 -0
  61. package/dist/protocol/pattern.js +82 -0
  62. package/dist/protocol/provision.d.ts +100 -0
  63. package/dist/protocol/provision.js +105 -0
  64. package/dist/protocol/route.d.ts +179 -0
  65. package/dist/protocol/route.js +135 -0
  66. package/dist/protocol/sync.d.ts +31 -0
  67. package/dist/protocol/sync.js +87 -0
  68. package/dist/protocol/tag.d.ts +12 -0
  69. package/dist/protocol/tag.js +26 -0
  70. package/dist/protocol/template/compile.d.ts +35 -0
  71. package/dist/protocol/template/compile.js +82 -0
  72. package/dist/protocol/template/interpret.d.ts +9 -0
  73. package/dist/protocol/template/interpret.js +82 -0
  74. package/dist/protocol/vector.d.ts +32 -0
  75. package/dist/protocol/vector.js +56 -0
  76. package/dist/protocol/view.d.ts +110 -0
  77. package/dist/protocol/view.js +201 -0
  78. package/dist/proxy/dictionary.d.ts +58 -0
  79. package/dist/proxy/dictionary.js +197 -0
  80. package/dist/proxy/eligible.d.ts +45 -0
  81. package/dist/proxy/eligible.js +96 -0
  82. package/dist/proxy/index.d.ts +5 -0
  83. package/dist/proxy/index.js +5 -0
  84. package/dist/proxy/proxy.d.ts +133 -0
  85. package/dist/proxy/proxy.js +507 -0
  86. package/dist/proxy/purge.d.ts +10 -0
  87. package/dist/proxy/purge.js +66 -0
  88. package/dist/proxy/tags.d.ts +8 -0
  89. package/dist/proxy/tags.js +36 -0
  90. package/dist/report/derive.d.ts +223 -0
  91. package/dist/report/derive.js +494 -0
  92. package/dist/report/economics.d.ts +14 -0
  93. package/dist/report/economics.js +24 -0
  94. package/dist/report/index.d.ts +9 -0
  95. package/dist/report/index.js +9 -0
  96. package/dist/report/inputs.d.ts +141 -0
  97. package/dist/report/inputs.js +216 -0
  98. package/dist/report/ledger.d.ts +169 -0
  99. package/dist/report/ledger.js +278 -0
  100. package/dist/report/measurement.d.ts +48 -0
  101. package/dist/report/measurement.js +117 -0
  102. package/dist/report/model.d.ts +42 -0
  103. package/dist/report/model.js +74 -0
  104. package/dist/report/thirdparty.d.ts +62 -0
  105. package/dist/report/thirdparty.js +107 -0
  106. package/dist/report/traffic.d.ts +140 -0
  107. package/dist/report/traffic.js +228 -0
  108. package/dist/report/trim.d.ts +81 -0
  109. package/dist/report/trim.js +105 -0
  110. package/dist/server/beacon.d.ts +20 -0
  111. package/dist/server/beacon.js +47 -0
  112. package/dist/server/embedded.d.ts +6 -0
  113. package/dist/server/embedded.js +6 -0
  114. package/dist/server/ga4.d.ts +70 -0
  115. package/dist/server/ga4.js +165 -0
  116. package/dist/server/identity.d.ts +12 -0
  117. package/dist/server/identity.js +17 -0
  118. package/dist/server/index.d.ts +7 -0
  119. package/dist/server/index.js +7 -0
  120. package/dist/server/sink.d.ts +56 -0
  121. package/dist/server/sink.js +133 -0
  122. package/dist/server/size.d.ts +7 -0
  123. package/dist/server/size.js +18 -0
  124. package/dist/server/spool.d.ts +97 -0
  125. package/dist/server/spool.js +463 -0
  126. package/dist/server/stub.d.ts +9 -0
  127. package/dist/server/stub.js +19 -0
  128. package/dist/server/sync.d.ts +12 -0
  129. package/dist/server/sync.js +48 -0
  130. package/dist/server/work.d.ts +22 -0
  131. package/dist/server/work.js +30 -0
  132. package/dist/stores/durable-object.d.ts +19 -0
  133. package/dist/stores/durable-object.js +53 -0
  134. package/dist/stores/index.d.ts +2 -0
  135. package/dist/stores/index.js +2 -0
  136. package/dist/stores/memory.d.ts +4 -0
  137. package/dist/stores/memory.js +10 -0
  138. package/dist/stores/version-store.d.ts +11 -0
  139. package/dist/stores/version-store.js +1 -0
  140. package/dist/sw/analytics.d.ts +81 -0
  141. package/dist/sw/analytics.js +275 -0
  142. package/dist/sw/collect.d.ts +8 -0
  143. package/dist/sw/collect.js +33 -0
  144. package/dist/sw/core.d.ts +1 -0
  145. package/dist/sw/core.js +3 -0
  146. package/dist/sw/freshness.d.ts +15 -0
  147. package/dist/sw/freshness.js +20 -0
  148. package/dist/sw/graph/index.d.ts +67 -0
  149. package/dist/sw/graph/index.js +170 -0
  150. package/dist/sw/graph/invalidate.d.ts +23 -0
  151. package/dist/sw/graph/invalidate.js +109 -0
  152. package/dist/sw/graph/projections.d.ts +7 -0
  153. package/dist/sw/graph/projections.js +9 -0
  154. package/dist/sw/graph/records.d.ts +7 -0
  155. package/dist/sw/graph/records.js +9 -0
  156. package/dist/sw/graph/store.d.ts +32 -0
  157. package/dist/sw/graph/store.js +52 -0
  158. package/dist/sw/idb.d.ts +8 -0
  159. package/dist/sw/idb.js +117 -0
  160. package/dist/sw/index.d.ts +2 -0
  161. package/dist/sw/index.js +1 -0
  162. package/dist/sw/partition.d.ts +21 -0
  163. package/dist/sw/partition.js +137 -0
  164. package/dist/sw/policy.d.ts +8 -0
  165. package/dist/sw/policy.js +24 -0
  166. package/dist/sw/provision.d.ts +39 -0
  167. package/dist/sw/provision.js +61 -0
  168. package/dist/sw/render.d.ts +21 -0
  169. package/dist/sw/render.js +53 -0
  170. package/dist/sw/replica.d.ts +77 -0
  171. package/dist/sw/replica.js +299 -0
  172. package/dist/sw/router.d.ts +42 -0
  173. package/dist/sw/router.js +48 -0
  174. package/dist/sw/routing.d.ts +23 -0
  175. package/dist/sw/routing.js +48 -0
  176. package/dist/sw/runtime.d.ts +112 -0
  177. package/dist/sw/runtime.js +1070 -0
  178. package/dist/sw/scheduler.d.ts +83 -0
  179. package/dist/sw/scheduler.js +149 -0
  180. package/dist/sw/seal.d.ts +37 -0
  181. package/dist/sw/seal.js +183 -0
  182. package/dist/sw/serve.d.ts +15 -0
  183. package/dist/sw/serve.js +63 -0
  184. package/dist/sw/shadow.d.ts +16 -0
  185. package/dist/sw/shadow.js +27 -0
  186. package/dist/sw/static.d.ts +17 -0
  187. package/dist/sw/static.js +81 -0
  188. package/dist/sw/storage.d.ts +64 -0
  189. package/dist/sw/storage.js +34 -0
  190. package/dist/sw/thirdparty.d.ts +17 -0
  191. package/dist/sw/thirdparty.js +36 -0
  192. package/dist/trim/css.d.ts +38 -0
  193. package/dist/trim/css.js +328 -0
  194. package/dist/trim/index.d.ts +3 -0
  195. package/dist/trim/index.js +3 -0
  196. package/dist/trim/manifest.d.ts +67 -0
  197. package/dist/trim/manifest.js +134 -0
  198. package/dist/trim/tokens.d.ts +47 -0
  199. package/dist/trim/tokens.js +142 -0
  200. package/dist/version.d.ts +2 -0
  201. package/dist/version.js +3 -0
  202. package/package.json +150 -5
@@ -0,0 +1,507 @@
1
+ import { resolveEmbedOptions, rewriteEmbeds } from '../media/embeds.js';
2
+ import { makePrivate, mergeList, signalsOf } from '../media/headers.js';
3
+ import { convertImage, createImageConverter, IMAGE_HINTS, imageRequest, originSearch } from '../media/image.js';
4
+ import { rewriteLazy } from '../media/lazy.js';
5
+ import { rewriteResponsive } from '../media/responsive.js';
6
+ import { resolveVideoOptions, rewriteVideo } from '../media/video.js';
7
+ import { PATHS } from '../protocol/config.js';
8
+ import { writeEnvelope } from '../protocol/envelope.js';
9
+ import { ProtocolError } from '../protocol/errors.js';
10
+ import { normalise } from '../protocol/normalise.js';
11
+ import { defineRoute, matchRoute } from '../protocol/route.js';
12
+ import { artifactTag } from '../protocol/sync.js';
13
+ import { createSpool } from '../server/spool.js';
14
+ import { guardPage, sourceHash, stripTrimParam, TRIM_PARAM, trimKey } from '../trim/index.js';
15
+ import { createDictionaries } from './dictionary.js';
16
+ import { decline, htmlCharset, isUtf8, setsPrivateCookie } from './eligible.js';
17
+ import { PURGE_PATH, respondPurge } from './purge.js';
18
+ import { pathTag, readTags, stripTags } from './tags.js';
19
+ const HOP_BY_HOP = [
20
+ 'connection',
21
+ 'keep-alive',
22
+ 'proxy-authenticate',
23
+ 'proxy-authorization',
24
+ 'te',
25
+ 'trailer',
26
+ 'transfer-encoding',
27
+ 'upgrade',
28
+ 'host'
29
+ ];
30
+ /** where the proxy writes a guard firing or a hash mismatch to the sink, as JSON `{ kind, ... }` */
31
+ export const TRIM_PATH = '/.spool/trim';
32
+ const isHtml = (response) => (response.headers.get('content-type') ?? '').toLowerCase().includes('text/html');
33
+ /** one character per byte, so an ASCII edit leaves every other byte as it was */
34
+ function fromBytes(bytes) {
35
+ let text = '';
36
+ for (let at = 0; at < bytes.length; at += 8192) {
37
+ text += String.fromCharCode(...bytes.subarray(at, at + 8192));
38
+ }
39
+ return text;
40
+ }
41
+ const toBytes = (text) => Uint8Array.from(text, (char) => char.charCodeAt(0));
42
+ /**
43
+ * A Fetch handler that sits in front of an origin that cannot load the library, such as PHP or a
44
+ * Drupal site with no module, and gives it local page views.
45
+ *
46
+ * `/.spool/*` and `/spool-sw.js` are answered here. Everything else goes to the origin with its
47
+ * method, headers and body, and redirects are passed through. A GET or HEAD without `Authorization`
48
+ * or a cookie outside `publicCookies`, answered 200 as `text/html` with no `Set-Cookie`, `private`
49
+ * or `no-store`, is decorated so the worker can keep it; anything else is forwarded untouched.
50
+ * The tags the origin sends in `Cache-Tag`, `Surrogate-Key`, `X-Drupal-Cache-Tags` or `xkey`
51
+ * become the page's dependencies, and a page with none depends on its own path.
52
+ *
53
+ * With `images` the proxy answers a JPEG or PNG request from a browser that accepts WebP with the
54
+ * converted file, and adds `Accept-CH` to HTML so later image requests carry their width. With
55
+ * `video` it rewrites `<video>` tags in HTML. Both run on the origin's response before the page is
56
+ * kept, so a held page carries the rewritten tags.
57
+ *
58
+ * Limits: the proxy sees finished HTML, so an artifact is the whole page, the replica holds no
59
+ * records, and two pages never share a fragment. A page is invalidated by `POST /.spool/purge`, or
60
+ * when the proxy sees its body change, and it stays on a device until its route's `maxStale` runs
61
+ * out and the next sync reports the change. Freshness is therefore bounded by `maxStale` plus the
62
+ * time a purge takes to arrive. The memory of last-seen bodies lives in the instance: a restart
63
+ * forgets it, and each instance of a scaled-out proxy keeps its own.
64
+ */
65
+ export function spoolProxy(options) {
66
+ const base = new URL(options.origin);
67
+ if (base.pathname !== '/' || base.search !== '' || base.hash !== '') {
68
+ throw new ProtocolError('bad-route', `origin ${base.href} must have no path`);
69
+ }
70
+ const gate = {
71
+ publicCookies: new Set(options.publicCookies ?? []),
72
+ maxTags: options.maxTags ?? 512,
73
+ maxBytes: options.maxBytes ?? 2 * 1024 * 1024
74
+ };
75
+ if (!Number.isInteger(gate.maxTags) || gate.maxTags < 1) {
76
+ throw new ProtocolError('bad-route', 'maxTags must be a positive integer');
77
+ }
78
+ if (!Number.isInteger(gate.maxBytes) || gate.maxBytes < 1) {
79
+ throw new ProtocolError('bad-route', 'maxBytes must be a positive integer');
80
+ }
81
+ const remembered = options.maxRemembered ?? 10_000;
82
+ if (!Number.isInteger(remembered) || remembered < 1) {
83
+ throw new ProtocolError('bad-route', 'maxRemembered must be a positive integer');
84
+ }
85
+ const call = options.fetch ?? ((request) => fetch(request));
86
+ const { store } = options.spool;
87
+ const { trim } = options;
88
+ const stats = { served: 0, mismatched: 0, guarded: 0 };
89
+ const images = options.images ? createImageConverter(options.images) : undefined;
90
+ const video = options.video ? resolveVideoOptions(options.video) : undefined;
91
+ const videoStats = { pages: 0, rewritten: 0 };
92
+ const imageStats = images?.stats ?? {
93
+ converted: 0,
94
+ bytesIn: 0,
95
+ bytesOut: 0,
96
+ cacheHits: 0,
97
+ coalesced: 0,
98
+ lazied: 0,
99
+ responsive: 0,
100
+ skipped: {}
101
+ };
102
+ const dictionaries = options.dictionary
103
+ ? createDictionaries(options.dictionary, gate)
104
+ : undefined;
105
+ const embeds = options.embeds ? resolveEmbedOptions(options.embeds) : undefined;
106
+ const embedStats = { pages: 0, rewritten: 0, skipped: {} };
107
+ const lazy = images?.options.lazy || undefined;
108
+ const responsive = images?.options.responsive || undefined;
109
+ const readsHtml = Boolean(trim || video || embeds || lazy || responsive || images?.options.hints);
110
+ /** the embed, responsive and lazy rewrites, which do not depend on the visitor; only the live path counts them */
111
+ function rewriteMarkup(text, headers, pageUrl, count) {
112
+ let out = text;
113
+ if (embeds) {
114
+ const result = rewriteEmbeds(out, embeds, {
115
+ csp: headers.has('content-security-policy')
116
+ });
117
+ out = result.html;
118
+ if (count) {
119
+ if (result.rewritten > 0) {
120
+ embedStats.pages++;
121
+ embedStats.rewritten += result.rewritten;
122
+ }
123
+ for (const [why, n] of Object.entries(result.skipped)) {
124
+ embedStats.skipped[why] = (embedStats.skipped[why] ?? 0) + n;
125
+ }
126
+ }
127
+ }
128
+ if (responsive && images) {
129
+ const result = rewriteResponsive(out, responsive, {
130
+ pageUrl,
131
+ paths: images.options.paths
132
+ });
133
+ out = result.html;
134
+ if (count)
135
+ imageStats.responsive += result.rewritten;
136
+ }
137
+ if (lazy) {
138
+ const result = rewriteLazy(out, lazy);
139
+ out = result.html;
140
+ if (count)
141
+ imageStats.lazied += result.rewritten;
142
+ }
143
+ return out;
144
+ }
145
+ function note(kind, detail) {
146
+ if (!options.spool.sink)
147
+ return;
148
+ try {
149
+ const written = options.spool.sink.write([
150
+ {
151
+ path: TRIM_PATH,
152
+ body: JSON.stringify({ kind, ...detail }),
153
+ contentType: 'application/json',
154
+ at: Date.now()
155
+ }
156
+ ]);
157
+ Promise.resolve(written).catch(() => undefined);
158
+ }
159
+ catch {
160
+ // a failing sink never breaks a page
161
+ }
162
+ }
163
+ /** the page with its trimmed stylesheets pointed at the full files when the guard fires */
164
+ function guarded(text, pageUrl) {
165
+ if (!trim)
166
+ return text;
167
+ const verdict = guardPage(text, trim, pageUrl);
168
+ if (!verdict.fired)
169
+ return text;
170
+ stats.guarded++;
171
+ note('guard', {
172
+ page: new URL(pageUrl).pathname,
173
+ reason: verdict.reason,
174
+ files: verdict.files
175
+ });
176
+ return verdict.html;
177
+ }
178
+ function weaken(headers) {
179
+ const etag = headers.get('etag');
180
+ if (etag && !etag.startsWith('W/'))
181
+ headers.set('etag', `W/${etag}`);
182
+ }
183
+ // the page a request resolved to, so `depends` and `load` do not fetch it again
184
+ const pages = new WeakMap();
185
+ const seen = new Map();
186
+ function originHeaders(source, publicUrl) {
187
+ const headers = new Headers(source);
188
+ for (const name of HOP_BY_HOP)
189
+ headers.delete(name);
190
+ for (const name of [...headers.keys()])
191
+ if (name.startsWith('spool-'))
192
+ headers.delete(name);
193
+ headers.set('x-forwarded-host', publicUrl.host);
194
+ headers.set('x-forwarded-proto', publicUrl.protocol.slice(0, -1));
195
+ return headers;
196
+ }
197
+ function forward(request, url, whole) {
198
+ const read = request.method === 'GET' || request.method === 'HEAD';
199
+ const headers = originHeaders(request.headers, url);
200
+ // a conversion needs the whole file, so the origin gets no conditional request
201
+ if (whole) {
202
+ headers.delete('if-none-match');
203
+ headers.delete('if-modified-since');
204
+ }
205
+ const search = images ? originSearch(images, url) : url.search;
206
+ return new Request(new URL(url.pathname + stripTrimParam(search), base), {
207
+ method: request.method,
208
+ headers,
209
+ redirect: 'manual',
210
+ signal: request.signal,
211
+ ...(read || !request.body ? {} : { body: request.body, duplex: 'half' })
212
+ });
213
+ }
214
+ /** the headers the browser gets: bodies arrive decoded, so the encoding and its length go */
215
+ function tidy(source, url) {
216
+ const headers = new Headers(source);
217
+ if (headers.has('content-encoding')) {
218
+ headers.delete('content-encoding');
219
+ headers.delete('content-length');
220
+ }
221
+ const location = headers.get('location');
222
+ if (location && /^[a-z][a-z0-9+.-]*:/i.test(location)) {
223
+ const target = new URL(location);
224
+ if (target.origin === base.origin) {
225
+ headers.set('location', new URL(target.pathname + target.search + target.hash, url).href);
226
+ }
227
+ }
228
+ if (!options.exposeTags)
229
+ stripTags(headers);
230
+ return headers;
231
+ }
232
+ async function digest(text) {
233
+ const bytes = new TextEncoder().encode(normalise(text, options.ignore ? { ignore: options.ignore } : {}));
234
+ const hash = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes));
235
+ return btoa(String.fromCharCode(...hash));
236
+ }
237
+ /** records a public page's body and bumps its tags when it differs from the last one seen */
238
+ async function settle(path, text, tags) {
239
+ const hash = await digest(text);
240
+ const prior = seen.get(path);
241
+ seen.delete(path);
242
+ seen.set(path, { hash, tags });
243
+ if (seen.size > remembered)
244
+ seen.delete(seen.keys().next().value);
245
+ if (prior && prior.hash !== hash) {
246
+ await spool.invalidate([...new Set([...prior.tags, ...tags])]);
247
+ }
248
+ }
249
+ async function fetchPage(ctx, data, principal) {
250
+ const path = ctx.url.pathname;
251
+ const headers = originHeaders(ctx.request.headers, new URL(ctx.request.url));
252
+ for (const name of [
253
+ 'range',
254
+ 'if-match',
255
+ 'if-none-match',
256
+ 'if-modified-since',
257
+ 'if-unmodified-since',
258
+ 'if-range',
259
+ 'content-length',
260
+ 'content-type'
261
+ ]) {
262
+ headers.delete(name);
263
+ }
264
+ headers.set('accept', data ? 'application/json' : 'text/html');
265
+ const response = await call(new Request(new URL(path, base), { method: 'GET', headers, redirect: 'manual' }));
266
+ const tags = readTags(response.headers);
267
+ const why = decline({
268
+ method: 'GET',
269
+ search: '',
270
+ requestHeaders: ctx.request.headers,
271
+ response,
272
+ tags,
273
+ data,
274
+ principal
275
+ }, gate);
276
+ if (why)
277
+ throw new ProtocolError('bad-artifact', `${path} is not kept (${why})`);
278
+ const bytes = new Uint8Array(await response.arrayBuffer());
279
+ if (!data && !isUtf8(htmlCharset(response.headers, bytes))) {
280
+ throw new ProtocolError('bad-artifact', `${path} is not kept (charset)`);
281
+ }
282
+ let text = guarded(new TextDecoder().decode(bytes), ctx.url.href);
283
+ // a signal-triggered rewrite is never part of a held copy
284
+ if (!data && video?.when === 'always') {
285
+ text = rewriteVideo(text, video, { saveData: false, slow: false }).html;
286
+ }
287
+ if (!data)
288
+ text = rewriteMarkup(text, response.headers, ctx.url.href, false);
289
+ if (text.length > gate.maxBytes) {
290
+ throw new ProtocolError('bad-artifact', `${path} is not kept (size)`);
291
+ }
292
+ const effective = tags.length > 0 ? tags : [pathTag(path)];
293
+ if (!principal)
294
+ await settle(path, text, effective);
295
+ return { text, tags: effective };
296
+ }
297
+ function pageFor(ctx, data, principal) {
298
+ const known = pages.get(ctx.request);
299
+ if (known && known.path === ctx.url.pathname)
300
+ return known.page;
301
+ const page = fetchPage(ctx, data, principal);
302
+ pages.set(ctx.request, { path: ctx.url.pathname, page });
303
+ return page;
304
+ }
305
+ function proxied(pattern, declaration) {
306
+ const data = declaration?.strategy === 'data';
307
+ const principal = declaration?.scope === 'principal';
308
+ if (principal && !options.spool.identity) {
309
+ throw new ProtocolError('bad-route', `${pattern}: a principal route needs spool.identity`);
310
+ }
311
+ const extra = declaration?.depends;
312
+ const depends = async (ctx) => [
313
+ ...(await pageFor(ctx, data, principal)).tags,
314
+ ...(extra ? await extra(ctx) : [])
315
+ ];
316
+ if (declaration?.strategy === 'data') {
317
+ return defineRoute(pattern, {
318
+ ...declaration,
319
+ strategy: 'data',
320
+ depends,
321
+ load: async (ctx) => {
322
+ const { text } = await pageFor(ctx, true, principal);
323
+ try {
324
+ return JSON.parse(text);
325
+ }
326
+ catch {
327
+ throw new ProtocolError('bad-artifact', `${pattern}: the origin sent no JSON`);
328
+ }
329
+ }
330
+ });
331
+ }
332
+ return defineRoute(pattern, {
333
+ ...declaration,
334
+ strategy: 'assembly',
335
+ depends,
336
+ load: async (ctx) => ({ shell: (await pageFor(ctx, false, principal)).text })
337
+ });
338
+ }
339
+ const routes = [
340
+ ...(options.routes ?? []).map((declaration) => proxied(declaration.pattern, declaration)),
341
+ proxied('/**')
342
+ ];
343
+ const spool = createSpool({ ...options.spool, routes });
344
+ /** a page the proxy declines tells the worker to drop its copy of that path, in the same response */
345
+ async function unfit(response, path, why, page) {
346
+ if (!page || why === 'query')
347
+ return response;
348
+ const headers = new Headers(response.headers);
349
+ writeEnvelope(headers, { invalidate: { [artifactTag(path)]: (await store.cursor()) + 1 } });
350
+ return new Response(response.body, {
351
+ status: response.status,
352
+ statusText: response.statusText,
353
+ headers
354
+ });
355
+ }
356
+ async function purge(tags) {
357
+ const named = new Set(tags);
358
+ for (const [path, entry] of seen) {
359
+ if (entry.tags.some((tag) => named.has(tag)))
360
+ seen.delete(path);
361
+ }
362
+ return spool.invalidate(tags);
363
+ }
364
+ async function handle(request) {
365
+ const url = new URL(request.url);
366
+ if (url.pathname === PURGE_PATH)
367
+ return respondPurge(request, options.purgeSecret, purge);
368
+ const answered = await spool.respond(request);
369
+ if (answered)
370
+ return answered;
371
+ if (url.pathname === PATHS.worker && request.method === 'GET')
372
+ return spool.stub();
373
+ const converting = images !== undefined && imageRequest(images, request);
374
+ let upstream;
375
+ try {
376
+ upstream = await call(forward(request, url, converting));
377
+ }
378
+ catch {
379
+ return new Response('bad gateway', {
380
+ status: 502,
381
+ headers: { 'content-type': 'text/plain', 'cache-control': 'no-store' }
382
+ });
383
+ }
384
+ if (images && converting)
385
+ upstream = await convertImage(images, request, upstream);
386
+ if (request.method === 'GET') {
387
+ const key = trim ? trimKey(url.href) : '';
388
+ const entry = trim?.styles[key];
389
+ if (trim &&
390
+ entry &&
391
+ url.searchParams.get(TRIM_PARAM) !== '0' &&
392
+ upstream.status === 200 &&
393
+ (upstream.headers.get('content-type') ?? '').toLowerCase().includes('text/css')) {
394
+ const bytes = new Uint8Array(await upstream.arrayBuffer());
395
+ const headers = new Headers(upstream.headers);
396
+ headers.delete('content-length');
397
+ const match = (await sourceHash(bytes)) === entry.source;
398
+ if (match) {
399
+ weaken(headers);
400
+ stats.served++;
401
+ }
402
+ else {
403
+ stats.mismatched++;
404
+ note('mismatch', { file: key });
405
+ }
406
+ upstream = new Response(match ? entry.css : bytes, {
407
+ status: 200,
408
+ statusText: upstream.statusText,
409
+ headers
410
+ });
411
+ }
412
+ else if (readsHtml &&
413
+ isHtml(upstream) &&
414
+ ![204, 205, 304].includes(upstream.status)) {
415
+ const bytes = new Uint8Array(await upstream.arrayBuffer());
416
+ // another charset is rewritten byte for byte; the edits insert ASCII only
417
+ const utf8 = isUtf8(htmlCharset(upstream.headers, bytes));
418
+ const text = utf8 ? new TextDecoder().decode(bytes) : fromBytes(bytes);
419
+ let verdict = guarded(text, url.href);
420
+ const headers = new Headers(upstream.headers);
421
+ headers.delete('content-length');
422
+ if (video) {
423
+ const result = rewriteVideo(verdict, video, signalsOf(request.headers));
424
+ if (result.rewritten > 0) {
425
+ verdict = result.html;
426
+ videoStats.pages++;
427
+ videoStats.rewritten += result.rewritten;
428
+ // a page rewritten for one visitor's connection is not another's copy
429
+ if (video.when !== 'always') {
430
+ makePrivate(headers);
431
+ mergeList(headers, 'vary', ['Save-Data', 'ECT']);
432
+ }
433
+ }
434
+ }
435
+ verdict = rewriteMarkup(verdict, upstream.headers, url.href, true);
436
+ if (images?.options.hints)
437
+ mergeList(headers, 'accept-ch', IMAGE_HINTS);
438
+ if (verdict !== text)
439
+ weaken(headers);
440
+ const body = verdict === text ? bytes : utf8 ? verdict : toBytes(verdict);
441
+ upstream = new Response(body, {
442
+ status: upstream.status,
443
+ statusText: upstream.statusText,
444
+ headers
445
+ });
446
+ }
447
+ }
448
+ const tags = readTags(upstream.headers);
449
+ const headers = tidy(upstream.headers, url);
450
+ const init = { status: upstream.status, statusText: upstream.statusText, headers };
451
+ // a new cookie may be a login or a logout (the held copies are another visitor's view)
452
+ if (setsPrivateCookie(upstream.headers, gate.publicCookies))
453
+ writeEnvelope(headers, { clear: ['*'] });
454
+ if (request.method !== 'GET' && request.method !== 'HEAD') {
455
+ return new Response(upstream.body, init);
456
+ }
457
+ if (dictionaries?.applies(request, upstream)) {
458
+ return dictionaries.apply(request, new Response(upstream.body, init));
459
+ }
460
+ const hit = matchRoute(routes, url.pathname);
461
+ const data = hit.route.strategy === 'data';
462
+ const principal = hit.route.scope === 'principal';
463
+ const page = upstream.status === 200 && (data || isHtml(upstream));
464
+ const why = decline({
465
+ method: request.method,
466
+ search: url.search,
467
+ requestHeaders: request.headers,
468
+ response: upstream,
469
+ tags,
470
+ data,
471
+ principal
472
+ }, gate);
473
+ if (why)
474
+ return unfit(new Response(upstream.body, init), url.pathname, why, page);
475
+ const bytes = new Uint8Array(await upstream.arrayBuffer());
476
+ if (!data && !isUtf8(htmlCharset(upstream.headers, bytes))) {
477
+ return unfit(new Response(bytes, init), url.pathname, 'charset', page);
478
+ }
479
+ const text = new TextDecoder().decode(bytes);
480
+ headers.delete('content-length');
481
+ if (text.length > gate.maxBytes) {
482
+ return unfit(new Response(text, init), url.pathname, 'size', page);
483
+ }
484
+ const effective = tags.length > 0 ? tags : [pathTag(url.pathname)];
485
+ if (request.method === 'GET' && !principal)
486
+ await settle(url.pathname, text, effective);
487
+ pages.set(request, {
488
+ path: url.pathname,
489
+ page: Promise.resolve({ text, tags: effective })
490
+ });
491
+ return spool.decorate(request, new Response(text, init));
492
+ }
493
+ return Object.assign(handle, {
494
+ spool,
495
+ trim: stats,
496
+ images: imageStats,
497
+ video: videoStats,
498
+ embeds: embedStats,
499
+ dictionary: dictionaries?.stats ?? {
500
+ registered: 0,
501
+ encoded: 0,
502
+ bytesIn: 0,
503
+ bytesOut: 0,
504
+ skipped: {}
505
+ }
506
+ });
507
+ }
@@ -0,0 +1,10 @@
1
+ /** where a CMS or a hook tells the proxy that tags changed */
2
+ export declare const PURGE_PATH = "/.spool/purge";
3
+ /** compares two secrets through HMAC digests of equal length, so neither the value nor its length shows in the time taken */
4
+ export declare function equalSecrets(a: string, b: string): Promise<boolean>;
5
+ /**
6
+ * Answers `POST /.spool/purge`. The caller sends `Authorization: Bearer <secret>` and a JSON body
7
+ * `{"tags": ["node:1", ...]}`. With no secret configured every call is refused, so a proxy that
8
+ * never asked for purges cannot be made to bump its vector.
9
+ */
10
+ export declare function respondPurge(request: Request, secret: string | undefined, apply: (tags: string[]) => Promise<number>): Promise<Response>;
@@ -0,0 +1,66 @@
1
+ import { json } from '../server/sync.js';
2
+ /** where a CMS or a hook tells the proxy that tags changed */
3
+ export const PURGE_PATH = '/.spool/purge';
4
+ const MAX_BODY = 65_536;
5
+ const MAX_TAGS = 1000;
6
+ const MAX_TAG_LENGTH = 1024;
7
+ const encoder = new TextEncoder();
8
+ let macKey;
9
+ /** compares two secrets through HMAC digests of equal length, so neither the value nor its length shows in the time taken */
10
+ export async function equalSecrets(a, b) {
11
+ macKey ??= crypto.subtle.importKey('raw', crypto.getRandomValues(new Uint8Array(32)), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
12
+ const key = await macKey;
13
+ const digest = async (value) => new Uint8Array(await crypto.subtle.sign('HMAC', key, encoder.encode(value)));
14
+ const left = await digest(a);
15
+ const right = await digest(b);
16
+ let diff = 0;
17
+ for (let i = 0; i < left.length; i++)
18
+ diff |= (left[i] ?? 0) ^ (right[i] ?? 1);
19
+ return diff === 0;
20
+ }
21
+ function bearer(request) {
22
+ const match = /^Bearer\s+(.+)$/i.exec(request.headers.get('authorization') ?? '');
23
+ return match ? match[1].trim() : null;
24
+ }
25
+ function tagsOf(text) {
26
+ let body;
27
+ try {
28
+ body = JSON.parse(text);
29
+ }
30
+ catch {
31
+ return null;
32
+ }
33
+ const tags = body?.tags;
34
+ if (!Array.isArray(tags) || tags.length === 0 || tags.length > MAX_TAGS)
35
+ return null;
36
+ for (const tag of tags) {
37
+ if (typeof tag !== 'string' || tag === '' || tag.length > MAX_TAG_LENGTH)
38
+ return null;
39
+ }
40
+ return [...new Set(tags)];
41
+ }
42
+ /**
43
+ * Answers `POST /.spool/purge`. The caller sends `Authorization: Bearer <secret>` and a JSON body
44
+ * `{"tags": ["node:1", ...]}`. With no secret configured every call is refused, so a proxy that
45
+ * never asked for purges cannot be made to bump its vector.
46
+ */
47
+ export async function respondPurge(request, secret, apply) {
48
+ if (request.method !== 'POST')
49
+ return json({ error: 'use POST' }, 405, { allow: 'POST' });
50
+ if (!secret)
51
+ return json({ error: 'purge is not configured' }, 403);
52
+ const given = bearer(request);
53
+ if (given === null || !(await equalSecrets(given, secret))) {
54
+ return json({ error: 'bad credentials' }, 401, { 'www-authenticate': 'Bearer' });
55
+ }
56
+ const length = Number(request.headers.get('content-length'));
57
+ if (Number.isFinite(length) && length > MAX_BODY)
58
+ return json({ error: 'body too large' }, 413);
59
+ const text = await request.text();
60
+ if (text.length > MAX_BODY)
61
+ return json({ error: 'body too large' }, 413);
62
+ const tags = tagsOf(text);
63
+ if (!tags)
64
+ return json({ error: 'send {"tags": [...]} with 1 to 1000 tags' }, 400);
65
+ return json({ vector: await apply(tags), tags: tags.length });
66
+ }
@@ -0,0 +1,8 @@
1
+ /** the tag headers {@link readTags} understands, lowercase */
2
+ export declare const TAG_HEADERS: readonly string[];
3
+ /** every tag the response names across all four headers, in first-seen order, without repeats */
4
+ export declare function readTags(headers: Headers): string[];
5
+ /** removes every tag header, so the browser never sees the site's cache vocabulary */
6
+ export declare function stripTags(headers: Headers): void;
7
+ /** the tag a page with no tag header depends on */
8
+ export declare function pathTag(path: string): string;
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Response headers that carry cache tags, with the separator each vendor documents:
3
+ * Cloudflare `Cache-Tag` takes commas, Fastly `Surrogate-Key` and Drupal `X-Drupal-Cache-Tags` take
4
+ * spaces, Varnish `xkey` takes spaces or commas. A header sent twice reaches `Headers.get` joined
5
+ * with a comma and a space, so the space-separated forms also split on that join.
6
+ */
7
+ const SEPARATORS = {
8
+ 'cache-tag': /\s*,\s*/,
9
+ 'surrogate-key': /,?\s+/,
10
+ 'x-drupal-cache-tags': /,?\s+/,
11
+ xkey: /[\s,]+/
12
+ };
13
+ /** the tag headers {@link readTags} understands, lowercase */
14
+ export const TAG_HEADERS = Object.keys(SEPARATORS);
15
+ /** every tag the response names across all four headers, in first-seen order, without repeats */
16
+ export function readTags(headers) {
17
+ const tags = new Set();
18
+ for (const [name, separator] of Object.entries(SEPARATORS)) {
19
+ const value = headers.get(name);
20
+ if (value === null)
21
+ continue;
22
+ for (const tag of value.trim().split(separator))
23
+ if (tag !== '')
24
+ tags.add(tag);
25
+ }
26
+ return [...tags];
27
+ }
28
+ /** removes every tag header, so the browser never sees the site's cache vocabulary */
29
+ export function stripTags(headers) {
30
+ for (const name of TAG_HEADERS)
31
+ headers.delete(name);
32
+ }
33
+ /** the tag a page with no tag header depends on */
34
+ export function pathTag(path) {
35
+ return `proxy:path:${path}`;
36
+ }