@drupflare/spool 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +444 -18
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/activity.d.ts +37 -0
  6. package/dist/client/activity.js +114 -0
  7. package/dist/client/entry.js +2 -1
  8. package/dist/client/index.d.ts +1 -0
  9. package/dist/client/index.js +1 -0
  10. package/dist/media/embeds.d.ts +60 -0
  11. package/dist/media/embeds.js +135 -0
  12. package/dist/media/headers.d.ts +13 -0
  13. package/dist/media/headers.js +44 -0
  14. package/dist/media/html.d.ts +27 -0
  15. package/dist/media/html.js +93 -0
  16. package/dist/media/image.d.ts +222 -0
  17. package/dist/media/image.js +413 -0
  18. package/dist/media/index.d.ts +6 -0
  19. package/dist/media/index.js +6 -0
  20. package/dist/media/lazy.d.ts +26 -0
  21. package/dist/media/lazy.js +74 -0
  22. package/dist/media/responsive.d.ts +45 -0
  23. package/dist/media/responsive.js +98 -0
  24. package/dist/media/video.d.ts +30 -0
  25. package/dist/media/video.js +75 -0
  26. package/dist/protocol/envelope.d.ts +2 -0
  27. package/dist/protocol/envelope.js +12 -2
  28. package/dist/protocol/errors.d.ts +1 -1
  29. package/dist/protocol/hash.d.ts +2 -0
  30. package/dist/protocol/hash.js +8 -0
  31. package/dist/protocol/index.d.ts +4 -0
  32. package/dist/protocol/index.js +4 -0
  33. package/dist/protocol/inline.d.ts +14 -0
  34. package/dist/protocol/inline.js +43 -0
  35. package/dist/protocol/navigation.d.ts +81 -0
  36. package/dist/protocol/navigation.js +249 -0
  37. package/dist/protocol/normalise.d.ts +4 -3
  38. package/dist/protocol/normalise.js +6 -4
  39. package/dist/protocol/provision.d.ts +6 -1
  40. package/dist/protocol/provision.js +15 -5
  41. package/dist/protocol/query.d.ts +80 -0
  42. package/dist/protocol/query.js +153 -0
  43. package/dist/protocol/route.d.ts +80 -4
  44. package/dist/protocol/route.js +63 -0
  45. package/dist/proxy/bases.d.ts +53 -0
  46. package/dist/proxy/bases.js +76 -0
  47. package/dist/proxy/dictionary.d.ts +58 -0
  48. package/dist/proxy/dictionary.js +197 -0
  49. package/dist/proxy/index.d.ts +2 -0
  50. package/dist/proxy/index.js +2 -0
  51. package/dist/proxy/proxy.d.ts +47 -0
  52. package/dist/proxy/proxy.js +140 -15
  53. package/dist/report/capacity.d.ts +59 -0
  54. package/dist/report/capacity.js +76 -0
  55. package/dist/report/channels.d.ts +60 -0
  56. package/dist/report/channels.js +65 -0
  57. package/dist/report/derive.d.ts +2 -0
  58. package/dist/report/derive.js +6 -2
  59. package/dist/report/index.d.ts +4 -0
  60. package/dist/report/index.js +4 -0
  61. package/dist/report/institution.d.ts +97 -0
  62. package/dist/report/institution.js +79 -0
  63. package/dist/report/ledger.d.ts +197 -0
  64. package/dist/report/ledger.js +308 -0
  65. package/dist/report/measurement.d.ts +1 -1
  66. package/dist/report/measurement.js +7 -1
  67. package/dist/report/traffic.d.ts +11 -2
  68. package/dist/report/traffic.js +13 -3
  69. package/dist/server/embedded.d.ts +3 -3
  70. package/dist/server/embedded.js +3 -3
  71. package/dist/server/index.d.ts +1 -0
  72. package/dist/server/index.js +1 -0
  73. package/dist/server/spool.d.ts +20 -0
  74. package/dist/server/spool.js +62 -9
  75. package/dist/sw/render.d.ts +3 -1
  76. package/dist/sw/render.js +14 -1
  77. package/dist/sw/replica.d.ts +2 -0
  78. package/dist/sw/replica.js +6 -0
  79. package/dist/sw/router.js +2 -2
  80. package/dist/sw/runtime.js +104 -9
  81. package/dist/trim/tokens.d.ts +4 -0
  82. package/dist/trim/tokens.js +1 -1
  83. package/package.json +10 -4
package/README.md CHANGED
@@ -36,6 +36,8 @@ has no runtime dependencies. A site with no server code can load it from two fil
36
36
  - [Shadow Mode](#-shadow-mode)
37
37
  - [Proxy Mode](#-proxy-mode)
38
38
  - [Trimmed Stylesheets](#-trimmed-stylesheets)
39
+ - [Images and Video](#-images-and-video)
40
+ - [Capacity Reports](#-capacity-reports)
39
41
  - [Out of Scope](#-out-of-scope)
40
42
  - [Browser Support](#-browser-support)
41
43
  - [Subpath Exports](#-subpath-exports)
@@ -194,6 +196,41 @@ The script registers `/spool-sw.js` and exposes `window.spool.ready`, which reso
194
196
  handle or to `null` where service workers do not exist. To bundle the runtime yourself, import
195
197
  `register` from `@drupflare/spool/client`.
196
198
 
199
+ ### Activity Hooks
200
+
201
+ A loop that keeps running while nobody can see its output costs the visitor's device power. The
202
+ page runtime exposes `window.spool.activity.register`, which lets a page hand spool a `pause` and
203
+ a `resume` for one of its own loops. Spool calls `pause` while the loop's `target` is off screen
204
+ and after the page has been hidden for `hiddenAfterMs`, and calls `resume` when neither holds.
205
+
206
+ ```ts
207
+ import { createActivity } from '@drupflare/spool/client';
208
+
209
+ const canvas = document.querySelector('canvas') as HTMLCanvasElement;
210
+ let frame = 0;
211
+
212
+ const draw = () => {
213
+ canvas.style.transform = `translateX(${(performance.now() / 10) % 200}px)`;
214
+ frame = requestAnimationFrame(draw);
215
+ };
216
+
217
+ draw();
218
+
219
+ export const stop = createActivity().register(
220
+ { pause: () => cancelAnimationFrame(frame), resume: draw },
221
+ { target: canvas, hiddenAfterMs: 10_000 }
222
+ );
223
+ ```
224
+
225
+ `target` is optional, and without it only the hidden-page rule applies. `hiddenAfterMs` defaults to
226
+ 10000, and 0 pauses at once. `register` returns a function that stops watching and resumes the loop
227
+ if it is paused. Spool does not touch `setInterval`, `requestAnimationFrame`, `fetch`, audio,
228
+ WebRTC or any element the page did not name, and a `pause` or `resume` that throws is logged once.
229
+ Where `IntersectionObserver` is missing the target counts as on screen. On the one desktop
230
+ measured, the off-screen pause lowered power for a page whose frame loop kept running, and the
231
+ hidden-page pause showed no saving for a once-a-second poll. Both readings are in
232
+ [docs/impact.md](docs/impact.md#metered-figures-beside-the-ledger).
233
+
197
234
  ---
198
235
 
199
236
  ## 🧩 Runtimes
@@ -861,6 +898,13 @@ export const news = defineRoute('/news/:slug', {
861
898
 
862
899
  A template that names a field it does not trust for markup fails when the route is declared.
863
900
 
901
+ A public `template` route can carry its own data in the page. `inlineMaxBytes` on `createSpool`
902
+ sets the largest artifact, in bytes, that `handle` and `decorate` write into the HTML as an
903
+ escaped `<script type="application/spool+json">` element. The worker reads it from the response
904
+ and skips the `/.spool/artifact` request. It is 0, which is off, because every network view of the
905
+ page carries the element, used or not. The page the visitor sees and the shadow comparison ignore
906
+ it.
907
+
864
908
  ### Static Files
865
909
 
866
910
  A `static` route has no `load` and no template. The application serves the file as it always did,
@@ -929,9 +973,78 @@ records included, against what a local view saves: reuse times future views time
929
973
  avoids, less the cost of the request. When that is not positive it answers `204` with
930
974
  `Spool-Provision: declined`, and the worker leaves that path to the network until it restarts.
931
975
  The priors are scenario assumptions: set them from your own telemetry in `provision`, or per route
932
- as above. A `data` route weighs against its own size, so it needs reuse times future views above
976
+ as above. A `data` route weighs against its own size, so its reuse times future views must exceed
977
+ one. The defaults of 0.5 and 2 give exactly one, which breaks even, and the route is left out.
978
+
979
+ ### Query Routes
980
+
981
+ A URL with a query string goes to the network. A public `template`, `projection` or `data` route
982
+ can declare the query strings it answers, and the device then answers a new filter, sort or page of
983
+ a collection it already holds with no request.
984
+
985
+ ```ts
986
+ import { datasetProof, defineRoute } from '@drupflare/spool/server';
987
+
988
+ const people = [
989
+ { name: 'Ada', department: 'math', age: 36 },
990
+ { name: 'Bo', department: 'cs', age: 25 }
991
+ ];
992
+
993
+ export const directory = defineRoute('/people', {
994
+ strategy: 'template',
995
+ template: {
996
+ source:
997
+ '<h1>{{#if query}}Page {{query.page}} of {{query.pages}}{{else}}Everyone{{/if}}</h1><ul>{{#each people}}<li>{{name}}</li>{{/each}}</ul>'
998
+ },
999
+ depends: () => ['people'],
1000
+ load: async () => ({ people, proof: await datasetProof(people) }),
1001
+ query: {
1002
+ list: 'people',
1003
+ filter: { department: 'string', age: 'number' },
1004
+ sort: ['name', 'age'],
1005
+ pageSize: 20
1006
+ },
1007
+ provision: { reuse: 0.8, futureViews: 6 }
1008
+ });
1009
+ ```
933
1010
 
934
- 1. With the defaults of 0.5 and 2 it breaks even and is left out.
1011
+ `list` names the key of the loaded data that holds the whole collection, an array of flat objects.
1012
+ Each key of `filter` is a query parameter that keeps the items whose field of the same name equals
1013
+ it, read as a `string` or a `number`. A repeated parameter matches any of its values, and different
1014
+ parameters combine with and. `sort` lists the fields the `sort` parameter may name, and `-name`
1015
+ sorts descending. Ties keep the stored order. Strings compare by UTF-16 code unit, with no locale
1016
+ and no case folding, so the application's own sort has to agree. A database that compares bytes,
1017
+ such as SQLite by default, agrees except for characters above U+FFFF, and a case-insensitive or
1018
+ locale collation does not. `pageSize` sets the items per page and `page` counts from 1; without it
1019
+ the route is one page of everything. A route with `principal` scope, or any
1020
+ other strategy, cannot declare `query`.
1021
+
1022
+ `load` returns the array at `list` and a `proof` made by `datasetProof`. The server refuses to
1023
+ build the artifact when the proof is missing or does not match the array. The worker computes the
1024
+ proof again from the stored copy before it answers a query. The proof shows that the device holds
1025
+ the array the server sent. That the array is the whole collection is the route's declaration:
1026
+ `list` has to hold every item the application's own answer could draw from, or a filter returns
1027
+ fewer rows than the server would.
1028
+
1029
+ The template, or the renderer of a `projection` route, receives the stored data with the array
1030
+ replaced by the page's items and one added key, `query`, holding `{ total, page, pages }`. `total`
1031
+ counts the items after filtering. A `data` route answers with that same object as JSON. The URL
1032
+ without a query string renders the stored data as it is and has no `query` key, which is what
1033
+ `{{#if query}}` tests above.
1034
+
1035
+ The worker answers locally only when the artifact for the path is held and fresh under the route's
1036
+ freshness contract, the proof matches and the query is one the route declares. Everything else goes
1037
+ to the network as before: a parameter the route does not name, a number filter that is not a
1038
+ finite number, a `sort` given twice or not listed, a `page` that is not a positive integer or
1039
+ lies past the last page, and an item that is not an object. A copy that fails its proof is dropped
1040
+ and fetched whole again. The worker never stores anything under a URL with a query string, and in
1041
+ shadow mode it compares its render with the network answer for the same URL.
1042
+
1043
+ A query route's artifact is the whole collection, so it is large. The provisioning rule weighs it
1044
+ like any other artifact, and `provision` on the route sets the priors. A collection repays its
1045
+ transfer after `entryBreakEven(bytes, documentBytes)` local answers per device: a 40 KB collection
1046
+ that replaces 17 KB documents needs 2.3 (L2). Pages of one visit come from one held copy until a
1047
+ sync replaces it, so paging never mixes two versions on a device.
935
1048
 
936
1049
  ---
937
1050
 
@@ -1071,9 +1184,20 @@ is larger. An entry is data for a `template`, `projection` or `data` route and n
1071
1184
  The site supplies the candidates. A route's `catalogue` returns the pages a visitor on it may open
1072
1185
  next, each with the share expected to. Take the share from your own `nextShare` over the sink
1073
1186
  (each `spool.view` names the route the visitor came from), from link position, or from a fixed
1074
- prior. `documentBytes` on the entry's route says what a hit saves, and the provisioning prior is
1187
+ prior. Pass `{ first: true }` to `nextShare` to count only first views, which is what an entry
1188
+ saves: the share over all views ran 1.5 to 7 times high in the bins that decide an entry in a replay
1189
+ of public visitors ([docs/impact.md](docs/impact.md#next-page-prediction)). `documentBytes` on the entry's route says what a hit saves, and the provisioning prior is
1075
1190
  used when it is absent.
1076
1191
 
1192
+ A share taken from counts can carry `n`, the observations behind it. `catalogueConfidence` on
1193
+ `createSpool` is a number of standard deviations, and with it above 0 an entry whose guess has an
1194
+ `n` is judged on the lower confidence bound of its `p`, so a share seen on 5 visits stops counting
1195
+ as one seen on 5,000. It defaults to 0, which judges every guess on its `p`. `NavModel` in
1196
+ `@drupflare/spool/protocol` keeps first-order counts with decay and a prior and returns a `p` with
1197
+ the weight to pass as `n`. In a replay of 989,818 public visitors, a model like that earned more
1198
+ per provisioned kilobyte than the most popular page
1199
+ ([docs/impact.md](docs/impact.md#next-page-prediction)).
1200
+
1077
1201
  ```ts
1078
1202
  import { createSpool, defineRoute } from '@drupflare/spool/server';
1079
1203
  import { memoryStore } from '@drupflare/spool/stores';
@@ -1311,7 +1435,11 @@ requests keep the browser's own path.
1311
1435
  or anything else that speaks HTTP. It returns a Fetch handler. The proxy answers `/.spool/*` and
1312
1436
  `/spool-sw.js`. Every other request goes to the origin with its method, headers and body, and a
1313
1437
  redirect is passed on, not followed. A page the proxy may keep is decorated like a `createSpool`
1314
- response, so the worker stores it and answers later views with no request.
1438
+ response, so the worker stores it and answers later views with no request. A cold visit costs one
1439
+ origin fetch: the proxy names a hash of the page in `Spool-Artifact`, and the worker keeps the
1440
+ page it already received as the artifact when its own hash matches. On any mismatch it asks
1441
+ `/.spool/artifact` as before. In the first-payload reading (L1, a synthetic 11,145-byte page) a
1442
+ cold visit made 1 request and 1 origin fetch where it made 2 and 2 before.
1315
1443
 
1316
1444
  `spool` takes the options `createSpool` takes, minus `routes`. The proxy adds one public
1317
1445
  `assembly` route that matches every path (`/**`) and uses the origin's finished HTML as the
@@ -1419,7 +1547,9 @@ in the instance: a restart forgets them, and each instance of a scaled-out proxy
1419
1547
  A whole page is a large artifact, and the provisioning rule declines one that costs more than the
1420
1548
  priors say it saves, so set `provision` on the spool for large pages. A `Content-Security-Policy`
1421
1549
  that allows only nonce or hash scripts blocks the injected client tag. A URL with a query string
1422
- always goes to the origin. [Trimmed Stylesheets](#-trimmed-stylesheets) covers the `trim` option.
1550
+ always goes to the origin, because the proxy holds finished HTML and has no collection to filter;
1551
+ [Query Routes](#query-routes) apply to `createSpool` only. [Trimmed Stylesheets](#-trimmed-stylesheets) covers the `trim` option
1552
+ and [Images and Video](#-images-and-video) the `images` and `video` options.
1423
1553
 
1424
1554
  ---
1425
1555
 
@@ -1475,6 +1605,301 @@ seen, never to a repeat or local view, and it is not part of the per-view headli
1475
1605
 
1476
1606
  ---
1477
1607
 
1608
+ ## 🎬 Images and Video
1609
+
1610
+ A browser that has fetched an image cannot make it smaller, so conversion happens where the file is
1611
+ served. `@drupflare/spool/media` gives `spoolProxy` three options, each off until set. `images`
1612
+ converts a JPEG or PNG response to WebP for a browser whose `Accept` lists it. `video` rewrites
1613
+ the `<video>` tags of an HTML response, such as adding `preload="none"`. `embeds` swaps a YouTube
1614
+ or Vimeo iframe for a link that loads the player on click.
1615
+
1616
+ ```ts
1617
+ import { type ImageEncoder } from '@drupflare/spool/media';
1618
+ import { spoolProxy } from '@drupflare/spool/proxy';
1619
+ import { memoryStore } from '@drupflare/spool/stores';
1620
+
1621
+ export function mediaProxy(encoder: ImageEncoder) {
1622
+ return spoolProxy({
1623
+ origin: 'https://origin.example.edu',
1624
+ spool: { store: memoryStore() },
1625
+ images: { encoder, width: { fallback: 1600 } },
1626
+ video: { preload: 'none', autoplay: 'defer', when: 'slow' }
1627
+ });
1628
+ }
1629
+ ```
1630
+
1631
+ An `ImageEncoder` takes the origin's bytes and `{ format: 'webp', quality, width }` and returns
1632
+ `{ bytes }`. It never upscales, so `width` is a ceiling, and it throws on input it cannot decode.
1633
+ spool has no runtime dependencies and ships an adapter for [tinyimg](https://tinyimg.gmitch215.dev),
1634
+ which decodes and encodes inside WebAssembly:
1635
+
1636
+ ```js
1637
+ import { readFile } from 'node:fs/promises';
1638
+ import { TinyImgModule, transform } from '@gmitch215/tinyimg';
1639
+ import { tinyimgEncoder } from '@drupflare/spool/media';
1640
+
1641
+ const wasm = new URL(import.meta.resolve('@gmitch215/tinyimg/tinyimg.wasm'));
1642
+ const module = await TinyImgModule.loadBytes(await readFile(wasm));
1643
+
1644
+ export const encoder = tinyimgEncoder({ module, transform });
1645
+ ```
1646
+
1647
+ Use tinyimg 1.1.1 or later. Version 1.1.0 and earlier drop the EXIF orientation tag without
1648
+ rotating the pixels, so a phone photo comes out sideways. `tinyimgEncoder` throws when the module
1649
+ reports an older version.
1650
+
1651
+ Quality is 80, 65 for `Save-Data: on` and 70 for an `ECT` of `slow-2g`, `2g` or `3g`. The width
1652
+ comes from `Sec-CH-Width`, then `Sec-CH-Viewport-Width` times `Sec-CH-DPR`, then
1653
+ `width.fallback`. The proxy converts a `GET` that matches `paths` (JPEG and PNG names by default),
1654
+ has no `Range` or `Authorization`, and gets a `200` JPEG or PNG without a cookie between
1655
+ `minBytes` (4096) and `maxBytes` (15,000,000). Anything else, and any file the encoder cannot
1656
+ shrink, goes out as the origin sent it. The proxy fetches the origin for every request, so a
1657
+ visitor only gets bytes the origin sent that visitor.
1658
+
1659
+ A converted response carries `Vary` for `Accept`, `Save-Data`, `Sec-CH-Width`, `Sec-CH-DPR`,
1660
+ `Sec-CH-Viewport-Width` and `ECT`, a weak `ETag` and `X-Spool-Image`. HTML responses gain
1661
+ `Accept-CH` for the three size hints and `ECT` unless `hints` is false, and the browser then
1662
+ sends them with later image requests to the same origin.
1663
+
1664
+ `maxWidth` (2560) is the width for a request that carries no hint, and `false` removes it. `png`
1665
+ is `'lossy'` by default. `'lossless'` encodes every PNG without loss, and `'auto'` encodes a PNG
1666
+ under 200 KB both ways and serves the smaller file, which suits screenshots and flat graphics.
1667
+ Requests for the same file, quality and width share one encoder call while it runs. The proxy
1668
+ waits `timeoutMs` (10,000) for the encoder and then serves the origin's file, and the late result
1669
+ is cached for the next request. `timeoutMs: false` waits without limit.
1670
+
1671
+ Drupal's video formatter has no `preload` setting, and `preload="none"` does not stop `autoplay`.
1672
+ `autoplay: 'defer'` removes the attribute, adds `controls` and marks the tag
1673
+ `data-spool-autoplay`. With `when: 'always'` the page the spool keeps carries the rewrite. With
1674
+ `'save-data'` or `'slow'` the rewritten page is private to that visitor: it gets `private` in
1675
+ `Cache-Control` and `Save-Data, ECT` in `Vary`, and the spool does not keep it. A device's stored
1676
+ copy of that page keeps the origin's tags.
1677
+
1678
+ `proxy.images` counts `converted`, `bytesIn` and `bytesOut` (converted responses only),
1679
+ `cacheHits`, `coalesced`, `lazied`, `responsive`, and `skipped` by reason. `proxy.video` counts
1680
+ `pages` and `rewritten`. Read the saving as `bytesIn - bytesOut` over real traffic on the site. A
1681
+ request for any other path counts under `skipped.path`, so that key follows total traffic.
1682
+ [docs/security.md](docs/security.md#images-and-video) covers the decoder.
1683
+
1684
+ ### Page Rewrites
1685
+
1686
+ Three options change the HTML the origin sends. Each edits start tags only and leaves every other
1687
+ byte as it was, and a rewritten page gets a weak `ETag`.
1688
+
1689
+ ```ts
1690
+ import { type ImageEncoder } from '@drupflare/spool/media';
1691
+ import { spoolProxy } from '@drupflare/spool/proxy';
1692
+ import { memoryStore } from '@drupflare/spool/stores';
1693
+
1694
+ export function rewriteProxy(encoder: ImageEncoder) {
1695
+ return spoolProxy({
1696
+ origin: 'https://origin.example.edu',
1697
+ spool: { store: memoryStore() },
1698
+ images: {
1699
+ encoder,
1700
+ png: 'auto',
1701
+ maxWidth: 1920,
1702
+ timeoutMs: 5000,
1703
+ lazy: { skip: 3 },
1704
+ responsive: { widths: [480, 960, 1440], sizes: '(min-width: 60em) 50vw, 100vw' }
1705
+ },
1706
+ embeds: { providers: ['youtube'], thumbnail: false }
1707
+ });
1708
+ }
1709
+ ```
1710
+
1711
+ `lazy` adds `loading="lazy"` and `decoding="async"` to each `<img>` after the first `skip` (2 by
1712
+ default), and `iframes: true` does the same for `<iframe>`. It leaves a tag that has `loading`,
1713
+ `fetchpriority` or a lazy-loading library's `data-src`, a tag inside a `<header>` or `<noscript>`,
1714
+ and an image that a `<link rel="preload" as="image">` names. A browser that defers the largest
1715
+ image of a page paints it later, so `skip` is the only guard and the option is off until set. In a
1716
+ crawl of 70 pages per profile on 16 sites, the default `skip` of 2 made the largest image lazy on 2
1717
+ of 23 desktop pages and 2 of 20 phone pages that had an image as their largest element, and no
1718
+ `skip` from 2 to 4 reached zero on both profiles.
1719
+
1720
+ `responsive` adds a `srcset` of `?spool-w=<width>` variants to a same-origin `<img>` that has none,
1721
+ one per entry of `widths` (320, 480, 640, 960, 1280 and 1920 by default), and a `sizes` of `100vw`
1722
+ unless the tag has one. The proxy removes `spool-w` before it asks the origin, answers only widths
1723
+ from the list and never sends a file larger than the origin's. Firefox and Safari send no
1724
+ `Sec-CH-Width`, and this is how they get a resized file. Name the layout width in `sizes`, since
1725
+ `100vw` makes the browser pick the widest candidate it can use. A tag with `srcset` or `data-src`,
1726
+ and any tag inside `<picture>`, stays as it is.
1727
+
1728
+ `embeds` replaces a server-rendered YouTube or Vimeo `<iframe>` with one whose `srcdoc` holds a
1729
+ link to the player. YouTube shows its thumbnail (`thumbnail: false` shows a plain frame) and the
1730
+ player loads on click. A page that sends `Content-Security-Policy` is left alone unless
1731
+ `ignoreCsp` is true, and so is an embed with `enablejsapi=1`. An embed that plays on its own by
1732
+ design stays too: a URL with `background=1`, or with `autoplay=1` beside `mute`, `muted` or `loop`,
1733
+ and an iframe whose `class` or `id` names a hero, ambient, background or banner. In a crawl of the
1734
+ server-rendered pages of 52 sites (L1, `measurements/embed-markup`), 2 of 4 Vimeo iframes were
1735
+ background loops. `proxy.embeds` counts `pages`, `rewritten` and `skipped` by reason. [docs/security.md](docs/security.md#embed-facades) covers the rules.
1736
+
1737
+ ### Dictionary Transport
1738
+
1739
+ A visitor who holds an earlier stylesheet or script of a directory can fetch the next one as the
1740
+ difference from it. `dictionary` implements this for Chromium 130 and later
1741
+ ([RFC 9842](https://www.rfc-editor.org/rfc/rfc9842)). The proxy marks public `.css` and `.js`
1742
+ responses with `Use-As-Dictionary` and answers a request that names one of them with
1743
+ `Content-Encoding: dcz`. spool has no compression library, so the caller supplies the `encoder`,
1744
+ and Node's `zlib` does zstd:
1745
+
1746
+ ```ts
1747
+ import { promisify } from 'node:util';
1748
+ import { zstdCompress } from 'node:zlib';
1749
+ import { spoolProxy } from '@drupflare/spool/proxy';
1750
+ import { memoryStore } from '@drupflare/spool/stores';
1751
+
1752
+ const compress = promisify(zstdCompress);
1753
+
1754
+ export const proxy = spoolProxy({
1755
+ origin: 'https://origin.example.edu',
1756
+ spool: { store: memoryStore() },
1757
+ dictionary: {
1758
+ encoder: async (dictionary, input) => new Uint8Array(await compress(input, { dictionary }))
1759
+ }
1760
+ });
1761
+ ```
1762
+
1763
+ The `encoder` gets the dictionary, the new file and the coding (`'dcz'`, or `'dcb'` when `formats`
1764
+ lists it) and returns the compressed stream. spool adds the coding's header. An encoder that
1765
+ throws, or whose output is no smaller, leaves the plain file. Only responses the proxy would keep
1766
+ as public ones, with no cookie, ever become dictionaries. `proxy.dictionary` counts `registered`,
1767
+ `encoded`, `bytesIn`, `bytesOut` and `skipped`.
1768
+ [docs/security.md](docs/security.md#dictionary-transport) covers the exposure.
1769
+
1770
+ `base` names the files that register as dictionaries, as a function of the pathname, a regular
1771
+ expression or a list of them. Without it the browser holds the latest file of each directory it
1772
+ fetched, and the sizing in [docs/impact.md](docs/impact.md#savings-today-and-at-maximum) shows that
1773
+ choice recovers little. With it the browser holds only the files you name, one per directory, and
1774
+ every other file of the directory is encoded against the base the request names.
1775
+
1776
+ `chooseBases` picks the bases from a crawl. It takes the site's stylesheets and scripts with their
1777
+ compressed sizes and a `delta` function that returns the size of one file compressed against
1778
+ another. For each directory and file kind it returns the file whose deltas against the others, each
1779
+ capped at the other file's plain size, add up smallest. That `held` policy scores a base for a
1780
+ visitor who already holds it. Pass `{ policy: 'first-visit' }` to add the base's own plain size
1781
+ to the sum, for visitors who load the whole group before they hold anything. Neither policy
1782
+ weighs how often a file is requested or how long browsers keep the base, so neither is a proven
1783
+ minimum of total transfer. A directory with one file has no base, and a tie goes to the URL that
1784
+ sorts first. `baseMatcher` turns the choices into a `base` function.
1785
+
1786
+ ```ts
1787
+ import { promisify } from 'node:util';
1788
+ import { brotliCompressSync, zstdCompress } from 'node:zlib';
1789
+ import { baseMatcher, chooseBases, spoolProxy } from '@drupflare/spool/proxy';
1790
+ import { memoryStore } from '@drupflare/spool/stores';
1791
+
1792
+ const compress = promisify(zstdCompress);
1793
+
1794
+ const crawl = [
1795
+ { url: 'https://origin.example.edu/css/a.css', kind: 'style', text: '.a{color:red}.b{margin:0}' },
1796
+ {
1797
+ url: 'https://origin.example.edu/css/b.css',
1798
+ kind: 'style',
1799
+ text: '.a{color:red}.c{margin:1px}'
1800
+ },
1801
+ {
1802
+ url: 'https://origin.example.edu/css/c.css',
1803
+ kind: 'style',
1804
+ text: '.a{color:red}.d{margin:2px}'
1805
+ }
1806
+ ] as const;
1807
+
1808
+ const bytes = (index: number) => Buffer.from(crawl[index]?.text ?? '');
1809
+
1810
+ const choices = await chooseBases(
1811
+ crawl.map(({ url, kind, text }) => ({ url, kind, plain: brotliCompressSync(text).length })),
1812
+ async (file, dictionary) =>
1813
+ (await compress(bytes(file), { dictionary: bytes(dictionary) })).length
1814
+ );
1815
+
1816
+ export const proxy = spoolProxy({
1817
+ origin: 'https://origin.example.edu',
1818
+ spool: { store: memoryStore() },
1819
+ dictionary: {
1820
+ encoder: async (dictionary, input) => new Uint8Array(await compress(input, { dictionary })),
1821
+ base: baseMatcher(choices)
1822
+ }
1823
+ });
1824
+ ```
1825
+
1826
+ On the 10 Drupal sites of the reading, a base chosen this way and already held by the browser saves
1827
+ 30% of the stylesheet and script bytes at the median site (L1, [docs/impact.md](docs/impact.md#savings-today-and-at-maximum)).
1828
+ `scripts/measure/dictionary-size.ts` records the same choice for a site's crawl.
1829
+
1830
+ ### Defaults
1831
+
1832
+ Every option is absent by default. The table shows what a feature does once its parent option is
1833
+ set and how to change it.
1834
+
1835
+ | Feature | Option | Default once the parent is set | Leave or enter |
1836
+ | ------------------------------------------ | ------------------- | ------------------------------------------- | ----------------------------------------------------- |
1837
+ | WebP conversion, `304` and `X-Spool-Image` | `images` | on, needs an `encoder` | leave `images` out |
1838
+ | `Accept-CH` hints and `Vary` | `images.hints` | on | `hints: false` |
1839
+ | Quality by `Save-Data` and `ECT` | `images.quality` | 80, 65 and 70 | give all three one number |
1840
+ | Width ceiling for a request with no width | `images.maxWidth` | 2560 | `maxWidth: false`, or another number |
1841
+ | Shared encodes and the encoder time limit | `images.timeoutMs` | 10,000 ms, identical encodes share one call | `timeoutMs: false` removes the limit |
1842
+ | PNG mode | `images.png` | `'lossy'` | `'auto'` or `'lossless'` |
1843
+ | `loading="lazy"` on images and iframes | `images.lazy` | off | `lazy: {}` or `{ skip, iframes }` |
1844
+ | `srcset` variants | `images.responsive` | off | `responsive: {}` or `{ widths, sizes }` |
1845
+ | `preload="none"` on video | `video.preload` | `'none'` | `preload: false` |
1846
+ | Video autoplay deferral | `video.autoplay` | off | `autoplay: 'defer'` |
1847
+ | YouTube and Vimeo facades | `embeds` | off | `embeds: {}` or `{ providers, thumbnail, ignoreCsp }` |
1848
+ | Dictionary transport | `dictionary` | off | `dictionary: { encoder }` |
1849
+ | Which files become dictionaries | `dictionary.base` | every `.css` and `.js` response | `base: /\/base\.js$/`, a list or a function |
1850
+
1851
+ ---
1852
+
1853
+ ## 📈 Capacity Reports
1854
+
1855
+ `@drupflare/spool/report` turns peak work and an institution's own limits into three separate
1856
+ results: potential, financial and physical. None is added to another, and any can be empty.
1857
+
1858
+ `capacityPlan` takes intervals of demand per resource with the part spool avoided, and what one
1859
+ node serves per second within the latency target. It returns the nodes needed before and after,
1860
+ as whole numbers, and how many could retire. `perNode.busyMs` is the milliseconds of handler work
1861
+ a node does per second, such as 1000 times its cores times a target utilisation. `spares`, `floor`,
1862
+ `regions` and `minPerRegion` raise the answer for redundancy, and a resource with no `avoided`, such
1863
+ as memory, does not shrink. `intervalsFromWork` builds the intervals from the `WorkSample`s the
1864
+ work series records.
1865
+
1866
+ `institutionReport` adds the plan, an `observed` before and after of the same workload from
1867
+ production telemetry, an invoice total before and after, and a ledger of retired equipment. The
1868
+ plan says what could change, `observed` what changed, the invoices what was paid, and the ledger
1869
+ what was removed. The physical result reads only the ledger, and an entry needs the evidence for
1870
+ its removal. With no ledger the statement reads `No physical capacity saving demonstrated.`
1871
+
1872
+ ```ts
1873
+ import { capacityPlan, institutionReport } from '@drupflare/spool/report';
1874
+
1875
+ const plan = capacityPlan(
1876
+ [
1877
+ {
1878
+ start: Date.UTC(2026, 8, 1, 14),
1879
+ seconds: 300,
1880
+ demand: { busyMs: 9000 },
1881
+ avoided: { busyMs: 6300 }
1882
+ },
1883
+ {
1884
+ start: Date.UTC(2026, 8, 1, 14, 5),
1885
+ seconds: 300,
1886
+ demand: { busyMs: 7000 },
1887
+ avoided: { busyMs: 4900 }
1888
+ }
1889
+ ],
1890
+ { perNode: { busyMs: 700 }, spares: 1, floor: 2 }
1891
+ );
1892
+
1893
+ export const report = institutionReport({ plan, planLayer: 'L3' });
1894
+ ```
1895
+
1896
+ The example needs 14 nodes without spool and 5 with it, a potential of 9, at layer L3 because the
1897
+ numbers are made up. `channelRows` lists the metered, attributional, capacity and avoided-demand
1898
+ channels as separate rows, and nothing in the module sums them. [docs/impact.md](docs/impact.md#channels)
1899
+ says what each one counts.
1900
+
1901
+ ---
1902
+
1478
1903
  ## 🚫 Out of Scope
1479
1904
 
1480
1905
  - spool never authorises on the client. The device holds only what the server already sent that
@@ -1507,19 +1932,20 @@ loads per view and reports the view as such.
1507
1932
 
1508
1933
  ## 🧭 Subpath Exports
1509
1934
 
1510
- | Import | What is in it |
1511
- | ---------------------------------------- | -------------------------------------------------------------------- |
1512
- | `@drupflare/spool` | the protocol and report modules |
1513
- | `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events |
1514
- | `@drupflare/spool/server` | `createSpool`, `defineRoute`, sinks, identity and work series |
1515
- | `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse` |
1516
- | `@drupflare/spool/proxy` | `spoolProxy`, the tag readers and the eligibility verdict |
1517
- | `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type |
1518
- | `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class |
1519
- | `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker |
1520
- | `@drupflare/spool/client` | `register` and the page handle |
1521
- | `@drupflare/spool/report` | the energy model |
1522
- | `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage` |
1935
+ | Import | What is in it |
1936
+ | ---------------------------------------- | ----------------------------------------------------------------------------- |
1937
+ | `@drupflare/spool` | the protocol and report modules |
1938
+ | `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events |
1939
+ | `@drupflare/spool/server` | `createSpool`, `defineRoute`, `datasetProof`, sinks, identity and work series |
1940
+ | `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse` |
1941
+ | `@drupflare/spool/proxy` | `spoolProxy`, `chooseBases`, the tag readers and the eligibility verdict |
1942
+ | `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type |
1943
+ | `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class |
1944
+ | `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker |
1945
+ | `@drupflare/spool/client` | `register` and the page handle |
1946
+ | `@drupflare/spool/report` | the energy model, capacity plans and institution reports |
1947
+ | `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage` |
1948
+ | `@drupflare/spool/media` | `tinyimgEncoder`, `imageVariant` and `rewriteVideo` |
1523
1949
 
1524
1950
  ---
1525
1951
 
@@ -1 +1 @@
1
- (()=>{var p="data-spool-client";function d(t,o,n=[]){let s=0;for(let{name:r}of t){if(n.includes(r))continue;try{if(new URL(r).origin!==o)s+=1}catch{continue}}return s}function u(t,o,n=[],s=32){let r=new Map;for(let{name:i}of t){if(n.includes(i))continue;try{let e=new URL(i).origin;if(e!==o&&e!=="null")r.set(e,(r.get(e)??0)+1)}catch{continue}}return Object.fromEntries([...r].sort((i,e)=>e[1]-i[1]||(i[0]<e[0]?-1:1)).slice(0,s))}var f=1;var l={sync:"/.spool/sync",artifact:"/.spool/artifact",beacon:"/.spool/beacon",worker:"/spool-sw.js"};function g(t,o=window){let n=()=>t.close();return o.addEventListener("pagehide",n),()=>o.removeEventListener("pagehide",n)}function a(t,o,n){return new Promise((s,r)=>{let i=new MessageChannel,e=setTimeout(()=>{i.port1.close(),r(Error("the worker did not answer"))},n);i.port1.onmessage=(c)=>{clearTimeout(e),i.port1.close(),s(c.data)},t.postMessage(o,[i.port2])})}async function y(t={}){let o=t.container??navigator.serviceWorker;if(!o)return null;let n=t.helloTimeoutMs??5000;o.addEventListener?.("message",(e)=>{let c=e.data;if(c?.type!=="spool:cleared")return;(t.target??window).dispatchEvent(new CustomEvent("spool:cleared",{detail:{reason:c.reason}}))}),await o.register(t.workerUrl??l.worker,{scope:t.scope,updateViaCache:"all"});let s=await o.ready,r=s.active;if(!r)throw Error("the worker is not active");let i=await a(r,{type:"spool:hello",protocol:f},n);return{registration:s,hello:i,status:()=>a(s.active??r,{type:"spool:status"},n),view:async(e=location.pathname)=>(await a(s.active??r,{type:"spool:view",path:e},n)).id,reportLoads:(e,c,b)=>(s.active??r).postMessage({type:"spool:loads",view:e,count:c,origins:b}),track:(e)=>g(e)}}var m=y().catch(()=>null);globalThis.spool={ready:m};m.then(async(t)=>{let o=await t?.view().catch(()=>null);if(!t||!o)return;let n=-1,s=()=>{let r=document.querySelector(`script[${p}]`)?.src,i=performance.getEntriesByType("resource"),e=d(i,location.origin,r?[r]:[]);if(e===n)return;n=e,t.reportLoads(o,e,u(i,location.origin,r?[r]:[]))};addEventListener("pagehide",s),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")s()})});})();
1
+ (()=>{var b="data-spool-client";function w(e=l()){return{register(n,o={}){let s=o.hiddenAfterMs??1e4,t=!0,i=!1,r=!1,c=!1,p=!1,d=null,g=(a)=>{try{n[a]()}catch(P){if(!p)p=!0,console.warn("spool.activity",P)}},u=()=>{if(c)return;let a=!t||i;if(a===r)return;r=a,g(a?"pause":"resume")},y=()=>{if(d===null)return;e.clearTimeout(d),d=null},m=()=>{if(c)return;if(!e.hidden()){y(),i=!1,u();return}if(i||d!==null)return;if(s<=0){i=!0,u();return}d=e.setTimeout(()=>{if(d=null,e.hidden())i=!0,u()},s)},T=e.onVisibilityChange(m),A=o.target?e.observe(o.target,(a)=>{t=a,u()}):()=>{return};return m(),()=>{if(c)return;if(c=!0,y(),T(),A(),r)r=!1,g("resume")}}}}function l(){return{hidden:()=>document.hidden,onVisibilityChange(e){return document.addEventListener("visibilitychange",e),()=>document.removeEventListener("visibilitychange",e)},observe(e,n){if(typeof IntersectionObserver>"u")return n(!0),()=>{return};let o=new IntersectionObserver((s)=>{let t=s.at(-1);if(t)n(t.isIntersecting||t.intersectionRatio>0)},{threshold:0});return o.observe(e),()=>o.disconnect()},setTimeout:(e,n)=>setTimeout(e,n),clearTimeout:(e)=>clearTimeout(e)}}function v(e,n,o=[]){let s=0;for(let{name:t}of e){if(o.includes(t))continue;try{if(new URL(t).origin!==n)s+=1}catch{continue}}return s}function j(e,n,o=[],s=32){let t=new Map;for(let{name:i}of e){if(o.includes(i))continue;try{let r=new URL(i).origin;if(r!==n&&r!=="null")t.set(r,(t.get(r)??0)+1)}catch{continue}}return Object.fromEntries([...t].sort((i,r)=>r[1]-i[1]||(i[0]<r[0]?-1:1)).slice(0,s))}var R=1;var h={sync:"/.spool/sync",artifact:"/.spool/artifact",beacon:"/.spool/beacon",worker:"/spool-sw.js"};function E(e,n=window){let o=()=>e.close();return n.addEventListener("pagehide",o),()=>n.removeEventListener("pagehide",o)}function f(e,n,o){return new Promise((s,t)=>{let i=new MessageChannel,r=setTimeout(()=>{i.port1.close(),t(Error("the worker did not answer"))},o);i.port1.onmessage=(c)=>{clearTimeout(r),i.port1.close(),s(c.data)},e.postMessage(n,[i.port2])})}async function k(e={}){let n=e.container??navigator.serviceWorker;if(!n)return null;let o=e.helloTimeoutMs??5000;n.addEventListener?.("message",(r)=>{let c=r.data;if(c?.type!=="spool:cleared")return;(e.target??window).dispatchEvent(new CustomEvent("spool:cleared",{detail:{reason:c.reason}}))}),await n.register(e.workerUrl??h.worker,{scope:e.scope,updateViaCache:"all"});let s=await n.ready,t=s.active;if(!t)throw Error("the worker is not active");let i=await f(t,{type:"spool:hello",protocol:R},o);return{registration:s,hello:i,status:()=>f(s.active??t,{type:"spool:status"},o),view:async(r=location.pathname)=>(await f(s.active??t,{type:"spool:view",path:r},o)).id,reportLoads:(r,c,p)=>(s.active??t).postMessage({type:"spool:loads",view:r,count:c,origins:p}),track:(r)=>E(r)}}var x=k().catch(()=>null);globalThis.spool={ready:x,activity:w(l())};x.then(async(e)=>{let n=await e?.view().catch(()=>null);if(!e||!n)return;let o=-1,s=()=>{let t=document.querySelector(`script[${b}]`)?.src,i=performance.getEntriesByType("resource"),r=v(i,location.origin,t?[t]:[]);if(r===o)return;o=r,e.reportLoads(n,r,j(i,location.origin,t?[t]:[]))};addEventListener("pagehide",s),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")s()})});})();
@@ -1,16 +1,16 @@
1
1
  {
2
2
  "schema": 2,
3
- "version": "0.1.0",
3
+ "version": "0.3.0",
4
4
  "files": {
5
5
  "client.js": {
6
- "bytes": 2155,
7
- "sha384": "0baa3c240fdec865d1b53a00e5db1f7d456ec910f12c11ea9b8ac356df108ca88a77e0bb4eb955a88bbcaa53fd32d04e",
8
- "integrity": "sha384-C6o8JA/eyGXRtToA5dsffUVuyRDxLBHqm4rDVt8QjKiKd+C7TrlVqIu8qlP9MtBO"
6
+ "bytes": 3263,
7
+ "sha384": "f48546b44a63503f6232f6edb70a2755172a2cccdcaaed61f9e8ebdb8d4ed7a040fe9c0817369c29667d56fb4afd2b91",
8
+ "integrity": "sha384-9IVGtEpjUD9iMvbttwonVRcqLMzcqu1h+ejr241O16BA/pwIFzacKWZ9VvtK/SuR"
9
9
  },
10
10
  "spool-sw.js": {
11
- "bytes": 60560,
12
- "sha384": "3adadb58e469f2c975ca67c755721d46af092bd9487fe9d673ce7e3a4a083153b955348822ed516c7c047fe80ac85c35",
13
- "integrity": "sha384-OtrbWORp8sl1ymfHVXIdRq8JK9lIf+nWc85+OkoIMVO5VTSIIu1RbHwEf+gKyFw1"
11
+ "bytes": 65371,
12
+ "sha384": "33aa92b7bc086e472b5c01a1b814e76df76654cd0c11cd63755f77c759d11adc405fed39ee10a5ab7e8c4da058196386",
13
+ "integrity": "sha384-M6qSt7wIbkcrXAGhuBTnbfdmVM0MEc1jdV93x1nRGtxAX+057hClq36MTaBYGWOG"
14
14
  }
15
15
  }
16
16
  }