@drupflare/spool 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +263 -41
  2. package/dist/cdn/client.js +1 -1
  3. package/dist/cdn/manifest.json +7 -7
  4. package/dist/cdn/spool-sw.js +1 -1
  5. package/dist/client/entry.js +33 -1
  6. package/dist/client/register.d.ts +23 -1
  7. package/dist/client/register.js +53 -7
  8. package/dist/protocol/config.d.ts +16 -1
  9. package/dist/protocol/config.js +3 -1
  10. package/dist/protocol/dataset.d.ts +183 -0
  11. package/dist/protocol/dataset.js +206 -0
  12. package/dist/protocol/envelope.d.ts +1 -0
  13. package/dist/protocol/envelope.js +2 -1
  14. package/dist/protocol/errors.d.ts +1 -1
  15. package/dist/protocol/graph.d.ts +23 -4
  16. package/dist/protocol/graph.js +34 -3
  17. package/dist/protocol/index.d.ts +3 -0
  18. package/dist/protocol/index.js +3 -0
  19. package/dist/protocol/plan.d.ts +100 -0
  20. package/dist/protocol/plan.js +661 -0
  21. package/dist/protocol/provision.d.ts +46 -0
  22. package/dist/protocol/provision.js +38 -0
  23. package/dist/protocol/query.d.ts +66 -5
  24. package/dist/protocol/query.js +103 -4
  25. package/dist/protocol/route.d.ts +6 -2
  26. package/dist/protocol/route.js +31 -3
  27. package/dist/protocol/write.d.ts +146 -0
  28. package/dist/protocol/write.js +101 -0
  29. package/dist/report/dataset.d.ts +48 -0
  30. package/dist/report/dataset.js +93 -0
  31. package/dist/report/derive.d.ts +285 -0
  32. package/dist/report/derive.js +676 -1
  33. package/dist/report/economics.d.ts +11 -1
  34. package/dist/report/economics.js +19 -1
  35. package/dist/report/index.d.ts +2 -0
  36. package/dist/report/index.js +2 -0
  37. package/dist/report/inputs.d.ts +97 -0
  38. package/dist/report/inputs.js +133 -0
  39. package/dist/report/institution.d.ts +188 -1
  40. package/dist/report/institution.js +146 -0
  41. package/dist/report/loops.d.ts +73 -0
  42. package/dist/report/loops.js +101 -0
  43. package/dist/report/measurement.d.ts +1 -1
  44. package/dist/report/measurement.js +3 -1
  45. package/dist/report/model.d.ts +2 -0
  46. package/dist/report/model.js +4 -0
  47. package/dist/report/thirdparty.d.ts +57 -0
  48. package/dist/report/thirdparty.js +111 -0
  49. package/dist/report/trim.d.ts +56 -0
  50. package/dist/report/trim.js +83 -1
  51. package/dist/server/dataset.d.ts +48 -0
  52. package/dist/server/dataset.js +308 -0
  53. package/dist/server/embedded.d.ts +3 -3
  54. package/dist/server/embedded.js +3 -3
  55. package/dist/server/index.d.ts +2 -0
  56. package/dist/server/index.js +2 -0
  57. package/dist/server/spool.d.ts +20 -0
  58. package/dist/server/spool.js +126 -18
  59. package/dist/server/write.d.ts +27 -0
  60. package/dist/server/write.js +127 -0
  61. package/dist/sw/analytics.d.ts +6 -0
  62. package/dist/sw/analytics.js +14 -0
  63. package/dist/sw/dataset-sync.d.ts +76 -0
  64. package/dist/sw/dataset-sync.js +149 -0
  65. package/dist/sw/dataset.d.ts +91 -0
  66. package/dist/sw/dataset.js +318 -0
  67. package/dist/sw/graph/index.d.ts +31 -1
  68. package/dist/sw/graph/index.js +57 -5
  69. package/dist/sw/graph/invalidate.d.ts +15 -3
  70. package/dist/sw/graph/invalidate.js +42 -4
  71. package/dist/sw/idb.js +78 -9
  72. package/dist/sw/partition.d.ts +6 -1
  73. package/dist/sw/partition.js +21 -2
  74. package/dist/sw/principal-dataset.d.ts +64 -0
  75. package/dist/sw/principal-dataset.js +254 -0
  76. package/dist/sw/render.d.ts +7 -2
  77. package/dist/sw/render.js +19 -5
  78. package/dist/sw/replica.d.ts +12 -3
  79. package/dist/sw/replica.js +40 -5
  80. package/dist/sw/runtime.d.ts +43 -2
  81. package/dist/sw/runtime.js +340 -24
  82. package/dist/sw/seal.d.ts +13 -0
  83. package/dist/sw/seal.js +13 -4
  84. package/dist/sw/static.d.ts +18 -0
  85. package/dist/sw/static.js +84 -1
  86. package/dist/sw/storage.d.ts +51 -1
  87. package/dist/sw/storage.js +33 -1
  88. package/dist/sw/write.d.ts +107 -0
  89. package/dist/sw/write.js +442 -0
  90. package/package.json +3 -2
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # 🧵 spool
2
2
 
3
- > An incremental application replica for the browser
3
+ > A browser replica that removes repeated demand from your server
4
4
 
5
5
  [![Build](https://github.com/drupflare/spool/actions/workflows/build.yml/badge.svg)](https://github.com/drupflare/spool/actions/workflows/build.yml)
6
6
  [![Browser](https://github.com/drupflare/spool/actions/workflows/browser.yml/badge.svg)](https://github.com/drupflare/spool/actions/workflows/browser.yml)
@@ -8,9 +8,16 @@
8
8
  [![codecov](https://codecov.io/gh/drupflare/spool/branch/master/graph/badge.svg)](https://codecov.io/gh/drupflare/spool)
9
9
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
10
10
 
11
- spool keeps a replica of what a page needs on the visitor's device. The server sends templates
12
- and authorised, versioned data. A service worker stores them and computes eligible pages on the
13
- device, so a repeat view makes no request and costs the server no work.
11
+ spool removes repeated execution demand from your server. It keeps a replica of what a page needs
12
+ on the visitor's device. The server sends templates and authorised, versioned data, and a service
13
+ worker computes eligible pages from them. A page answered there is a request that never reaches
14
+ your Worker or origin, and server work that never runs. The contract is a count: a test asserts
15
+ the server hits of a local view, and in Chromium and Firefox the count is zero.
16
+
17
+ spool has three uses. In a drupflare deployment it takes eligible public views off the Worker.
18
+ For a team on a free or constrained plan it lowers the requests the plan has to carry, by an
19
+ amount that depends on the site's traffic ([Request Arithmetic](#request-arithmetic)). It is also
20
+ a research system: every figure it reports carries a layer and a method.
14
21
 
15
22
  It runs on any runtime that speaks `Request` and `Response`, bridges `node:http` for the rest, and
16
23
  has no runtime dependencies. A site with no server code can load it from two files and a manifest.
@@ -28,7 +35,9 @@ has no runtime dependencies. A site with no server code can load it from two fil
28
35
  - [Static Sites](#-static-sites)
29
36
  - [Strategies](#-strategies)
30
37
  - [Freshness Contracts](#-freshness-contracts)
38
+ - [Datasets](#-datasets)
31
39
  - [Authenticated Data](#-authenticated-data)
40
+ - [Declared Writes](#-declared-writes)
32
41
  - [Catalogue](#-catalogue)
33
42
  - [Analytics Batching](#-analytics-batching)
34
43
  - [Third-Party Files](#-third-party-files)
@@ -38,10 +47,14 @@ has no runtime dependencies. A site with no server code can load it from two fil
38
47
  - [Trimmed Stylesheets](#-trimmed-stylesheets)
39
48
  - [Images and Video](#-images-and-video)
40
49
  - [Capacity Reports](#-capacity-reports)
50
+ - [Security](#-security)
51
+ - [Energy Reporting](#-energy-reporting)
52
+ - [Prior Art](#-prior-art)
41
53
  - [Out of Scope](#-out-of-scope)
42
54
  - [Browser Support](#-browser-support)
43
55
  - [Subpath Exports](#-subpath-exports)
44
56
  - [Testing](#-testing)
57
+ - [Documentation](#-documentation)
45
58
  - [Related Repositories](#-related-repositories)
46
59
  - [License](#-license)
47
60
 
@@ -49,16 +62,25 @@ has no runtime dependencies. A site with no server code can load it from two fil
49
62
 
50
63
  ## 🎯 Why Spool
51
64
 
65
+ A local view removes, in order, a request that would have reached the Worker, the server work
66
+ behind it (rendering, queries, Durable Object and database reads), and the bytes of a document the
67
+ device already holds. Energy follows from those and is one objective an administrator may weigh.
68
+
52
69
  A response is a deterministic transformation of a set of authorised, versioned dependencies. The
53
70
  server half names those dependencies and sends the snapshot the visitor may hold. The worker keeps
54
71
  a graph of records, projections and responses, with their versions, and syncs only what the next
55
72
  page needs.
56
73
 
57
- Your application keeps every write, every access decision and every page it cannot reproduce
58
- safely on the device. A route the worker cannot answer goes to the network, so adopting spool
59
- changes no page that was not declared.
74
+ Your application keeps every access decision and every page it cannot reproduce safely on the
75
+ device. It keeps every write too, except the last-value writes you declare, which ship off. A route
76
+ the worker cannot answer goes to the network, so adopting spool changes no page that was not
77
+ declared.
78
+
79
+ A collection of thousands of rows, such as a directory or a course catalogue, can be held as a
80
+ [dataset](#-datasets). The device seeds it once and then receives only the rows that changed.
60
81
 
61
- [docs/impact.md](docs/impact.md) holds the energy model and the measurements behind it.
82
+ [docs/impact.md](docs/impact.md) holds the request, cost and energy figures and the measurements
83
+ behind them. See [Energy Reporting](#-energy-reporting).
62
84
 
63
85
  ---
64
86
 
@@ -193,8 +215,9 @@ it against the hash, so a changed file never runs. Release files are never overw
193
215
  script fails to load, the page works as it did before.
194
216
 
195
217
  The script registers `/spool-sw.js` and exposes `window.spool.ready`, which resolves to the client
196
- handle or to `null` where service workers do not exist. To bundle the runtime yourself, import
197
- `register` from `@drupflare/spool/client`.
218
+ handle or to `null` where service workers do not exist. It also exposes `window.spool.activity`,
219
+ described below, and the write calls of [Declared Writes](#-declared-writes). To bundle the runtime
220
+ yourself, import `register` from `@drupflare/spool/client`.
198
221
 
199
222
  ### Activity Hooks
200
223
 
@@ -744,7 +767,7 @@ from one directory:
744
767
  - `client.js`, the page script, from the same place.
745
768
  - `spool.json`, the list of files the worker may keep.
746
769
 
747
- Each page loads the script with the tag from [CDN Script](#-install), or from the site's own copy.
770
+ Each page loads the script with the tag from [CDN Script](#cdn-script), or from the site's own copy.
748
771
  The worker has no server to talk to, so it reads `spool.json` from the directory its own script
749
772
  sits in.
750
773
 
@@ -1016,8 +1039,9 @@ sorts descending. Ties keep the stored order. Strings compare by UTF-16 code uni
1016
1039
  and no case folding, so the application's own sort has to agree. A database that compares bytes,
1017
1040
  such as SQLite by default, agrees except for characters above U+FFFF, and a case-insensitive or
1018
1041
  locale collation does not. `pageSize` sets the items per page and `page` counts from 1; without it
1019
- the route is one page of everything. A route with `principal` scope, or any
1020
- other strategy, cannot declare `query`.
1042
+ the route is one page of everything. A route with `static` or `assembly` strategy cannot declare
1043
+ `query`. A route with `principal` scope can declare it only with `query.dataset`, which
1044
+ [Datasets](#-datasets) describes.
1021
1045
 
1022
1046
  `load` returns the array at `list` and a `proof` made by `datasetProof`. The server refuses to
1023
1047
  build the artifact when the proof is missing or does not match the array. The worker computes the
@@ -1090,6 +1114,72 @@ invalidations, so a sync does not always need a request of its own.
1090
1114
 
1091
1115
  ---
1092
1116
 
1117
+ ## 💾 Datasets
1118
+
1119
+ A route that lists thousands of rows, such as a staff directory, makes a device fetch the whole
1120
+ collection again after each change. A dataset lets the device hold the rows and receive only the
1121
+ rows that changed. The device seeds a dataset once, then asks for a patch from the cursor it holds.
1122
+ A patch carries the rows to put, the keys to delete and a proof the device checks before it stores
1123
+ anything.
1124
+
1125
+ `defineDataset(name, declaration)` from `@drupflare/spool/server` declares a dataset, and
1126
+ `createSpool({ datasets })` serves it at `GET /.spool/dataset`. The declaration gives the
1127
+ functions `key`, `seed`, `read`, `changes`, `horizon` and `head`, a `schema` string, and the
1128
+ `maxRows` and `maxBytes` of a seed. A dataset has its own integer cursor, which your backend holds.
1129
+ It does not use the site's version vector. `changes(since)` returns the net change after a cursor
1130
+ and decides which rows are in scope. `read(ids)` is a keyed read that never runs a route's `load`.
1131
+ A device whose cursor is below `horizon`, or whose `schema` differs, seeds again.
1132
+
1133
+ The proof is the number of rows and the sum of every row's hash. A sum does not depend on order, so
1134
+ a patch moves it by the rows it touches. The device applies a seed or a patch to a staging copy,
1135
+ compares the result with the proof, and writes the rows and their metadata in one transaction. A
1136
+ failed proof clears that dataset and seeds it once more. The proof catches corruption and a
1137
+ half-applied patch. It does not defend against a server that sends rows and a digest that agree.
1138
+
1139
+ A route reads a dataset in two ways. `query.dataset` names it in place of `list`: the worker takes
1140
+ the held rows in key order and applies the route's filter, sort and page, and goes to the network
1141
+ while the dataset is not held. `ctx.row(dataset, key)` puts a reference to one row in the route's
1142
+ data. The reference resolves when the page renders, so a changed row needs no new artifact.
1143
+
1144
+ A route with `query.dataset` may also declare `query.plan`, a read-only plan written as JSON with
1145
+ joins, conditions, groups and aggregates over held datasets. It replaces `filter`, `sort` and
1146
+ `pageSize`. The worker compares values the way SQLite compares a column with no declared type, and
1147
+ sends the query to the network when a plan could give another answer than SQLite would. The server
1148
+ never evaluates a plan.
1149
+
1150
+ A dataset declared with `provision: 'auto'` reaches a device only while it beats two controls: the
1151
+ held page under v0.3, and one plain resource that a browser cache revalidates with an `ETag`.
1152
+ Otherwise the seed request gets `409` with `Spool-Provision: declined`, and the worker holds
1153
+ nothing and does not ask again in that session. The default, `'always'`, answers every seed. The worker also
1154
+ refuses a seed whose rows would not fit the headroom that `navigator.storage.estimate()` reports,
1155
+ and the page runtime asks the browser for persistent storage once the first dataset is held.
1156
+
1157
+ On synthetic rows (L1, no real site measured), one changed row of 500 costs 228 bytes as a patch and
1158
+ 19,981 bytes as a v0.3 list refetch, and the first load costs 867 bytes more than under v0.3
1159
+ ([docs/impact.md](docs/impact.md#dataset-patches)). [docs/protocol.md](docs/protocol.md#datasets)
1160
+ has the declaration rules, the wire format, query plans and the device store.
1161
+
1162
+ ### Principal Datasets
1163
+
1164
+ Principal datasets are off by default and unreviewed: no external review of this feature has
1165
+ happened. They hold rows that differ for each signed-in principal. Two switches turn them on, and
1166
+ both are needed. `defineDataset(name, declaration, { principalDatasets: true })` accepts
1167
+ `scope: 'principal'`, and `createSpool({ principalDatasets: true, identity })` serves the dataset.
1168
+ A principal dataset declares `maxStaleMs`, which has no default, and cannot use
1169
+ `provision: 'auto'`.
1170
+
1171
+ The server takes the identity from the request and passes it to every dataset function as
1172
+ `ctx.principal`. Which rows a principal may see is your code, and spool does not check what the
1173
+ functions return. The device keeps the rows sealed in the principal's partition, as
1174
+ [Authenticated Data](#-authenticated-data) describes, and wipes them with it. Held rows answer a
1175
+ query for `maxStaleMs` after the last verified sync, so a principal removed on the server can read
1176
+ held rows for up to that long. A principal route can declare `query` only with `query.dataset` and
1177
+ no plan, and a route and its dataset need the same scope.
1178
+ [docs/security.md](docs/security.md#principal-datasets) lists the threat modes and the test that
1179
+ holds each one.
1180
+
1181
+ ---
1182
+
1093
1183
  ## 🔐 Authenticated Data
1094
1184
 
1095
1185
  Set `scope: 'principal'` on a route that depends on who is signed in, and give `createSpool` an
@@ -1167,6 +1257,38 @@ that page reads.
1167
1257
 
1168
1258
  ---
1169
1259
 
1260
+ ## 📝 Declared Writes
1261
+
1262
+ Declared writes are off by default and unreviewed: no external review of this feature has
1263
+ happened. A declared write queues one kind of edit on the device and sends it at a network moment.
1264
+ The last value for a key wins. `createSpool({ identity, writes: { enabled: true, classes } })`
1265
+ turns them on. With the flag off the worker queues nothing, the page calls answer `writes-off`, and
1266
+ `PUT /.spool/write` answers `403`.
1267
+
1268
+ `defineWrite` from `@drupflare/spool/server` declares a class. Its `scope` has to be `principal`,
1269
+ and its `path` a declared principal route. `key` says which key this principal may write, and
1270
+ `read` gives that key's current version. `apply` writes the value only when the key is still at the
1271
+ version the device edited from, in one atomic step, and returns the new version or the conflict.
1272
+ `maxBytes` and `maxPending` bound one value and the queue. The only `kind` is `last-value`, so a
1273
+ declaration cannot queue a comment, a cart, a payment, a counter, an inventory change or an audit
1274
+ record.
1275
+
1276
+ The page calls `window.spool.write(name, key, value, version)`, `window.spool.flush()`,
1277
+ `window.spool.writes()` and `window.spool.resolve(name, key, 'local' | 'server')`. Each answers
1278
+ `{ ok: true }` with its result or `{ ok: false, code, message }`, and none throws. Edits of one key
1279
+ replace each other, so several edits cost one request. A flush sends `PUT /.spool/write` with the
1280
+ version in `If-Match` and a `Spool-Write-Id` that stays the same across retries. The page runtime
1281
+ flushes when the page is hidden. A stale version gets `412`, and the record waits as `conflicted`
1282
+ until `resolve` settles it.
1283
+
1284
+ The queue is sealed in the principal's partition and deleted with it. A value that no flush has
1285
+ sent is lost at logout, so call `window.spool.flush()` before the logout navigation. The server
1286
+ remembers recent write ids in memory, 10,000 by default, so a site with several instances stores
1287
+ `ctx.writeId` with the value in `apply`. [docs/protocol.md](docs/protocol.md#declared-writes) has
1288
+ the request and its answers, and [docs/security.md](docs/security.md#declared-writes) the limits.
1289
+
1290
+ ---
1291
+
1170
1292
  ## 📚 Catalogue
1171
1293
 
1172
1294
  A catalogue sends the data of pages a visitor may open next, so that opening one makes no
@@ -1185,7 +1307,7 @@ The site supplies the candidates. A route's `catalogue` returns the pages a visi
1185
1307
  next, each with the share expected to. Take the share from your own `nextShare` over the sink
1186
1308
  (each `spool.view` names the route the visitor came from), from link position, or from a fixed
1187
1309
  prior. Pass `{ first: true }` to `nextShare` to count only first views, which is what an entry
1188
- saves: the share over all views ran 1.5 to 7 times high in the bins that decide an entry in a replay
1310
+ saves: the share over all views ran 1.5 to 6.6 times high in the bins that decide an entry in a replay
1189
1311
  of public visitors ([docs/impact.md](docs/impact.md#next-page-prediction)). `documentBytes` on the entry's route says what a hit saves, and the provisioning prior is
1190
1312
  used when it is absent.
1191
1313
 
@@ -1383,13 +1505,10 @@ that. `passthrough` leaves assets to the browser's cache and never touches them.
1383
1505
  asks the browser through `InstallEvent.addRoutes` to send only navigations, beacons and routed
1384
1506
  paths to the worker and the rest of the origin to the network, and acts as `passthrough` where the
1385
1507
  API does not exist: Chromium has it, Playwright's WebKit has it on macOS and not on Linux, and
1386
- Firefox does not. Both options cut the service worker's fetch events (Static Routing took them from
1387
- 32 to 3 per view on a desktop Chromium rig); neither changes server-side Worker invocations.
1388
- Neither is measured to save energy. In the same run, new-tab views drew 2.78 J,
1389
- 2.83 J and 2.84 J of RAPL package energy above idle with CacheStorage, navigation-only handling
1390
- and Static Routing (8 rounds each), and the ranges overlap. Under Static Routing a same-origin write no
1391
- longer reaches the worker, so it is not a network moment and its response cannot carry
1392
- invalidations; navigations still are.
1508
+ Firefox does not. Static Routing keeps the requests it does not route out of the worker's fetch
1509
+ handler, and neither option changes server-side Worker invocations. Neither is measured to save
1510
+ energy, and [docs/impact.md](docs/impact.md) holds no reading for them. Under Static Routing a same-origin write no longer reaches the worker,
1511
+ so it is not a network moment and its response cannot carry invalidations; navigations still are.
1393
1512
 
1394
1513
  ```ts
1395
1514
  import { createSpool } from '@drupflare/spool/server';
@@ -1447,7 +1566,8 @@ artifact. `routes` lists overrides, such as a shorter freshness bound for one pa
1447
1566
  match first and take the same fields as `defineRoute`, without `load`.
1448
1567
 
1449
1568
  On Node the handler goes through `toNodeListener`. `origin` is the address of the real server
1450
- and has no path.
1569
+ and has no path. `fetch` replaces the global `fetch` the proxy calls the origin with, such as a
1570
+ Workers service binding.
1451
1571
 
1452
1572
  ```ts
1453
1573
  import { createServer } from 'node:http';
@@ -1823,8 +1943,10 @@ export const proxy = spoolProxy({
1823
1943
  });
1824
1944
  ```
1825
1945
 
1826
- On the 10 Drupal sites of the reading, a base chosen this way and already held by the browser saves
1827
- 30% of the stylesheet and script bytes at the median site (L1, [docs/impact.md](docs/impact.md#savings-today-and-at-maximum)).
1946
+ On the 10 Drupal sites of the reading, a base chosen this way and held from an earlier visit saves
1947
+ 29.9% of the brotli 11 bytes of new stylesheets and scripts at the median site with `dcz`, and a
1948
+ visit that has to fetch the base first saves 3.9%
1949
+ (L1, [docs/impact.md](docs/impact.md#savings-today-and-at-maximum)).
1828
1950
  `scripts/measure/dictionary-size.ts` records the same choice for a site's crawl.
1829
1951
 
1830
1952
  ### Defaults
@@ -1867,7 +1989,12 @@ work series records.
1867
1989
  production telemetry, an invoice total before and after, and a ledger of retired equipment. The
1868
1990
  plan says what could change, `observed` what changed, the invoices what was paid, and the ledger
1869
1991
  what was removed. The physical result reads only the ledger, and an entry needs the evidence for
1870
- its removal. With no ledger the statement reads `No physical capacity saving demonstrated.`
1992
+ its removal. With no ledger the statement reads `No physical capacity saving demonstrated.` The
1993
+ report also returns `economics`, `storage`, `trust` and `research` blocks. Each is null without its
1994
+ input: an `institution` profile for `economics` and `research`, `storage` for the held bytes, and
1995
+ `views` for `trust`.
1996
+ [docs/impact.md](docs/impact.md#an-institutions-figures) prints them for three example
1997
+ institutions.
1871
1998
 
1872
1999
  ```ts
1873
2000
  import { capacityPlan, institutionReport } from '@drupflare/spool/report';
@@ -1898,6 +2025,87 @@ numbers are made up. `channelRows` lists the metered, attributional, capacity an
1898
2025
  channels as separate rows, and nothing in the module sums them. [docs/impact.md](docs/impact.md#channels)
1899
2026
  says what each one counts.
1900
2027
 
2028
+ ### Request Arithmetic
2029
+
2030
+ On a plan that limits requests, the question is how many views the plan carries. Name three
2031
+ inputs, each a scenario (L3) until a site's traffic supplies it:
2032
+
2033
+ - `h`, the share of views answered on the device
2034
+ - `r`, the requests a network view makes to the platform, and `rho`, the part of them a local view
2035
+ removes (one for the document when the page's API calls still leave)
2036
+ - `s`, the requests spool sends itself per local view: syncs, beacon flushes, catalogue asks and
2037
+ worker script checks
2038
+
2039
+ The platform sees `r - h * rho + h * s` requests per view, so a limit `L` carries
2040
+ `L / (r - h * rho + h * s)` views. With `r = rho = 1` this is `freeTierCeiling` in
2041
+ `@drupflare/spool/report`. Three things bound `h` below 1:
2042
+
2043
+ - A device's first view of a site, a view after the browser evicted the replica, and a view past a
2044
+ freshness bound all reach the network. A device that makes `v` views in a freshness window
2045
+ answers at most `v - 1` of them locally, so `h <= 1 - 1/v` for that device.
2046
+ - A client that does not register the worker, such as a crawler, answers none.
2047
+ - A request outside every route family always leaves.
2048
+
2049
+ A limit on CPU per request, such as the 10 ms of the Workers Free plan, does not change with `h`.
2050
+ The docs' free-tier table uses one sync per 20 local views as an illustration. No multiplier
2051
+ follows from the formula until a site's `v`, its share of clients that cannot answer locally and
2052
+ `s` are measured.
2053
+
2054
+ ---
2055
+
2056
+ ## 🔒 Security
2057
+
2058
+ Authorisation stays on the server. The device holds only what the server already sent that
2059
+ principal, and the worker has no access rule. Principal data sits in a per-partition IndexedDB
2060
+ database sealed with AES-GCM, and deleting the key at logout is crypto-shredding of data at rest.
2061
+ That does not protect against script running on your origin. A revoked permission can still show
2062
+ for up to a route's `maxStale`, because nothing pushes a revocation to a device. Tag names are
2063
+ readable through `/.spool/sync`, so use opaque ids for anything sensitive. Principal datasets and
2064
+ declared writes ship off and have had no external review.
2065
+
2066
+ [docs/security.md](docs/security.md) is the threat model. It names the file and the test behind
2067
+ each claim and lists the open limitations in one table. Report a vulnerability as
2068
+ [SECURITY.md](https://github.com/drupflare/spool/blob/master/SECURITY.md) describes.
2069
+
2070
+ ---
2071
+
2072
+ ## 🔋 Energy Reporting
2073
+
2074
+ Energy is one objective among those spool reports, and the administrator decides how much weight
2075
+ it gets. The energy of one deployment is small against an institution's total, and small against
2076
+ the idle and hosting energy drupflare removes per site. The figures below are research figures.
2077
+
2078
+ [docs/impact.md](docs/impact.md) is generated from the readings in `measurements/` by
2079
+ `bun run impact`, and `bun run impact:check` fails when the page differs from them. Energy is
2080
+ reported in kWh per million views, and nothing sets a target intensity. Every figure carries a
2081
+ layer: L0 published, L1 measured here, L2 derived, L3 scenario. A per-view figure that spreads a
2082
+ published intensity over bytes is an allocation of that figure, and the page prints the metered
2083
+ reading beside it. The savings from trimming, images and dictionaries apply to network views and
2084
+ never add to the local-view figure.
2085
+
2086
+ [An Institution's Figures](docs/impact.md#an-institutions-figures) leads with economics, storage
2087
+ and trust and treats energy as a research figure. [What This Does Not
2088
+ Claim](docs/impact.md#what-this-does-not-claim) lists what the page leaves unsaid.
2089
+
2090
+ ---
2091
+
2092
+ ## 🧬 Prior Art
2093
+
2094
+ spool is a server-authoritative replica whose freshness policy lets a provisioned view render with
2095
+ no per-view validation request, with shadow mode and a fall back to the server for anything it
2096
+ cannot answer. HTTP caching, service-worker precaching and earlier replication systems cover parts
2097
+ of it. What is shared and what differs:
2098
+
2099
+ | System | What it does | Shared with spool | Difference |
2100
+ | ------------------ | ------------------------------------------------------------- | ----------------------------------------------- | ----------------------------------------------------------------------------------------- |
2101
+ | HTTP caching | reuses a response within `max-age` with no validation | a repeat view with no request | one response at a time; no per-dependency invalidation and no held collections |
2102
+ | Workbox precaching | answers a precached navigation from the cache with no request | a worker answering a navigation with no request | the list and its revisions are fixed at build time and refresh when a new worker installs |
2103
+ | Fawkes (NSDI 2020) | caches static page templates and sends a patch of DOM updates | a cached template, the server authoritative | the patch is requested on every load, so a cached template never answers a view alone |
2104
+ | Speculation Rules | prefetches or prerenders pages the visitor may open next | acting on a predicted next view | the origin answers every speculated request, used or not |
2105
+
2106
+ Sync Kit (WWW 2010) also keeps templates and data in the browser to reduce server load. The
2107
+ [technical report](TECHNICAL_REPORT.md#executive-summary) states the narrow claim.
2108
+
1901
2109
  ---
1902
2110
 
1903
2111
  ## 🚫 Out of Scope
@@ -1905,7 +2113,8 @@ says what each one counts.
1905
2113
  - spool never authorises on the client. The device holds only what the server already sent that
1906
2114
  principal.
1907
2115
  - It never shows a hard route past its bound without validating it first.
1908
- - It never writes. Your application owns every write, and `invalidate` only announces one.
2116
+ - It never writes on its own. Your application owns every write, and `invalidate` only announces
2117
+ one. The one exception is a declared write class, which is off until you enable it.
1909
2118
  - It never infers dependencies from opaque HTML. A route names its tags, or the records it
1910
2119
  reads, and in proxy mode the origin names them in its cache tag headers.
1911
2120
  - It never replaces a published release file on the CDN.
@@ -1932,20 +2141,20 @@ loads per view and reports the view as such.
1932
2141
 
1933
2142
  ## 🧭 Subpath Exports
1934
2143
 
1935
- | Import | What is in it |
1936
- | ---------------------------------------- | ----------------------------------------------------------------------------- |
1937
- | `@drupflare/spool` | the protocol and report modules |
1938
- | `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events |
1939
- | `@drupflare/spool/server` | `createSpool`, `defineRoute`, `datasetProof`, sinks, identity and work series |
1940
- | `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse` |
1941
- | `@drupflare/spool/proxy` | `spoolProxy`, `chooseBases`, the tag readers and the eligibility verdict |
1942
- | `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type |
1943
- | `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class |
1944
- | `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker |
1945
- | `@drupflare/spool/client` | `register` and the page handle |
1946
- | `@drupflare/spool/report` | the energy model, capacity plans and institution reports |
1947
- | `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage` |
1948
- | `@drupflare/spool/media` | `tinyimgEncoder`, `imageVariant` and `rewriteVideo` |
2144
+ | Import | What is in it |
2145
+ | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------- |
2146
+ | `@drupflare/spool` | the protocol and report modules |
2147
+ | `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events |
2148
+ | `@drupflare/spool/server` | `createSpool`, `defineRoute`, `defineDataset`, `defineWrite`, `datasetProof`, sinks, identity and work series |
2149
+ | `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse` |
2150
+ | `@drupflare/spool/proxy` | `spoolProxy`, `chooseBases`, the tag readers and the eligibility verdict |
2151
+ | `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type |
2152
+ | `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class |
2153
+ | `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker |
2154
+ | `@drupflare/spool/client` | `register`, the page handle and `createActivity` |
2155
+ | `@drupflare/spool/report` | the energy model, capacity plans and institution reports |
2156
+ | `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage` |
2157
+ | `@drupflare/spool/media` | `tinyimgEncoder`, `imageVariant`, `rewriteVideo` and `rewriteEmbeds` |
1949
2158
 
1950
2159
  ---
1951
2160
 
@@ -1957,17 +2166,30 @@ bun run test
1957
2166
  bun run test:coverage
1958
2167
  bun run test:browser
1959
2168
  bun run prettier:check
2169
+ bun run impact:check
1960
2170
  bun run docs:build
1961
2171
  bun run build
1962
2172
  ```
1963
2173
 
1964
- `test` runs the unit project in workerd and the node project, which includes a test that packs the
2174
+ `impact:check` fails when `docs/impact.md` differs from the readings in `measurements/`. A node
2175
+ test fails when `src/server/embedded.ts` differs from the source under `src/protocol`, `src/sw`
2176
+ and `src/client`, and `bun run build:sw` regenerates it. `test` runs the unit project in workerd and the node project, which includes a test that packs the
1965
2177
  package and imports every subpath in plain Node. `test:browser` runs Chromium, Firefox and WebKit
1966
2178
  against real servers and counts the requests that reach them. `bunx playwright install chromium
1967
2179
  firefox webkit` fetches the browsers.
1968
2180
 
1969
2181
  ---
1970
2182
 
2183
+ ## 📖 Documentation
2184
+
2185
+ - [docs/protocol.md](docs/protocol.md) holds the paths, headers, JSON bodies and limits for anyone
2186
+ who writes an adapter or a second server. A test pins what `Spool-Protocol: 1` freezes, and the
2187
+ page lists what may change without a new number.
2188
+ - [docs/security.md](docs/security.md) holds the threat model and the open limitations.
2189
+ - [docs/impact.md](docs/impact.md) holds the energy model, the readings and what they leave unsaid.
2190
+
2191
+ ---
2192
+
1971
2193
  ## 🔗 Related Repositories
1972
2194
 
1973
2195
  | Repository | What it is |
@@ -1 +1 @@
1
- (()=>{var b="data-spool-client";function w(e=l()){return{register(n,o={}){let s=o.hiddenAfterMs??1e4,t=!0,i=!1,r=!1,c=!1,p=!1,d=null,g=(a)=>{try{n[a]()}catch(P){if(!p)p=!0,console.warn("spool.activity",P)}},u=()=>{if(c)return;let a=!t||i;if(a===r)return;r=a,g(a?"pause":"resume")},y=()=>{if(d===null)return;e.clearTimeout(d),d=null},m=()=>{if(c)return;if(!e.hidden()){y(),i=!1,u();return}if(i||d!==null)return;if(s<=0){i=!0,u();return}d=e.setTimeout(()=>{if(d=null,e.hidden())i=!0,u()},s)},T=e.onVisibilityChange(m),A=o.target?e.observe(o.target,(a)=>{t=a,u()}):()=>{return};return m(),()=>{if(c)return;if(c=!0,y(),T(),A(),r)r=!1,g("resume")}}}}function l(){return{hidden:()=>document.hidden,onVisibilityChange(e){return document.addEventListener("visibilitychange",e),()=>document.removeEventListener("visibilitychange",e)},observe(e,n){if(typeof IntersectionObserver>"u")return n(!0),()=>{return};let o=new IntersectionObserver((s)=>{let t=s.at(-1);if(t)n(t.isIntersecting||t.intersectionRatio>0)},{threshold:0});return o.observe(e),()=>o.disconnect()},setTimeout:(e,n)=>setTimeout(e,n),clearTimeout:(e)=>clearTimeout(e)}}function v(e,n,o=[]){let s=0;for(let{name:t}of e){if(o.includes(t))continue;try{if(new URL(t).origin!==n)s+=1}catch{continue}}return s}function j(e,n,o=[],s=32){let t=new Map;for(let{name:i}of e){if(o.includes(i))continue;try{let r=new URL(i).origin;if(r!==n&&r!=="null")t.set(r,(t.get(r)??0)+1)}catch{continue}}return Object.fromEntries([...t].sort((i,r)=>r[1]-i[1]||(i[0]<r[0]?-1:1)).slice(0,s))}var R=1;var h={sync:"/.spool/sync",artifact:"/.spool/artifact",beacon:"/.spool/beacon",worker:"/spool-sw.js"};function E(e,n=window){let o=()=>e.close();return n.addEventListener("pagehide",o),()=>n.removeEventListener("pagehide",o)}function f(e,n,o){return new Promise((s,t)=>{let i=new MessageChannel,r=setTimeout(()=>{i.port1.close(),t(Error("the worker did not answer"))},o);i.port1.onmessage=(c)=>{clearTimeout(r),i.port1.close(),s(c.data)},e.postMessage(n,[i.port2])})}async function k(e={}){let n=e.container??navigator.serviceWorker;if(!n)return null;let o=e.helloTimeoutMs??5000;n.addEventListener?.("message",(r)=>{let c=r.data;if(c?.type!=="spool:cleared")return;(e.target??window).dispatchEvent(new CustomEvent("spool:cleared",{detail:{reason:c.reason}}))}),await n.register(e.workerUrl??h.worker,{scope:e.scope,updateViaCache:"all"});let s=await n.ready,t=s.active;if(!t)throw Error("the worker is not active");let i=await f(t,{type:"spool:hello",protocol:R},o);return{registration:s,hello:i,status:()=>f(s.active??t,{type:"spool:status"},o),view:async(r=location.pathname)=>(await f(s.active??t,{type:"spool:view",path:r},o)).id,reportLoads:(r,c,p)=>(s.active??t).postMessage({type:"spool:loads",view:r,count:c,origins:p}),track:(r)=>E(r)}}var x=k().catch(()=>null);globalThis.spool={ready:x,activity:w(l())};x.then(async(e)=>{let n=await e?.view().catch(()=>null);if(!e||!n)return;let o=-1,s=()=>{let t=document.querySelector(`script[${b}]`)?.src,i=performance.getEntriesByType("resource"),r=v(i,location.origin,t?[t]:[]);if(r===o)return;o=r,e.reportLoads(n,r,j(i,location.origin,t?[t]:[]))};addEventListener("pagehide",s),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")s()})});})();
1
+ (()=>{var R="data-spool-client";function v(e=k()){return{register(t,r={}){let i=r.hiddenAfterMs??1e4,n=!0,o=!1,a=!1,u=!1,d=!1,p=null,f=(g)=>{try{t[g]()}catch(L){if(!d)d=!0,console.warn("spool.activity",L)}},s=()=>{if(u)return;let g=!n||o;if(g===a)return;a=g,f(g?"pause":"resume")},c=()=>{if(p===null)return;e.clearTimeout(p),p=null},l=()=>{if(u)return;if(!e.hidden()){c(),o=!1,s();return}if(o||p!==null)return;if(i<=0){o=!0,s();return}p=e.setTimeout(()=>{if(p=null,e.hidden())o=!0,s()},i)},m=e.onVisibilityChange(l),h=r.target?e.observe(r.target,(g)=>{n=g,s()}):()=>{return};return l(),()=>{if(u)return;if(u=!0,c(),m(),h(),a)a=!1,f("resume")}}}}function k(){return{hidden:()=>document.hidden,onVisibilityChange(e){return document.addEventListener("visibilitychange",e),()=>document.removeEventListener("visibilitychange",e)},observe(e,t){if(typeof IntersectionObserver>"u")return t(!0),()=>{return};let r=new IntersectionObserver((i)=>{let n=i.at(-1);if(n)t(n.isIntersecting||n.intersectionRatio>0)},{threshold:0});return r.observe(e),()=>r.disconnect()},setTimeout:(e,t)=>setTimeout(e,t),clearTimeout:(e)=>clearTimeout(e)}}function x(e,t,r=[]){let i=0;for(let{name:n}of e){if(r.includes(n))continue;try{if(new URL(n).origin!==t)i+=1}catch{continue}}return i}function S(e,t,r=[],i=32){let n=new Map;for(let{name:o}of e){if(r.includes(o))continue;try{let a=new URL(o).origin;if(a!==t&&a!=="null")n.set(a,(n.get(a)??0)+1)}catch{continue}}return Object.fromEntries([...n].sort((o,a)=>a[1]-o[1]||(o[0]<a[0]?-1:1)).slice(0,i))}var F=1n<<256n,D="0".repeat(64);var P=1;var j={sync:"/.spool/sync",artifact:"/.spool/artifact",dataset:"/.spool/dataset",beacon:"/.spool/beacon",write:"/.spool/write",worker:"/spool-sw.js"};function E(e,t=window){let r=()=>e.close();return t.addEventListener("pagehide",r),()=>t.removeEventListener("pagehide",r)}var A=30000;function w(e,t,r){return new Promise((i,n)=>{let o=new MessageChannel,a=setTimeout(()=>{o.port1.close(),n(Error("the worker did not answer"))},r);o.port1.onmessage=(u)=>{clearTimeout(a),o.port1.close(),i(u.data)},e.postMessage(t,[o.port2])})}async function T(e={}){let t=e.container??navigator.serviceWorker;if(!t)return null;let r=e.helloTimeoutMs??5000,i=null,n=!1,o,a=async()=>{if(n)return;n=!0;let s=e.storage??navigator.storage;try{i=await s?.persisted?.()===!0||await s?.persist?.()===!0}catch{i=!1}(e.target??window).dispatchEvent(new CustomEvent("spool:persisted",{detail:{granted:i}})),(o??t.controller)?.postMessage({type:"spool:persisted",granted:i})};t.addEventListener?.("message",(s)=>{let c=s.data;if(c?.type==="spool:persist")a();if(c?.type==="spool:written"){let{queued:l,flushed:m,conflicted:h}=c;(e.target??window).dispatchEvent(new CustomEvent("spool:written",{detail:{queued:l,flushed:m,conflicted:h}}))}if(c?.type!=="spool:cleared")return;(e.target??window).dispatchEvent(new CustomEvent("spool:cleared",{detail:{reason:c.reason}}))}),await t.register(e.workerUrl??j.worker,{scope:e.scope,updateViaCache:"all"});let u=await t.ready,d=u.active;if(!d)throw Error("the worker is not active");o=d;let p=await w(d,{type:"spool:hello",protocol:P},r),f=async(s,c=r)=>{try{let l=await w(u.active??d,s,c);if(l?.type===s.type)return l.result}catch{}return{ok:!1,code:"unavailable",message:"the worker did not answer"}};return{registration:u,hello:p,status:()=>w(u.active??d,{type:"spool:status"},r),view:async(s=location.pathname)=>(await w(u.active??d,{type:"spool:view",path:s},r)).id,reportLoads:(s,c,l)=>(u.active??d).postMessage({type:"spool:loads",view:s,count:c,origins:l}),persisted:()=>i,track:(s)=>E(s),write:(s,c,l,m)=>f({type:"spool:write",name:s,key:c,value:l,version:m}),flush:()=>f({type:"spool:flush"},Math.max(r,A)),writes:()=>f({type:"spool:writes"}),resolve:(s,c,l)=>f({type:"spool:resolve",name:s,key:c,use:l},Math.max(r,A))}}var y=T().catch(()=>null),b={ok:!1,code:"unavailable",message:"there is no worker on this page"},C=!1,M={write:async(...e)=>{let t=await y.then((r)=>r?.write(...e)??b);if(t.ok)C=!0;return t},flush:()=>y.then((e)=>e?.flush()??b),writes:()=>y.then((e)=>e?.writes()??b),resolve:(...e)=>y.then((t)=>t?.resolve(...e)??b)};globalThis.spool={ready:y,activity:v(k()),...M};var O=()=>{if(C)M.flush()};y.then((e)=>{if(!e)return;addEventListener("pagehide",O),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")O()})});y.then(async(e)=>{let t=await e?.view().catch(()=>null);if(!e||!t)return;let r=-1,i=()=>{let n=document.querySelector(`script[${R}]`)?.src,o=performance.getEntriesByType("resource"),a=x(o,location.origin,n?[n]:[]);if(a===r)return;r=a,e.reportLoads(t,a,S(o,location.origin,n?[n]:[]))};addEventListener("pagehide",i),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")i()})});})();
@@ -1,16 +1,16 @@
1
1
  {
2
2
  "schema": 2,
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "files": {
5
5
  "client.js": {
6
- "bytes": 3263,
7
- "sha384": "f48546b44a63503f6232f6edb70a2755172a2cccdcaaed61f9e8ebdb8d4ed7a040fe9c0817369c29667d56fb4afd2b91",
8
- "integrity": "sha384-9IVGtEpjUD9iMvbttwonVRcqLMzcqu1h+ejr241O16BA/pwIFzacKWZ9VvtK/SuR"
6
+ "bytes": 4769,
7
+ "sha384": "1e5b2f9377a9b6a2aae6419e69bc67878fd9c4c0a5568c05c7340c7aaa9a0fc866d5e4f29b9130da711a218c5563240e",
8
+ "integrity": "sha384-Hlsvk3eptqKq5kGeabxnh4/ZxMClVowFxzQMeqqaD8hm1eTym5Ew2nEaIYxVYyQO"
9
9
  },
10
10
  "spool-sw.js": {
11
- "bytes": 65371,
12
- "sha384": "33aa92b7bc086e472b5c01a1b814e76df76654cd0c11cd63755f77c759d11adc405fed39ee10a5ab7e8c4da058196386",
13
- "integrity": "sha384-M6qSt7wIbkcrXAGhuBTnbfdmVM0MEc1jdV93x1nRGtxAX+057hClq36MTaBYGWOG"
11
+ "bytes": 104735,
12
+ "sha384": "9a9a89745420549f1421d1201ab60bb4a723cfe6c3afad048a342d293082321f81716dd5dc44090c62801214a8d16a05",
13
+ "integrity": "sha384-mpqJdFQgVJ8UIdEgGrYLtKcjz+bDr60EijQtKTCCMh+BcW3V3EQJDGKAEhSo0WoF"
14
14
  }
15
15
  }
16
16
  }