@drupflare/spool 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +263 -41
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/entry.js +33 -1
- package/dist/client/register.d.ts +23 -1
- package/dist/client/register.js +53 -7
- package/dist/protocol/config.d.ts +16 -1
- package/dist/protocol/config.js +3 -1
- package/dist/protocol/dataset.d.ts +183 -0
- package/dist/protocol/dataset.js +206 -0
- package/dist/protocol/envelope.d.ts +1 -0
- package/dist/protocol/envelope.js +2 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index.d.ts +3 -0
- package/dist/protocol/index.js +3 -0
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/route.d.ts +6 -2
- package/dist/protocol/route.js +31 -3
- package/dist/protocol/write.d.ts +146 -0
- package/dist/protocol/write.js +101 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +11 -1
- package/dist/report/economics.js +19 -1
- package/dist/report/index.d.ts +2 -0
- package/dist/report/index.js +2 -0
- package/dist/report/inputs.d.ts +97 -0
- package/dist/report/inputs.js +133 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +3 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/dataset.d.ts +48 -0
- package/dist/server/dataset.js +308 -0
- package/dist/server/embedded.d.ts +3 -3
- package/dist/server/embedded.js +3 -3
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +2 -0
- package/dist/server/spool.d.ts +20 -0
- package/dist/server/spool.js +126 -18
- package/dist/server/write.d.ts +27 -0
- package/dist/server/write.js +127 -0
- package/dist/sw/analytics.d.ts +6 -0
- package/dist/sw/analytics.js +14 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.js +78 -9
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/runtime.d.ts +43 -2
- package/dist/sw/runtime.js +340 -24
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +107 -0
- package/dist/sw/write.js +442 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# 🧵 spool
|
|
2
2
|
|
|
3
|
-
>
|
|
3
|
+
> A browser replica that removes repeated demand from your server
|
|
4
4
|
|
|
5
5
|
[](https://github.com/drupflare/spool/actions/workflows/build.yml)
|
|
6
6
|
[](https://github.com/drupflare/spool/actions/workflows/browser.yml)
|
|
@@ -8,9 +8,16 @@
|
|
|
8
8
|
[](https://codecov.io/gh/drupflare/spool)
|
|
9
9
|
[](LICENSE)
|
|
10
10
|
|
|
11
|
-
spool keeps a replica of what a page needs
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
spool removes repeated execution demand from your server. It keeps a replica of what a page needs
|
|
12
|
+
on the visitor's device. The server sends templates and authorised, versioned data, and a service
|
|
13
|
+
worker computes eligible pages from them. A page answered there is a request that never reaches
|
|
14
|
+
your Worker or origin, and server work that never runs. The contract is a count: a test asserts
|
|
15
|
+
the server hits of a local view, and in Chromium and Firefox the count is zero.
|
|
16
|
+
|
|
17
|
+
spool has three uses. In a drupflare deployment it takes eligible public views off the Worker.
|
|
18
|
+
For a team on a free or constrained plan it lowers the requests the plan has to carry, by an
|
|
19
|
+
amount that depends on the site's traffic ([Request Arithmetic](#request-arithmetic)). It is also
|
|
20
|
+
a research system: every figure it reports carries a layer and a method.
|
|
14
21
|
|
|
15
22
|
It runs on any runtime that speaks `Request` and `Response`, bridges `node:http` for the rest, and
|
|
16
23
|
has no runtime dependencies. A site with no server code can load it from two files and a manifest.
|
|
@@ -28,7 +35,9 @@ has no runtime dependencies. A site with no server code can load it from two fil
|
|
|
28
35
|
- [Static Sites](#-static-sites)
|
|
29
36
|
- [Strategies](#-strategies)
|
|
30
37
|
- [Freshness Contracts](#-freshness-contracts)
|
|
38
|
+
- [Datasets](#-datasets)
|
|
31
39
|
- [Authenticated Data](#-authenticated-data)
|
|
40
|
+
- [Declared Writes](#-declared-writes)
|
|
32
41
|
- [Catalogue](#-catalogue)
|
|
33
42
|
- [Analytics Batching](#-analytics-batching)
|
|
34
43
|
- [Third-Party Files](#-third-party-files)
|
|
@@ -38,10 +47,14 @@ has no runtime dependencies. A site with no server code can load it from two fil
|
|
|
38
47
|
- [Trimmed Stylesheets](#-trimmed-stylesheets)
|
|
39
48
|
- [Images and Video](#-images-and-video)
|
|
40
49
|
- [Capacity Reports](#-capacity-reports)
|
|
50
|
+
- [Security](#-security)
|
|
51
|
+
- [Energy Reporting](#-energy-reporting)
|
|
52
|
+
- [Prior Art](#-prior-art)
|
|
41
53
|
- [Out of Scope](#-out-of-scope)
|
|
42
54
|
- [Browser Support](#-browser-support)
|
|
43
55
|
- [Subpath Exports](#-subpath-exports)
|
|
44
56
|
- [Testing](#-testing)
|
|
57
|
+
- [Documentation](#-documentation)
|
|
45
58
|
- [Related Repositories](#-related-repositories)
|
|
46
59
|
- [License](#-license)
|
|
47
60
|
|
|
@@ -49,16 +62,25 @@ has no runtime dependencies. A site with no server code can load it from two fil
|
|
|
49
62
|
|
|
50
63
|
## 🎯 Why Spool
|
|
51
64
|
|
|
65
|
+
A local view removes, in order, a request that would have reached the Worker, the server work
|
|
66
|
+
behind it (rendering, queries, Durable Object and database reads), and the bytes of a document the
|
|
67
|
+
device already holds. Energy follows from those and is one objective an administrator may weigh.
|
|
68
|
+
|
|
52
69
|
A response is a deterministic transformation of a set of authorised, versioned dependencies. The
|
|
53
70
|
server half names those dependencies and sends the snapshot the visitor may hold. The worker keeps
|
|
54
71
|
a graph of records, projections and responses, with their versions, and syncs only what the next
|
|
55
72
|
page needs.
|
|
56
73
|
|
|
57
|
-
Your application keeps every
|
|
58
|
-
|
|
59
|
-
changes no page that was not
|
|
74
|
+
Your application keeps every access decision and every page it cannot reproduce safely on the
|
|
75
|
+
device. It keeps every write too, except the last-value writes you declare, which ship off. A route
|
|
76
|
+
the worker cannot answer goes to the network, so adopting spool changes no page that was not
|
|
77
|
+
declared.
|
|
78
|
+
|
|
79
|
+
A collection of thousands of rows, such as a directory or a course catalogue, can be held as a
|
|
80
|
+
[dataset](#-datasets). The device seeds it once and then receives only the rows that changed.
|
|
60
81
|
|
|
61
|
-
[docs/impact.md](docs/impact.md) holds the energy
|
|
82
|
+
[docs/impact.md](docs/impact.md) holds the request, cost and energy figures and the measurements
|
|
83
|
+
behind them. See [Energy Reporting](#-energy-reporting).
|
|
62
84
|
|
|
63
85
|
---
|
|
64
86
|
|
|
@@ -193,8 +215,9 @@ it against the hash, so a changed file never runs. Release files are never overw
|
|
|
193
215
|
script fails to load, the page works as it did before.
|
|
194
216
|
|
|
195
217
|
The script registers `/spool-sw.js` and exposes `window.spool.ready`, which resolves to the client
|
|
196
|
-
handle or to `null` where service workers do not exist.
|
|
197
|
-
|
|
218
|
+
handle or to `null` where service workers do not exist. It also exposes `window.spool.activity`,
|
|
219
|
+
described below, and the write calls of [Declared Writes](#-declared-writes). To bundle the runtime
|
|
220
|
+
yourself, import `register` from `@drupflare/spool/client`.
|
|
198
221
|
|
|
199
222
|
### Activity Hooks
|
|
200
223
|
|
|
@@ -744,7 +767,7 @@ from one directory:
|
|
|
744
767
|
- `client.js`, the page script, from the same place.
|
|
745
768
|
- `spool.json`, the list of files the worker may keep.
|
|
746
769
|
|
|
747
|
-
Each page loads the script with the tag from [CDN Script](
|
|
770
|
+
Each page loads the script with the tag from [CDN Script](#cdn-script), or from the site's own copy.
|
|
748
771
|
The worker has no server to talk to, so it reads `spool.json` from the directory its own script
|
|
749
772
|
sits in.
|
|
750
773
|
|
|
@@ -1016,8 +1039,9 @@ sorts descending. Ties keep the stored order. Strings compare by UTF-16 code uni
|
|
|
1016
1039
|
and no case folding, so the application's own sort has to agree. A database that compares bytes,
|
|
1017
1040
|
such as SQLite by default, agrees except for characters above U+FFFF, and a case-insensitive or
|
|
1018
1041
|
locale collation does not. `pageSize` sets the items per page and `page` counts from 1; without it
|
|
1019
|
-
the route is one page of everything. A route with `
|
|
1020
|
-
|
|
1042
|
+
the route is one page of everything. A route with `static` or `assembly` strategy cannot declare
|
|
1043
|
+
`query`. A route with `principal` scope can declare it only with `query.dataset`, which
|
|
1044
|
+
[Datasets](#-datasets) describes.
|
|
1021
1045
|
|
|
1022
1046
|
`load` returns the array at `list` and a `proof` made by `datasetProof`. The server refuses to
|
|
1023
1047
|
build the artifact when the proof is missing or does not match the array. The worker computes the
|
|
@@ -1090,6 +1114,72 @@ invalidations, so a sync does not always need a request of its own.
|
|
|
1090
1114
|
|
|
1091
1115
|
---
|
|
1092
1116
|
|
|
1117
|
+
## 💾 Datasets
|
|
1118
|
+
|
|
1119
|
+
A route that lists thousands of rows, such as a staff directory, makes a device fetch the whole
|
|
1120
|
+
collection again after each change. A dataset lets the device hold the rows and receive only the
|
|
1121
|
+
rows that changed. The device seeds a dataset once, then asks for a patch from the cursor it holds.
|
|
1122
|
+
A patch carries the rows to put, the keys to delete and a proof the device checks before it stores
|
|
1123
|
+
anything.
|
|
1124
|
+
|
|
1125
|
+
`defineDataset(name, declaration)` from `@drupflare/spool/server` declares a dataset, and
|
|
1126
|
+
`createSpool({ datasets })` serves it at `GET /.spool/dataset`. The declaration gives the
|
|
1127
|
+
functions `key`, `seed`, `read`, `changes`, `horizon` and `head`, a `schema` string, and the
|
|
1128
|
+
`maxRows` and `maxBytes` of a seed. A dataset has its own integer cursor, which your backend holds.
|
|
1129
|
+
It does not use the site's version vector. `changes(since)` returns the net change after a cursor
|
|
1130
|
+
and decides which rows are in scope. `read(ids)` is a keyed read that never runs a route's `load`.
|
|
1131
|
+
A device whose cursor is below `horizon`, or whose `schema` differs, seeds again.
|
|
1132
|
+
|
|
1133
|
+
The proof is the number of rows and the sum of every row's hash. A sum does not depend on order, so
|
|
1134
|
+
a patch moves it by the rows it touches. The device applies a seed or a patch to a staging copy,
|
|
1135
|
+
compares the result with the proof, and writes the rows and their metadata in one transaction. A
|
|
1136
|
+
failed proof clears that dataset and seeds it once more. The proof catches corruption and a
|
|
1137
|
+
half-applied patch. It does not defend against a server that sends rows and a digest that agree.
|
|
1138
|
+
|
|
1139
|
+
A route reads a dataset in two ways. `query.dataset` names it in place of `list`: the worker takes
|
|
1140
|
+
the held rows in key order and applies the route's filter, sort and page, and goes to the network
|
|
1141
|
+
while the dataset is not held. `ctx.row(dataset, key)` puts a reference to one row in the route's
|
|
1142
|
+
data. The reference resolves when the page renders, so a changed row needs no new artifact.
|
|
1143
|
+
|
|
1144
|
+
A route with `query.dataset` may also declare `query.plan`, a read-only plan written as JSON with
|
|
1145
|
+
joins, conditions, groups and aggregates over held datasets. It replaces `filter`, `sort` and
|
|
1146
|
+
`pageSize`. The worker compares values the way SQLite compares a column with no declared type, and
|
|
1147
|
+
sends the query to the network when a plan could give another answer than SQLite would. The server
|
|
1148
|
+
never evaluates a plan.
|
|
1149
|
+
|
|
1150
|
+
A dataset declared with `provision: 'auto'` reaches a device only while it beats two controls: the
|
|
1151
|
+
held page under v0.3, and one plain resource that a browser cache revalidates with an `ETag`.
|
|
1152
|
+
Otherwise the seed request gets `409` with `Spool-Provision: declined`, and the worker holds
|
|
1153
|
+
nothing and does not ask again in that session. The default, `'always'`, answers every seed. The worker also
|
|
1154
|
+
refuses a seed whose rows would not fit the headroom that `navigator.storage.estimate()` reports,
|
|
1155
|
+
and the page runtime asks the browser for persistent storage once the first dataset is held.
|
|
1156
|
+
|
|
1157
|
+
On synthetic rows (L1, no real site measured), one changed row of 500 costs 228 bytes as a patch and
|
|
1158
|
+
19,981 bytes as a v0.3 list refetch, and the first load costs 867 bytes more than under v0.3
|
|
1159
|
+
([docs/impact.md](docs/impact.md#dataset-patches)). [docs/protocol.md](docs/protocol.md#datasets)
|
|
1160
|
+
has the declaration rules, the wire format, query plans and the device store.
|
|
1161
|
+
|
|
1162
|
+
### Principal Datasets
|
|
1163
|
+
|
|
1164
|
+
Principal datasets are off by default and unreviewed: no external review of this feature has
|
|
1165
|
+
happened. They hold rows that differ for each signed-in principal. Two switches turn them on, and
|
|
1166
|
+
both are needed. `defineDataset(name, declaration, { principalDatasets: true })` accepts
|
|
1167
|
+
`scope: 'principal'`, and `createSpool({ principalDatasets: true, identity })` serves the dataset.
|
|
1168
|
+
A principal dataset declares `maxStaleMs`, which has no default, and cannot use
|
|
1169
|
+
`provision: 'auto'`.
|
|
1170
|
+
|
|
1171
|
+
The server takes the identity from the request and passes it to every dataset function as
|
|
1172
|
+
`ctx.principal`. Which rows a principal may see is your code, and spool does not check what the
|
|
1173
|
+
functions return. The device keeps the rows sealed in the principal's partition, as
|
|
1174
|
+
[Authenticated Data](#-authenticated-data) describes, and wipes them with it. Held rows answer a
|
|
1175
|
+
query for `maxStaleMs` after the last verified sync, so a principal removed on the server can read
|
|
1176
|
+
held rows for up to that long. A principal route can declare `query` only with `query.dataset` and
|
|
1177
|
+
no plan, and a route and its dataset need the same scope.
|
|
1178
|
+
[docs/security.md](docs/security.md#principal-datasets) lists the threat modes and the test that
|
|
1179
|
+
holds each one.
|
|
1180
|
+
|
|
1181
|
+
---
|
|
1182
|
+
|
|
1093
1183
|
## 🔐 Authenticated Data
|
|
1094
1184
|
|
|
1095
1185
|
Set `scope: 'principal'` on a route that depends on who is signed in, and give `createSpool` an
|
|
@@ -1167,6 +1257,38 @@ that page reads.
|
|
|
1167
1257
|
|
|
1168
1258
|
---
|
|
1169
1259
|
|
|
1260
|
+
## 📝 Declared Writes
|
|
1261
|
+
|
|
1262
|
+
Declared writes are off by default and unreviewed: no external review of this feature has
|
|
1263
|
+
happened. A declared write queues one kind of edit on the device and sends it at a network moment.
|
|
1264
|
+
The last value for a key wins. `createSpool({ identity, writes: { enabled: true, classes } })`
|
|
1265
|
+
turns them on. With the flag off the worker queues nothing, the page calls answer `writes-off`, and
|
|
1266
|
+
`PUT /.spool/write` answers `403`.
|
|
1267
|
+
|
|
1268
|
+
`defineWrite` from `@drupflare/spool/server` declares a class. Its `scope` has to be `principal`,
|
|
1269
|
+
and its `path` a declared principal route. `key` says which key this principal may write, and
|
|
1270
|
+
`read` gives that key's current version. `apply` writes the value only when the key is still at the
|
|
1271
|
+
version the device edited from, in one atomic step, and returns the new version or the conflict.
|
|
1272
|
+
`maxBytes` and `maxPending` bound one value and the queue. The only `kind` is `last-value`, so a
|
|
1273
|
+
declaration cannot queue a comment, a cart, a payment, a counter, an inventory change or an audit
|
|
1274
|
+
record.
|
|
1275
|
+
|
|
1276
|
+
The page calls `window.spool.write(name, key, value, version)`, `window.spool.flush()`,
|
|
1277
|
+
`window.spool.writes()` and `window.spool.resolve(name, key, 'local' | 'server')`. Each answers
|
|
1278
|
+
`{ ok: true }` with its result or `{ ok: false, code, message }`, and none throws. Edits of one key
|
|
1279
|
+
replace each other, so several edits cost one request. A flush sends `PUT /.spool/write` with the
|
|
1280
|
+
version in `If-Match` and a `Spool-Write-Id` that stays the same across retries. The page runtime
|
|
1281
|
+
flushes when the page is hidden. A stale version gets `412`, and the record waits as `conflicted`
|
|
1282
|
+
until `resolve` settles it.
|
|
1283
|
+
|
|
1284
|
+
The queue is sealed in the principal's partition and deleted with it. A value that no flush has
|
|
1285
|
+
sent is lost at logout, so call `window.spool.flush()` before the logout navigation. The server
|
|
1286
|
+
remembers recent write ids in memory, 10,000 by default, so a site with several instances stores
|
|
1287
|
+
`ctx.writeId` with the value in `apply`. [docs/protocol.md](docs/protocol.md#declared-writes) has
|
|
1288
|
+
the request and its answers, and [docs/security.md](docs/security.md#declared-writes) the limits.
|
|
1289
|
+
|
|
1290
|
+
---
|
|
1291
|
+
|
|
1170
1292
|
## 📚 Catalogue
|
|
1171
1293
|
|
|
1172
1294
|
A catalogue sends the data of pages a visitor may open next, so that opening one makes no
|
|
@@ -1185,7 +1307,7 @@ The site supplies the candidates. A route's `catalogue` returns the pages a visi
|
|
|
1185
1307
|
next, each with the share expected to. Take the share from your own `nextShare` over the sink
|
|
1186
1308
|
(each `spool.view` names the route the visitor came from), from link position, or from a fixed
|
|
1187
1309
|
prior. Pass `{ first: true }` to `nextShare` to count only first views, which is what an entry
|
|
1188
|
-
saves: the share over all views ran 1.5 to
|
|
1310
|
+
saves: the share over all views ran 1.5 to 6.6 times high in the bins that decide an entry in a replay
|
|
1189
1311
|
of public visitors ([docs/impact.md](docs/impact.md#next-page-prediction)). `documentBytes` on the entry's route says what a hit saves, and the provisioning prior is
|
|
1190
1312
|
used when it is absent.
|
|
1191
1313
|
|
|
@@ -1383,13 +1505,10 @@ that. `passthrough` leaves assets to the browser's cache and never touches them.
|
|
|
1383
1505
|
asks the browser through `InstallEvent.addRoutes` to send only navigations, beacons and routed
|
|
1384
1506
|
paths to the worker and the rest of the origin to the network, and acts as `passthrough` where the
|
|
1385
1507
|
API does not exist: Chromium has it, Playwright's WebKit has it on macOS and not on Linux, and
|
|
1386
|
-
Firefox does not.
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
and Static Routing (8 rounds each), and the ranges overlap. Under Static Routing a same-origin write no
|
|
1391
|
-
longer reaches the worker, so it is not a network moment and its response cannot carry
|
|
1392
|
-
invalidations; navigations still are.
|
|
1508
|
+
Firefox does not. Static Routing keeps the requests it does not route out of the worker's fetch
|
|
1509
|
+
handler, and neither option changes server-side Worker invocations. Neither is measured to save
|
|
1510
|
+
energy, and [docs/impact.md](docs/impact.md) holds no reading for them. Under Static Routing a same-origin write no longer reaches the worker,
|
|
1511
|
+
so it is not a network moment and its response cannot carry invalidations; navigations still are.
|
|
1393
1512
|
|
|
1394
1513
|
```ts
|
|
1395
1514
|
import { createSpool } from '@drupflare/spool/server';
|
|
@@ -1447,7 +1566,8 @@ artifact. `routes` lists overrides, such as a shorter freshness bound for one pa
|
|
|
1447
1566
|
match first and take the same fields as `defineRoute`, without `load`.
|
|
1448
1567
|
|
|
1449
1568
|
On Node the handler goes through `toNodeListener`. `origin` is the address of the real server
|
|
1450
|
-
and has no path.
|
|
1569
|
+
and has no path. `fetch` replaces the global `fetch` the proxy calls the origin with, such as a
|
|
1570
|
+
Workers service binding.
|
|
1451
1571
|
|
|
1452
1572
|
```ts
|
|
1453
1573
|
import { createServer } from 'node:http';
|
|
@@ -1823,8 +1943,10 @@ export const proxy = spoolProxy({
|
|
|
1823
1943
|
});
|
|
1824
1944
|
```
|
|
1825
1945
|
|
|
1826
|
-
On the 10 Drupal sites of the reading, a base chosen this way and
|
|
1827
|
-
|
|
1946
|
+
On the 10 Drupal sites of the reading, a base chosen this way and held from an earlier visit saves
|
|
1947
|
+
29.9% of the brotli 11 bytes of new stylesheets and scripts at the median site with `dcz`, and a
|
|
1948
|
+
visit that has to fetch the base first saves 3.9%
|
|
1949
|
+
(L1, [docs/impact.md](docs/impact.md#savings-today-and-at-maximum)).
|
|
1828
1950
|
`scripts/measure/dictionary-size.ts` records the same choice for a site's crawl.
|
|
1829
1951
|
|
|
1830
1952
|
### Defaults
|
|
@@ -1867,7 +1989,12 @@ work series records.
|
|
|
1867
1989
|
production telemetry, an invoice total before and after, and a ledger of retired equipment. The
|
|
1868
1990
|
plan says what could change, `observed` what changed, the invoices what was paid, and the ledger
|
|
1869
1991
|
what was removed. The physical result reads only the ledger, and an entry needs the evidence for
|
|
1870
|
-
its removal. With no ledger the statement reads `No physical capacity saving demonstrated.`
|
|
1992
|
+
its removal. With no ledger the statement reads `No physical capacity saving demonstrated.` The
|
|
1993
|
+
report also returns `economics`, `storage`, `trust` and `research` blocks. Each is null without its
|
|
1994
|
+
input: an `institution` profile for `economics` and `research`, `storage` for the held bytes, and
|
|
1995
|
+
`views` for `trust`.
|
|
1996
|
+
[docs/impact.md](docs/impact.md#an-institutions-figures) prints them for three example
|
|
1997
|
+
institutions.
|
|
1871
1998
|
|
|
1872
1999
|
```ts
|
|
1873
2000
|
import { capacityPlan, institutionReport } from '@drupflare/spool/report';
|
|
@@ -1898,6 +2025,87 @@ numbers are made up. `channelRows` lists the metered, attributional, capacity an
|
|
|
1898
2025
|
channels as separate rows, and nothing in the module sums them. [docs/impact.md](docs/impact.md#channels)
|
|
1899
2026
|
says what each one counts.
|
|
1900
2027
|
|
|
2028
|
+
### Request Arithmetic
|
|
2029
|
+
|
|
2030
|
+
On a plan that limits requests, the question is how many views the plan carries. Name three
|
|
2031
|
+
inputs, each a scenario (L3) until a site's traffic supplies it:
|
|
2032
|
+
|
|
2033
|
+
- `h`, the share of views answered on the device
|
|
2034
|
+
- `r`, the requests a network view makes to the platform, and `rho`, the part of them a local view
|
|
2035
|
+
removes (one for the document when the page's API calls still leave)
|
|
2036
|
+
- `s`, the requests spool sends itself per local view: syncs, beacon flushes, catalogue asks and
|
|
2037
|
+
worker script checks
|
|
2038
|
+
|
|
2039
|
+
The platform sees `r - h * rho + h * s` requests per view, so a limit `L` carries
|
|
2040
|
+
`L / (r - h * rho + h * s)` views. With `r = rho = 1` this is `freeTierCeiling` in
|
|
2041
|
+
`@drupflare/spool/report`. Three things bound `h` below 1:
|
|
2042
|
+
|
|
2043
|
+
- A device's first view of a site, a view after the browser evicted the replica, and a view past a
|
|
2044
|
+
freshness bound all reach the network. A device that makes `v` views in a freshness window
|
|
2045
|
+
answers at most `v - 1` of them locally, so `h <= 1 - 1/v` for that device.
|
|
2046
|
+
- A client that does not register the worker, such as a crawler, answers none.
|
|
2047
|
+
- A request outside every route family always leaves.
|
|
2048
|
+
|
|
2049
|
+
A limit on CPU per request, such as the 10 ms of the Workers Free plan, does not change with `h`.
|
|
2050
|
+
The docs' free-tier table uses one sync per 20 local views as an illustration. No multiplier
|
|
2051
|
+
follows from the formula until a site's `v`, its share of clients that cannot answer locally and
|
|
2052
|
+
`s` are measured.
|
|
2053
|
+
|
|
2054
|
+
---
|
|
2055
|
+
|
|
2056
|
+
## 🔒 Security
|
|
2057
|
+
|
|
2058
|
+
Authorisation stays on the server. The device holds only what the server already sent that
|
|
2059
|
+
principal, and the worker has no access rule. Principal data sits in a per-partition IndexedDB
|
|
2060
|
+
database sealed with AES-GCM, and deleting the key at logout is crypto-shredding of data at rest.
|
|
2061
|
+
That does not protect against script running on your origin. A revoked permission can still show
|
|
2062
|
+
for up to a route's `maxStale`, because nothing pushes a revocation to a device. Tag names are
|
|
2063
|
+
readable through `/.spool/sync`, so use opaque ids for anything sensitive. Principal datasets and
|
|
2064
|
+
declared writes ship off and have had no external review.
|
|
2065
|
+
|
|
2066
|
+
[docs/security.md](docs/security.md) is the threat model. It names the file and the test behind
|
|
2067
|
+
each claim and lists the open limitations in one table. Report a vulnerability as
|
|
2068
|
+
[SECURITY.md](https://github.com/drupflare/spool/blob/master/SECURITY.md) describes.
|
|
2069
|
+
|
|
2070
|
+
---
|
|
2071
|
+
|
|
2072
|
+
## 🔋 Energy Reporting
|
|
2073
|
+
|
|
2074
|
+
Energy is one objective among those spool reports, and the administrator decides how much weight
|
|
2075
|
+
it gets. The energy of one deployment is small against an institution's total, and small against
|
|
2076
|
+
the idle and hosting energy drupflare removes per site. The figures below are research figures.
|
|
2077
|
+
|
|
2078
|
+
[docs/impact.md](docs/impact.md) is generated from the readings in `measurements/` by
|
|
2079
|
+
`bun run impact`, and `bun run impact:check` fails when the page differs from them. Energy is
|
|
2080
|
+
reported in kWh per million views, and nothing sets a target intensity. Every figure carries a
|
|
2081
|
+
layer: L0 published, L1 measured here, L2 derived, L3 scenario. A per-view figure that spreads a
|
|
2082
|
+
published intensity over bytes is an allocation of that figure, and the page prints the metered
|
|
2083
|
+
reading beside it. The savings from trimming, images and dictionaries apply to network views and
|
|
2084
|
+
never add to the local-view figure.
|
|
2085
|
+
|
|
2086
|
+
[An Institution's Figures](docs/impact.md#an-institutions-figures) leads with economics, storage
|
|
2087
|
+
and trust and treats energy as a research figure. [What This Does Not
|
|
2088
|
+
Claim](docs/impact.md#what-this-does-not-claim) lists what the page leaves unsaid.
|
|
2089
|
+
|
|
2090
|
+
---
|
|
2091
|
+
|
|
2092
|
+
## 🧬 Prior Art
|
|
2093
|
+
|
|
2094
|
+
spool is a server-authoritative replica whose freshness policy lets a provisioned view render with
|
|
2095
|
+
no per-view validation request, with shadow mode and a fall back to the server for anything it
|
|
2096
|
+
cannot answer. HTTP caching, service-worker precaching and earlier replication systems cover parts
|
|
2097
|
+
of it. What is shared and what differs:
|
|
2098
|
+
|
|
2099
|
+
| System | What it does | Shared with spool | Difference |
|
|
2100
|
+
| ------------------ | ------------------------------------------------------------- | ----------------------------------------------- | ----------------------------------------------------------------------------------------- |
|
|
2101
|
+
| HTTP caching | reuses a response within `max-age` with no validation | a repeat view with no request | one response at a time; no per-dependency invalidation and no held collections |
|
|
2102
|
+
| Workbox precaching | answers a precached navigation from the cache with no request | a worker answering a navigation with no request | the list and its revisions are fixed at build time and refresh when a new worker installs |
|
|
2103
|
+
| Fawkes (NSDI 2020) | caches static page templates and sends a patch of DOM updates | a cached template, the server authoritative | the patch is requested on every load, so a cached template never answers a view alone |
|
|
2104
|
+
| Speculation Rules | prefetches or prerenders pages the visitor may open next | acting on a predicted next view | the origin answers every speculated request, used or not |
|
|
2105
|
+
|
|
2106
|
+
Sync Kit (WWW 2010) also keeps templates and data in the browser to reduce server load. The
|
|
2107
|
+
[technical report](TECHNICAL_REPORT.md#executive-summary) states the narrow claim.
|
|
2108
|
+
|
|
1901
2109
|
---
|
|
1902
2110
|
|
|
1903
2111
|
## 🚫 Out of Scope
|
|
@@ -1905,7 +2113,8 @@ says what each one counts.
|
|
|
1905
2113
|
- spool never authorises on the client. The device holds only what the server already sent that
|
|
1906
2114
|
principal.
|
|
1907
2115
|
- It never shows a hard route past its bound without validating it first.
|
|
1908
|
-
- It never writes. Your application owns every write, and `invalidate` only announces
|
|
2116
|
+
- It never writes on its own. Your application owns every write, and `invalidate` only announces
|
|
2117
|
+
one. The one exception is a declared write class, which is off until you enable it.
|
|
1909
2118
|
- It never infers dependencies from opaque HTML. A route names its tags, or the records it
|
|
1910
2119
|
reads, and in proxy mode the origin names them in its cache tag headers.
|
|
1911
2120
|
- It never replaces a published release file on the CDN.
|
|
@@ -1932,20 +2141,20 @@ loads per view and reports the view as such.
|
|
|
1932
2141
|
|
|
1933
2142
|
## 🧭 Subpath Exports
|
|
1934
2143
|
|
|
1935
|
-
| Import | What is in it
|
|
1936
|
-
| ---------------------------------------- |
|
|
1937
|
-
| `@drupflare/spool` | the protocol and report modules
|
|
1938
|
-
| `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events
|
|
1939
|
-
| `@drupflare/spool/server` | `createSpool`, `defineRoute`, `datasetProof`, sinks, identity and work series |
|
|
1940
|
-
| `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse`
|
|
1941
|
-
| `@drupflare/spool/proxy` | `spoolProxy`, `chooseBases`, the tag readers and the eligibility verdict
|
|
1942
|
-
| `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type
|
|
1943
|
-
| `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class
|
|
1944
|
-
| `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker
|
|
1945
|
-
| `@drupflare/spool/client` | `register
|
|
1946
|
-
| `@drupflare/spool/report` | the energy model, capacity plans and institution reports
|
|
1947
|
-
| `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage`
|
|
1948
|
-
| `@drupflare/spool/media` | `tinyimgEncoder`, `imageVariant` and `
|
|
2144
|
+
| Import | What is in it |
|
|
2145
|
+
| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------- |
|
|
2146
|
+
| `@drupflare/spool` | the protocol and report modules |
|
|
2147
|
+
| `@drupflare/spool/protocol` | envelopes, artifacts, patterns, templates, the manifest, view events |
|
|
2148
|
+
| `@drupflare/spool/server` | `createSpool`, `defineRoute`, `defineDataset`, `defineWrite`, `datasetProof`, sinks, identity and work series |
|
|
2149
|
+
| `@drupflare/spool/node` | `toNodeListener`, `toRequest`, `writeResponse` |
|
|
2150
|
+
| `@drupflare/spool/proxy` | `spoolProxy`, `chooseBases`, the tag readers and the eligibility verdict |
|
|
2151
|
+
| `@drupflare/spool/stores` | `memoryStore` and the `VersionStore` type |
|
|
2152
|
+
| `@drupflare/spool/stores/durable-object` | `durableObjectStore` and the `SpoolVersions` class |
|
|
2153
|
+
| `@drupflare/spool/sw` | `serve`, for a site that bundles its own worker |
|
|
2154
|
+
| `@drupflare/spool/client` | `register`, the page handle and `createActivity` |
|
|
2155
|
+
| `@drupflare/spool/report` | the energy model, capacity plans and institution reports |
|
|
2156
|
+
| `@drupflare/spool/trim` | `buildTrimManifest`, `trimCss` and `guardPage` |
|
|
2157
|
+
| `@drupflare/spool/media` | `tinyimgEncoder`, `imageVariant`, `rewriteVideo` and `rewriteEmbeds` |
|
|
1949
2158
|
|
|
1950
2159
|
---
|
|
1951
2160
|
|
|
@@ -1957,17 +2166,30 @@ bun run test
|
|
|
1957
2166
|
bun run test:coverage
|
|
1958
2167
|
bun run test:browser
|
|
1959
2168
|
bun run prettier:check
|
|
2169
|
+
bun run impact:check
|
|
1960
2170
|
bun run docs:build
|
|
1961
2171
|
bun run build
|
|
1962
2172
|
```
|
|
1963
2173
|
|
|
1964
|
-
`
|
|
2174
|
+
`impact:check` fails when `docs/impact.md` differs from the readings in `measurements/`. A node
|
|
2175
|
+
test fails when `src/server/embedded.ts` differs from the source under `src/protocol`, `src/sw`
|
|
2176
|
+
and `src/client`, and `bun run build:sw` regenerates it. `test` runs the unit project in workerd and the node project, which includes a test that packs the
|
|
1965
2177
|
package and imports every subpath in plain Node. `test:browser` runs Chromium, Firefox and WebKit
|
|
1966
2178
|
against real servers and counts the requests that reach them. `bunx playwright install chromium
|
|
1967
2179
|
firefox webkit` fetches the browsers.
|
|
1968
2180
|
|
|
1969
2181
|
---
|
|
1970
2182
|
|
|
2183
|
+
## 📖 Documentation
|
|
2184
|
+
|
|
2185
|
+
- [docs/protocol.md](docs/protocol.md) holds the paths, headers, JSON bodies and limits for anyone
|
|
2186
|
+
who writes an adapter or a second server. A test pins what `Spool-Protocol: 1` freezes, and the
|
|
2187
|
+
page lists what may change without a new number.
|
|
2188
|
+
- [docs/security.md](docs/security.md) holds the threat model and the open limitations.
|
|
2189
|
+
- [docs/impact.md](docs/impact.md) holds the energy model, the readings and what they leave unsaid.
|
|
2190
|
+
|
|
2191
|
+
---
|
|
2192
|
+
|
|
1971
2193
|
## 🔗 Related Repositories
|
|
1972
2194
|
|
|
1973
2195
|
| Repository | What it is |
|
package/dist/cdn/client.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
(()=>{var
|
|
1
|
+
(()=>{var R="data-spool-client";function v(e=k()){return{register(t,r={}){let i=r.hiddenAfterMs??1e4,n=!0,o=!1,a=!1,u=!1,d=!1,p=null,f=(g)=>{try{t[g]()}catch(L){if(!d)d=!0,console.warn("spool.activity",L)}},s=()=>{if(u)return;let g=!n||o;if(g===a)return;a=g,f(g?"pause":"resume")},c=()=>{if(p===null)return;e.clearTimeout(p),p=null},l=()=>{if(u)return;if(!e.hidden()){c(),o=!1,s();return}if(o||p!==null)return;if(i<=0){o=!0,s();return}p=e.setTimeout(()=>{if(p=null,e.hidden())o=!0,s()},i)},m=e.onVisibilityChange(l),h=r.target?e.observe(r.target,(g)=>{n=g,s()}):()=>{return};return l(),()=>{if(u)return;if(u=!0,c(),m(),h(),a)a=!1,f("resume")}}}}function k(){return{hidden:()=>document.hidden,onVisibilityChange(e){return document.addEventListener("visibilitychange",e),()=>document.removeEventListener("visibilitychange",e)},observe(e,t){if(typeof IntersectionObserver>"u")return t(!0),()=>{return};let r=new IntersectionObserver((i)=>{let n=i.at(-1);if(n)t(n.isIntersecting||n.intersectionRatio>0)},{threshold:0});return r.observe(e),()=>r.disconnect()},setTimeout:(e,t)=>setTimeout(e,t),clearTimeout:(e)=>clearTimeout(e)}}function x(e,t,r=[]){let i=0;for(let{name:n}of e){if(r.includes(n))continue;try{if(new URL(n).origin!==t)i+=1}catch{continue}}return i}function S(e,t,r=[],i=32){let n=new Map;for(let{name:o}of e){if(r.includes(o))continue;try{let a=new URL(o).origin;if(a!==t&&a!=="null")n.set(a,(n.get(a)??0)+1)}catch{continue}}return Object.fromEntries([...n].sort((o,a)=>a[1]-o[1]||(o[0]<a[0]?-1:1)).slice(0,i))}var F=1n<<256n,D="0".repeat(64);var P=1;var j={sync:"/.spool/sync",artifact:"/.spool/artifact",dataset:"/.spool/dataset",beacon:"/.spool/beacon",write:"/.spool/write",worker:"/spool-sw.js"};function E(e,t=window){let r=()=>e.close();return t.addEventListener("pagehide",r),()=>t.removeEventListener("pagehide",r)}var A=30000;function w(e,t,r){return new Promise((i,n)=>{let o=new MessageChannel,a=setTimeout(()=>{o.port1.close(),n(Error("the worker did not answer"))},r);o.port1.onmessage=(u)=>{clearTimeout(a),o.port1.close(),i(u.data)},e.postMessage(t,[o.port2])})}async function T(e={}){let t=e.container??navigator.serviceWorker;if(!t)return null;let r=e.helloTimeoutMs??5000,i=null,n=!1,o,a=async()=>{if(n)return;n=!0;let s=e.storage??navigator.storage;try{i=await s?.persisted?.()===!0||await s?.persist?.()===!0}catch{i=!1}(e.target??window).dispatchEvent(new CustomEvent("spool:persisted",{detail:{granted:i}})),(o??t.controller)?.postMessage({type:"spool:persisted",granted:i})};t.addEventListener?.("message",(s)=>{let c=s.data;if(c?.type==="spool:persist")a();if(c?.type==="spool:written"){let{queued:l,flushed:m,conflicted:h}=c;(e.target??window).dispatchEvent(new CustomEvent("spool:written",{detail:{queued:l,flushed:m,conflicted:h}}))}if(c?.type!=="spool:cleared")return;(e.target??window).dispatchEvent(new CustomEvent("spool:cleared",{detail:{reason:c.reason}}))}),await t.register(e.workerUrl??j.worker,{scope:e.scope,updateViaCache:"all"});let u=await t.ready,d=u.active;if(!d)throw Error("the worker is not active");o=d;let p=await w(d,{type:"spool:hello",protocol:P},r),f=async(s,c=r)=>{try{let l=await w(u.active??d,s,c);if(l?.type===s.type)return l.result}catch{}return{ok:!1,code:"unavailable",message:"the worker did not answer"}};return{registration:u,hello:p,status:()=>w(u.active??d,{type:"spool:status"},r),view:async(s=location.pathname)=>(await w(u.active??d,{type:"spool:view",path:s},r)).id,reportLoads:(s,c,l)=>(u.active??d).postMessage({type:"spool:loads",view:s,count:c,origins:l}),persisted:()=>i,track:(s)=>E(s),write:(s,c,l,m)=>f({type:"spool:write",name:s,key:c,value:l,version:m}),flush:()=>f({type:"spool:flush"},Math.max(r,A)),writes:()=>f({type:"spool:writes"}),resolve:(s,c,l)=>f({type:"spool:resolve",name:s,key:c,use:l},Math.max(r,A))}}var y=T().catch(()=>null),b={ok:!1,code:"unavailable",message:"there is no worker on this page"},C=!1,M={write:async(...e)=>{let t=await y.then((r)=>r?.write(...e)??b);if(t.ok)C=!0;return t},flush:()=>y.then((e)=>e?.flush()??b),writes:()=>y.then((e)=>e?.writes()??b),resolve:(...e)=>y.then((t)=>t?.resolve(...e)??b)};globalThis.spool={ready:y,activity:v(k()),...M};var O=()=>{if(C)M.flush()};y.then((e)=>{if(!e)return;addEventListener("pagehide",O),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")O()})});y.then(async(e)=>{let t=await e?.view().catch(()=>null);if(!e||!t)return;let r=-1,i=()=>{let n=document.querySelector(`script[${R}]`)?.src,o=performance.getEntriesByType("resource"),a=x(o,location.origin,n?[n]:[]);if(a===r)return;r=a,e.reportLoads(t,a,S(o,location.origin,n?[n]:[]))};addEventListener("pagehide",i),document.addEventListener("visibilitychange",()=>{if(document.visibilityState==="hidden")i()})});})();
|
package/dist/cdn/manifest.json
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema": 2,
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"files": {
|
|
5
5
|
"client.js": {
|
|
6
|
-
"bytes":
|
|
7
|
-
"sha384": "
|
|
8
|
-
"integrity": "sha384-
|
|
6
|
+
"bytes": 4769,
|
|
7
|
+
"sha384": "1e5b2f9377a9b6a2aae6419e69bc67878fd9c4c0a5568c05c7340c7aaa9a0fc866d5e4f29b9130da711a218c5563240e",
|
|
8
|
+
"integrity": "sha384-Hlsvk3eptqKq5kGeabxnh4/ZxMClVowFxzQMeqqaD8hm1eTym5Ew2nEaIYxVYyQO"
|
|
9
9
|
},
|
|
10
10
|
"spool-sw.js": {
|
|
11
|
-
"bytes":
|
|
12
|
-
"sha384": "
|
|
13
|
-
"integrity": "sha384-
|
|
11
|
+
"bytes": 104735,
|
|
12
|
+
"sha384": "9a9a89745420549f1421d1201ab60bb4a723cfe6c3afad048a342d293082321f81716dd5dc44090c62801214a8d16a05",
|
|
13
|
+
"integrity": "sha384-mpqJdFQgVJ8UIdEgGrYLtKcjz+bDr60EijQtKTCCMh+BcW3V3EQJDGKAEhSo0WoF"
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
}
|