@drupflare/spool 0.3.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +279 -1607
  2. package/dist/angular/index.d.ts +13 -0
  3. package/dist/angular/index.js +25 -0
  4. package/dist/angular/pages.d.ts +28 -0
  5. package/dist/angular/pages.js +85 -0
  6. package/dist/cdn/client.js +1 -1
  7. package/dist/cdn/location.d.ts +6 -0
  8. package/dist/cdn/location.js +8 -0
  9. package/dist/cdn/manifest.json +7 -7
  10. package/dist/cdn/spool-sw.js +1 -1
  11. package/dist/client/bootstrap.d.ts +57 -0
  12. package/dist/client/bootstrap.js +307 -0
  13. package/dist/client/contact.d.ts +24 -0
  14. package/dist/client/contact.js +66 -0
  15. package/dist/client/declared.d.ts +23 -0
  16. package/dist/client/declared.js +68 -0
  17. package/dist/client/entry.js +123 -2
  18. package/dist/client/gate.d.ts +1 -0
  19. package/dist/client/gate.js +148 -0
  20. package/dist/client/index.d.ts +25 -0
  21. package/dist/client/index.js +1 -0
  22. package/dist/client/live.d.ts +66 -0
  23. package/dist/client/live.js +225 -0
  24. package/dist/client/navigate-entry.d.ts +1 -0
  25. package/dist/client/navigate-entry.js +4 -0
  26. package/dist/client/navigate.d.ts +54 -0
  27. package/dist/client/navigate.js +272 -0
  28. package/dist/client/persist.d.ts +2 -0
  29. package/dist/client/persist.js +9 -0
  30. package/dist/client/register.d.ts +34 -2
  31. package/dist/client/register.js +64 -7
  32. package/dist/express/index.d.ts +11 -0
  33. package/dist/express/index.js +209 -0
  34. package/dist/hono/index.d.ts +19 -0
  35. package/dist/hono/index.js +49 -0
  36. package/dist/next/index.d.ts +33 -0
  37. package/dist/next/index.js +41 -0
  38. package/dist/next/pages.d.ts +22 -0
  39. package/dist/next/pages.js +39 -0
  40. package/dist/node/listener.d.ts +3 -0
  41. package/dist/node/listener.js +57 -2
  42. package/dist/nuxt/index.d.ts +22 -0
  43. package/dist/nuxt/index.js +104 -0
  44. package/dist/nuxt/pages.d.ts +26 -0
  45. package/dist/nuxt/pages.js +48 -0
  46. package/dist/nuxt/runtime/index.d.ts +24 -0
  47. package/dist/nuxt/runtime/index.js +58 -0
  48. package/dist/plugins/coverage.d.ts +14 -0
  49. package/dist/plugins/coverage.js +27 -0
  50. package/dist/plugins/mode.d.ts +19 -0
  51. package/dist/plugins/mode.js +20 -0
  52. package/dist/plugins/path.d.ts +16 -0
  53. package/dist/plugins/path.js +139 -0
  54. package/dist/plugins/report.d.ts +22 -0
  55. package/dist/plugins/report.js +18 -0
  56. package/dist/plugins/routes.d.ts +38 -0
  57. package/dist/plugins/routes.js +52 -0
  58. package/dist/plugins/static.d.ts +42 -0
  59. package/dist/plugins/static.js +97 -0
  60. package/dist/protocol/bundle.d.ts +111 -0
  61. package/dist/protocol/bundle.js +428 -0
  62. package/dist/protocol/config.d.ts +80 -1
  63. package/dist/protocol/config.js +31 -1
  64. package/dist/protocol/dataset.d.ts +189 -0
  65. package/dist/protocol/dataset.js +212 -0
  66. package/dist/protocol/envelope.d.ts +34 -1
  67. package/dist/protocol/envelope.js +16 -1
  68. package/dist/protocol/errors.d.ts +1 -1
  69. package/dist/protocol/graph.d.ts +23 -4
  70. package/dist/protocol/graph.js +34 -3
  71. package/dist/protocol/index-rows.d.ts +22 -0
  72. package/dist/protocol/index-rows.js +53 -0
  73. package/dist/protocol/index.d.ts +5 -0
  74. package/dist/protocol/index.js +5 -0
  75. package/dist/protocol/normalise.d.ts +4 -4
  76. package/dist/protocol/normalise.js +9 -5
  77. package/dist/protocol/plan.d.ts +100 -0
  78. package/dist/protocol/plan.js +661 -0
  79. package/dist/protocol/provision.d.ts +46 -0
  80. package/dist/protocol/provision.js +38 -0
  81. package/dist/protocol/query.d.ts +66 -5
  82. package/dist/protocol/query.js +103 -4
  83. package/dist/protocol/ring.d.ts +58 -0
  84. package/dist/protocol/ring.js +6 -0
  85. package/dist/protocol/route.d.ts +63 -13
  86. package/dist/protocol/route.js +126 -6
  87. package/dist/protocol/tag.d.ts +22 -2
  88. package/dist/protocol/tag.js +25 -4
  89. package/dist/protocol/view.d.ts +2 -0
  90. package/dist/protocol/view.js +8 -2
  91. package/dist/protocol/views.d.ts +6 -0
  92. package/dist/protocol/views.js +15 -0
  93. package/dist/protocol/write.d.ts +234 -0
  94. package/dist/protocol/write.js +343 -0
  95. package/dist/proxy/eligible.d.ts +22 -1
  96. package/dist/proxy/eligible.js +62 -15
  97. package/dist/proxy/proxy.js +28 -18
  98. package/dist/react-router/index.d.ts +20 -0
  99. package/dist/react-router/index.js +60 -0
  100. package/dist/react-router/pages.d.ts +34 -0
  101. package/dist/react-router/pages.js +83 -0
  102. package/dist/report/dataset.d.ts +48 -0
  103. package/dist/report/dataset.js +93 -0
  104. package/dist/report/demand.d.ts +49 -0
  105. package/dist/report/demand.js +60 -0
  106. package/dist/report/derive.d.ts +285 -0
  107. package/dist/report/derive.js +676 -1
  108. package/dist/report/economics.d.ts +66 -1
  109. package/dist/report/economics.js +208 -1
  110. package/dist/report/headline.d.ts +158 -0
  111. package/dist/report/headline.js +473 -0
  112. package/dist/report/index.d.ts +4 -0
  113. package/dist/report/index.js +4 -0
  114. package/dist/report/inputs.d.ts +176 -0
  115. package/dist/report/inputs.js +248 -0
  116. package/dist/report/institution.d.ts +188 -1
  117. package/dist/report/institution.js +146 -0
  118. package/dist/report/loops.d.ts +73 -0
  119. package/dist/report/loops.js +101 -0
  120. package/dist/report/measurement.d.ts +1 -1
  121. package/dist/report/measurement.js +7 -1
  122. package/dist/report/model.d.ts +2 -0
  123. package/dist/report/model.js +4 -0
  124. package/dist/report/thirdparty.d.ts +57 -0
  125. package/dist/report/thirdparty.js +111 -0
  126. package/dist/report/traffic.d.ts +8 -0
  127. package/dist/report/traffic.js +7 -0
  128. package/dist/report/trim.d.ts +56 -0
  129. package/dist/report/trim.js +83 -1
  130. package/dist/server/bundle.d.ts +55 -0
  131. package/dist/server/bundle.js +172 -0
  132. package/dist/server/dataset.d.ts +54 -0
  133. package/dist/server/dataset.js +355 -0
  134. package/dist/server/embedded.d.ts +7 -3
  135. package/dist/server/embedded.js +7 -3
  136. package/dist/server/index.d.ts +6 -0
  137. package/dist/server/index.js +6 -0
  138. package/dist/server/live-sockets.d.ts +54 -0
  139. package/dist/server/live-sockets.js +131 -0
  140. package/dist/server/live.d.ts +146 -0
  141. package/dist/server/live.js +228 -0
  142. package/dist/server/ring.d.ts +77 -0
  143. package/dist/server/ring.js +311 -0
  144. package/dist/server/spool.d.ts +158 -3
  145. package/dist/server/spool.js +578 -41
  146. package/dist/server/static.d.ts +42 -0
  147. package/dist/server/static.js +29 -0
  148. package/dist/server/stub.d.ts +43 -5
  149. package/dist/server/stub.js +126 -8
  150. package/dist/server/write.d.ts +49 -0
  151. package/dist/server/write.js +223 -0
  152. package/dist/stores/durable-object.d.ts +46 -0
  153. package/dist/stores/durable-object.js +114 -0
  154. package/dist/sw/analytics.d.ts +7 -1
  155. package/dist/sw/analytics.js +16 -2
  156. package/dist/sw/coverage.d.ts +6 -0
  157. package/dist/sw/coverage.js +34 -0
  158. package/dist/sw/dataset-sync.d.ts +76 -0
  159. package/dist/sw/dataset-sync.js +149 -0
  160. package/dist/sw/dataset.d.ts +91 -0
  161. package/dist/sw/dataset.js +318 -0
  162. package/dist/sw/family.d.ts +15 -0
  163. package/dist/sw/family.js +48 -0
  164. package/dist/sw/graph/index.d.ts +31 -1
  165. package/dist/sw/graph/index.js +57 -5
  166. package/dist/sw/graph/invalidate.d.ts +15 -3
  167. package/dist/sw/graph/invalidate.js +42 -4
  168. package/dist/sw/idb.d.ts +2 -0
  169. package/dist/sw/idb.js +80 -11
  170. package/dist/sw/index.d.ts +2 -0
  171. package/dist/sw/index.js +1 -0
  172. package/dist/sw/pack/store.d.ts +128 -0
  173. package/dist/sw/pack/store.js +737 -0
  174. package/dist/sw/partition.d.ts +6 -1
  175. package/dist/sw/partition.js +21 -2
  176. package/dist/sw/policy.d.ts +2 -0
  177. package/dist/sw/policy.js +11 -0
  178. package/dist/sw/principal-dataset.d.ts +64 -0
  179. package/dist/sw/principal-dataset.js +254 -0
  180. package/dist/sw/probe.d.ts +16 -0
  181. package/dist/sw/probe.js +25 -0
  182. package/dist/sw/render.d.ts +7 -2
  183. package/dist/sw/render.js +19 -5
  184. package/dist/sw/replica.d.ts +12 -3
  185. package/dist/sw/replica.js +40 -5
  186. package/dist/sw/router.d.ts +3 -2
  187. package/dist/sw/router.js +15 -2
  188. package/dist/sw/runtime.d.ts +90 -2
  189. package/dist/sw/runtime.js +857 -61
  190. package/dist/sw/scheduler.d.ts +6 -1
  191. package/dist/sw/scheduler.js +22 -5
  192. package/dist/sw/seal.d.ts +13 -0
  193. package/dist/sw/seal.js +13 -4
  194. package/dist/sw/static.d.ts +18 -0
  195. package/dist/sw/static.js +84 -1
  196. package/dist/sw/storage.d.ts +51 -1
  197. package/dist/sw/storage.js +33 -1
  198. package/dist/sw/write.d.ts +147 -0
  199. package/dist/sw/write.js +625 -0
  200. package/dist/version.d.ts +1 -1
  201. package/dist/version.js +1 -2
  202. package/dist/vite/index.d.ts +26 -0
  203. package/dist/vite/index.js +66 -0
  204. package/dist/vue/index.d.ts +36 -0
  205. package/dist/vue/index.js +53 -0
  206. package/package.json +65 -4
@@ -1,4 +1,6 @@
1
1
  import { DurableObject } from 'cloudflare:workers';
2
+ import { decodeOpen, encodeOpen, hibernatingSockets, SOCKET_OPEN_HEADER } from '../server/live-sockets.js';
3
+ import { memoryHub } from '../server/live.js';
2
4
  /**
3
5
  * The version vector in a SQLite-backed Durable Object: one row per tag and one cursor row, so it
4
6
  * survives eviction and restarts. Bind it as a Durable Object class and pass the namespace to
@@ -51,3 +53,115 @@ export function durableObjectStore(namespace, name = 'spool') {
51
53
  diff: (from) => stub().diff(from)
52
54
  };
53
55
  }
56
+ /**
57
+ * The live streams of every isolate, held by one Durable Object: a page's `/.spool/live` request
58
+ * is forwarded here and its response is the stream, so a bump or a revocation from any isolate
59
+ * reaches every open page. The object stays in memory while a stream is open and is billed for
60
+ * that duration. One instance serves every stream of a site, which is its ceiling. Bind it as a
61
+ * Durable Object class and pass the namespace to {@link durableObjectHub}.
62
+ *
63
+ * A WebSocket upgrade is accepted with the Hibernation API when the hub sets `hibernate`. Those
64
+ * sockets hold no timer, so while no SSE stream is open the object hibernates between a publish, a
65
+ * revoke, a maximum-age alarm and a message that is not a ping.
66
+ */
67
+ export class SpoolLive extends DurableObject {
68
+ #hub = memoryHub();
69
+ #sockets = hibernatingSockets({
70
+ pair: () => {
71
+ const pair = new WebSocketPair();
72
+ return [pair[0], pair[1]];
73
+ },
74
+ accept: (server, tags) => this.ctx.acceptWebSocket(server, tags),
75
+ sockets: (tag) => this.ctx.getWebSockets(tag),
76
+ autoResponse: (request, response) => this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair(request, response)),
77
+ alarm: {
78
+ get: () => this.ctx.storage.getAlarm(),
79
+ set: (at) => this.ctx.storage.setAlarm(at)
80
+ }
81
+ });
82
+ constructor(ctx, env) {
83
+ super(ctx, env);
84
+ this.#sockets.restore();
85
+ }
86
+ async fetch(request) {
87
+ if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') {
88
+ const init = decodeOpen(request.headers.get(SOCKET_OPEN_HEADER));
89
+ if (!init)
90
+ return new Response('bad opening request', { status: 400 });
91
+ const client = (await this.#sockets.open(init));
92
+ return new Response(null, { status: 101, webSocket: client });
93
+ }
94
+ const init = (await request.json());
95
+ return this.#hub.open({ ...init, signal: request.signal });
96
+ }
97
+ /** sends a delta to every open stream and socket */
98
+ publish(delta) {
99
+ this.#hub.publish(delta);
100
+ this.#sockets.publish(delta);
101
+ }
102
+ /** sends `revoked` to every stream and socket of `partition` and closes them */
103
+ revoke(partition) {
104
+ this.#hub.revoke(partition);
105
+ this.#sockets.revoke(partition);
106
+ }
107
+ webSocketMessage(ws, message) {
108
+ this.#sockets.message(ws, message);
109
+ }
110
+ webSocketClose(ws, code) {
111
+ // older compatibility dates need the reply; a newer runtime has sent it and ignores this
112
+ try {
113
+ if (code >= 1000 && code < 5000 && ![1004, 1005, 1006, 1015].includes(code))
114
+ ws.close(code);
115
+ else
116
+ ws.close();
117
+ }
118
+ catch {
119
+ // already closed
120
+ }
121
+ }
122
+ webSocketError(ws) {
123
+ try {
124
+ ws.close(1011, 'error');
125
+ }
126
+ catch {
127
+ // already closed
128
+ }
129
+ }
130
+ async alarm() {
131
+ await this.#sockets.alarm();
132
+ }
133
+ /** what this object has counted since it started */
134
+ stats() {
135
+ return this.#hub.stats();
136
+ }
137
+ }
138
+ /**
139
+ * A {@link LiveHub} over one named instance of the {@link SpoolLive} namespace. The hub cannot
140
+ * call back into the app, so a session the app ends is revoked with `spool.revoke`, and one it
141
+ * forgets to revoke is cut off when its stream reaches `maxAge`. The second argument is the
142
+ * instance name or {@link DurableObjectHubOptions}.
143
+ */
144
+ export function durableObjectHub(namespace, options = {}) {
145
+ const { name = 'spool', hibernate = false } = typeof options === 'string' ? { name: options } : options;
146
+ const stub = () => namespace.getByName(name);
147
+ return {
148
+ sockets: hibernate,
149
+ open: (init) => {
150
+ if (hibernate && init.socket) {
151
+ const header = encodeOpen(init);
152
+ if (header === null)
153
+ return new Response('opening frames too large', { status: 413 });
154
+ return stub().fetch(new Request('https://spool.live/socket', {
155
+ headers: { upgrade: 'websocket', [SOCKET_OPEN_HEADER]: header }
156
+ }));
157
+ }
158
+ const { partition, first, leaseMs, keepaliveMs, maxAgeMs } = init;
159
+ return stub().fetch(new Request('https://spool.live/open', {
160
+ method: 'POST',
161
+ body: JSON.stringify({ partition, first, leaseMs, keepaliveMs, maxAgeMs })
162
+ }));
163
+ },
164
+ publish: (delta) => stub().publish(delta),
165
+ revoke: (partition) => stub().revoke(partition)
166
+ };
167
+ }
@@ -36,7 +36,7 @@ export type QueueDeps = {
36
36
  * within {@link CATALOGUE_REASK_MS}. A due ask makes {@link BeaconQueue.flushAtMoment} flush
37
37
  * and nothing else does, so it adds no request on a local view or with nothing held
38
38
  */
39
- catalogue?(): CatalogueAsk | undefined;
39
+ catalogue?(): CatalogueAsk | undefined | Promise<CatalogueAsk | undefined>;
40
40
  };
41
41
  /**
42
42
  * Holds beacon requests on the device. A beacon that reaches the network on its own wakes the
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
78
78
  * browser's. A request that fails, or a network that is down, loses the event for good.
79
79
  */
80
80
  sendSameOrigin(timeoutMs: number): Promise<void>;
81
+ /**
82
+ * Drops the held same-origin requests without sending them, for a response that names another
83
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
84
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
85
+ */
86
+ dropSameOrigin(): Promise<void>;
81
87
  }
@@ -100,7 +100,7 @@ export class BeaconQueue {
100
100
  const others = held.some((b) => !own(b.event.path));
101
101
  const views = held.filter((b) => b.event.path === VIEW_PATH).length;
102
102
  const wanted = ((await this.#deps.want?.()) ?? []).length > 0;
103
- const asking = (await this.#due(this.#deps.catalogue?.())) !== undefined;
103
+ const asking = (await this.#due(await this.#deps.catalogue?.())) !== undefined;
104
104
  const oldest = Math.min(...held.map((b) => b.event.at));
105
105
  const aged = this.#deps.now() - oldest >= VIEW_HOLD_MS;
106
106
  if (others || wanted || asking || aged || views >= VIEW_BATCH)
@@ -199,6 +199,20 @@ export class BeaconQueue {
199
199
  }
200
200
  await this.#report(lost);
201
201
  }
202
+ /**
203
+ * Drops the held same-origin requests without sending them, for a response that names another
204
+ * partition or ends the session. The browser already holds the new session's cookies by then, so a
205
+ * replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
206
+ */
207
+ async dropSameOrigin() {
208
+ if (this.#flushing)
209
+ await this.#flushing.catch(() => false);
210
+ const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
211
+ if (mine.length === 0)
212
+ return;
213
+ await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
214
+ await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
215
+ }
202
216
  async #run() {
203
217
  const { storage, send, replay, want, catalogue } = this.#deps;
204
218
  await this.#expire();
@@ -221,7 +235,7 @@ export class BeaconQueue {
221
235
  }
222
236
  const batched = held.filter((h) => inBatch(h.event));
223
237
  let wanted = (await want?.()) ?? [];
224
- let ask = await this.#due(catalogue?.());
238
+ let ask = await this.#due(await catalogue?.());
225
239
  if (batched.length === 0 && wanted.length === 0)
226
240
  return ok;
227
241
  let done = true;
@@ -0,0 +1,6 @@
1
+ import type { SerializedRoute } from '../protocol/route.js';
2
+ /** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
3
+ export declare function coverageReport(requests: readonly {
4
+ method: string;
5
+ url: string;
6
+ }[], families: readonly SerializedRoute[], origin: string): string[];
@@ -0,0 +1,34 @@
1
+ import { matchPattern } from '../protocol/pattern.js';
2
+ /** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
3
+ export function coverageReport(requests, families, origin) {
4
+ const uncovered = new Set();
5
+ for (const { method, url } of requests) {
6
+ if (method !== 'GET')
7
+ continue;
8
+ const parsed = new URL(url);
9
+ if (!families.some((route) => covers(route, parsed, origin)))
10
+ uncovered.add(url);
11
+ }
12
+ return [...uncovered];
13
+ }
14
+ /**
15
+ * A data family covers a GET of its origin (the page's own when it names none) whose path fits its
16
+ * pattern. A query string is covered only by the parameters the family's query takes, the rule
17
+ * `evaluateQuery` applies when the worker reads.
18
+ */
19
+ function covers(route, url, origin) {
20
+ if (route.strategy !== 'data')
21
+ return false;
22
+ if (route.family === undefined && route.origin === undefined)
23
+ return false;
24
+ if ((route.origin ?? origin) !== url.origin)
25
+ return false;
26
+ if (matchPattern(route.pattern, url.pathname) === null)
27
+ return false;
28
+ if (url.search === '')
29
+ return true;
30
+ if (route.query === undefined)
31
+ return false;
32
+ const filter = route.query.filter ?? {};
33
+ return [...url.searchParams.keys()].every((key) => key === 'sort' || key === 'page' || Object.hasOwn(filter, key));
34
+ }
@@ -0,0 +1,76 @@
1
+ import type { DatasetStore } from './dataset.js';
2
+ /** why a dataset was not held or not brought current */
3
+ export type DatasetRefusal =
4
+ /** the rows would use more than the storage the device has to give */
5
+ 'budget'
6
+ /** the proof failed again after a patch and one reseed */
7
+ | 'proof'
8
+ /** the server has no such dataset, or refused the request */
9
+ | 'unavailable'
10
+ /** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
11
+ | 'declined'
12
+ /** the server's answer was not a dataset response, or its paging did not advance */
13
+ | 'bad-response'
14
+ /** the replica was wiped while the apply ran */
15
+ | 'epoch'
16
+ /** a principal dataset's answer named another partition than the one held, or none */
17
+ | 'partition'
18
+ /** the server answered a principal dataset with a sign-in request: the session ended */
19
+ | 'ended'
20
+ /** the server forbids this principal the dataset: its held rows were dropped */
21
+ | 'revoked'
22
+ /** the storage transaction failed, for example on quota */
23
+ | 'storage';
24
+ /**
25
+ * What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
26
+ * changed them, `current` found nothing to change. `offline` means no answer came and the held
27
+ * rows, if any, are as they were; `refused` names why a dataset is not held.
28
+ */
29
+ export type DatasetOutcome = {
30
+ name: string;
31
+ requests: number;
32
+ } & ({
33
+ status: 'seeded' | 'reseeded' | 'patched' | 'current';
34
+ cursor: number;
35
+ rows: number;
36
+ } | {
37
+ status: 'offline';
38
+ } | {
39
+ status: 'refused';
40
+ reason: DatasetRefusal;
41
+ by?: string | null;
42
+ });
43
+ /** what a sync needs from the worker around it */
44
+ export type DatasetSyncDeps = {
45
+ store: DatasetStore;
46
+ fetch(url: string, init?: RequestInit): Promise<Response>;
47
+ origin: string;
48
+ /** the replica's wipe count */
49
+ epoch(): number;
50
+ /** the most bytes of rows one dataset may use; undefined when the device cannot tell */
51
+ budget(): Promise<number | undefined>;
52
+ /**
53
+ * Set for a principal dataset: reads what a response says about whose rows it carries before
54
+ * anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
55
+ */
56
+ guard?(response: Response): 'ok' | DatasetGuardVerdict;
57
+ };
58
+ /** why a guard stopped a response */
59
+ export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
60
+ /**
61
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
62
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
63
+ */
64
+ export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
65
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
66
+ export declare const DATASET_MAX_PAGES = 1000;
67
+ /**
68
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
69
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
70
+ * rows are applied to a staging area and written in one transaction after the collection proof
71
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
72
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
73
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
74
+ * an outcome.
75
+ */
76
+ export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;
@@ -0,0 +1,149 @@
1
+ import { PATHS } from '../protocol/config.js';
2
+ import { encodeDatasetRequest, parseDatasetResponse } from '../protocol/dataset.js';
3
+ import { HEADERS } from '../protocol/envelope.js';
4
+ import { ProtocolError } from '../protocol/errors.js';
5
+ /**
6
+ * The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
7
+ * dataset, and a success that does not name exactly this partition is another principal's or none.
8
+ */
9
+ export function principalGuard(partition) {
10
+ return (response) => {
11
+ if (response.status === 401)
12
+ return 'ended';
13
+ if (response.status === 403)
14
+ return 'revoked';
15
+ if (response.ok && response.headers.get(HEADERS.partition) !== partition)
16
+ return 'partition';
17
+ return 'ok';
18
+ };
19
+ }
20
+ /** seed pages followed before a sync gives up on a server whose paging does not end */
21
+ export const DATASET_MAX_PAGES = 1000;
22
+ class Refused extends Error {
23
+ reason;
24
+ by;
25
+ constructor(reason, by) {
26
+ super(reason);
27
+ this.reason = reason;
28
+ this.by = by;
29
+ }
30
+ }
31
+ /**
32
+ * Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
33
+ * is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
34
+ * rows are applied to a staging area and written in one transaction after the collection proof
35
+ * matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
36
+ * which covers rows that moved while the pages were read; a proof that still fails clears the
37
+ * dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
38
+ * an outcome.
39
+ */
40
+ export async function syncDataset(deps, name) {
41
+ const { store, origin } = deps;
42
+ const epoch = deps.epoch();
43
+ let requests = 0;
44
+ const get = async (request) => {
45
+ requests += 1;
46
+ const response = await deps.fetch(`${origin}${PATHS.dataset}?${encodeDatasetRequest(request)}`, { cache: 'no-store', credentials: 'same-origin' });
47
+ const verdict = deps.guard?.(response) ?? 'ok';
48
+ if (verdict !== 'ok')
49
+ throw new Refused(verdict);
50
+ if (response.status === 409)
51
+ throw new Refused('declined', response.headers.get(HEADERS.provisionReason));
52
+ if (response.status === 400 || response.status === 404)
53
+ throw new Refused('unavailable');
54
+ if (!response.ok)
55
+ throw new TypeError(`dataset answered ${response.status}`);
56
+ const parsed = parseDatasetResponse(await response.json());
57
+ if (parsed.name !== name)
58
+ throw new Refused('bad-response');
59
+ return parsed;
60
+ };
61
+ const write = async (staging, at, status) => {
62
+ try {
63
+ if (deps.epoch() !== epoch)
64
+ throw new Refused('epoch');
65
+ if (!(await store.commit(staging, at, epoch)))
66
+ throw new Refused('epoch');
67
+ }
68
+ catch (error) {
69
+ if (error instanceof Refused)
70
+ throw error;
71
+ throw new Refused('storage');
72
+ }
73
+ return { name, requests, status, cursor: at.cursor, rows: staging.rows.size };
74
+ };
75
+ const seed = async (first, status, retry) => {
76
+ let page = first ?? (await get({ name }));
77
+ const { cursor, schema } = page;
78
+ const budget = await deps.budget();
79
+ const staging = store.stage(name);
80
+ let matches;
81
+ let after;
82
+ for (let pages = 1;; pages += 1) {
83
+ if (deps.epoch() !== epoch)
84
+ throw new Refused('epoch');
85
+ matches = await staging.add(page);
86
+ if (budget !== undefined && staging.bytes > budget)
87
+ throw new Refused('budget');
88
+ if (page.next === undefined)
89
+ break;
90
+ if (pages >= DATASET_MAX_PAGES || (after !== undefined && page.next <= after)) {
91
+ throw new Refused('bad-response');
92
+ }
93
+ after = page.next;
94
+ page = await get({ name, after });
95
+ }
96
+ if (matches === true)
97
+ return write(staging, { cursor, schema }, status);
98
+ // rows moved while the pages were read: the changes since the first page repair the copy
99
+ const patch = await get({ name, since: cursor, schema });
100
+ if (!patch.reset && (await staging.add(patch)) === true) {
101
+ return write(staging, { cursor: patch.cursor, schema }, status);
102
+ }
103
+ if (retry)
104
+ return seed(undefined, 'reseeded', false);
105
+ throw new Refused('proof');
106
+ };
107
+ try {
108
+ const held = await store.get(name);
109
+ if (!held)
110
+ return await seed(undefined, 'seeded', true);
111
+ const { cursor, schema, digest, count } = held.meta;
112
+ const response = await get({ name, since: cursor, schema });
113
+ if (response.reset)
114
+ return await seed(response, 'reseeded', true);
115
+ if (response.cursor < cursor)
116
+ return { name, requests, status: 'current', cursor, rows: count };
117
+ const proof = response.proof;
118
+ if (response.cursor === cursor) {
119
+ // nothing moved; a proof that disagrees with the held rows means the copy is damaged
120
+ if (proof && (proof.count !== count || proof.digest !== digest)) {
121
+ await store.drop(name);
122
+ return await seed(undefined, 'reseeded', false);
123
+ }
124
+ if (!(await store.touch(name, epoch)))
125
+ throw new Refused('epoch');
126
+ return { name, requests, status: 'current', cursor, rows: count };
127
+ }
128
+ const staging = store.stage(name, held);
129
+ if ((await staging.add(response)) === true) {
130
+ return await write(staging, { cursor: response.cursor, schema }, 'patched');
131
+ }
132
+ // the patched rows do not match the server's proof: nothing wrong is kept
133
+ await store.drop(name);
134
+ return await seed(undefined, 'reseeded', false);
135
+ }
136
+ catch (error) {
137
+ if (error instanceof Refused) {
138
+ // the server stopped admitting this principal to the dataset, so no row of it stays
139
+ if (error.reason === 'revoked' && deps.epoch() === epoch) {
140
+ await store.drop(name).catch(() => undefined);
141
+ }
142
+ return { name, requests, status: 'refused', reason: error.reason, by: error.by };
143
+ }
144
+ if (error instanceof ProtocolError) {
145
+ return { name, requests, status: 'refused', reason: 'bad-response' };
146
+ }
147
+ return { name, requests, status: 'offline' };
148
+ }
149
+ }
@@ -0,0 +1,91 @@
1
+ import { type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
2
+ import { type PlanTables } from '../protocol/plan.js';
3
+ import type { DatasetMeta, DatasetRowRecord, ReplicaStorage } from './storage.js';
4
+ /** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
5
+ export declare const rowBytes: (row: DatasetRow) => number;
6
+ /** a dataset in memory, as a verified load or a commit left it */
7
+ export type Held = {
8
+ meta: DatasetMeta;
9
+ rows: Map<string, DatasetRow>;
10
+ /** the hash of every row, so a patch never hashes a row it replaces or deletes */
11
+ hashes: Map<string, string>;
12
+ };
13
+ /** what the store needs from the replica around it */
14
+ export type DatasetStoreDeps = {
15
+ storage: ReplicaStorage;
16
+ now(): number;
17
+ /** the replica's wipe count; a commit that began under another one is dropped */
18
+ epoch(): number;
19
+ /** the row hash; {@link rowHash}. A test counts calls through it */
20
+ hash?: (key: string, row: DatasetRow) => Promise<string>;
21
+ /** told after every commit which row keys it put and which it removed */
22
+ onChange?(name: string, puts: string[], deletes: string[]): Promise<void> | void;
23
+ };
24
+ /**
25
+ * One dataset's state while a response or a run of seed pages is applied, before anything is
26
+ * written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
27
+ * touches storage, so a failed or dropped apply leaves the store as it was.
28
+ */
29
+ export declare class Staging {
30
+ #private;
31
+ readonly name: string;
32
+ /** whether the commit replaces every held row */
33
+ clear: boolean;
34
+ rows: Map<string, DatasetRow>;
35
+ hashes: Map<string, string>;
36
+ digest: string;
37
+ bytes: number;
38
+ constructor(deps: DatasetStoreDeps, name: string, base?: Held);
39
+ /** applies a response; returns whether the result equals the response's proof, null without one */
40
+ add(response: DatasetResponse): Promise<boolean | null>;
41
+ /** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
42
+ writes(): {
43
+ puts: DatasetRowRecord[];
44
+ deletes: string[];
45
+ };
46
+ }
47
+ /**
48
+ * The datasets a device holds. Rows live once in memory after a verified load and once in
49
+ * storage, and a response is applied to a {@link Staging} first and committed in one storage
50
+ * transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
51
+ * never interleave.
52
+ */
53
+ export declare class DatasetStore {
54
+ #private;
55
+ /** datasets found damaged on a load and cleared, since the worker started */
56
+ evicted: number;
57
+ constructor(deps: DatasetStoreDeps);
58
+ /** the meta of every held dataset */
59
+ list(): Promise<DatasetMeta[]>;
60
+ /**
61
+ * Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
62
+ * hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
63
+ * like, is cleared and reads as absent. No row is hashed.
64
+ */
65
+ get(name: string): Promise<Held | undefined>;
66
+ /** the held rows in ascending key order, or undefined when `name` is not loaded and held */
67
+ rows(name: string): Promise<DatasetRow[] | undefined>;
68
+ /** the base table and each joined table a plan reads, or undefined when any of them is not held */
69
+ tables(base: string, joins: readonly string[]): Promise<PlanTables | undefined>;
70
+ /** whether `name` is held and in memory */
71
+ loaded(name: string): boolean;
72
+ /** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
73
+ row(name: string, key: string): DatasetRow | undefined;
74
+ /** a staging area over the held rows of `name`, or an empty one for a seed */
75
+ stage(name: string, base?: Held): Staging;
76
+ /**
77
+ * Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
78
+ * Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
79
+ * with the held rows and the memory copy untouched, when the storage transaction fails.
80
+ */
81
+ commit(staging: Staging, at: {
82
+ cursor: number;
83
+ schema: string;
84
+ }, epoch: number): Promise<boolean>;
85
+ /** records that a sync found the held rows current, with no change to them */
86
+ touch(name: string, epoch: number): Promise<boolean>;
87
+ /** clears one dataset's rows and meta, and nothing else */
88
+ drop(name: string): Promise<void>;
89
+ /** clears every dataset, for a wipe of the replica */
90
+ clear(): Promise<void>;
91
+ }