@drupflare/spool 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -1607
- package/dist/angular/index.d.ts +13 -0
- package/dist/angular/index.js +25 -0
- package/dist/angular/pages.d.ts +28 -0
- package/dist/angular/pages.js +85 -0
- package/dist/cdn/client.js +1 -1
- package/dist/cdn/location.d.ts +6 -0
- package/dist/cdn/location.js +8 -0
- package/dist/cdn/manifest.json +7 -7
- package/dist/cdn/spool-sw.js +1 -1
- package/dist/client/bootstrap.d.ts +57 -0
- package/dist/client/bootstrap.js +307 -0
- package/dist/client/contact.d.ts +24 -0
- package/dist/client/contact.js +66 -0
- package/dist/client/declared.d.ts +23 -0
- package/dist/client/declared.js +68 -0
- package/dist/client/entry.js +123 -2
- package/dist/client/gate.d.ts +1 -0
- package/dist/client/gate.js +148 -0
- package/dist/client/index.d.ts +25 -0
- package/dist/client/index.js +1 -0
- package/dist/client/live.d.ts +66 -0
- package/dist/client/live.js +225 -0
- package/dist/client/navigate-entry.d.ts +1 -0
- package/dist/client/navigate-entry.js +4 -0
- package/dist/client/navigate.d.ts +54 -0
- package/dist/client/navigate.js +272 -0
- package/dist/client/persist.d.ts +2 -0
- package/dist/client/persist.js +9 -0
- package/dist/client/register.d.ts +34 -2
- package/dist/client/register.js +64 -7
- package/dist/express/index.d.ts +11 -0
- package/dist/express/index.js +209 -0
- package/dist/hono/index.d.ts +19 -0
- package/dist/hono/index.js +49 -0
- package/dist/next/index.d.ts +33 -0
- package/dist/next/index.js +41 -0
- package/dist/next/pages.d.ts +22 -0
- package/dist/next/pages.js +39 -0
- package/dist/node/listener.d.ts +3 -0
- package/dist/node/listener.js +57 -2
- package/dist/nuxt/index.d.ts +22 -0
- package/dist/nuxt/index.js +104 -0
- package/dist/nuxt/pages.d.ts +26 -0
- package/dist/nuxt/pages.js +48 -0
- package/dist/nuxt/runtime/index.d.ts +24 -0
- package/dist/nuxt/runtime/index.js +58 -0
- package/dist/plugins/coverage.d.ts +14 -0
- package/dist/plugins/coverage.js +27 -0
- package/dist/plugins/mode.d.ts +19 -0
- package/dist/plugins/mode.js +20 -0
- package/dist/plugins/path.d.ts +16 -0
- package/dist/plugins/path.js +139 -0
- package/dist/plugins/report.d.ts +22 -0
- package/dist/plugins/report.js +18 -0
- package/dist/plugins/routes.d.ts +38 -0
- package/dist/plugins/routes.js +52 -0
- package/dist/plugins/static.d.ts +42 -0
- package/dist/plugins/static.js +97 -0
- package/dist/protocol/bundle.d.ts +111 -0
- package/dist/protocol/bundle.js +428 -0
- package/dist/protocol/config.d.ts +80 -1
- package/dist/protocol/config.js +31 -1
- package/dist/protocol/dataset.d.ts +189 -0
- package/dist/protocol/dataset.js +212 -0
- package/dist/protocol/envelope.d.ts +34 -1
- package/dist/protocol/envelope.js +16 -1
- package/dist/protocol/errors.d.ts +1 -1
- package/dist/protocol/graph.d.ts +23 -4
- package/dist/protocol/graph.js +34 -3
- package/dist/protocol/index-rows.d.ts +22 -0
- package/dist/protocol/index-rows.js +53 -0
- package/dist/protocol/index.d.ts +5 -0
- package/dist/protocol/index.js +5 -0
- package/dist/protocol/normalise.d.ts +4 -4
- package/dist/protocol/normalise.js +9 -5
- package/dist/protocol/plan.d.ts +100 -0
- package/dist/protocol/plan.js +661 -0
- package/dist/protocol/provision.d.ts +46 -0
- package/dist/protocol/provision.js +38 -0
- package/dist/protocol/query.d.ts +66 -5
- package/dist/protocol/query.js +103 -4
- package/dist/protocol/ring.d.ts +58 -0
- package/dist/protocol/ring.js +6 -0
- package/dist/protocol/route.d.ts +63 -13
- package/dist/protocol/route.js +126 -6
- package/dist/protocol/tag.d.ts +22 -2
- package/dist/protocol/tag.js +25 -4
- package/dist/protocol/view.d.ts +2 -0
- package/dist/protocol/view.js +8 -2
- package/dist/protocol/views.d.ts +6 -0
- package/dist/protocol/views.js +15 -0
- package/dist/protocol/write.d.ts +234 -0
- package/dist/protocol/write.js +343 -0
- package/dist/proxy/eligible.d.ts +22 -1
- package/dist/proxy/eligible.js +62 -15
- package/dist/proxy/proxy.js +28 -18
- package/dist/react-router/index.d.ts +20 -0
- package/dist/react-router/index.js +60 -0
- package/dist/react-router/pages.d.ts +34 -0
- package/dist/react-router/pages.js +83 -0
- package/dist/report/dataset.d.ts +48 -0
- package/dist/report/dataset.js +93 -0
- package/dist/report/demand.d.ts +49 -0
- package/dist/report/demand.js +60 -0
- package/dist/report/derive.d.ts +285 -0
- package/dist/report/derive.js +676 -1
- package/dist/report/economics.d.ts +66 -1
- package/dist/report/economics.js +208 -1
- package/dist/report/headline.d.ts +158 -0
- package/dist/report/headline.js +473 -0
- package/dist/report/index.d.ts +4 -0
- package/dist/report/index.js +4 -0
- package/dist/report/inputs.d.ts +176 -0
- package/dist/report/inputs.js +248 -0
- package/dist/report/institution.d.ts +188 -1
- package/dist/report/institution.js +146 -0
- package/dist/report/loops.d.ts +73 -0
- package/dist/report/loops.js +101 -0
- package/dist/report/measurement.d.ts +1 -1
- package/dist/report/measurement.js +7 -1
- package/dist/report/model.d.ts +2 -0
- package/dist/report/model.js +4 -0
- package/dist/report/thirdparty.d.ts +57 -0
- package/dist/report/thirdparty.js +111 -0
- package/dist/report/traffic.d.ts +8 -0
- package/dist/report/traffic.js +7 -0
- package/dist/report/trim.d.ts +56 -0
- package/dist/report/trim.js +83 -1
- package/dist/server/bundle.d.ts +55 -0
- package/dist/server/bundle.js +172 -0
- package/dist/server/dataset.d.ts +54 -0
- package/dist/server/dataset.js +355 -0
- package/dist/server/embedded.d.ts +7 -3
- package/dist/server/embedded.js +7 -3
- package/dist/server/index.d.ts +6 -0
- package/dist/server/index.js +6 -0
- package/dist/server/live-sockets.d.ts +54 -0
- package/dist/server/live-sockets.js +131 -0
- package/dist/server/live.d.ts +146 -0
- package/dist/server/live.js +228 -0
- package/dist/server/ring.d.ts +77 -0
- package/dist/server/ring.js +311 -0
- package/dist/server/spool.d.ts +158 -3
- package/dist/server/spool.js +578 -41
- package/dist/server/static.d.ts +42 -0
- package/dist/server/static.js +29 -0
- package/dist/server/stub.d.ts +43 -5
- package/dist/server/stub.js +126 -8
- package/dist/server/write.d.ts +49 -0
- package/dist/server/write.js +223 -0
- package/dist/stores/durable-object.d.ts +46 -0
- package/dist/stores/durable-object.js +114 -0
- package/dist/sw/analytics.d.ts +7 -1
- package/dist/sw/analytics.js +16 -2
- package/dist/sw/coverage.d.ts +6 -0
- package/dist/sw/coverage.js +34 -0
- package/dist/sw/dataset-sync.d.ts +76 -0
- package/dist/sw/dataset-sync.js +149 -0
- package/dist/sw/dataset.d.ts +91 -0
- package/dist/sw/dataset.js +318 -0
- package/dist/sw/family.d.ts +15 -0
- package/dist/sw/family.js +48 -0
- package/dist/sw/graph/index.d.ts +31 -1
- package/dist/sw/graph/index.js +57 -5
- package/dist/sw/graph/invalidate.d.ts +15 -3
- package/dist/sw/graph/invalidate.js +42 -4
- package/dist/sw/idb.d.ts +2 -0
- package/dist/sw/idb.js +80 -11
- package/dist/sw/index.d.ts +2 -0
- package/dist/sw/index.js +1 -0
- package/dist/sw/pack/store.d.ts +128 -0
- package/dist/sw/pack/store.js +737 -0
- package/dist/sw/partition.d.ts +6 -1
- package/dist/sw/partition.js +21 -2
- package/dist/sw/policy.d.ts +2 -0
- package/dist/sw/policy.js +11 -0
- package/dist/sw/principal-dataset.d.ts +64 -0
- package/dist/sw/principal-dataset.js +254 -0
- package/dist/sw/probe.d.ts +16 -0
- package/dist/sw/probe.js +25 -0
- package/dist/sw/render.d.ts +7 -2
- package/dist/sw/render.js +19 -5
- package/dist/sw/replica.d.ts +12 -3
- package/dist/sw/replica.js +40 -5
- package/dist/sw/router.d.ts +3 -2
- package/dist/sw/router.js +15 -2
- package/dist/sw/runtime.d.ts +90 -2
- package/dist/sw/runtime.js +857 -61
- package/dist/sw/scheduler.d.ts +6 -1
- package/dist/sw/scheduler.js +22 -5
- package/dist/sw/seal.d.ts +13 -0
- package/dist/sw/seal.js +13 -4
- package/dist/sw/static.d.ts +18 -0
- package/dist/sw/static.js +84 -1
- package/dist/sw/storage.d.ts +51 -1
- package/dist/sw/storage.js +33 -1
- package/dist/sw/write.d.ts +147 -0
- package/dist/sw/write.js +625 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -2
- package/dist/vite/index.d.ts +26 -0
- package/dist/vite/index.js +66 -0
- package/dist/vue/index.d.ts +36 -0
- package/dist/vue/index.js +53 -0
- package/package.json +65 -4
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
|
+
import { decodeOpen, encodeOpen, hibernatingSockets, SOCKET_OPEN_HEADER } from '../server/live-sockets.js';
|
|
3
|
+
import { memoryHub } from '../server/live.js';
|
|
2
4
|
/**
|
|
3
5
|
* The version vector in a SQLite-backed Durable Object: one row per tag and one cursor row, so it
|
|
4
6
|
* survives eviction and restarts. Bind it as a Durable Object class and pass the namespace to
|
|
@@ -51,3 +53,115 @@ export function durableObjectStore(namespace, name = 'spool') {
|
|
|
51
53
|
diff: (from) => stub().diff(from)
|
|
52
54
|
};
|
|
53
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* The live streams of every isolate, held by one Durable Object: a page's `/.spool/live` request
|
|
58
|
+
* is forwarded here and its response is the stream, so a bump or a revocation from any isolate
|
|
59
|
+
* reaches every open page. The object stays in memory while a stream is open and is billed for
|
|
60
|
+
* that duration. One instance serves every stream of a site, which is its ceiling. Bind it as a
|
|
61
|
+
* Durable Object class and pass the namespace to {@link durableObjectHub}.
|
|
62
|
+
*
|
|
63
|
+
* A WebSocket upgrade is accepted with the Hibernation API when the hub sets `hibernate`. Those
|
|
64
|
+
* sockets hold no timer, so while no SSE stream is open the object hibernates between a publish, a
|
|
65
|
+
* revoke, a maximum-age alarm and a message that is not a ping.
|
|
66
|
+
*/
|
|
67
|
+
export class SpoolLive extends DurableObject {
|
|
68
|
+
#hub = memoryHub();
|
|
69
|
+
#sockets = hibernatingSockets({
|
|
70
|
+
pair: () => {
|
|
71
|
+
const pair = new WebSocketPair();
|
|
72
|
+
return [pair[0], pair[1]];
|
|
73
|
+
},
|
|
74
|
+
accept: (server, tags) => this.ctx.acceptWebSocket(server, tags),
|
|
75
|
+
sockets: (tag) => this.ctx.getWebSockets(tag),
|
|
76
|
+
autoResponse: (request, response) => this.ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair(request, response)),
|
|
77
|
+
alarm: {
|
|
78
|
+
get: () => this.ctx.storage.getAlarm(),
|
|
79
|
+
set: (at) => this.ctx.storage.setAlarm(at)
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
constructor(ctx, env) {
|
|
83
|
+
super(ctx, env);
|
|
84
|
+
this.#sockets.restore();
|
|
85
|
+
}
|
|
86
|
+
async fetch(request) {
|
|
87
|
+
if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') {
|
|
88
|
+
const init = decodeOpen(request.headers.get(SOCKET_OPEN_HEADER));
|
|
89
|
+
if (!init)
|
|
90
|
+
return new Response('bad opening request', { status: 400 });
|
|
91
|
+
const client = (await this.#sockets.open(init));
|
|
92
|
+
return new Response(null, { status: 101, webSocket: client });
|
|
93
|
+
}
|
|
94
|
+
const init = (await request.json());
|
|
95
|
+
return this.#hub.open({ ...init, signal: request.signal });
|
|
96
|
+
}
|
|
97
|
+
/** sends a delta to every open stream and socket */
|
|
98
|
+
publish(delta) {
|
|
99
|
+
this.#hub.publish(delta);
|
|
100
|
+
this.#sockets.publish(delta);
|
|
101
|
+
}
|
|
102
|
+
/** sends `revoked` to every stream and socket of `partition` and closes them */
|
|
103
|
+
revoke(partition) {
|
|
104
|
+
this.#hub.revoke(partition);
|
|
105
|
+
this.#sockets.revoke(partition);
|
|
106
|
+
}
|
|
107
|
+
webSocketMessage(ws, message) {
|
|
108
|
+
this.#sockets.message(ws, message);
|
|
109
|
+
}
|
|
110
|
+
webSocketClose(ws, code) {
|
|
111
|
+
// older compatibility dates need the reply; a newer runtime has sent it and ignores this
|
|
112
|
+
try {
|
|
113
|
+
if (code >= 1000 && code < 5000 && ![1004, 1005, 1006, 1015].includes(code))
|
|
114
|
+
ws.close(code);
|
|
115
|
+
else
|
|
116
|
+
ws.close();
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// already closed
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
webSocketError(ws) {
|
|
123
|
+
try {
|
|
124
|
+
ws.close(1011, 'error');
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
// already closed
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
async alarm() {
|
|
131
|
+
await this.#sockets.alarm();
|
|
132
|
+
}
|
|
133
|
+
/** what this object has counted since it started */
|
|
134
|
+
stats() {
|
|
135
|
+
return this.#hub.stats();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* A {@link LiveHub} over one named instance of the {@link SpoolLive} namespace. The hub cannot
|
|
140
|
+
* call back into the app, so a session the app ends is revoked with `spool.revoke`, and one it
|
|
141
|
+
* forgets to revoke is cut off when its stream reaches `maxAge`. The second argument is the
|
|
142
|
+
* instance name or {@link DurableObjectHubOptions}.
|
|
143
|
+
*/
|
|
144
|
+
export function durableObjectHub(namespace, options = {}) {
|
|
145
|
+
const { name = 'spool', hibernate = false } = typeof options === 'string' ? { name: options } : options;
|
|
146
|
+
const stub = () => namespace.getByName(name);
|
|
147
|
+
return {
|
|
148
|
+
sockets: hibernate,
|
|
149
|
+
open: (init) => {
|
|
150
|
+
if (hibernate && init.socket) {
|
|
151
|
+
const header = encodeOpen(init);
|
|
152
|
+
if (header === null)
|
|
153
|
+
return new Response('opening frames too large', { status: 413 });
|
|
154
|
+
return stub().fetch(new Request('https://spool.live/socket', {
|
|
155
|
+
headers: { upgrade: 'websocket', [SOCKET_OPEN_HEADER]: header }
|
|
156
|
+
}));
|
|
157
|
+
}
|
|
158
|
+
const { partition, first, leaseMs, keepaliveMs, maxAgeMs } = init;
|
|
159
|
+
return stub().fetch(new Request('https://spool.live/open', {
|
|
160
|
+
method: 'POST',
|
|
161
|
+
body: JSON.stringify({ partition, first, leaseMs, keepaliveMs, maxAgeMs })
|
|
162
|
+
}));
|
|
163
|
+
},
|
|
164
|
+
publish: (delta) => stub().publish(delta),
|
|
165
|
+
revoke: (partition) => stub().revoke(partition)
|
|
166
|
+
};
|
|
167
|
+
}
|
package/dist/sw/analytics.d.ts
CHANGED
|
@@ -36,7 +36,7 @@ export type QueueDeps = {
|
|
|
36
36
|
* within {@link CATALOGUE_REASK_MS}. A due ask makes {@link BeaconQueue.flushAtMoment} flush
|
|
37
37
|
* and nothing else does, so it adds no request on a local view or with nothing held
|
|
38
38
|
*/
|
|
39
|
-
catalogue?(): CatalogueAsk | undefined
|
|
39
|
+
catalogue?(): CatalogueAsk | undefined | Promise<CatalogueAsk | undefined>;
|
|
40
40
|
};
|
|
41
41
|
/**
|
|
42
42
|
* Holds beacon requests on the device. A beacon that reaches the network on its own wakes the
|
|
@@ -78,4 +78,10 @@ export declare class BeaconQueue {
|
|
|
78
78
|
* browser's. A request that fails, or a network that is down, loses the event for good.
|
|
79
79
|
*/
|
|
80
80
|
sendSameOrigin(timeoutMs: number): Promise<void>;
|
|
81
|
+
/**
|
|
82
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
83
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
84
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
85
|
+
*/
|
|
86
|
+
dropSameOrigin(): Promise<void>;
|
|
81
87
|
}
|
package/dist/sw/analytics.js
CHANGED
|
@@ -100,7 +100,7 @@ export class BeaconQueue {
|
|
|
100
100
|
const others = held.some((b) => !own(b.event.path));
|
|
101
101
|
const views = held.filter((b) => b.event.path === VIEW_PATH).length;
|
|
102
102
|
const wanted = ((await this.#deps.want?.()) ?? []).length > 0;
|
|
103
|
-
const asking = (await this.#due(this.#deps.catalogue?.())) !== undefined;
|
|
103
|
+
const asking = (await this.#due(await this.#deps.catalogue?.())) !== undefined;
|
|
104
104
|
const oldest = Math.min(...held.map((b) => b.event.at));
|
|
105
105
|
const aged = this.#deps.now() - oldest >= VIEW_HOLD_MS;
|
|
106
106
|
if (others || wanted || asking || aged || views >= VIEW_BATCH)
|
|
@@ -199,6 +199,20 @@ export class BeaconQueue {
|
|
|
199
199
|
}
|
|
200
200
|
await this.#report(lost);
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Drops the held same-origin requests without sending them, for a response that names another
|
|
204
|
+
* partition or ends the session. The browser already holds the new session's cookies by then, so a
|
|
205
|
+
* replay would carry them. Counted under {@link SAME_ORIGIN_VENDOR}.
|
|
206
|
+
*/
|
|
207
|
+
async dropSameOrigin() {
|
|
208
|
+
if (this.#flushing)
|
|
209
|
+
await this.#flushing.catch(() => false);
|
|
210
|
+
const mine = (await this.#deps.storage.readBeacons()).filter(({ event }) => !inBatch(event) && event.origin === undefined);
|
|
211
|
+
if (mine.length === 0)
|
|
212
|
+
return;
|
|
213
|
+
await this.#deps.storage.removeBeacons(mine.map((b) => b.key));
|
|
214
|
+
await this.#report(mine.map(() => SAME_ORIGIN_VENDOR));
|
|
215
|
+
}
|
|
202
216
|
async #run() {
|
|
203
217
|
const { storage, send, replay, want, catalogue } = this.#deps;
|
|
204
218
|
await this.#expire();
|
|
@@ -221,7 +235,7 @@ export class BeaconQueue {
|
|
|
221
235
|
}
|
|
222
236
|
const batched = held.filter((h) => inBatch(h.event));
|
|
223
237
|
let wanted = (await want?.()) ?? [];
|
|
224
|
-
let ask = await this.#due(catalogue?.());
|
|
238
|
+
let ask = await this.#due(await catalogue?.());
|
|
225
239
|
if (batched.length === 0 && wanted.length === 0)
|
|
226
240
|
return ok;
|
|
227
241
|
let done = true;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { SerializedRoute } from '../protocol/route.js';
|
|
2
|
+
/** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
|
|
3
|
+
export declare function coverageReport(requests: readonly {
|
|
4
|
+
method: string;
|
|
5
|
+
url: string;
|
|
6
|
+
}[], families: readonly SerializedRoute[], origin: string): string[];
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { matchPattern } from '../protocol/pattern.js';
|
|
2
|
+
/** the GETs a page made that no data family answers, as absolute URLs in first-seen order */
|
|
3
|
+
export function coverageReport(requests, families, origin) {
|
|
4
|
+
const uncovered = new Set();
|
|
5
|
+
for (const { method, url } of requests) {
|
|
6
|
+
if (method !== 'GET')
|
|
7
|
+
continue;
|
|
8
|
+
const parsed = new URL(url);
|
|
9
|
+
if (!families.some((route) => covers(route, parsed, origin)))
|
|
10
|
+
uncovered.add(url);
|
|
11
|
+
}
|
|
12
|
+
return [...uncovered];
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* A data family covers a GET of its origin (the page's own when it names none) whose path fits its
|
|
16
|
+
* pattern. A query string is covered only by the parameters the family's query takes, the rule
|
|
17
|
+
* `evaluateQuery` applies when the worker reads.
|
|
18
|
+
*/
|
|
19
|
+
function covers(route, url, origin) {
|
|
20
|
+
if (route.strategy !== 'data')
|
|
21
|
+
return false;
|
|
22
|
+
if (route.family === undefined && route.origin === undefined)
|
|
23
|
+
return false;
|
|
24
|
+
if ((route.origin ?? origin) !== url.origin)
|
|
25
|
+
return false;
|
|
26
|
+
if (matchPattern(route.pattern, url.pathname) === null)
|
|
27
|
+
return false;
|
|
28
|
+
if (url.search === '')
|
|
29
|
+
return true;
|
|
30
|
+
if (route.query === undefined)
|
|
31
|
+
return false;
|
|
32
|
+
const filter = route.query.filter ?? {};
|
|
33
|
+
return [...url.searchParams.keys()].every((key) => key === 'sort' || key === 'page' || Object.hasOwn(filter, key));
|
|
34
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { DatasetStore } from './dataset.js';
|
|
2
|
+
/** why a dataset was not held or not brought current */
|
|
3
|
+
export type DatasetRefusal =
|
|
4
|
+
/** the rows would use more than the storage the device has to give */
|
|
5
|
+
'budget'
|
|
6
|
+
/** the proof failed again after a patch and one reseed */
|
|
7
|
+
| 'proof'
|
|
8
|
+
/** the server has no such dataset, or refused the request */
|
|
9
|
+
| 'unavailable'
|
|
10
|
+
/** the server declined the seed: the dataset does not beat v0.3 and native HTTP (`provision: 'auto'`) */
|
|
11
|
+
| 'declined'
|
|
12
|
+
/** the server's answer was not a dataset response, or its paging did not advance */
|
|
13
|
+
| 'bad-response'
|
|
14
|
+
/** the replica was wiped while the apply ran */
|
|
15
|
+
| 'epoch'
|
|
16
|
+
/** a principal dataset's answer named another partition than the one held, or none */
|
|
17
|
+
| 'partition'
|
|
18
|
+
/** the server answered a principal dataset with a sign-in request: the session ended */
|
|
19
|
+
| 'ended'
|
|
20
|
+
/** the server forbids this principal the dataset: its held rows were dropped */
|
|
21
|
+
| 'revoked'
|
|
22
|
+
/** the storage transaction failed, for example on quota */
|
|
23
|
+
| 'storage';
|
|
24
|
+
/**
|
|
25
|
+
* What one sync of one dataset ended as. `seeded` and `reseeded` replaced the rows, `patched`
|
|
26
|
+
* changed them, `current` found nothing to change. `offline` means no answer came and the held
|
|
27
|
+
* rows, if any, are as they were; `refused` names why a dataset is not held.
|
|
28
|
+
*/
|
|
29
|
+
export type DatasetOutcome = {
|
|
30
|
+
name: string;
|
|
31
|
+
requests: number;
|
|
32
|
+
} & ({
|
|
33
|
+
status: 'seeded' | 'reseeded' | 'patched' | 'current';
|
|
34
|
+
cursor: number;
|
|
35
|
+
rows: number;
|
|
36
|
+
} | {
|
|
37
|
+
status: 'offline';
|
|
38
|
+
} | {
|
|
39
|
+
status: 'refused';
|
|
40
|
+
reason: DatasetRefusal;
|
|
41
|
+
by?: string | null;
|
|
42
|
+
});
|
|
43
|
+
/** what a sync needs from the worker around it */
|
|
44
|
+
export type DatasetSyncDeps = {
|
|
45
|
+
store: DatasetStore;
|
|
46
|
+
fetch(url: string, init?: RequestInit): Promise<Response>;
|
|
47
|
+
origin: string;
|
|
48
|
+
/** the replica's wipe count */
|
|
49
|
+
epoch(): number;
|
|
50
|
+
/** the most bytes of rows one dataset may use; undefined when the device cannot tell */
|
|
51
|
+
budget(): Promise<number | undefined>;
|
|
52
|
+
/**
|
|
53
|
+
* Set for a principal dataset: reads what a response says about whose rows it carries before
|
|
54
|
+
* anything in it is parsed. Anything but `ok` ends the sync with that refusal and applies nothing.
|
|
55
|
+
*/
|
|
56
|
+
guard?(response: Response): 'ok' | DatasetGuardVerdict;
|
|
57
|
+
};
|
|
58
|
+
/** why a guard stopped a response */
|
|
59
|
+
export type DatasetGuardVerdict = 'partition' | 'ended' | 'revoked';
|
|
60
|
+
/**
|
|
61
|
+
* The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
|
|
62
|
+
* dataset, and a success that does not name exactly this partition is another principal's or none.
|
|
63
|
+
*/
|
|
64
|
+
export declare function principalGuard(partition: string): NonNullable<DatasetSyncDeps['guard']>;
|
|
65
|
+
/** seed pages followed before a sync gives up on a server whose paging does not end */
|
|
66
|
+
export declare const DATASET_MAX_PAGES = 1000;
|
|
67
|
+
/**
|
|
68
|
+
* Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
|
|
69
|
+
* is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
|
|
70
|
+
* rows are applied to a staging area and written in one transaction after the collection proof
|
|
71
|
+
* matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
|
|
72
|
+
* which covers rows that moved while the pages were read; a proof that still fails clears the
|
|
73
|
+
* dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
|
|
74
|
+
* an outcome.
|
|
75
|
+
*/
|
|
76
|
+
export declare function syncDataset(deps: DatasetSyncDeps, name: string): Promise<DatasetOutcome>;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { PATHS } from '../protocol/config.js';
|
|
2
|
+
import { encodeDatasetRequest, parseDatasetResponse } from '../protocol/dataset.js';
|
|
3
|
+
import { HEADERS } from '../protocol/envelope.js';
|
|
4
|
+
import { ProtocolError } from '../protocol/errors.js';
|
|
5
|
+
/**
|
|
6
|
+
* The guard of a principal dataset held in `partition`: `401` ends the session, `403` revokes the
|
|
7
|
+
* dataset, and a success that does not name exactly this partition is another principal's or none.
|
|
8
|
+
*/
|
|
9
|
+
export function principalGuard(partition) {
|
|
10
|
+
return (response) => {
|
|
11
|
+
if (response.status === 401)
|
|
12
|
+
return 'ended';
|
|
13
|
+
if (response.status === 403)
|
|
14
|
+
return 'revoked';
|
|
15
|
+
if (response.ok && response.headers.get(HEADERS.partition) !== partition)
|
|
16
|
+
return 'partition';
|
|
17
|
+
return 'ok';
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/** seed pages followed before a sync gives up on a server whose paging does not end */
|
|
21
|
+
export const DATASET_MAX_PAGES = 1000;
|
|
22
|
+
class Refused extends Error {
|
|
23
|
+
reason;
|
|
24
|
+
by;
|
|
25
|
+
constructor(reason, by) {
|
|
26
|
+
super(reason);
|
|
27
|
+
this.reason = reason;
|
|
28
|
+
this.by = by;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Brings dataset `name` current. A held dataset asks for the changes since its cursor; one that
|
|
33
|
+
* is not held, or that the server resets (past its horizon, another schema), seeds in pages. The
|
|
34
|
+
* rows are applied to a staging area and written in one transaction after the collection proof
|
|
35
|
+
* matches. A proof that fails after a seed is repaired by one patch from the first page's cursor,
|
|
36
|
+
* which covers rows that moved while the pages were read; a proof that still fails clears the
|
|
37
|
+
* dataset and reseeds once, and a second failure leaves it not held. Never throws: every end is
|
|
38
|
+
* an outcome.
|
|
39
|
+
*/
|
|
40
|
+
export async function syncDataset(deps, name) {
|
|
41
|
+
const { store, origin } = deps;
|
|
42
|
+
const epoch = deps.epoch();
|
|
43
|
+
let requests = 0;
|
|
44
|
+
const get = async (request) => {
|
|
45
|
+
requests += 1;
|
|
46
|
+
const response = await deps.fetch(`${origin}${PATHS.dataset}?${encodeDatasetRequest(request)}`, { cache: 'no-store', credentials: 'same-origin' });
|
|
47
|
+
const verdict = deps.guard?.(response) ?? 'ok';
|
|
48
|
+
if (verdict !== 'ok')
|
|
49
|
+
throw new Refused(verdict);
|
|
50
|
+
if (response.status === 409)
|
|
51
|
+
throw new Refused('declined', response.headers.get(HEADERS.provisionReason));
|
|
52
|
+
if (response.status === 400 || response.status === 404)
|
|
53
|
+
throw new Refused('unavailable');
|
|
54
|
+
if (!response.ok)
|
|
55
|
+
throw new TypeError(`dataset answered ${response.status}`);
|
|
56
|
+
const parsed = parseDatasetResponse(await response.json());
|
|
57
|
+
if (parsed.name !== name)
|
|
58
|
+
throw new Refused('bad-response');
|
|
59
|
+
return parsed;
|
|
60
|
+
};
|
|
61
|
+
const write = async (staging, at, status) => {
|
|
62
|
+
try {
|
|
63
|
+
if (deps.epoch() !== epoch)
|
|
64
|
+
throw new Refused('epoch');
|
|
65
|
+
if (!(await store.commit(staging, at, epoch)))
|
|
66
|
+
throw new Refused('epoch');
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (error instanceof Refused)
|
|
70
|
+
throw error;
|
|
71
|
+
throw new Refused('storage');
|
|
72
|
+
}
|
|
73
|
+
return { name, requests, status, cursor: at.cursor, rows: staging.rows.size };
|
|
74
|
+
};
|
|
75
|
+
const seed = async (first, status, retry) => {
|
|
76
|
+
let page = first ?? (await get({ name }));
|
|
77
|
+
const { cursor, schema } = page;
|
|
78
|
+
const budget = await deps.budget();
|
|
79
|
+
const staging = store.stage(name);
|
|
80
|
+
let matches;
|
|
81
|
+
let after;
|
|
82
|
+
for (let pages = 1;; pages += 1) {
|
|
83
|
+
if (deps.epoch() !== epoch)
|
|
84
|
+
throw new Refused('epoch');
|
|
85
|
+
matches = await staging.add(page);
|
|
86
|
+
if (budget !== undefined && staging.bytes > budget)
|
|
87
|
+
throw new Refused('budget');
|
|
88
|
+
if (page.next === undefined)
|
|
89
|
+
break;
|
|
90
|
+
if (pages >= DATASET_MAX_PAGES || (after !== undefined && page.next <= after)) {
|
|
91
|
+
throw new Refused('bad-response');
|
|
92
|
+
}
|
|
93
|
+
after = page.next;
|
|
94
|
+
page = await get({ name, after });
|
|
95
|
+
}
|
|
96
|
+
if (matches === true)
|
|
97
|
+
return write(staging, { cursor, schema }, status);
|
|
98
|
+
// rows moved while the pages were read: the changes since the first page repair the copy
|
|
99
|
+
const patch = await get({ name, since: cursor, schema });
|
|
100
|
+
if (!patch.reset && (await staging.add(patch)) === true) {
|
|
101
|
+
return write(staging, { cursor: patch.cursor, schema }, status);
|
|
102
|
+
}
|
|
103
|
+
if (retry)
|
|
104
|
+
return seed(undefined, 'reseeded', false);
|
|
105
|
+
throw new Refused('proof');
|
|
106
|
+
};
|
|
107
|
+
try {
|
|
108
|
+
const held = await store.get(name);
|
|
109
|
+
if (!held)
|
|
110
|
+
return await seed(undefined, 'seeded', true);
|
|
111
|
+
const { cursor, schema, digest, count } = held.meta;
|
|
112
|
+
const response = await get({ name, since: cursor, schema });
|
|
113
|
+
if (response.reset)
|
|
114
|
+
return await seed(response, 'reseeded', true);
|
|
115
|
+
if (response.cursor < cursor)
|
|
116
|
+
return { name, requests, status: 'current', cursor, rows: count };
|
|
117
|
+
const proof = response.proof;
|
|
118
|
+
if (response.cursor === cursor) {
|
|
119
|
+
// nothing moved; a proof that disagrees with the held rows means the copy is damaged
|
|
120
|
+
if (proof && (proof.count !== count || proof.digest !== digest)) {
|
|
121
|
+
await store.drop(name);
|
|
122
|
+
return await seed(undefined, 'reseeded', false);
|
|
123
|
+
}
|
|
124
|
+
if (!(await store.touch(name, epoch)))
|
|
125
|
+
throw new Refused('epoch');
|
|
126
|
+
return { name, requests, status: 'current', cursor, rows: count };
|
|
127
|
+
}
|
|
128
|
+
const staging = store.stage(name, held);
|
|
129
|
+
if ((await staging.add(response)) === true) {
|
|
130
|
+
return await write(staging, { cursor: response.cursor, schema }, 'patched');
|
|
131
|
+
}
|
|
132
|
+
// the patched rows do not match the server's proof: nothing wrong is kept
|
|
133
|
+
await store.drop(name);
|
|
134
|
+
return await seed(undefined, 'reseeded', false);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (error instanceof Refused) {
|
|
138
|
+
// the server stopped admitting this principal to the dataset, so no row of it stays
|
|
139
|
+
if (error.reason === 'revoked' && deps.epoch() === epoch) {
|
|
140
|
+
await store.drop(name).catch(() => undefined);
|
|
141
|
+
}
|
|
142
|
+
return { name, requests, status: 'refused', reason: error.reason, by: error.by };
|
|
143
|
+
}
|
|
144
|
+
if (error instanceof ProtocolError) {
|
|
145
|
+
return { name, requests, status: 'refused', reason: 'bad-response' };
|
|
146
|
+
}
|
|
147
|
+
return { name, requests, status: 'offline' };
|
|
148
|
+
}
|
|
149
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { type DatasetResponse, type DatasetRow } from '../protocol/dataset.js';
|
|
2
|
+
import { type PlanTables } from '../protocol/plan.js';
|
|
3
|
+
import type { DatasetMeta, DatasetRowRecord, ReplicaStorage } from './storage.js';
|
|
4
|
+
/** bytes of a row's JSON, the unit `maxBytes` and the storage budget count in */
|
|
5
|
+
export declare const rowBytes: (row: DatasetRow) => number;
|
|
6
|
+
/** a dataset in memory, as a verified load or a commit left it */
|
|
7
|
+
export type Held = {
|
|
8
|
+
meta: DatasetMeta;
|
|
9
|
+
rows: Map<string, DatasetRow>;
|
|
10
|
+
/** the hash of every row, so a patch never hashes a row it replaces or deletes */
|
|
11
|
+
hashes: Map<string, string>;
|
|
12
|
+
};
|
|
13
|
+
/** what the store needs from the replica around it */
|
|
14
|
+
export type DatasetStoreDeps = {
|
|
15
|
+
storage: ReplicaStorage;
|
|
16
|
+
now(): number;
|
|
17
|
+
/** the replica's wipe count; a commit that began under another one is dropped */
|
|
18
|
+
epoch(): number;
|
|
19
|
+
/** the row hash; {@link rowHash}. A test counts calls through it */
|
|
20
|
+
hash?: (key: string, row: DatasetRow) => Promise<string>;
|
|
21
|
+
/** told after every commit which row keys it put and which it removed */
|
|
22
|
+
onChange?(name: string, puts: string[], deletes: string[]): Promise<void> | void;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* One dataset's state while a response or a run of seed pages is applied, before anything is
|
|
26
|
+
* written. A staged seed replaces the held rows; a staged patch changes them. Nothing here
|
|
27
|
+
* touches storage, so a failed or dropped apply leaves the store as it was.
|
|
28
|
+
*/
|
|
29
|
+
export declare class Staging {
|
|
30
|
+
#private;
|
|
31
|
+
readonly name: string;
|
|
32
|
+
/** whether the commit replaces every held row */
|
|
33
|
+
clear: boolean;
|
|
34
|
+
rows: Map<string, DatasetRow>;
|
|
35
|
+
hashes: Map<string, string>;
|
|
36
|
+
digest: string;
|
|
37
|
+
bytes: number;
|
|
38
|
+
constructor(deps: DatasetStoreDeps, name: string, base?: Held);
|
|
39
|
+
/** applies a response; returns whether the result equals the response's proof, null without one */
|
|
40
|
+
add(response: DatasetResponse): Promise<boolean | null>;
|
|
41
|
+
/** the rows a commit writes: all of them for a replacement, the touched ones for a patch */
|
|
42
|
+
writes(): {
|
|
43
|
+
puts: DatasetRowRecord[];
|
|
44
|
+
deletes: string[];
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The datasets a device holds. Rows live once in memory after a verified load and once in
|
|
49
|
+
* storage, and a response is applied to a {@link Staging} first and committed in one storage
|
|
50
|
+
* transaction. Reads and writes of the held set go through one queue, so a commit and a wipe
|
|
51
|
+
* never interleave.
|
|
52
|
+
*/
|
|
53
|
+
export declare class DatasetStore {
|
|
54
|
+
#private;
|
|
55
|
+
/** datasets found damaged on a load and cleared, since the worker started */
|
|
56
|
+
evicted: number;
|
|
57
|
+
constructor(deps: DatasetStoreDeps);
|
|
58
|
+
/** the meta of every held dataset */
|
|
59
|
+
list(): Promise<DatasetMeta[]>;
|
|
60
|
+
/**
|
|
61
|
+
* Loads one dataset into memory and checks it: the stored row count and the sum of the stored row
|
|
62
|
+
* hashes must equal the meta. A dataset that fails, which is what a partly evicted store looks
|
|
63
|
+
* like, is cleared and reads as absent. No row is hashed.
|
|
64
|
+
*/
|
|
65
|
+
get(name: string): Promise<Held | undefined>;
|
|
66
|
+
/** the held rows in ascending key order, or undefined when `name` is not loaded and held */
|
|
67
|
+
rows(name: string): Promise<DatasetRow[] | undefined>;
|
|
68
|
+
/** the base table and each joined table a plan reads, or undefined when any of them is not held */
|
|
69
|
+
tables(base: string, joins: readonly string[]): Promise<PlanTables | undefined>;
|
|
70
|
+
/** whether `name` is held and in memory */
|
|
71
|
+
loaded(name: string): boolean;
|
|
72
|
+
/** a held row, looked up in memory; undefined unless {@link DatasetStore.get} loaded the dataset */
|
|
73
|
+
row(name: string, key: string): DatasetRow | undefined;
|
|
74
|
+
/** a staging area over the held rows of `name`, or an empty one for a seed */
|
|
75
|
+
stage(name: string, base?: Held): Staging;
|
|
76
|
+
/**
|
|
77
|
+
* Writes a staged apply. `cursor` and `schema` are what the response brought the rows to.
|
|
78
|
+
* Returns false, writing nothing, when the replica was wiped since `epoch` was read. Rejects,
|
|
79
|
+
* with the held rows and the memory copy untouched, when the storage transaction fails.
|
|
80
|
+
*/
|
|
81
|
+
commit(staging: Staging, at: {
|
|
82
|
+
cursor: number;
|
|
83
|
+
schema: string;
|
|
84
|
+
}, epoch: number): Promise<boolean>;
|
|
85
|
+
/** records that a sync found the held rows current, with no change to them */
|
|
86
|
+
touch(name: string, epoch: number): Promise<boolean>;
|
|
87
|
+
/** clears one dataset's rows and meta, and nothing else */
|
|
88
|
+
drop(name: string): Promise<void>;
|
|
89
|
+
/** clears every dataset, for a wipe of the replica */
|
|
90
|
+
clear(): Promise<void>;
|
|
91
|
+
}
|