@zubra/sync 5.5.0 → 5.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -4
- package/src/connect.d.ts +8 -0
- package/src/connect.js +21 -11
- package/src/durable-core.js +38 -0
- package/src/durable-log.d.ts +14 -0
- package/src/durable-log.js +28 -0
- package/src/durable-stub.js +47 -0
- package/src/durable.d.ts +47 -0
- package/src/durable.js +55 -0
- package/src/index.js +33 -2
- package/src/server.d.ts +48 -3
- package/src/server.js +291 -32
- package/src/sql-log.js +86 -0
- package/src/sqlite.d.ts +7 -2
- package/src/sqlite.js +28 -35
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zubra/sync",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.7.0",
|
|
4
4
|
"description": "Local-first state for Zubra components: 'use collection' keeps a component's state in IndexedDB, across reloads, offline, and in step across tabs.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
@@ -21,14 +21,18 @@
|
|
|
21
21
|
"./sqlite": {
|
|
22
22
|
"types": "./src/sqlite.d.ts",
|
|
23
23
|
"default": "./src/sqlite.js"
|
|
24
|
+
},
|
|
25
|
+
"./durable": {
|
|
26
|
+
"types": "./src/durable.d.ts",
|
|
27
|
+
"default": "./src/durable.js"
|
|
24
28
|
}
|
|
25
29
|
},
|
|
26
30
|
"scripts": {
|
|
27
|
-
"test": "node --test
|
|
28
|
-
"test:binary": "ZUBRA_SYNC_BACKEND=binary node --test
|
|
31
|
+
"test": "node --test \"test/*.test.js\"",
|
|
32
|
+
"test:binary": "ZUBRA_SYNC_BACKEND=binary node --test test/browser.test.js"
|
|
29
33
|
},
|
|
30
34
|
"peerDependencies": {
|
|
31
|
-
"@zubra/runtime": "^5.
|
|
35
|
+
"@zubra/runtime": "^5.7.0"
|
|
32
36
|
},
|
|
33
37
|
"devDependencies": {
|
|
34
38
|
"@zubra/compiler": "file:../compiler",
|
package/src/connect.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { Scope } from './server.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Share 'use collection' documents through a log at `url` (default `/_zb/sync`), pulling every
|
|
3
5
|
* `interval` ms (default 2000). Writes stay on the device first and go out when the network allows.
|
|
@@ -13,6 +15,12 @@ export function connect(options?: {
|
|
|
13
15
|
* server carries introductions, goodbyes, signaling, and presence to anyone else.
|
|
14
16
|
*/
|
|
15
17
|
webrtc?: boolean | { iceServers?: RTCIceServer[] };
|
|
18
|
+
/**
|
|
19
|
+
* Only what this page shows: per collection, `true` or a key prefix (or a list of them), e.g.
|
|
20
|
+
* `{ seats: 'showing-12/', brands: true }`. Changes and presence outside it aren't sent. Each
|
|
21
|
+
* scope keeps its own cursor. Leave it out for everything.
|
|
22
|
+
*/
|
|
23
|
+
scope?: Scope;
|
|
16
24
|
}): {
|
|
17
25
|
flush(): Promise<void>;
|
|
18
26
|
pull(): Promise<void>;
|
package/src/connect.js
CHANGED
|
@@ -7,13 +7,13 @@
|
|
|
7
7
|
//
|
|
8
8
|
// The protocol (plan/sync-design.md), plain HTTP so any server can keep the log:
|
|
9
9
|
// POST <url>/push { changes: [Change] } → { ok: true }
|
|
10
|
-
// GET <url>/pull?since=<cursor>
|
|
10
|
+
// GET <url>/pull?since=<cursor>[&scope=<json>] → { changes: [Change], cursor }
|
|
11
11
|
// Change = { collection, key, field, value, clock } (clock: a hybrid logical clock, a string
|
|
12
12
|
// that sorts by time, then counter, then client id)
|
|
13
13
|
//
|
|
14
14
|
// Merging is last-writer-wins per field, by clock. A device that was offline pushes what it
|
|
15
15
|
// wrote when it's back; whichever write has the later clock wins, on every device alike.
|
|
16
|
-
import { onWrite, receive, currentStore, onPresence, receivePresence, peerId } from './index.js';
|
|
16
|
+
import { onWrite, receive, currentStore, onPresence, receivePresence, peerId, startingPull } from './index.js';
|
|
17
17
|
import { mesh } from './webrtc.js';
|
|
18
18
|
|
|
19
19
|
const CLOCKS = '$clocks';
|
|
@@ -64,17 +64,23 @@ const keep = (name, value) => { try { localStorage.setItem(name, value); } catch
|
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
66
|
* Start sharing. `url`: where the log is (`/_zb/sync`). `interval`: how often to pull, in ms
|
|
67
|
-
* (default 2000).
|
|
67
|
+
* (default 2000). `scope`: only what this page shows, per collection `true` or a key prefix (or a
|
|
68
|
+
* list of them), e.g. `{ seats: 'showing-12/', brands: true }`; leave it out for everything.
|
|
69
|
+
* Returns `{ flush, pull, stop }`.
|
|
68
70
|
*/
|
|
69
|
-
export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrtc = false } = {}) {
|
|
71
|
+
export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrtc = false, scope = null } = {}) {
|
|
70
72
|
const base = url.replace(/\/$/, '');
|
|
73
|
+
// The scope goes with every pull and live request. Each scope keeps its own cursor: a page that
|
|
74
|
+
// moves to another showing reads that showing from the start, not from where the last one was.
|
|
75
|
+
const scoped = scope ? `&scope=${encodeURIComponent(JSON.stringify(scope))}` : '';
|
|
76
|
+
const cursorName = scope ? `zubra-sync:cursor:${JSON.stringify(scope)}` : 'zubra-sync:cursor';
|
|
71
77
|
const client = remember('zubra-sync:client', () => {
|
|
72
78
|
const id = Math.random().toString(36).slice(2, 10);
|
|
73
79
|
keep('zubra-sync:client', id);
|
|
74
80
|
return id;
|
|
75
81
|
});
|
|
76
82
|
const clocks = clock(client);
|
|
77
|
-
let cursor = remember(
|
|
83
|
+
let cursor = remember(cursorName, () => '0');
|
|
78
84
|
const outbox = outboxOf(JSON.parse(remember('zubra-sync:outbox', () => '[]')), (pending) =>
|
|
79
85
|
keep('zubra-sync:outbox', JSON.stringify(pending)));
|
|
80
86
|
|
|
@@ -116,7 +122,7 @@ export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrt
|
|
|
116
122
|
async function pull() {
|
|
117
123
|
let body;
|
|
118
124
|
try {
|
|
119
|
-
const response = await fetch(`${base}/pull?since=${encodeURIComponent(cursor)}`);
|
|
125
|
+
const response = await fetch(`${base}/pull?since=${encodeURIComponent(cursor)}${scoped}`);
|
|
120
126
|
if (!response.ok) return;
|
|
121
127
|
body = await response.json();
|
|
122
128
|
} catch {
|
|
@@ -151,10 +157,11 @@ export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrt
|
|
|
151
157
|
}
|
|
152
158
|
if (Object.keys(fields).length) await receive(collection, key, { ...fields, [CLOCKS]: known });
|
|
153
159
|
}
|
|
154
|
-
// Never move the cursor back (a live backlog can arrive after a newer push)
|
|
160
|
+
// Never move the cursor back (a live backlog can arrive after a newer push), unless the server
|
|
161
|
+
// says this device's cursor was past the end of its log: then it starts over from what it sent.
|
|
155
162
|
const next = String(body.cursor);
|
|
156
|
-
if (!(/^\d+$/.test(next) && /^\d+$/.test(cursor) && Number(next) < Number(cursor))) cursor = next;
|
|
157
|
-
keep(
|
|
163
|
+
if (body.reset || !(/^\d+$/.test(next) && /^\d+$/.test(cursor) && Number(next) < Number(cursor))) cursor = next;
|
|
164
|
+
keep(cursorName, cursor);
|
|
158
165
|
}
|
|
159
166
|
|
|
160
167
|
// Live: a server-sent event stream brings changes as they're pushed, and other viewers'
|
|
@@ -164,7 +171,7 @@ export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrt
|
|
|
164
171
|
let retry = null;
|
|
165
172
|
const listen = () => {
|
|
166
173
|
if (!live || typeof EventSource === 'undefined' || stopped) return;
|
|
167
|
-
source = new EventSource(`${base}/live?since=${encodeURIComponent(cursor)}&peer=${encodeURIComponent(peerId())}`);
|
|
174
|
+
source = new EventSource(`${base}/live?since=${encodeURIComponent(cursor)}&peer=${encodeURIComponent(peerId())}${scoped}`);
|
|
168
175
|
source.addEventListener('open', () => (open = true));
|
|
169
176
|
source.addEventListener('changes', (event) => apply(JSON.parse(event.data)));
|
|
170
177
|
source.addEventListener('presence', (event) => {
|
|
@@ -215,7 +222,10 @@ export function connect({ url = '/_zb/sync', interval = 2000, live = true, webrt
|
|
|
215
222
|
const loop = setInterval(() => flush().then(() => (open ? undefined : pull())), interval);
|
|
216
223
|
const online = () => flush().then(pull);
|
|
217
224
|
addEventListener('online', online);
|
|
218
|
-
|
|
225
|
+
// Components wait for this first pull before applying what's stored (index.js: fresh first).
|
|
226
|
+
const first = flush().then(pull);
|
|
227
|
+
startingPull(first);
|
|
228
|
+
first.then(listen);
|
|
219
229
|
return {
|
|
220
230
|
flush,
|
|
221
231
|
pull,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// What a SyncObject is, without Workers' modules: a sync server on the object's storage, its
|
|
2
|
+
// `authorize` asked of the object, and a snapshot of the documents for the Worker to render from.
|
|
3
|
+
// durable.js wraps this in a DurableObject; tests run it on Node against a stand-in storage.
|
|
4
|
+
import { createSyncServer, MOUNT } from './server.js';
|
|
5
|
+
import { durableObjectLog } from './durable-log.js';
|
|
6
|
+
|
|
7
|
+
export { MOUNT };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* `owner` is the object: its `authorize(…)`, if it has one, decides as `createSyncServer`'s does.
|
|
11
|
+
* `heartbeat` is shorter than on Node (2 s, not 15): a Durable Object isn't told when a page's
|
|
12
|
+
* connection is cut, only that a keep-alive couldn't be written, which on workerd is the third
|
|
13
|
+
* after the cut. So a crashed page's presence lingers about three keep-alives (6 s). A page that
|
|
14
|
+
* closes normally says it's leaving and is gone at once.
|
|
15
|
+
*/
|
|
16
|
+
export function syncObject(ctx, owner, { mount = MOUNT, heartbeat = 2_000, device = 'server', table = 'changes', origins = [] } = {}) {
|
|
17
|
+
const sync = createSyncServer({
|
|
18
|
+
log: durableObjectLog(ctx.storage, { table }),
|
|
19
|
+
mount,
|
|
20
|
+
heartbeat,
|
|
21
|
+
device,
|
|
22
|
+
origins,
|
|
23
|
+
authorize: (asked) => (typeof owner.authorize === 'function' ? owner.authorize(asked) : true),
|
|
24
|
+
});
|
|
25
|
+
return {
|
|
26
|
+
async fetch(request) {
|
|
27
|
+
return (await sync.fetch(request)) ?? new Response(JSON.stringify({ error: 'Not found' }), { status: 404, headers: { 'content-type': 'application/json' } });
|
|
28
|
+
},
|
|
29
|
+
documents: sync.documents,
|
|
30
|
+
write: sync.write,
|
|
31
|
+
/** `{ collection: [{ key, ...fields }] }` for the collections named, or every one: plain data, for RPC. */
|
|
32
|
+
async snapshot(collections = []) {
|
|
33
|
+
const docs = await sync.documents();
|
|
34
|
+
const names = collections.length ? collections : docs.collections();
|
|
35
|
+
return Object.fromEntries(names.map((name) => [name, docs.all(name)]));
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Log } from './server.js';
|
|
2
|
+
|
|
3
|
+
/** The parts of a Durable Object's `ctx.storage` the log uses. */
|
|
4
|
+
export interface DurableObjectStorageLike {
|
|
5
|
+
sql: { exec(query: string, ...bindings: unknown[]): { toArray(): any[]; one(): any } };
|
|
6
|
+
transactionSync<T>(fn: () => T): T;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A sync log in a Durable Object's SQLite: the tables and cursors `sqliteLog` uses. `keep:
|
|
11
|
+
* 'latest'` keeps each field's latest value only. `table` names the table (default `changes`, or
|
|
12
|
+
* `latest`), if the object keeps tables of its own.
|
|
13
|
+
*/
|
|
14
|
+
export function durableObjectLog(storage: DurableObjectStorageLike, options?: { table?: string; keep?: 'all' | 'latest' }): Log;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// A sync log in a Cloudflare Durable Object's own SQLite (`ctx.storage.sql`): the same tables and
|
|
2
|
+
// cursors as `sqliteLog`, so a log can move between Node and a Durable Object. It runs anywhere a
|
|
3
|
+
// Durable Object's storage is passed in; nothing here imports Workers' modules.
|
|
4
|
+
//
|
|
5
|
+
// import { createSyncServer } from '@zubra/sync/server';
|
|
6
|
+
// import { durableObjectLog } from '@zubra/sync/durable';
|
|
7
|
+
// const sync = createSyncServer({ log: durableObjectLog(ctx.storage, { keep: 'latest' }) });
|
|
8
|
+
//
|
|
9
|
+
// `keep: 'latest'` keeps each field's latest value only (see sql-log.js); by default every change
|
|
10
|
+
// is kept. `table` names the table, if the object keeps tables of its own (default `changes`, or
|
|
11
|
+
// `latest`). Appends are one transaction (`transactionSync`): a batch is written whole or not at all.
|
|
12
|
+
import { sqlLog } from './sql-log.js';
|
|
13
|
+
|
|
14
|
+
export function durableObjectLog(storage, { table, keep = 'all' } = {}) {
|
|
15
|
+
if (!storage?.sql || typeof storage.transactionSync !== 'function') {
|
|
16
|
+
throw new TypeError("durableObjectLog(storage): pass a Durable Object's `ctx.storage` (SQLite-backed)");
|
|
17
|
+
}
|
|
18
|
+
const { sql } = storage;
|
|
19
|
+
return sqlLog(
|
|
20
|
+
{
|
|
21
|
+
exec: (query) => sql.exec(query),
|
|
22
|
+
run: (query, ...args) => sql.exec(query, ...args),
|
|
23
|
+
all: (query, ...args) => sql.exec(query, ...args).toArray(),
|
|
24
|
+
transaction: (fn) => storage.transactionSync(fn),
|
|
25
|
+
},
|
|
26
|
+
{ keep, table, label: 'durableObjectLog' },
|
|
27
|
+
);
|
|
28
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// The Worker's side of a SyncObject: forward sync requests to it, read its documents, write as
|
|
2
|
+
// the server. Plain JavaScript (the namespace is a binding the Worker is given), so it runs on
|
|
3
|
+
// Node in tests too.
|
|
4
|
+
import { MOUNT, documentsView } from './server.js';
|
|
5
|
+
|
|
6
|
+
/** A snapshot (`{ collection: [{ key, ...fields }] }`) read like `sync.documents()`. */
|
|
7
|
+
export function viewOf(snapshot) {
|
|
8
|
+
const split = ({ key, ...fields }) => ({ key, fields });
|
|
9
|
+
return documentsView({
|
|
10
|
+
find: (collection, key) => {
|
|
11
|
+
const doc = (snapshot[collection] ?? []).find((each) => each.key === key);
|
|
12
|
+
return doc && split(doc).fields;
|
|
13
|
+
},
|
|
14
|
+
list: (collection) => (snapshot[collection] ?? []).map(split),
|
|
15
|
+
names: () => Object.keys(snapshot),
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* `namespace` is the Durable Object binding (`env.SYNC`); `name` picks the object (one per app:
|
|
21
|
+
* the log's cursor is one order). The stub's own methods (`stub.placeOrder(…)`) are the object's,
|
|
22
|
+
* over RPC: put a read-then-write that mustn't interleave there, not in the Worker.
|
|
23
|
+
*/
|
|
24
|
+
export function syncStub(namespace, { name = 'sync', mount = MOUNT } = {}) {
|
|
25
|
+
if (!namespace || typeof namespace.idFromName !== 'function') {
|
|
26
|
+
throw new TypeError('syncStub(env.SYNC): pass the Durable Object namespace binding');
|
|
27
|
+
}
|
|
28
|
+
const stub = namespace.get(namespace.idFromName(name));
|
|
29
|
+
const base = mount.replace(/\/$/, '');
|
|
30
|
+
return {
|
|
31
|
+
stub,
|
|
32
|
+
/** The object's answer to a sync request, or null when the request isn't one. */
|
|
33
|
+
async fetch(request) {
|
|
34
|
+
return new URL(request.url).pathname.startsWith(`${base}/`) ? stub.fetch(request) : null;
|
|
35
|
+
},
|
|
36
|
+
/**
|
|
37
|
+
* The documents as they are now, for rendering: a copy, not kept current, and reading it then
|
|
38
|
+
* writing from the Worker can interleave with other requests. Name the collections you need
|
|
39
|
+
* to send less.
|
|
40
|
+
*/
|
|
41
|
+
async documents(...collections) {
|
|
42
|
+
return viewOf(await stub.snapshot(collections));
|
|
43
|
+
},
|
|
44
|
+
/** Write as the server, through the object: to its log and every live page. */
|
|
45
|
+
write: (collection, key, fields) => stub.write(collection, key, fields),
|
|
46
|
+
};
|
|
47
|
+
}
|
package/src/durable.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { Change, Documents } from './server.js';
|
|
2
|
+
export { durableObjectLog, type DurableObjectStorageLike } from './durable-log.js';
|
|
3
|
+
|
|
4
|
+
/** `{ collection: [{ key, ...fields }] }` */
|
|
5
|
+
export type Snapshot = Record<string, Array<{ key: string } & Record<string, unknown>>>;
|
|
6
|
+
|
|
7
|
+
export interface SyncObjectOptions {
|
|
8
|
+
/** Default `/_zb/sync`; the Worker's `syncStub` must use the same. */
|
|
9
|
+
mount?: string;
|
|
10
|
+
/** Keep-alive interval (ms, default 2000). A page whose connection is cut is noticed about three keep-alives later. */
|
|
11
|
+
heartbeat?: number;
|
|
12
|
+
/** The server's name in its clocks (default `server`). */
|
|
13
|
+
device?: string;
|
|
14
|
+
/** The log's table (default `changes`). */
|
|
15
|
+
table?: string;
|
|
16
|
+
/** Other origins allowed to push. */
|
|
17
|
+
origins?: string[];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Extend it with your Durable Object class. Methods you add are the stub's, over RPC. */
|
|
21
|
+
export class SyncObject<Env = unknown> {
|
|
22
|
+
static options?: SyncObjectOptions;
|
|
23
|
+
constructor(ctx: any, env: Env);
|
|
24
|
+
protected ctx: any;
|
|
25
|
+
protected env: Env;
|
|
26
|
+
/** Refuse a write or hide a change; as `createSyncServer`'s. Allows everything if not defined. */
|
|
27
|
+
authorize?(asked: { request: Request; collection: string; key: string; write: boolean; kind: 'change' | 'presence' | 'read' }): boolean | Promise<boolean>;
|
|
28
|
+
fetch(request: Request): Promise<Response>;
|
|
29
|
+
documents(): Promise<Documents>;
|
|
30
|
+
write(collection: string, key: string, fields: Record<string, unknown>): Promise<Change[]>;
|
|
31
|
+
snapshot(collections?: string[]): Promise<Snapshot>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function viewOf(snapshot: Snapshot): Documents;
|
|
35
|
+
|
|
36
|
+
export function syncStub<Stub = any>(
|
|
37
|
+
namespace: { idFromName(name: string): unknown; get(id: unknown): Stub },
|
|
38
|
+
options?: { name?: string; mount?: string },
|
|
39
|
+
): {
|
|
40
|
+
/** The object's stub: your SyncObject's own methods, over RPC. */
|
|
41
|
+
stub: Stub;
|
|
42
|
+
/** The object's answer to a sync request, or null when it isn't one. */
|
|
43
|
+
fetch(request: Request): Promise<Response | null>;
|
|
44
|
+
/** A copy of the documents (the collections named, or all), for rendering. Not kept current. */
|
|
45
|
+
documents(...collections: string[]): Promise<Documents>;
|
|
46
|
+
write(collection: string, key: string, fields: Record<string, unknown>): Promise<Change[]>;
|
|
47
|
+
};
|
package/src/durable.js
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// @zubra/sync on Cloudflare Workers. A Durable Object holds the sync server (one per app), with
|
|
2
|
+
// the log in its own SQLite; the Worker forwards sync requests to it and reads and writes through
|
|
3
|
+
// it. Extend SyncObject, and put what must not interleave (check stock, then take it) in its
|
|
4
|
+
// methods: the object runs one call at a time.
|
|
5
|
+
//
|
|
6
|
+
// import { SyncObject, syncStub } from '@zubra/sync/durable';
|
|
7
|
+
//
|
|
8
|
+
// export class Market extends SyncObject {
|
|
9
|
+
// static options = { device: 'market' }; // createSyncServer's, and `table`
|
|
10
|
+
// authorize({ request, collection, key, kind }) { … } // as on Node; secrets are this.env's
|
|
11
|
+
// async take(key, qty) {
|
|
12
|
+
// const left = (await this.documents()).get('stock', key)?.qty ?? 0;
|
|
13
|
+
// if (left < qty) return { error: 'Sold out' };
|
|
14
|
+
// await this.write('stock', key, { qty: left - qty });
|
|
15
|
+
// return { ok: true };
|
|
16
|
+
// }
|
|
17
|
+
// }
|
|
18
|
+
//
|
|
19
|
+
// export default {
|
|
20
|
+
// async fetch(request, env) {
|
|
21
|
+
// const sync = syncStub(env.MARKET);
|
|
22
|
+
// return (await sync.fetch(request)) ?? render(request, await sync.documents('stock'));
|
|
23
|
+
// },
|
|
24
|
+
// };
|
|
25
|
+
// // wrangler.jsonc: durable_objects.bindings [{ name: 'MARKET', class_name: 'Market' }],
|
|
26
|
+
// // migrations [{ tag: 'v1', new_sqlite_classes: ['Market'] }]
|
|
27
|
+
import { DurableObject } from 'cloudflare:workers';
|
|
28
|
+
import { syncObject } from './durable-core.js';
|
|
29
|
+
|
|
30
|
+
export { durableObjectLog } from './durable-log.js';
|
|
31
|
+
export { syncStub, viewOf } from './durable-stub.js';
|
|
32
|
+
|
|
33
|
+
export class SyncObject extends DurableObject {
|
|
34
|
+
#sync;
|
|
35
|
+
constructor(ctx, env) {
|
|
36
|
+
super(ctx, env);
|
|
37
|
+
this.#sync = syncObject(ctx, this, this.constructor.options);
|
|
38
|
+
}
|
|
39
|
+
/** Sync's requests (`/_zb/sync/*`), forwarded by the Worker's `syncStub(…).fetch`. */
|
|
40
|
+
fetch(request) {
|
|
41
|
+
return this.#sync.fetch(request);
|
|
42
|
+
}
|
|
43
|
+
/** Every document, kept current (in the object's own methods; the Worker uses the stub's). */
|
|
44
|
+
documents() {
|
|
45
|
+
return this.#sync.documents();
|
|
46
|
+
}
|
|
47
|
+
/** Write as the server: to the log and every live page. */
|
|
48
|
+
write(collection, key, fields) {
|
|
49
|
+
return this.#sync.write(collection, key, fields);
|
|
50
|
+
}
|
|
51
|
+
/** Plain data for the stub's `documents()`. */
|
|
52
|
+
snapshot(collections) {
|
|
53
|
+
return this.#sync.snapshot(collections);
|
|
54
|
+
}
|
|
55
|
+
}
|
package/src/index.js
CHANGED
|
@@ -48,10 +48,41 @@ export async function receive(collection, key, fields) {
|
|
|
48
48
|
/** @internal The store in use. */
|
|
49
49
|
export const currentStore = () => theStore();
|
|
50
50
|
|
|
51
|
+
// Fresh first: a page connected to a log reads it before a component applies what this device
|
|
52
|
+
// stored last time, so a returning visitor never sees last visit's state over the server's (or
|
|
53
|
+
// over a page the server just rendered). Offline, the pull fails at once and stored values show.
|
|
54
|
+
const FRESH_WAIT = 1500; // and never longer than this
|
|
55
|
+
let firstPull = null;
|
|
56
|
+
/** @internal connect() hands over its first pull, as it starts. */
|
|
57
|
+
export const startingPull = (pull) => {
|
|
58
|
+
firstPull = Promise.resolve(pull).catch(() => {});
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Before reading a stored document: wait until the page's own scripts have run (they finish before
|
|
62
|
+
* DOMContentLoaded, and connect() is one of them), then for the first pull if there is a connection.
|
|
63
|
+
*/
|
|
64
|
+
const fresh = () =>
|
|
65
|
+
scriptsDone().then(() => (firstPull ? Promise.race([firstPull, new Promise((done) => setTimeout(done, FRESH_WAIT))]) : undefined));
|
|
66
|
+
let scripts = null;
|
|
67
|
+
const scriptsDone = () =>
|
|
68
|
+
(scripts ??= (() => {
|
|
69
|
+
if (typeof document === 'undefined' || document.readyState === 'complete') return Promise.resolve();
|
|
70
|
+
// Navigation timing says whether DOMContentLoaded has already started (readyState is
|
|
71
|
+
// 'interactive' both while module scripts run and after).
|
|
72
|
+
const navigation = globalThis.performance?.getEntriesByType?.('navigation')?.[0];
|
|
73
|
+
if (navigation && navigation.domContentLoadedEventStart > 0) return Promise.resolve();
|
|
74
|
+
return new Promise((done) => {
|
|
75
|
+
document.addEventListener('DOMContentLoaded', done, { once: true });
|
|
76
|
+
setTimeout(done, FRESH_WAIT);
|
|
77
|
+
});
|
|
78
|
+
})());
|
|
79
|
+
|
|
51
80
|
// Every bound element, by document, so a write reaches the others (this tab and, below, others).
|
|
52
81
|
const bound = new Map();
|
|
53
82
|
const docId = (collection, key) => `${collection}\u0000${key}`;
|
|
54
83
|
const channel = typeof BroadcastChannel === 'undefined' ? null : new BroadcastChannel('zubra-sync');
|
|
84
|
+
// Outside a browser (Node, a test), an open channel would keep the process from exiting.
|
|
85
|
+
channel?.unref?.();
|
|
55
86
|
channel?.addEventListener('message', ({ data }) =>
|
|
56
87
|
data.kind === 'presence' ? receivePresence(data) : deliver(data.collection, data.key, data.fields, null));
|
|
57
88
|
|
|
@@ -140,8 +171,8 @@ export function collection(name, keys, props, fields, presenceFields = null, pee
|
|
|
140
171
|
let cancelled = false;
|
|
141
172
|
// What the fields hold as loading starts, to tell what someone changed while it loaded.
|
|
142
173
|
const before = Signal.subtle.untrack(values);
|
|
143
|
-
|
|
144
|
-
.get(name, key)
|
|
174
|
+
fresh()
|
|
175
|
+
.then(() => theStore().get(name, key))
|
|
145
176
|
.then((doc = {}) => {
|
|
146
177
|
if (cancelled) return;
|
|
147
178
|
// An edit made before the document arrived is newer than it: keep the edit (and save
|
package/src/server.d.ts
CHANGED
|
@@ -10,22 +10,67 @@ export interface Change {
|
|
|
10
10
|
export interface Log {
|
|
11
11
|
append(changes: Change[]): Promise<unknown>;
|
|
12
12
|
after(cursor: string, limit?: number): Promise<{ changes: Change[]; cursor: number | string }>;
|
|
13
|
+
/**
|
|
14
|
+
* The cursor after the last change. Optional: with it, a device whose cursor is past the end
|
|
15
|
+
* (the log was replaced) is answered from the start, with `reset: true`.
|
|
16
|
+
*/
|
|
17
|
+
head?(): Promise<number>;
|
|
13
18
|
}
|
|
14
19
|
|
|
15
|
-
|
|
20
|
+
/** A log in memory. `keep: 'latest'` keeps each field's latest value only. */
|
|
21
|
+
export function memoryLog(options?: { keep?: 'all' | 'latest' }): Log;
|
|
22
|
+
|
|
23
|
+
export interface Documents {
|
|
24
|
+
/** A document's fields (a copy), or undefined when nothing has been written to it. */
|
|
25
|
+
get(collection: string, key: string): Record<string, unknown> | undefined;
|
|
26
|
+
/** Every document in a collection, each with its key. */
|
|
27
|
+
all(collection: string): Array<{ key: string } & Record<string, unknown>>;
|
|
28
|
+
/** The collections that have documents, in the order they were first written to. */
|
|
29
|
+
collections(): string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Per collection, `true` (every document) or a key prefix, or a list of them. */
|
|
33
|
+
export type Scope = Record<string, true | string | string[]>;
|
|
34
|
+
|
|
35
|
+
/** A scope from a request's `?scope=` (JSON), or null for everything. Throws on anything else. */
|
|
36
|
+
export function parseScope(text: string | null | undefined): Scope | null;
|
|
37
|
+
/** Whether a document is in a scope (null is everything). */
|
|
38
|
+
export function inScope(scope: Scope | null, collection: string, key: string): boolean;
|
|
16
39
|
|
|
17
40
|
export function createSyncServer(options?: {
|
|
18
41
|
log?: Log;
|
|
19
42
|
/** Default `/_zb/sync`. */
|
|
20
43
|
mount?: string;
|
|
21
|
-
/**
|
|
22
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Refuse a write, or hide a change from a reader, by returning false. `kind` is what's asked:
|
|
46
|
+
* 'change' (a push), 'presence' (joining or signaling in a document's room; nothing is stored)
|
|
47
|
+
* or 'read' (a change about to be sent to this request). `write` is true for the first two.
|
|
48
|
+
*/
|
|
49
|
+
authorize?: (from: { request: Request; collection: string; key: string; write: boolean; kind: 'change' | 'presence' | 'read' }) => boolean | Promise<boolean>;
|
|
23
50
|
/** Other origins allowed to push. */
|
|
51
|
+
/**
|
|
52
|
+
* The scope a request may see and write (tenancy: a business's keys under its own prefix), or
|
|
53
|
+
* null for everything. A page's own `scope` can only narrow it; pushes and joins outside it are
|
|
54
|
+
* refused.
|
|
55
|
+
*/
|
|
56
|
+
scope?: (from: { request: Request }) => Scope | null | Promise<Scope | null>;
|
|
24
57
|
origins?: string[];
|
|
25
58
|
/** How often the live stream sends a keep-alive comment (ms, default 15000). */
|
|
26
59
|
heartbeat?: number;
|
|
60
|
+
/** The server's name in the clocks of its own writes (default `server`). */
|
|
61
|
+
device?: string;
|
|
27
62
|
}): {
|
|
28
63
|
fetch(request: Request): Promise<Response | null>;
|
|
64
|
+
/**
|
|
65
|
+
* Every document as the devices merge it (last writer wins, per field), read from the log the
|
|
66
|
+
* first time and kept current after. Its reads are synchronous.
|
|
67
|
+
*/
|
|
68
|
+
documents(): Promise<Documents>;
|
|
69
|
+
/**
|
|
70
|
+
* Write fields to a document as the server, with a clock later than any it has seen: to the
|
|
71
|
+
* documents view at once, then to the log and every live page. Resolves to the changes.
|
|
72
|
+
*/
|
|
73
|
+
write(collection: string, key: string, fields: Record<string, unknown>): Promise<Change[]>;
|
|
29
74
|
/** Node's http: answers sync requests (streaming the live ones) and resolves true, else false. */
|
|
30
75
|
handle(req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse): Promise<boolean>;
|
|
31
76
|
};
|
package/src/server.js
CHANGED
|
@@ -7,17 +7,44 @@
|
|
|
7
7
|
// const sync = createSyncServer();
|
|
8
8
|
// const response = await sync.fetch(request); // a Response, or null if not ours
|
|
9
9
|
//
|
|
10
|
-
// `authorize({ request, collection, key, write })` may refuse a change or hide one (return
|
|
10
|
+
// `authorize({ request, collection, key, write, kind })` may refuse a change or hide one (return
|
|
11
|
+
// false). `kind` says what's asked: 'read' (a change being sent to this request), 'change' (one
|
|
12
|
+
// it pushed) or 'presence' (joining a document's room, or signaling in it: nothing is stored).
|
|
13
|
+
// `write` is true for the last two.
|
|
14
|
+
//
|
|
15
|
+
// The server can take part too. `await sync.documents()` is every document as the devices see
|
|
16
|
+
// it (last writer wins, per field), kept current; `sync.write(collection, key, fields)` writes as
|
|
17
|
+
// the server, to the log and to every live page:
|
|
18
|
+
//
|
|
19
|
+
// const docs = await sync.documents();
|
|
20
|
+
// const left = docs.get('stock', 'eggs')?.qty ?? 0; // read and write with no await between,
|
|
21
|
+
// if (left > 0) sync.write('stock', 'eggs', { qty: left - 1 }); // and no request interleaves
|
|
22
|
+
//
|
|
23
|
+
// Scope: a page asks only for what it shows. `?scope=` on pull and live is JSON, per collection
|
|
24
|
+
// `true` or a key prefix (or a list of them): `{ "seats": "showing-12/", "brands": true }`.
|
|
25
|
+
// Changes and presence outside it aren't sent. `scope({ request })` imposes one from the server
|
|
26
|
+
// (tenancy: each business's keys under its own prefix), which the page's can only narrow; writes
|
|
27
|
+
// and joins outside it are refused.
|
|
28
|
+
//
|
|
29
|
+
// A cursor past the end of the log (the log was replaced, or compacted, or this device's cursor
|
|
30
|
+
// is from another log) starts over: pull and live answer from the beginning, with `reset: true`,
|
|
31
|
+
// and the device takes the new cursor. Logs that can say where they end have `head()`.
|
|
11
32
|
//
|
|
12
33
|
// Live: `GET <mount>/live?since=<cursor>&peer=<id>` is a server-sent event stream: `changes` as
|
|
13
34
|
// they're pushed, and `presence` from the other viewers (POST <mount>/presence). Subscribers are
|
|
14
35
|
// this process's; several server processes need a shared log and a pub/sub between them.
|
|
15
36
|
//
|
|
16
37
|
// On Node, `handle(req, res)` answers sync requests (streaming the live ones) and returns true.
|
|
17
|
-
const MOUNT = '/_zb/sync';
|
|
38
|
+
export const MOUNT = '/_zb/sync';
|
|
18
39
|
|
|
19
|
-
/**
|
|
20
|
-
|
|
40
|
+
/**
|
|
41
|
+
* A log in memory: `append(changes)` and `after(cursor)`. `keep: 'latest'` keeps each field's
|
|
42
|
+
* latest value only (a newer clock replaces it and takes the next cursor, an older one is dropped),
|
|
43
|
+
* as `sqliteLog(path, { keep: 'latest' })` does.
|
|
44
|
+
*/
|
|
45
|
+
export function memoryLog({ keep = 'all' } = {}) {
|
|
46
|
+
if (keep === 'latest') return latestInMemory();
|
|
47
|
+
if (keep !== 'all') throw new TypeError("memoryLog: `keep` is 'all' or 'latest'");
|
|
21
48
|
const changes = [];
|
|
22
49
|
return {
|
|
23
50
|
async append(more) {
|
|
@@ -28,17 +55,193 @@ export function memoryLog() {
|
|
|
28
55
|
const from = Number(cursor) || 0;
|
|
29
56
|
return { changes: changes.slice(from, from + limit), cursor: Math.min(changes.length, from + limit) };
|
|
30
57
|
},
|
|
58
|
+
head: async () => changes.length,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function latestInMemory() {
|
|
63
|
+
const rows = new Map(); // collection, key, field → { change, seq }
|
|
64
|
+
let seq = 0;
|
|
65
|
+
return {
|
|
66
|
+
async append(more) {
|
|
67
|
+
for (const change of more) {
|
|
68
|
+
const id = JSON.stringify([change.collection, change.key, change.field]);
|
|
69
|
+
const had = rows.get(id);
|
|
70
|
+
if (had && had.change.clock >= change.clock) continue;
|
|
71
|
+
rows.set(id, { change: structuredClone(change), seq: ++seq });
|
|
72
|
+
}
|
|
73
|
+
return seq;
|
|
74
|
+
},
|
|
75
|
+
async after(cursor, limit = 1000) {
|
|
76
|
+
const from = Number(cursor) || 0;
|
|
77
|
+
const page = [...rows.values()].filter((row) => row.seq > from).sort((a, b) => a.seq - b.seq).slice(0, limit);
|
|
78
|
+
return { changes: page.map((row) => structuredClone(row.change)), cursor: page.length ? page[page.length - 1].seq : from };
|
|
79
|
+
},
|
|
80
|
+
head: async () => seq,
|
|
31
81
|
};
|
|
32
82
|
}
|
|
33
83
|
|
|
84
|
+
/**
|
|
85
|
+
* A scope from a request's `?scope=`: `{ collection: true | prefix | [prefixes] }`, or null for
|
|
86
|
+
* everything. Throws on anything else.
|
|
87
|
+
*/
|
|
88
|
+
export function parseScope(text) {
|
|
89
|
+
if (text === null || text === undefined || text === '') return null;
|
|
90
|
+
let scope;
|
|
91
|
+
try {
|
|
92
|
+
scope = JSON.parse(text);
|
|
93
|
+
} catch {
|
|
94
|
+
throw new TypeError('scope is JSON');
|
|
95
|
+
}
|
|
96
|
+
if (scope === null) return null;
|
|
97
|
+
checkScope(scope);
|
|
98
|
+
return scope;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function checkScope(scope) {
|
|
102
|
+
const prefix = (each) => typeof each === 'string' && each.length <= 200;
|
|
103
|
+
const ok =
|
|
104
|
+
scope !== null && typeof scope === 'object' && !Array.isArray(scope) && Object.keys(scope).length <= 100 &&
|
|
105
|
+
Object.values(scope).every((each) => each === true || prefix(each) || (Array.isArray(each) && each.length <= 50 && each.every(prefix)));
|
|
106
|
+
if (!ok) throw new TypeError('scope is { collection: true | "key prefix" | ["prefixes"] }');
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Whether a document is in a scope (null is everything). */
|
|
110
|
+
export function inScope(scope, collection, key) {
|
|
111
|
+
if (!scope) return true;
|
|
112
|
+
const rule = scope[collection]; // inherited properties (constructor…) are none of these
|
|
113
|
+
if (rule === true) return true;
|
|
114
|
+
if (typeof rule === 'string') return key.startsWith(rule);
|
|
115
|
+
if (Array.isArray(rule)) return rule.some((prefix) => key.startsWith(prefix));
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
|
|
34
119
|
const json = (status, value) => new Response(JSON.stringify(value), { status, headers: { 'content-type': 'application/json' } });
|
|
35
120
|
|
|
121
|
+
// A clock is `<15-digit ms>-<5-digit counter>-<device>`, so clocks compare as strings.
|
|
122
|
+
const CLOCK = /^(\d{15})-(\d{5})-[\w-]+$/;
|
|
123
|
+
const clock = {
|
|
124
|
+
format: (time, counter, device) => `${String(time).padStart(15, '0')}-${String(counter).padStart(5, '0')}-${device}`,
|
|
125
|
+
/** `{ time, counter }`, or null when it isn't a clock. */
|
|
126
|
+
parse(text) {
|
|
127
|
+
const match = typeof text === 'string' && CLOCK.exec(text);
|
|
128
|
+
return match ? { time: Number(match[1]), counter: Number(match[2]) } : null;
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
|
|
36
132
|
const valid = (change) =>
|
|
37
133
|
change && typeof change.collection === 'string' && typeof change.key === 'string' && typeof change.field === 'string' &&
|
|
38
|
-
|
|
134
|
+
clock.parse(change.clock) !== null && change.field !== '$clocks';
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* How documents are read, wherever they're kept: `find(collection, key)` → fields or undefined,
|
|
138
|
+
* `list(collection)` → `[{ key, fields }]`, `names()` → the collections. What a reader gets is a
|
|
139
|
+
* copy, so changing it changes nothing.
|
|
140
|
+
*/
|
|
141
|
+
export function documentsView({ find, list, names }) {
|
|
142
|
+
return {
|
|
143
|
+
/** A document's fields (a copy), or undefined when nothing has been written to it. */
|
|
144
|
+
get(collection, key) {
|
|
145
|
+
const fields = find(collection, key);
|
|
146
|
+
return fields && structuredClone(fields);
|
|
147
|
+
},
|
|
148
|
+
/** Every document in a collection: `[{ key, ...fields }]`. */
|
|
149
|
+
all: (collection) => list(collection).map(({ key, fields }) => ({ key, ...structuredClone(fields) })),
|
|
150
|
+
/** The collections that have documents, in the order they were first written to. */
|
|
151
|
+
collections: () => names(),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** The documents a log describes, merged by each field's latest clock, as every device merges them. */
|
|
156
|
+
function documentsOf() {
|
|
157
|
+
const docs = new Map();
|
|
158
|
+
const id = (collection, key) => `${collection}\u0000${key}`;
|
|
159
|
+
return {
|
|
160
|
+
/** Applies a change if its clock is the field's latest; returns what undoes it, if it did. */
|
|
161
|
+
apply(change) {
|
|
162
|
+
let doc = docs.get(id(change.collection, change.key));
|
|
163
|
+
if (!doc) docs.set(id(change.collection, change.key), (doc = { collection: change.collection, key: change.key, fields: {}, clocks: {} }));
|
|
164
|
+
const known = doc.clocks[change.field];
|
|
165
|
+
if (known && known >= change.clock) return null;
|
|
166
|
+
const had = change.field in doc.fields;
|
|
167
|
+
const before = doc.fields[change.field];
|
|
168
|
+
doc.clocks[change.field] = change.clock;
|
|
169
|
+
doc.fields[change.field] = change.value;
|
|
170
|
+
return () => {
|
|
171
|
+
if (doc.clocks[change.field] !== change.clock) return; // something newer came since: it stands
|
|
172
|
+
if (had) [doc.clocks[change.field], doc.fields[change.field]] = [known, before];
|
|
173
|
+
else delete doc.clocks[change.field], delete doc.fields[change.field];
|
|
174
|
+
if (!Object.keys(doc.fields).length) docs.delete(id(change.collection, change.key));
|
|
175
|
+
};
|
|
176
|
+
},
|
|
177
|
+
view: documentsView({
|
|
178
|
+
find: (collection, key) => docs.get(id(collection, key))?.fields,
|
|
179
|
+
list: (collection) => [...docs.values()].filter((doc) => doc.collection === collection),
|
|
180
|
+
names: () => [...new Set([...docs.values()].map((doc) => doc.collection))],
|
|
181
|
+
}),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
39
184
|
|
|
40
|
-
export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize = () => true, origins = [], heartbeat = 15_000 } = {}) {
|
|
185
|
+
export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize = () => true, scope: serverScope = () => null, origins = [], heartbeat = 15_000, device = 'server' } = {}) {
|
|
186
|
+
if (!/^[\w-]{1,40}$/.test(device)) throw new Error('createSyncServer: `device` names the server in its clocks: letters, digits, - and _');
|
|
41
187
|
const base = mount.replace(/\/$/, '');
|
|
188
|
+
// Appends, one after another, so the log's order is the order they were made in.
|
|
189
|
+
let appending = Promise.resolve();
|
|
190
|
+
// The server's clock: later than any clock it has seen, so its writes win over what came
|
|
191
|
+
// before them (and lose to what devices write after, as any device's would).
|
|
192
|
+
let lastTime = 0;
|
|
193
|
+
let counter = 0;
|
|
194
|
+
const tick = () => {
|
|
195
|
+
const now = Math.max(Date.now(), lastTime);
|
|
196
|
+
counter = now === lastTime ? counter + 1 : 0;
|
|
197
|
+
lastTime = now;
|
|
198
|
+
return clock.format(now, counter, device);
|
|
199
|
+
};
|
|
200
|
+
const see = (change) => {
|
|
201
|
+
const { time, counter: seen } = clock.parse(change.clock);
|
|
202
|
+
if (time > lastTime) {
|
|
203
|
+
lastTime = time;
|
|
204
|
+
counter = seen;
|
|
205
|
+
} else if (time === lastTime) {
|
|
206
|
+
counter = Math.max(counter, seen);
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
// The documents, built from the log the first time they're asked for, then kept current.
|
|
210
|
+
// While they're being built, what's recorded goes into the view being built too (applying a
|
|
211
|
+
// change twice changes nothing), so nothing that arrives during the read is missed.
|
|
212
|
+
let docs = null;
|
|
213
|
+
let built = null;
|
|
214
|
+
let building = null;
|
|
215
|
+
/** Puts changes in the documents; returns what takes them back out (for a write the log refused). */
|
|
216
|
+
const record = (changes) => {
|
|
217
|
+
const undo = [];
|
|
218
|
+
for (const change of changes) {
|
|
219
|
+
see(change);
|
|
220
|
+
const taken = (docs ?? built)?.apply(change);
|
|
221
|
+
if (taken) undo.push(taken);
|
|
222
|
+
}
|
|
223
|
+
return () => undo.reverse().forEach((take) => take());
|
|
224
|
+
};
|
|
225
|
+
async function documents() {
|
|
226
|
+
if (docs) return docs.view;
|
|
227
|
+
building ??= (async () => {
|
|
228
|
+
built = documentsOf();
|
|
229
|
+
for (let cursor = 0; ; ) {
|
|
230
|
+
const page = await log.after(cursor, 5000);
|
|
231
|
+
for (const change of page.changes) {
|
|
232
|
+
see(change);
|
|
233
|
+
built.apply(change);
|
|
234
|
+
}
|
|
235
|
+
if (!page.changes.length || String(page.cursor) === String(cursor)) break;
|
|
236
|
+
cursor = page.cursor;
|
|
237
|
+
}
|
|
238
|
+
docs = built;
|
|
239
|
+
return built.view;
|
|
240
|
+
})();
|
|
241
|
+
return building;
|
|
242
|
+
}
|
|
243
|
+
// One that fails rejects for its own caller only; the next still runs.
|
|
244
|
+
const append = (changes) => (appending = appending.catch(() => {}).then(() => log.append(changes)));
|
|
42
245
|
// Live subscribers: { peer, request, send(event, data), rooms }.
|
|
43
246
|
const listeners = new Set();
|
|
44
247
|
const encoder = new TextEncoder();
|
|
@@ -46,22 +249,48 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
46
249
|
const at = room.indexOf('/');
|
|
47
250
|
return at > 0 ? { collection: room.slice(0, at), key: room.slice(at + 1) } : null;
|
|
48
251
|
};
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
252
|
+
/**
|
|
253
|
+
* What a request may see: inside the server's scope for it and the page's own, and allowed by
|
|
254
|
+
* authorize. `where` is `{ request, scopes: [server, page] }`.
|
|
255
|
+
*/
|
|
256
|
+
const sees = async ({ request, scopes }, collection, key) =>
|
|
257
|
+
scopes.every((scope) => inScope(scope, collection, key)) && (await authorize({ request, collection, key, write: false, kind: 'read' }));
|
|
258
|
+
async function visible(where, changes) {
|
|
259
|
+
const shown = [];
|
|
260
|
+
for (const change of changes) if (await sees(where, change.collection, change.key)) shown.push(change);
|
|
261
|
+
return shown;
|
|
262
|
+
}
|
|
263
|
+
/** The server's scope for a request and the page's (`?scope=`), or a Response refusing it. */
|
|
264
|
+
async function scopesOf(request, url) {
|
|
265
|
+
let page;
|
|
266
|
+
try {
|
|
267
|
+
page = parseScope(url.searchParams.get('scope'));
|
|
268
|
+
} catch (error) {
|
|
269
|
+
return json(400, { error: error.message });
|
|
56
270
|
}
|
|
271
|
+
const server = (await serverScope({ request })) ?? null;
|
|
272
|
+
if (server) checkScope(server);
|
|
273
|
+
return [server, page];
|
|
274
|
+
}
|
|
275
|
+
/** Where to read from for a device at `since`: from the start if it's past the end of the log. */
|
|
276
|
+
async function startFrom(since) {
|
|
277
|
+
const head = await log.head?.();
|
|
278
|
+
const at = Number(since);
|
|
279
|
+
if (head === undefined || !/^\d+$/.test(String(since)) || at <= Number(head)) return { since, reset: false };
|
|
280
|
+
return { since: '0', reset: true };
|
|
281
|
+
}
|
|
282
|
+
async function broadcastChanges(changes, cursor) {
|
|
283
|
+
for (const listener of listeners) listener.send('changes', { changes: await visible(listener, changes), cursor });
|
|
57
284
|
}
|
|
58
285
|
async function broadcastPresence(message) {
|
|
59
286
|
const { collection, key } = roomOf(message.room);
|
|
60
287
|
for (const listener of listeners) {
|
|
61
288
|
if (listener.peer === message.peer) continue;
|
|
62
|
-
if (await
|
|
289
|
+
if (await sees(listener, collection, key)) listener.send('presence', message);
|
|
63
290
|
}
|
|
64
291
|
}
|
|
292
|
+
/** Whether the server's scope lets this request write to (or join) a document. */
|
|
293
|
+
const mayWrite = async (request, collection, key) => inScope((await serverScope({ request })) ?? null, collection, key);
|
|
65
294
|
const sameOrigin = (request) => {
|
|
66
295
|
const site = request.headers.get('sec-fetch-site');
|
|
67
296
|
if (site) return site === 'same-origin' || site === 'none';
|
|
@@ -82,10 +311,12 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
82
311
|
const changes = Array.isArray(body?.changes) ? body.changes : null;
|
|
83
312
|
if (!changes || !changes.every(valid)) return json(400, { error: 'Expected { changes: [{ collection, key, field, value, clock }] }' });
|
|
84
313
|
for (const change of changes) {
|
|
85
|
-
if (!(await
|
|
314
|
+
if (!(await mayWrite(request, change.collection, change.key))) return json(403, { error: 'Not yours to change' });
|
|
315
|
+
if (!(await authorize({ request, collection: change.collection, key: change.key, write: true, kind: 'change' }))) return json(403, { error: 'Not yours to change' });
|
|
86
316
|
}
|
|
87
317
|
const appended = changes.map(({ collection, key, field, value, clock }) => ({ collection, key, field, value, clock }));
|
|
88
|
-
const cursor = await
|
|
318
|
+
const cursor = await append(appended);
|
|
319
|
+
record(appended); // once it's logged: the documents never hold what the log doesn't
|
|
89
320
|
await broadcastChanges(appended, cursor);
|
|
90
321
|
return json(200, { ok: true });
|
|
91
322
|
}
|
|
@@ -103,7 +334,7 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
103
334
|
if (!room || !okPeer || !okState || JSON.stringify(message.state).length > 4096) {
|
|
104
335
|
return json(400, { error: 'Expected { room: "<collection>/<key>", peer, state: {…} | null }' });
|
|
105
336
|
}
|
|
106
|
-
if (!(await authorize({ request, ...room, write: true }))) return json(403, { error: 'Not yours to join' });
|
|
337
|
+
if (!(await mayWrite(request, room.collection, room.key)) || !(await authorize({ request, ...room, write: true, kind: 'presence' }))) return json(403, { error: 'Not yours to join' });
|
|
107
338
|
const clean = { room: message.room, peer: message.peer, state: message.state };
|
|
108
339
|
for (const listener of listeners) if (listener.peer === clean.peer) {
|
|
109
340
|
if (clean.state === null) listener.rooms.delete(clean.room);
|
|
@@ -128,7 +359,7 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
128
359
|
if (!room || !peerOk(message.from) || !peerOk(message.to) || !dataOk) {
|
|
129
360
|
return json(400, { error: 'Expected { room, from, to, data: {…} }' });
|
|
130
361
|
}
|
|
131
|
-
if (!(await authorize({ request, ...room, write: true }))) return json(403, { error: 'Not yours to join' });
|
|
362
|
+
if (!(await mayWrite(request, room.collection, room.key)) || !(await authorize({ request, ...room, write: true, kind: 'presence' }))) return json(403, { error: 'Not yours to join' });
|
|
132
363
|
const clean = { room: message.room, from: message.from, to: message.to, data: message.data };
|
|
133
364
|
for (const listener of listeners) if (listener.peer === clean.to) listener.send('signal', clean);
|
|
134
365
|
return json(200, { ok: true });
|
|
@@ -136,7 +367,9 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
136
367
|
if (url.pathname === `${base}/live` && request.method === 'GET') {
|
|
137
368
|
const peer = url.searchParams.get('peer') ?? '';
|
|
138
369
|
if (!/^[\w-]{0,40}$/.test(peer)) return json(400, { error: 'Bad peer id' });
|
|
139
|
-
const
|
|
370
|
+
const scopes = await scopesOf(request, url);
|
|
371
|
+
if (scopes instanceof Response) return scopes;
|
|
372
|
+
const { since, reset } = await startFrom(url.searchParams.get('since') ?? '0');
|
|
140
373
|
let listener;
|
|
141
374
|
let timer;
|
|
142
375
|
const stream = new ReadableStream({
|
|
@@ -155,7 +388,7 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
155
388
|
for (const room of listener.rooms) broadcastPresence({ room, peer, state: null });
|
|
156
389
|
try { controller.close(); } catch {}
|
|
157
390
|
};
|
|
158
|
-
listener = { peer, request, rooms: new Set(), send: (event, data) => write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`), close };
|
|
391
|
+
listener = { peer, request, scopes, rooms: new Set(), send: (event, data) => write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`), close };
|
|
159
392
|
listeners.add(listener);
|
|
160
393
|
request.signal?.addEventListener('abort', close);
|
|
161
394
|
write('retry: 1000\n\n');
|
|
@@ -168,11 +401,7 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
168
401
|
for (let first = true; ; first = false) {
|
|
169
402
|
const backlog = await log.after(from);
|
|
170
403
|
if (!backlog.changes.length && !first) break;
|
|
171
|
-
|
|
172
|
-
for (const change of backlog.changes) {
|
|
173
|
-
if (await authorize({ request, collection: change.collection, key: change.key, write: false })) visible.push(change);
|
|
174
|
-
}
|
|
175
|
-
listener.send('changes', { changes: visible, cursor: backlog.cursor });
|
|
404
|
+
listener.send('changes', { changes: await visible(listener, backlog.changes), cursor: backlog.cursor, ...(reset && first ? { reset: true } : {}) });
|
|
176
405
|
if (!backlog.changes.length || String(backlog.cursor) === String(from)) break;
|
|
177
406
|
from = backlog.cursor;
|
|
178
407
|
}
|
|
@@ -186,12 +415,11 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
186
415
|
});
|
|
187
416
|
}
|
|
188
417
|
if (url.pathname === `${base}/pull` && request.method === 'GET') {
|
|
189
|
-
const
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
194
|
-
return json(200, { changes: visible, cursor });
|
|
418
|
+
const scopes = await scopesOf(request, url);
|
|
419
|
+
if (scopes instanceof Response) return scopes;
|
|
420
|
+
const { since, reset } = await startFrom(url.searchParams.get('since') ?? '0');
|
|
421
|
+
const { changes, cursor } = await log.after(since);
|
|
422
|
+
return json(200, { changes: await visible({ request, scopes }, changes), cursor, ...(reset ? { reset: true } : {}) });
|
|
195
423
|
}
|
|
196
424
|
if (url.pathname.startsWith(`${base}/`)) return json(404, { error: 'Not found' });
|
|
197
425
|
return null;
|
|
@@ -228,5 +456,36 @@ export function createSyncServer({ log = memoryLog(), mount = MOUNT, authorize =
|
|
|
228
456
|
return true;
|
|
229
457
|
}
|
|
230
458
|
|
|
231
|
-
|
|
459
|
+
/**
|
|
460
|
+
* Write `fields` to a document as the server: to the documents view at once (so a check
|
|
461
|
+
* before it, with no await between, can't be interleaved), then to the log and every live
|
|
462
|
+
* page. Resolves to the changes written.
|
|
463
|
+
*/
|
|
464
|
+
async function write(collection, key, fields) {
|
|
465
|
+
if (typeof collection !== 'string' || typeof key !== 'string' || !fields || typeof fields !== 'object' || Array.isArray(fields)) {
|
|
466
|
+
throw new TypeError('sync.write(collection, key, { field: value, … })');
|
|
467
|
+
}
|
|
468
|
+
if ('$clocks' in fields) throw new TypeError('sync.write: `$clocks` is not a field');
|
|
469
|
+
// A server that hasn't read its log yet (it just started, or a Durable Object was evicted)
|
|
470
|
+
// hasn't seen the clocks in it: read it first, so this write comes after all of them. Once
|
|
471
|
+
// the documents exist this doesn't wait, so a check and a write with no await between stay
|
|
472
|
+
// one step.
|
|
473
|
+
if (!docs) await documents();
|
|
474
|
+
const changes = Object.entries(fields).map(([field, value]) => ({ collection, key, field, value, clock: tick() }));
|
|
475
|
+
if (!changes.length) return [];
|
|
476
|
+
// In the documents at once, so a check before this call and the write are one step; taken
|
|
477
|
+
// back out if the log refuses it.
|
|
478
|
+
const undo = record(changes);
|
|
479
|
+
let cursor;
|
|
480
|
+
try {
|
|
481
|
+
cursor = await append(changes);
|
|
482
|
+
} catch (error) {
|
|
483
|
+
undo();
|
|
484
|
+
throw error;
|
|
485
|
+
}
|
|
486
|
+
await broadcastChanges(changes, cursor);
|
|
487
|
+
return changes;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
return { fetch, handle, documents, write };
|
|
232
491
|
}
|
package/src/sql-log.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// A sync log in SQL, over any SQLite: Node's and Bun's (sqliteLog) and a Durable Object's
|
|
2
|
+
// (durableObjectLog) hand it the same three calls. Two ways to keep it:
|
|
3
|
+
//
|
|
4
|
+
// - `keep: 'all'` (the default): every change, in the order it came. A table `changes`, cursor =
|
|
5
|
+
// row id. What `sqliteLog()` has always written; the zubra-sync binary reads the same table.
|
|
6
|
+
// - `keep: 'latest'`: each field's latest value only. A table `latest`, one row per field: a
|
|
7
|
+
// change with a newer clock replaces the row and takes the next cursor, an older one is dropped.
|
|
8
|
+
// `after(cursor)` hands out each field's latest value once, so a new device reads what's there
|
|
9
|
+
// now, not the whole evening's history. Opening it over a database that has a `changes` table
|
|
10
|
+
// and no `latest` one carries each field's latest value across, numbered after the old log's
|
|
11
|
+
// last cursor, so a device that had read the old log reads the carried values again and misses
|
|
12
|
+
// nothing.
|
|
13
|
+
//
|
|
14
|
+
// `db` is `{ exec(sql), run(sql, ...args), all(sql, ...args), transaction(fn) }`.
|
|
15
|
+
const NAME = /^[A-Za-z_][\w]{0,62}$/;
|
|
16
|
+
|
|
17
|
+
export function sqlLog(db, { keep = 'all', table, from = 'changes', label = 'log' } = {}) {
|
|
18
|
+
if (keep !== 'all' && keep !== 'latest') throw new TypeError(`${label}: \`keep\` is 'all' or 'latest'`);
|
|
19
|
+
table ??= keep === 'all' ? 'changes' : 'latest';
|
|
20
|
+
for (const name of [table, from]) if (!NAME.test(name)) throw new TypeError(`${label}: a table is a plain SQL name (letters, digits, _)`);
|
|
21
|
+
const json = (value) => JSON.stringify(value ?? null);
|
|
22
|
+
const read = (rows) => rows.map(({ collection, key, field, value, clock }) => ({ collection, key, field, value: JSON.parse(value), clock }));
|
|
23
|
+
const last = () => Number(db.all(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${table}`)[0].seq);
|
|
24
|
+
|
|
25
|
+
if (keep === 'all') {
|
|
26
|
+
db.exec(`CREATE TABLE IF NOT EXISTS ${table} (
|
|
27
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
28
|
+
collection TEXT NOT NULL,
|
|
29
|
+
key TEXT NOT NULL,
|
|
30
|
+
field TEXT NOT NULL,
|
|
31
|
+
value TEXT,
|
|
32
|
+
clock TEXT NOT NULL
|
|
33
|
+
)`);
|
|
34
|
+
} else {
|
|
35
|
+
const exists = (name) => db.all(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`, name).length > 0;
|
|
36
|
+
const fresh = !exists(table);
|
|
37
|
+
db.exec(`CREATE TABLE IF NOT EXISTS ${table} (
|
|
38
|
+
collection TEXT NOT NULL,
|
|
39
|
+
key TEXT NOT NULL,
|
|
40
|
+
field TEXT NOT NULL,
|
|
41
|
+
value TEXT,
|
|
42
|
+
clock TEXT NOT NULL,
|
|
43
|
+
seq INTEGER NOT NULL UNIQUE,
|
|
44
|
+
PRIMARY KEY (collection, key, field)
|
|
45
|
+
)`);
|
|
46
|
+
if (fresh && from !== table && exists(from)) {
|
|
47
|
+
// Each field's latest change (the later clock; the later row if two share one), in log
|
|
48
|
+
// order, numbered after the old log's last cursor.
|
|
49
|
+
db.transaction(() =>
|
|
50
|
+
db.run(`INSERT INTO ${table} (collection, key, field, value, clock, seq)
|
|
51
|
+
SELECT collection, key, field, value, clock,
|
|
52
|
+
(SELECT COALESCE(MAX(seq), 0) FROM ${from}) + ROW_NUMBER() OVER (ORDER BY seq)
|
|
53
|
+
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY collection, key, field ORDER BY clock DESC, seq DESC) AS rank FROM ${from})
|
|
54
|
+
WHERE rank = 1`),
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
async append(changes) {
|
|
61
|
+
return db.transaction(() => {
|
|
62
|
+
for (const { collection, key, field, value, clock } of changes) {
|
|
63
|
+
if (keep === 'all') {
|
|
64
|
+
db.run(`INSERT INTO ${table} (collection, key, field, value, clock) VALUES (?, ?, ?, ?, ?)`, collection, key, field, json(value), clock);
|
|
65
|
+
} else {
|
|
66
|
+
// A newer clock takes the row and the next cursor; an older or equal one changes nothing.
|
|
67
|
+
db.run(
|
|
68
|
+
`INSERT INTO ${table} (collection, key, field, value, clock, seq) VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(seq), 0) + 1 FROM ${table}))
|
|
69
|
+
ON CONFLICT (collection, key, field) DO UPDATE SET value = excluded.value, clock = excluded.clock, seq = excluded.seq
|
|
70
|
+
WHERE excluded.clock > ${table}.clock`,
|
|
71
|
+
collection, key, field, json(value), clock,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return last();
|
|
76
|
+
});
|
|
77
|
+
},
|
|
78
|
+
async after(cursor, limit = 1000) {
|
|
79
|
+
const from = Number(cursor) || 0;
|
|
80
|
+
const rows = db.all(`SELECT seq, collection, key, field, value, clock FROM ${table} WHERE seq > ? ORDER BY seq LIMIT ?`, from, limit);
|
|
81
|
+
return { changes: read(rows), cursor: rows.length ? Number(rows[rows.length - 1].seq) : from };
|
|
82
|
+
},
|
|
83
|
+
/** The cursor after the last change. */
|
|
84
|
+
head: async () => last(),
|
|
85
|
+
};
|
|
86
|
+
}
|
package/src/sqlite.d.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import type { Log } from './server.js';
|
|
2
2
|
|
|
3
|
-
/**
|
|
4
|
-
|
|
3
|
+
/**
|
|
4
|
+
* A sync log in SQLite (`node:sqlite` on Node 22.5+, `bun:sqlite` on Bun). `:memory:` by default.
|
|
5
|
+
* `keep: 'latest'` keeps each field's latest value only: a newer clock replaces it and takes the
|
|
6
|
+
* next cursor, an older one is dropped. Switching an existing file over carries each field's
|
|
7
|
+
* latest value across. By default every change is kept.
|
|
8
|
+
*/
|
|
9
|
+
export function sqliteLog(path?: string, options?: { keep?: 'all' | 'latest' }): Promise<Log & { close(): void }>;
|
package/src/sqlite.js
CHANGED
|
@@ -1,46 +1,39 @@
|
|
|
1
|
-
// A sync log that survives a restart:
|
|
2
|
-
//
|
|
1
|
+
// A sync log that survives a restart: SQLite, through the runtime's own (Node's `node:sqlite`,
|
|
2
|
+
// Bun's `bun:sqlite`), so there's nothing to install.
|
|
3
3
|
//
|
|
4
4
|
// import { createSyncServer } from '@zubra/sync/server';
|
|
5
5
|
// import { sqliteLog } from '@zubra/sync/sqlite';
|
|
6
6
|
// const sync = createSyncServer({ log: await sqliteLog('sync.db') });
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
|
|
8
|
+
// `keep: 'latest'` keeps each field's latest value only, so the log doesn't grow with every
|
|
9
|
+
// change and a new device reads what's there now (see sql-log.js). By default it keeps every
|
|
10
|
+
// change, in a table `changes` that the zubra-sync binary reads too.
|
|
11
|
+
import { sqlLog } from './sql-log.js';
|
|
12
|
+
|
|
13
|
+
export async function sqliteLog(path = ':memory:', { keep = 'all' } = {}) {
|
|
10
14
|
const db = await open(path);
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
insert.run(collection, key, field, JSON.stringify(value ?? null), clock);
|
|
15
|
+
const statements = new Map();
|
|
16
|
+
const statement = (sql) => statements.get(sql) ?? statements.set(sql, db.prepare(sql)).get(sql);
|
|
17
|
+
const log = sqlLog(
|
|
18
|
+
{
|
|
19
|
+
exec: (sql) => db.exec(sql),
|
|
20
|
+
run: (sql, ...args) => statement(sql).run(...args),
|
|
21
|
+
all: (sql, ...args) => statement(sql).all(...args),
|
|
22
|
+
transaction(fn) {
|
|
23
|
+
db.exec('BEGIN');
|
|
24
|
+
try {
|
|
25
|
+
const result = fn();
|
|
26
|
+
db.exec('COMMIT');
|
|
27
|
+
return result;
|
|
28
|
+
} catch (error) {
|
|
29
|
+
db.exec('ROLLBACK');
|
|
30
|
+
throw error;
|
|
28
31
|
}
|
|
29
|
-
|
|
30
|
-
} catch (error) {
|
|
31
|
-
db.exec('ROLLBACK');
|
|
32
|
-
throw error;
|
|
33
|
-
}
|
|
34
|
-
return Number(last.get().seq);
|
|
35
|
-
},
|
|
36
|
-
async after(cursor, limit = 1000) {
|
|
37
|
-
const rows = select.all(Number(cursor) || 0, limit);
|
|
38
|
-
const changes = rows.map(({ collection, key, field, value, clock }) => ({ collection, key, field, value: JSON.parse(value), clock }));
|
|
39
|
-
const cursorNow = rows.length ? Number(rows[rows.length - 1].seq) : Number(cursor) || 0;
|
|
40
|
-
return { changes, cursor: cursorNow };
|
|
32
|
+
},
|
|
41
33
|
},
|
|
42
|
-
|
|
43
|
-
|
|
34
|
+
{ keep, label: 'sqliteLog' },
|
|
35
|
+
);
|
|
36
|
+
return { ...log, close: () => db.close() };
|
|
44
37
|
}
|
|
45
38
|
|
|
46
39
|
async function open(path) {
|