@ultimat3/realtime 20.2.1 → 22.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +300 -952
- package/README.md +192 -131
- package/package.json +7 -4
- package/src/apply-patches.ts +1 -1
- package/src/boot.ts +72 -0
- package/src/browser-socket.ts +42 -0
- package/src/changefeed.ts +14 -1
- package/src/channel-authz.ts +52 -0
- package/src/channel-bridge.ts +34 -0
- package/src/channel-decl.ts +155 -0
- package/src/channel-describe.ts +35 -0
- package/src/channel-gaps.ts +57 -0
- package/src/channel-logs.ts +134 -0
- package/src/channel-presence.ts +68 -0
- package/src/channel-records.ts +87 -0
- package/src/channel-ref.ts +83 -0
- package/src/channel-registry.ts +35 -0
- package/src/channel-render.ts +37 -0
- package/src/channel-ring.ts +75 -0
- package/src/channel-wire.ts +66 -0
- package/src/channel.ts +147 -157
- package/src/client-channels.ts +359 -0
- package/src/client-contract.ts +35 -65
- package/src/client-frames.ts +42 -110
- package/src/client.ts +150 -195
- package/src/cursor.ts +7 -2
- package/src/errors.ts +55 -101
- package/src/frame-lanes.ts +9 -5
- package/src/idb-fake.ts +133 -0
- package/src/idb-types.ts +48 -0
- package/src/index.ts +80 -75
- package/src/json.ts +5 -0
- package/src/live-contract.ts +5 -0
- package/src/live-definition.ts +15 -4
- package/src/live-fanout.ts +81 -6
- package/src/live-query.ts +11 -0
- package/src/live-record-type.ts +19 -0
- package/src/live-replicator.ts +160 -0
- package/src/live-rows.ts +70 -67
- package/src/local-store-idb.ts +324 -0
- package/src/matcher-bridge.ts +5 -0
- package/src/nats-fake.ts +10 -1
- package/src/nats-jetstream.ts +36 -14
- package/src/nats-transport.ts +2 -2
- package/src/offline-queue.ts +85 -39
- package/src/outbox-slot.ts +31 -0
- package/src/page-errors.ts +124 -0
- package/src/page-outbox.ts +312 -0
- package/src/page-socket.ts +139 -0
- package/src/page-store.ts +138 -0
- package/src/pg-entity-row.ts +37 -184
- package/src/pg-preflight.ts +24 -2
- package/src/pg-replication.ts +28 -8
- package/src/pg-wire.ts +51 -15
- package/src/pgoutput.ts +37 -2
- package/src/policy-fake.ts +14 -0
- package/src/presence.ts +17 -9
- package/src/query-window.ts +38 -21
- package/src/reactivity.ts +70 -0
- package/src/realtime-error.ts +1 -1
- package/src/record-await.ts +102 -0
- package/src/record-key.ts +34 -0
- package/src/record-names.ts +45 -0
- package/src/record-persister.ts +156 -0
- package/src/record-store.ts +364 -0
- package/src/record-synced.ts +100 -0
- package/src/record-tx.ts +145 -0
- package/src/replicator.ts +20 -4
- package/src/server.ts +10 -11
- package/src/socket-drops.ts +30 -0
- package/src/socket-engine.ts +344 -0
- package/src/socket-host.ts +225 -0
- package/src/socket-idle.ts +21 -0
- package/src/socket-port.ts +55 -0
- package/src/socket-routes.ts +170 -0
- package/src/socket.ts +91 -49
- package/src/subscriber-gate.ts +92 -3
- package/src/sync-auth.ts +2 -2
- package/src/sync-frames.ts +41 -114
- package/src/sync-meta.ts +42 -0
- package/src/sync-node-contract.ts +100 -0
- package/src/sync-node.ts +26 -114
- package/src/sync-protocol.ts +63 -212
- package/src/sync-worker.ts +12 -0
- package/src/thundering-herd.ts +31 -12
- package/src/transport-env.ts +55 -14
- package/src/type-pins.ts +30 -61
- package/src/use-channel.ts +88 -0
- package/src/use-connection.ts +59 -0
- package/src/use-mutation.ts +227 -0
- package/src/use-query.ts +260 -0
- package/src/use-record.ts +121 -0
- package/src/wire-channel.ts +116 -0
- package/src/wire-read.ts +86 -0
- package/src/wire-version.ts +44 -0
- package/src/client-mutations.ts +0 -114
- package/src/client-topics.ts +0 -54
- package/src/hooks.ts +0 -277
- package/src/identity-map.ts +0 -141
- package/src/local-store.ts +0 -241
- package/src/query-hook.ts +0 -56
- package/src/rebase.ts +0 -263
- package/src/server-render-client.ts +0 -96
package/src/json.ts
CHANGED
|
@@ -34,6 +34,11 @@ export interface RowPatch {
|
|
|
34
34
|
readonly row: JsonObject | null;
|
|
35
35
|
readonly lsn: string;
|
|
36
36
|
readonly index?: number;
|
|
37
|
+
/**
|
|
38
|
+
* The row's RECORD key when it is not its `id` — the entity's primary key, rendered by its
|
|
39
|
+
* projection on the server. Absent means the key is the `id`.
|
|
40
|
+
*/
|
|
41
|
+
readonly key?: string;
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
export function isJsonObject(value: unknown): value is JsonObject {
|
package/src/live-contract.ts
CHANGED
|
@@ -41,6 +41,11 @@ export interface LiveQueryDefinition<R extends Row = Row> {
|
|
|
41
41
|
* rows private to that one subscription rather than guessing.
|
|
42
42
|
*/
|
|
43
43
|
rowEntity?(input: JsonValue): string | null;
|
|
44
|
+
/**
|
|
45
|
+
* The record key each row travels under, resolved with `rowEntity`: the entity's own projection.
|
|
46
|
+
* `null` (or absent) is a plain table, whose rows are keyed by `id`.
|
|
47
|
+
*/
|
|
48
|
+
rowKey?(input: JsonValue): ((row: Row) => string) | null;
|
|
44
49
|
/**
|
|
45
50
|
* Resolve whatever this input needs before an entry is built. `matcher` is synchronous by
|
|
46
51
|
* design — a change event must not await anything — so a definition that has to compile a
|
package/src/live-definition.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { type AnyQuery, queryHash, queryName } from '@ultimat3/query';
|
|
|
14
14
|
import { LiveRowUnidentifiedError } from './errors';
|
|
15
15
|
import { isRow, type JsonValue, type Row } from './json';
|
|
16
16
|
import type { LiveQueryDefinition, SnapshotResult } from './live-contract';
|
|
17
|
+
import { liveRecords } from './live-record-type';
|
|
17
18
|
import { type IncrementalMatcher, matcherFor, type Projection } from './matcher-bridge';
|
|
18
19
|
import { authorizeWithPolicy, visibleWithPolicy } from './policy-gate';
|
|
19
20
|
|
|
@@ -47,6 +48,8 @@ interface SharedWindow {
|
|
|
47
48
|
readonly matcher: IncrementalMatcher;
|
|
48
49
|
/** The compiled shape's root entity — the client's identity scope for every row of this read. */
|
|
49
50
|
readonly rowEntity: string;
|
|
51
|
+
/** Its projection's record key; `null` for a plain table keyed by `id`. */
|
|
52
|
+
readonly rowKey: ((row: Row) => string) | null;
|
|
50
53
|
read(): Promise<readonly Row[]>;
|
|
51
54
|
}
|
|
52
55
|
|
|
@@ -109,11 +112,14 @@ export function liveQueryDefinition(
|
|
|
109
112
|
// a second subject-less copy — which is what this did — paid for the parse and the `sql()`
|
|
110
113
|
// twice per query id and left two descriptions of one read that agreed only by luck.
|
|
111
114
|
const projection = learnProjection();
|
|
115
|
+
const records = liveRecords(live.shape.entity);
|
|
112
116
|
const built: SharedWindow = {
|
|
113
117
|
matcher: matcherFor(live, projection.read),
|
|
114
|
-
// `assertMatchable` already refused a shape without one
|
|
115
|
-
//
|
|
116
|
-
|
|
118
|
+
// `assertMatchable` already refused a shape without one. The shape names the TABLE (what a
|
|
119
|
+
// `ChangeEvent` carries); the client store keys records by ENTITY name, so the snapshot tells
|
|
120
|
+
// it the record type, and every frame carries the key the entity's projection renders.
|
|
121
|
+
rowEntity: records.type,
|
|
122
|
+
rowKey: records.key,
|
|
117
123
|
read: async () => {
|
|
118
124
|
const rows = rowsOf(name, await live.execute());
|
|
119
125
|
projection.teach(rows);
|
|
@@ -136,12 +142,17 @@ export function liveQueryDefinition(
|
|
|
136
142
|
},
|
|
137
143
|
snapshot: async ({ input }): Promise<SnapshotResult> => {
|
|
138
144
|
const window = await resolve(input);
|
|
139
|
-
|
|
145
|
+
// The position is taken BEFORE the rows: the rows are then at least that new, so the claim
|
|
146
|
+
// is true. Taken after, a commit landing mid-read was claimed and missing — and every change
|
|
147
|
+
// up to it is dropped downstream as already folded.
|
|
148
|
+
const lsn = options.lsn?.() ?? '';
|
|
149
|
+
return { rows: await window.read(), lsn };
|
|
140
150
|
},
|
|
141
151
|
matcher: (input) => windows.get(queryHash(name, input))?.matcher ?? UNRESOLVED,
|
|
142
152
|
// Read off the same resolved window as the matcher, so the scope the client keys rows under and
|
|
143
153
|
// the entity the matcher patches them from can never be two different names.
|
|
144
154
|
rowEntity: (input) => windows.get(queryHash(name, input))?.rowEntity ?? null,
|
|
155
|
+
rowKey: (input) => windows.get(queryHash(name, input))?.rowKey ?? null,
|
|
145
156
|
// The two per-subscriber gates, both through the package's one authz seam. Neither result is
|
|
146
157
|
// memoised anywhere: `authorize` runs on every subscribe, `visible` on every row of every
|
|
147
158
|
// delivery, and there is no key here an actor could share with another actor.
|
package/src/live-fanout.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { Row, RowPatch } from './json';
|
|
|
10
10
|
import type { LiveSubscription } from './live-contract';
|
|
11
11
|
import { applyToWindow, bridgeChange } from './matcher-bridge';
|
|
12
12
|
import { type QueryEntry, refillWindowInLane } from './query-window';
|
|
13
|
-
import type
|
|
13
|
+
import { type Subscriber, type SubscriberGate, windowIndex } from './subscriber-gate';
|
|
14
14
|
import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
15
15
|
|
|
16
16
|
export interface FanoutDeps {
|
|
@@ -38,14 +38,25 @@ export async function fanoutChange(
|
|
|
38
38
|
): Promise<FanoutResult> {
|
|
39
39
|
// A window that missed a change must be replaced before it is patched again, and it can only be
|
|
40
40
|
// replaced here — a fanout holds this entry's lane, and `fillWindow` takes the same one.
|
|
41
|
+
// No read has landed in this window yet: there is nothing to patch, and a patch folded into the
|
|
42
|
+
// empty rows would move `entry.lsn` past the read in flight and get that read discarded as older
|
|
43
|
+
// than the window — every row but the patched one lost, permanently. Marked stale instead, so the
|
|
44
|
+
// read that lands is applied and followed by one that includes this change.
|
|
45
|
+
if (entry.applied === 0) {
|
|
46
|
+
entry.stale = true;
|
|
47
|
+
return { sent: 0, stale: 0 };
|
|
48
|
+
}
|
|
49
|
+
if (change.op === 'truncate') return await truncated(deps, entry, change);
|
|
41
50
|
if (entry.stale) await refillWindowInLane(entry);
|
|
42
51
|
// The consume-side twin of the replicator's own duplicate guard, which had none. `entry.lsn =
|
|
43
52
|
// change.lsn` was unconditional, so a change the window already holds — a redelivery, or one
|
|
44
53
|
// that arrived behind the snapshot that already included it — rewound every subscriber's cursor
|
|
45
54
|
// to it and asked them to fold state they had already folded over newer rows.
|
|
46
55
|
if (entry.lsn !== '' && change.lsn <= entry.lsn) return { sent: 0, stale: 1 };
|
|
47
|
-
const
|
|
48
|
-
if (!
|
|
56
|
+
const bridged = bridgeChange(entry.shape, entry.matcher, change, entry.rows);
|
|
57
|
+
if (!bridged) return { sent: 0, stale: 0 };
|
|
58
|
+
// Keyed ONCE, here, before the retained window stores them: a resume replays the same key.
|
|
59
|
+
const result = { ...bridged, patches: keyPatches(entry, change, bridged.patches) };
|
|
49
60
|
entry.lsn = change.lsn;
|
|
50
61
|
entry.rows = applyToWindow(entry.rows, result.patches);
|
|
51
62
|
// The window lost its tail, so what it holds is a guess — the next delivery re-reads it rather
|
|
@@ -55,6 +66,8 @@ export async function fanoutChange(
|
|
|
55
66
|
for (const patch of result.patches) deps.source.append(entry.qid, patch);
|
|
56
67
|
|
|
57
68
|
let sent = 0;
|
|
69
|
+
// Indexed once for every subscriber below, never searched per subscriber per patch.
|
|
70
|
+
const index = windowIndex(entry.rows);
|
|
58
71
|
for (const subscription of entry.subscribers.values()) {
|
|
59
72
|
if (result.refill) {
|
|
60
73
|
// The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
|
|
@@ -82,7 +95,8 @@ export async function fanoutChange(
|
|
|
82
95
|
entry,
|
|
83
96
|
who,
|
|
84
97
|
result.patches,
|
|
85
|
-
|
|
98
|
+
heldBy(subscription.cursor),
|
|
99
|
+
index,
|
|
86
100
|
);
|
|
87
101
|
} catch {
|
|
88
102
|
// Already counted and reported as a gate failure. Degrade this one subscriber the way a
|
|
@@ -110,6 +124,43 @@ export async function fanoutChange(
|
|
|
110
124
|
return { sent, stale: 0 };
|
|
111
125
|
}
|
|
112
126
|
|
|
127
|
+
/**
|
|
128
|
+
* Every row of a relation this window reads is gone. There is nothing to patch from — a truncate
|
|
129
|
+
* names no row — so the window is re-read now, in the lane, and every subscriber is re-snapshotted
|
|
130
|
+
* out of what came back: a window that kept the truncated rows until its next change would serve
|
|
131
|
+
* them to every new subscriber in the meantime.
|
|
132
|
+
*/
|
|
133
|
+
async function truncated(
|
|
134
|
+
deps: FanoutDeps,
|
|
135
|
+
entry: QueryEntry,
|
|
136
|
+
change: ChangeEvent,
|
|
137
|
+
): Promise<FanoutResult> {
|
|
138
|
+
if (!entry.shape.entities.includes(change.entity)) return { sent: 0, stale: 0 };
|
|
139
|
+
await refillWindowInLane(entry);
|
|
140
|
+
if (change.lsn > entry.lsn) entry.lsn = change.lsn;
|
|
141
|
+
let sent = 0;
|
|
142
|
+
for (const subscription of entry.subscribers.values()) {
|
|
143
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
144
|
+
if (await resnapshot(deps, entry, subscription)) sent += 1;
|
|
145
|
+
}
|
|
146
|
+
return { sent, stale: 0 };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* A cursor's ids as a set, built once per ids ARRAY rather than once per subscriber per change.
|
|
151
|
+
* `advance` hands an update's cursor the same array it had (an update moves no id), so the cache
|
|
152
|
+
* holds across the common change and dies with the array; an insert or delete makes a new one.
|
|
153
|
+
*/
|
|
154
|
+
const heldSets = new WeakMap<readonly string[], ReadonlySet<string>>();
|
|
155
|
+
|
|
156
|
+
function heldBy(cursor: LiveCursor): ReadonlySet<string> {
|
|
157
|
+
const cached = heldSets.get(cursor.ids);
|
|
158
|
+
if (cached !== undefined) return cached;
|
|
159
|
+
const held = new Set(cursor.ids);
|
|
160
|
+
heldSets.set(cursor.ids, held);
|
|
161
|
+
return held;
|
|
162
|
+
}
|
|
163
|
+
|
|
113
164
|
/**
|
|
114
165
|
* The repair for one diverged subscriber, out of the window the lane is already holding — no DB
|
|
115
166
|
* read, one frame. Its cursor is rebuilt from what this subscriber may actually see, exactly as
|
|
@@ -138,7 +189,11 @@ async function resnapshot(
|
|
|
138
189
|
return true;
|
|
139
190
|
}
|
|
140
191
|
|
|
141
|
-
/**
|
|
192
|
+
/**
|
|
193
|
+
* The one place a snapshot frame is built, so the identity scope and the record keys cannot be
|
|
194
|
+
* told to one caller only. `keys` rides only when some key differs from its row's `id` — an entity
|
|
195
|
+
* keyed by `id` sends the frame it always sent.
|
|
196
|
+
*/
|
|
142
197
|
export function snapshotFrame(
|
|
143
198
|
entry: QueryEntry,
|
|
144
199
|
sid: string,
|
|
@@ -146,5 +201,25 @@ export function snapshotFrame(
|
|
|
146
201
|
cursor: LiveCursor,
|
|
147
202
|
): Frame {
|
|
148
203
|
const base = { type: 'snapshot', v: PROTOCOL_VERSION, sid, rows, cursor } as const;
|
|
149
|
-
|
|
204
|
+
const scoped = entry.rowEntity === null ? base : { ...base, entity: entry.rowEntity };
|
|
205
|
+
const keyOf = entry.rowKey;
|
|
206
|
+
if (keyOf === null) return scoped;
|
|
207
|
+
const keys = rows.map((row) => keyOf(row));
|
|
208
|
+
return keys.every((key, index) => key === rows[index]?.id) ? scoped : { ...scoped, keys };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* A patch's record key, from the change's WHOLE row — an update patch carries only the changed
|
|
213
|
+
* columns, and a key needs every primary-key column. Only stamped where it differs from `id`.
|
|
214
|
+
*/
|
|
215
|
+
function keyPatches(
|
|
216
|
+
entry: QueryEntry,
|
|
217
|
+
change: ChangeEvent,
|
|
218
|
+
patches: readonly RowPatch[],
|
|
219
|
+
): readonly RowPatch[] {
|
|
220
|
+
const keyOf = entry.rowKey;
|
|
221
|
+
const whole = change.after ?? change.before;
|
|
222
|
+
if (keyOf === null || whole === null) return patches;
|
|
223
|
+
const key = keyOf(whole);
|
|
224
|
+
return patches.map((patch) => (key === patch.id ? patch : { ...patch, key }));
|
|
150
225
|
}
|
package/src/live-query.ts
CHANGED
|
@@ -70,6 +70,7 @@ export class LiveQueryRegistry {
|
|
|
70
70
|
readonly #maxEntries: number;
|
|
71
71
|
/** What one lane needs, and nothing this class holds beyond it. */
|
|
72
72
|
readonly #fanout: FanoutDeps;
|
|
73
|
+
#lastLsn = '';
|
|
73
74
|
#staleChanges = 0;
|
|
74
75
|
|
|
75
76
|
constructor(options: LiveQueryRegistryOptions) {
|
|
@@ -363,7 +364,17 @@ export class LiveQueryRegistry {
|
|
|
363
364
|
* never per node — awaiting one entry before entering the next made one slow policy pass the
|
|
364
365
|
* whole node's pace, and let a lane that threw skip every entry behind it with nobody desynced.
|
|
365
366
|
*/
|
|
367
|
+
/**
|
|
368
|
+
* The newest change position this registry has been handed. What a node's snapshot may claim
|
|
369
|
+
* (`liveQueryDefinition`'s `lsn`): a read begun after it holds at least that change, and every
|
|
370
|
+
* later change is above it. `''` before the first.
|
|
371
|
+
*/
|
|
372
|
+
get lastLsn(): string {
|
|
373
|
+
return this.#lastLsn;
|
|
374
|
+
}
|
|
375
|
+
|
|
366
376
|
async deliver(change: ChangeEvent): Promise<number> {
|
|
377
|
+
if (change.lsn > this.#lastLsn) this.#lastLsn = change.lsn;
|
|
367
378
|
const lanes = [...this.#entries.values()].map(async (entry) => {
|
|
368
379
|
try {
|
|
369
380
|
const result = await entry.lock.run(() => fanoutChange(this.#fanout, entry, change));
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Which record type a live query's rows are, and the key each row travels under. A changefeed and
|
|
2
|
+
// a snapshot speak TABLES; the page store keys records by the entity's NAME and PRIMARY KEY. Both
|
|
3
|
+
// come from the entity's own projection here, on the server — the browser never derives a key.
|
|
4
|
+
|
|
5
|
+
import type { Row } from '@ultimat3/core/page';
|
|
6
|
+
import { recordProjectionForTable } from '@ultimat3/entity/record';
|
|
7
|
+
|
|
8
|
+
export interface LiveRecords {
|
|
9
|
+
/** The entity name — or the table itself when no registered entity owns it. */
|
|
10
|
+
readonly type: string;
|
|
11
|
+
/** The row's record key, or `null` for a plain table, whose rows are keyed by `id`. */
|
|
12
|
+
readonly key: ((row: Row) => string) | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function liveRecords(table: string): LiveRecords {
|
|
16
|
+
const projection = recordProjectionForTable(table);
|
|
17
|
+
if (projection === undefined) return { type: table, key: null };
|
|
18
|
+
return { type: projection.type, key: (row) => projection.key(row) };
|
|
19
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// The in-process replicator: committed row changes in, `ChangeEvent`s out, fanned into the node.
|
|
2
|
+
//
|
|
3
|
+
// Production decodes the write-ahead log. PGlite has no walsender and the memory driver has no log,
|
|
4
|
+
// so a test process had no change source at all — which is what left the `subscribe` fixture with
|
|
5
|
+
// no driver, and its five tests in `examples/dummy` asserting against a snapshot that never moved.
|
|
6
|
+
//
|
|
7
|
+
// WHAT IS REAL HERE, and it is everything downstream of the decoder: the matcher, the shared window,
|
|
8
|
+
// the per-subscriber `visible` gate, the cursor, the frames. This substitutes for the WAL DECODER
|
|
9
|
+
// and for nothing else — `@ultimat3/entity`'s `setRowObserver` reports what a repository wrote, in
|
|
10
|
+
// this process, and the events are shaped exactly as `PgLogicalReplicationFeed` shapes them.
|
|
11
|
+
//
|
|
12
|
+
// WHAT IS NOT: a write another process made is invisible, because nothing here reads a log. That is
|
|
13
|
+
// the honest bound, and it is why `selectChangeFeed` never picks it — the boot that installs it
|
|
14
|
+
// decides, and only under an embedded database.
|
|
15
|
+
//
|
|
16
|
+
// Lives in `@ultimat3/realtime/server` since 2026-09-23: it imports only `entity` and `realtime`,
|
|
17
|
+
// so it is a second change source beside the WAL decoder, and `x dev` booting it out of
|
|
18
|
+
// `@ultimat3/testing` put the test harness in every dev process's graph. `@ultimat3/testing`
|
|
19
|
+
// re-exports it until 22.0.0.
|
|
20
|
+
//
|
|
21
|
+
// `x dev` is the one boot that installs it outside a test, `As of 2026-09-05` (`@ultimat3/cli`'s
|
|
22
|
+
// `dev-live-feed.ts`), and only under the EMBEDDED database: PGlite has no walsender, every role
|
|
23
|
+
// runs in that one process, so the bound above holds by construction — and a real `DATABASE_URL`
|
|
24
|
+
// gets the decoder instead, never both.
|
|
25
|
+
|
|
26
|
+
import type { RowBulkChange, RowChange, RowObserver } from '@ultimat3/entity';
|
|
27
|
+
import type { ChangeEvent, ChangeOp } from './changefeed';
|
|
28
|
+
import type { Row } from './json';
|
|
29
|
+
import type { LiveQueryRegistry } from './live-query';
|
|
30
|
+
|
|
31
|
+
/** What a caller does with a change nobody could deliver. */
|
|
32
|
+
export interface LiveReplicatorOptions {
|
|
33
|
+
readonly registry: LiveQueryRegistry;
|
|
34
|
+
/**
|
|
35
|
+
* The node's declared channels, fed the same `ChangeEvent` — what a real node's change
|
|
36
|
+
* subscription does beside `registry.deliver` (`sync-node.ts`). Without it a channel's `records`
|
|
37
|
+
* frames never carried a write made under `x dev`, and every other tab stayed on the old row.
|
|
38
|
+
*/
|
|
39
|
+
readonly channels?: { deliverChange(change: ChangeEvent): unknown };
|
|
40
|
+
/** Tenant column, hoisted out of the row so fanout filters without parsing it. */
|
|
41
|
+
readonly tenantColumn?: string;
|
|
42
|
+
readonly onError?: (error: unknown) => void;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface LiveReplicator {
|
|
46
|
+
/** Resolves when every change observed so far has been fanned out. Never a sleep. */
|
|
47
|
+
settled(): Promise<void>;
|
|
48
|
+
/** Changes this replicator has delivered — the number a test asserts a patch count against. */
|
|
49
|
+
readonly delivered: number;
|
|
50
|
+
stop(): void;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* A `ChangeEvent` row is `Row` — a JSON object carrying an `id`. Every row a repository stores has
|
|
55
|
+
* one; the cast is what says so to a compiler that only sees `Record<string, unknown>`, and a row
|
|
56
|
+
* that genuinely has none fails downstream in `idOf`, with the entity named, exactly as a row off
|
|
57
|
+
* the wire would.
|
|
58
|
+
*/
|
|
59
|
+
const asRow = (value: Readonly<Record<string, unknown>> | null): Row | null =>
|
|
60
|
+
value === null ? null : (value as unknown as Row);
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Lexicographically comparable, which is the whole contract of an lsn — `formatLsn` in
|
|
64
|
+
* `@ultimat3/realtime` produces the same shape from a real WAL position. A counter is enough here
|
|
65
|
+
* because one process observes its own writes in the order it made them.
|
|
66
|
+
*/
|
|
67
|
+
const lsnOf = (position: number): string => position.toString(16).padStart(16, '0');
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Install the replicator for the length of one test. It takes over the process row observer and
|
|
71
|
+
* hands back whatever was installed before, because `bun test` shares one process across files and
|
|
72
|
+
* an unconditional clear would take an outer harness's observer with it.
|
|
73
|
+
*/
|
|
74
|
+
export async function startLiveReplicator(options: LiveReplicatorOptions): Promise<LiveReplicator> {
|
|
75
|
+
// Awaited BEFORE the observer exists, so installation is the last thing this function does and
|
|
76
|
+
// no write between the call and the install can slip past unobserved.
|
|
77
|
+
const entity = await import('@ultimat3/entity');
|
|
78
|
+
const { registry } = options;
|
|
79
|
+
const tenant = options.tenantColumn ?? 'orgId';
|
|
80
|
+
let position = 0;
|
|
81
|
+
let delivered = 0;
|
|
82
|
+
// One promise chain, because ORDERING is the guarantee the whole pipeline is built on — the same
|
|
83
|
+
// reason `InMemoryChangeFeed` serializes its deliveries rather than firing them concurrently.
|
|
84
|
+
let tail: Promise<void> = Promise.resolve();
|
|
85
|
+
let stopped = false;
|
|
86
|
+
|
|
87
|
+
const enqueue = (work: () => Promise<void>): void => {
|
|
88
|
+
tail = tail.then(work).catch((error: unknown) => {
|
|
89
|
+
// Never rethrown into the chain: one failed fanout must not silence every change behind it,
|
|
90
|
+
// and a rejection with nobody to hand it to ends the Bun process.
|
|
91
|
+
options.onError?.(error);
|
|
92
|
+
});
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const observer: RowObserver = {
|
|
96
|
+
onChange(change: RowChange): void {
|
|
97
|
+
if (stopped) return;
|
|
98
|
+
position += 1;
|
|
99
|
+
const at = position;
|
|
100
|
+
const row = change.after ?? change.before;
|
|
101
|
+
const orgId = typeof row?.[tenant] === 'string' ? (row[tenant] as string) : null;
|
|
102
|
+
const event: ChangeEvent = {
|
|
103
|
+
entity: change.entity,
|
|
104
|
+
// A repository's three row ops are three of the feed's four; a truncate never comes this way.
|
|
105
|
+
op: change.op satisfies ChangeOp,
|
|
106
|
+
before: asRow(change.before),
|
|
107
|
+
after: asRow(change.after),
|
|
108
|
+
lsn: lsnOf(at),
|
|
109
|
+
txid: String(at),
|
|
110
|
+
orgId,
|
|
111
|
+
// Deliberately not a clock read: the preload freezes `Date.now()`, and a change's commit
|
|
112
|
+
// time is not something any assertion in this repo reads. `at` keeps it monotonic anyway.
|
|
113
|
+
at,
|
|
114
|
+
// The keyed write it belongs to, read off the request scope — what the WAL decoder reads
|
|
115
|
+
// off the transaction's opening message — so a channel frame names it here as it would there.
|
|
116
|
+
...(change.write === undefined ? {} : { write: change.write }),
|
|
117
|
+
};
|
|
118
|
+
enqueue(async () => {
|
|
119
|
+
// Channels first, as the node does: a live query's fanout that throws must not also cost
|
|
120
|
+
// every declared channel the change.
|
|
121
|
+
options.channels?.deliverChange(event);
|
|
122
|
+
delivered += await registry.deliver(event);
|
|
123
|
+
});
|
|
124
|
+
},
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* A filtered write names rows this seam never saw, so there is no event to shape. Every window
|
|
128
|
+
* on the node is marked stale instead and re-read on the next change — `invalidate()` is the
|
|
129
|
+
* node's own answer to "the change stream skipped something", used here for the one write that
|
|
130
|
+
* genuinely does. Silence would be the alternative, and a subscriber told nothing happened
|
|
131
|
+
* diverges with nobody ever asking again.
|
|
132
|
+
*/
|
|
133
|
+
onBulk(_change: RowBulkChange): void {
|
|
134
|
+
if (stopped) return;
|
|
135
|
+
registry.invalidate();
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const previous = entity.setRowObserver(observer);
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
get delivered() {
|
|
143
|
+
return delivered;
|
|
144
|
+
},
|
|
145
|
+
settled: async () => {
|
|
146
|
+
// Twice: a fanout can enqueue nothing, but the writes that produced these changes may still
|
|
147
|
+
// be resolving their own promises when a test asks. Awaiting the chain, letting the
|
|
148
|
+
// microtask queue drain, then awaiting it again covers a change observed in between.
|
|
149
|
+
await tail;
|
|
150
|
+
for (let turn = 0; turn < 8; turn += 1) await Promise.resolve();
|
|
151
|
+
await tail;
|
|
152
|
+
},
|
|
153
|
+
stop: () => {
|
|
154
|
+
stopped = true;
|
|
155
|
+
// Restored, never cleared: one process runs every test file, and an outer harness's observer
|
|
156
|
+
// must survive an inner fixture finishing.
|
|
157
|
+
entity.setRowObserver(previous);
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
package/src/live-rows.ts
CHANGED
|
@@ -1,90 +1,101 @@
|
|
|
1
|
-
// One live subscription's window
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// One live subscription's window over the page's record store. The registration owns the ORDER
|
|
2
|
+
// (its ids) and the store owns the VALUES — which is what makes post #7 one object however many
|
|
3
|
+
// queries returned it, and what makes a write through any of them reach all of them.
|
|
4
4
|
|
|
5
|
+
import type { Row } from '@ultimat3/core/page';
|
|
5
6
|
import { orderAfterPatches } from './apply-patches';
|
|
6
7
|
import type { LiveCursor } from './cursor';
|
|
7
|
-
import
|
|
8
|
-
import type
|
|
8
|
+
import type { JsonValue, RowPatch } from './json';
|
|
9
|
+
import { type RecordKey, type RecordStore, recordKey } from './record-store';
|
|
9
10
|
|
|
10
|
-
export type LiveState = 'loading' | 'live' | 'stale' | 'offline';
|
|
11
|
+
export type LiveState = 'loading' | 'live' | 'stale' | 'offline' | 'failed';
|
|
11
12
|
|
|
12
13
|
/** One live query this client holds. Mutable: the ids and cursor a frame advances live here. */
|
|
13
14
|
export interface Registration {
|
|
14
15
|
readonly sid: string;
|
|
15
16
|
readonly name: string;
|
|
16
17
|
readonly input: JsonValue;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
/** Where this window's rows live in the map: the entity the server named, or a private scope. */
|
|
21
|
-
scope: RowScope;
|
|
22
|
-
/** Membership and order. The values are the map's — never a second copy of them. */
|
|
18
|
+
/** The record type the server named for this window; `unnamedType(name)` until it does. */
|
|
19
|
+
type: string;
|
|
20
|
+
/** Membership and order. The values are the store's — never a second copy of them. */
|
|
23
21
|
ids: readonly string[];
|
|
24
22
|
cursor: LiveCursor | null;
|
|
23
|
+
state: LiveState;
|
|
24
|
+
/** What the node answered when it refused this subscription. Set with `state: 'failed'`. */
|
|
25
|
+
error: unknown;
|
|
26
|
+
/** Called after anything a reader of this window renders has moved. */
|
|
27
|
+
readonly notify: () => void;
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
/**
|
|
28
|
-
*
|
|
29
|
-
*
|
|
31
|
+
* Where a window's rows live when the server named no record type: `?` starts no entity name, so
|
|
32
|
+
* two unnamed windows sharing an id never merge two entities' rows. Not a record type — only a
|
|
33
|
+
* snapshot from a node that cannot name the entity lands here.
|
|
34
|
+
*/
|
|
35
|
+
export function unnamedType(queryName: string): string {
|
|
36
|
+
return `?query:${queryName}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Every open window over the one store. It is the only writer of `Registration.ids`, so the
|
|
41
|
+
* retain/release pairs that keep the store from growing without end cannot be forgotten.
|
|
30
42
|
*/
|
|
31
43
|
export class RowWindows {
|
|
32
|
-
readonly #
|
|
33
|
-
/** The window a write is running for, so its own listener does not
|
|
44
|
+
readonly #store: RecordStore;
|
|
45
|
+
/** The window a write is running for, so its own listener does not notify it twice. */
|
|
34
46
|
#writing: Registration | null = null;
|
|
35
47
|
|
|
36
|
-
constructor(
|
|
37
|
-
this.#
|
|
48
|
+
constructor(store: RecordStore) {
|
|
49
|
+
this.#store = store;
|
|
38
50
|
}
|
|
39
51
|
|
|
40
|
-
/**
|
|
41
|
-
* Start rendering this registration out of the map. The returned close releases its rows and
|
|
42
|
-
* drops its listener — an unsubscribed component must stop holding rows and stop hearing about
|
|
43
|
-
* them in the same call, or one of the two outlives the other.
|
|
44
|
-
*/
|
|
52
|
+
/** Render this registration out of the store; the returned close releases every row it held. */
|
|
45
53
|
open(registration: Registration): () => void {
|
|
46
|
-
const unsubscribe = this.#
|
|
54
|
+
const unsubscribe = this.#store.subscribe((changed) => {
|
|
47
55
|
if (this.#writing === registration) return;
|
|
48
|
-
if (
|
|
49
|
-
this.#emit(registration);
|
|
56
|
+
if (holds(registration, changed)) registration.notify();
|
|
50
57
|
});
|
|
51
58
|
return () => {
|
|
52
59
|
unsubscribe();
|
|
53
|
-
this.#
|
|
54
|
-
for (const id of registration.ids) this.#
|
|
60
|
+
this.#store.batch(() => {
|
|
61
|
+
for (const id of registration.ids) this.#store.release(registration.type, id);
|
|
55
62
|
registration.ids = [];
|
|
56
63
|
});
|
|
57
64
|
};
|
|
58
65
|
}
|
|
59
66
|
|
|
60
|
-
/**
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
+
/** A snapshot: server truth for the whole window, under the record type the server named. */
|
|
68
|
+
snapshot(
|
|
69
|
+
registration: Registration,
|
|
70
|
+
type: string | null,
|
|
71
|
+
rows: readonly Row[],
|
|
72
|
+
keys?: readonly string[],
|
|
73
|
+
): void {
|
|
74
|
+
const next = type ?? registration.type;
|
|
75
|
+
// The server's record key where it sent one; a row's `id` IS its key everywhere else.
|
|
76
|
+
const keyed = rows.map((row, index) => [keys?.[index] ?? String(row['id']), row] as const);
|
|
67
77
|
this.#reseat(
|
|
68
78
|
registration,
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
(
|
|
72
|
-
for (const row of
|
|
73
|
-
if (held.has(row.id)) this.#identity.merge(scope, row.id, row);
|
|
74
|
-
}
|
|
79
|
+
next,
|
|
80
|
+
keyed.map(([id]) => id),
|
|
81
|
+
() => {
|
|
82
|
+
for (const [id, row] of keyed) this.#store.merge(next, id, row);
|
|
75
83
|
},
|
|
76
84
|
);
|
|
77
85
|
}
|
|
78
86
|
|
|
79
|
-
/** A patch list: values merged into the
|
|
80
|
-
patch(registration: Registration,
|
|
81
|
-
const
|
|
82
|
-
// A
|
|
83
|
-
|
|
84
|
-
|
|
87
|
+
/** A patch list: values merged into the store, membership and order folded over the ids. */
|
|
88
|
+
patch(registration: Registration, sent: readonly RowPatch[]): void {
|
|
89
|
+
const type = registration.type;
|
|
90
|
+
// A window holds RECORD keys: a patch the server keyed is folded under its key, not its id.
|
|
91
|
+
const patches = sent.map((patch) =>
|
|
92
|
+
patch.key === undefined ? patch : { ...patch, id: patch.key },
|
|
93
|
+
);
|
|
94
|
+
// A `delete` is this window losing the row, never the store losing it: another holder keeps it.
|
|
95
|
+
this.#reseat(registration, type, orderAfterPatches(registration.ids, patches), (held) => {
|
|
85
96
|
for (const patch of patches) {
|
|
86
97
|
if (patch.op === 'delete' || patch.row === null) continue;
|
|
87
|
-
if (held.has(patch.id)) this.#
|
|
98
|
+
if (held.has(patch.id)) this.#store.merge(type, patch.id, patch.row);
|
|
88
99
|
}
|
|
89
100
|
});
|
|
90
101
|
}
|
|
@@ -93,51 +104,43 @@ export class RowWindows {
|
|
|
93
104
|
rows(registration: Registration): readonly Row[] {
|
|
94
105
|
const out: Row[] = [];
|
|
95
106
|
for (const id of registration.ids) {
|
|
96
|
-
const row = this.#
|
|
107
|
+
const row = this.#store.peek(registration.type, id);
|
|
97
108
|
if (row !== undefined) out.push(row);
|
|
98
109
|
}
|
|
99
110
|
return out;
|
|
100
111
|
}
|
|
101
112
|
|
|
102
113
|
/**
|
|
103
|
-
* Move the window to `nextIds` under `
|
|
104
|
-
* and one emit for the window that caused it.
|
|
105
|
-
*
|
|
114
|
+
* Move the window to `nextIds` under `type`, writing values in between — one batch, one notify.
|
|
106
115
|
* The retain comes before the write and the release after it, so a row this window keeps across
|
|
107
|
-
* the move never reaches zero holds and
|
|
108
|
-
* given. `write` only touches ids the window ends up holding — a value nobody holds is a value
|
|
109
|
-
* no release will ever reclaim.
|
|
116
|
+
* the move never reaches zero holds and is evicted out from under the value it is being given.
|
|
110
117
|
*/
|
|
111
118
|
#reseat(
|
|
112
119
|
registration: Registration,
|
|
113
|
-
|
|
120
|
+
type: string,
|
|
114
121
|
nextIds: readonly string[],
|
|
115
122
|
write: (held: ReadonlySet<string>) => void,
|
|
116
123
|
): void {
|
|
117
124
|
const previous = this.#writing;
|
|
118
125
|
this.#writing = registration;
|
|
119
126
|
try {
|
|
120
|
-
this.#
|
|
121
|
-
for (const id of nextIds) this.#
|
|
127
|
+
this.#store.batch(() => {
|
|
128
|
+
for (const id of nextIds) this.#store.retain(type, id);
|
|
122
129
|
write(new Set(nextIds));
|
|
123
|
-
for (const id of registration.ids) this.#
|
|
124
|
-
registration.
|
|
130
|
+
for (const id of registration.ids) this.#store.release(registration.type, id);
|
|
131
|
+
registration.type = type;
|
|
125
132
|
registration.ids = nextIds;
|
|
126
133
|
});
|
|
127
134
|
} finally {
|
|
128
135
|
this.#writing = previous;
|
|
129
136
|
}
|
|
130
|
-
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
#emit(registration: Registration): void {
|
|
134
|
-
registration.setRows(this.rows(registration));
|
|
137
|
+
registration.notify();
|
|
135
138
|
}
|
|
136
139
|
}
|
|
137
140
|
|
|
138
|
-
function holds(registration: Registration, changed: ReadonlySet<
|
|
141
|
+
function holds(registration: Registration, changed: ReadonlySet<RecordKey>): boolean {
|
|
139
142
|
for (const id of registration.ids) {
|
|
140
|
-
if (changed.has(
|
|
143
|
+
if (changed.has(recordKey(registration.type, id))) return true;
|
|
141
144
|
}
|
|
142
145
|
return false;
|
|
143
146
|
}
|