@ultimat3/realtime 21.0.0 → 22.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +302 -1009
- package/README.md +130 -26
- package/package.json +4 -4
- package/src/changefeed.ts +7 -1
- package/src/channel-authz.ts +23 -4
- package/src/channel-decl.ts +16 -5
- package/src/channel-describe.ts +7 -5
- package/src/channel-logs.ts +19 -1
- package/src/channel-records.ts +8 -0
- package/src/client-channels.ts +75 -5
- package/src/client.ts +14 -2
- package/src/cursor.ts +5 -0
- package/src/errors.ts +43 -1
- package/src/idb-fake.ts +24 -4
- package/src/idb-types.ts +7 -0
- package/src/index.ts +1 -1
- package/src/live-definition.ts +5 -1
- package/src/live-fanout.ts +51 -2
- package/src/live-query.ts +11 -0
- package/src/live-replicator.ts +160 -0
- package/src/local-store-idb.ts +89 -15
- package/src/matcher-bridge.ts +5 -0
- package/src/nats-fake.ts +10 -1
- package/src/nats-jetstream.ts +36 -14
- package/src/nats-transport.ts +2 -2
- package/src/offline-queue.ts +76 -21
- package/src/page-outbox.ts +80 -10
- package/src/page-socket.ts +39 -8
- package/src/pg-entity-row.ts +37 -184
- package/src/pg-identifier.ts +23 -0
- package/src/pg-preflight.ts +32 -45
- package/src/pg-publication.ts +95 -0
- package/src/pg-replication.ts +21 -7
- package/src/pg-socket.ts +139 -53
- package/src/pg-tls.ts +124 -0
- package/src/pg-wire.ts +65 -16
- package/src/policy-fake.ts +14 -0
- package/src/query-window.ts +35 -21
- package/src/replication-errors.ts +29 -16
- package/src/replicator.ts +13 -3
- package/src/server.ts +8 -3
- package/src/socket-drops.ts +30 -0
- package/src/socket-engine.ts +15 -3
- package/src/socket-host.ts +103 -4
- package/src/socket-idle.ts +21 -0
- package/src/socket.ts +41 -38
- package/src/subscriber-gate.ts +92 -3
- package/src/sync-node-contract.ts +6 -0
- package/src/sync-node.ts +3 -7
- package/src/sync-origin.ts +33 -0
- package/src/sync-upgrade.ts +33 -9
- package/src/thundering-herd.ts +21 -11
- package/src/transport-env.ts +55 -14
- package/src/use-mutation.ts +13 -0
- package/src/use-query.ts +10 -5
package/src/cursor.ts
CHANGED
|
@@ -161,6 +161,11 @@ export function advance(
|
|
|
161
161
|
lsn: string,
|
|
162
162
|
now: number,
|
|
163
163
|
): LiveCursor {
|
|
164
|
+
// An update moves no id in or out, and it is the common change: the ids are reused as they are,
|
|
165
|
+
// so a fan-out to N subscribers of a W-row window is not N rebuilds of a W-id set per change.
|
|
166
|
+
if (!patches.some((patch) => patch.op !== 'update')) {
|
|
167
|
+
return { qid: cursor.qid, lsn, ids: cursor.ids, at: now };
|
|
168
|
+
}
|
|
164
169
|
const ids = new Set(cursor.ids);
|
|
165
170
|
for (const patch of patches) {
|
|
166
171
|
if (patch.op === 'delete') ids.delete(patch.id);
|
package/src/errors.ts
CHANGED
|
@@ -33,6 +33,9 @@ export const REALTIME_OWNED_ERROR_CODES = [
|
|
|
33
33
|
'X_LIVE_REPLICA_IDENTITY',
|
|
34
34
|
'X_SOCKET_UNAUTHENTICATED',
|
|
35
35
|
'X_SOCKET_AUTH_UNAVAILABLE',
|
|
36
|
+
'X_REALTIME_TOPOLOGY',
|
|
37
|
+
'X_REPLICATION_TLS',
|
|
38
|
+
'X_SOCKET_ORIGIN_REFUSED',
|
|
36
39
|
] as const;
|
|
37
40
|
|
|
38
41
|
/**
|
|
@@ -137,9 +140,12 @@ export const REALTIME_ERROR_TITLES: Readonly<Record<RealtimeOwnedErrorCode, stri
|
|
|
137
140
|
X_LIVE_SERVER_RENDER: 'a browser-only live operation ran during a server render',
|
|
138
141
|
X_LIVE_ROW_UNIDENTIFIED: 'a live query returned a row with no id',
|
|
139
142
|
X_LIVE_QUERY_UNKNOWN: 'no live query is registered under the name a subscribe frame asked for',
|
|
140
|
-
X_LIVE_REPLICA_IDENTITY: 'a replicated table
|
|
143
|
+
X_LIVE_REPLICA_IDENTITY: 'a replicated table has no replica identity',
|
|
141
144
|
X_SOCKET_UNAUTHENTICATED: 'the sync upgrade carried no credential this app accepts',
|
|
142
145
|
X_SOCKET_AUTH_UNAVAILABLE: 'the sync node could not decide who a connecting socket is',
|
|
146
|
+
X_REALTIME_TOPOLOGY: 'a sync node boots on a real database with no reachable change feed',
|
|
147
|
+
X_REPLICATION_TLS: 'the replication connection failed TLS',
|
|
148
|
+
X_SOCKET_ORIGIN_REFUSED: 'the websocket upgrade came from another origin',
|
|
143
149
|
};
|
|
144
150
|
|
|
145
151
|
// One unconditional call, so a second package claiming one of realtime's codes throws
|
|
@@ -170,6 +176,7 @@ export {
|
|
|
170
176
|
ReplicaIdentityError,
|
|
171
177
|
ReplicationFailedError,
|
|
172
178
|
ReplicationProtocolError,
|
|
179
|
+
ReplicationTlsError,
|
|
173
180
|
ReplicatorSlotHeldError,
|
|
174
181
|
} from './replication-errors';
|
|
175
182
|
|
|
@@ -255,6 +262,25 @@ export class TransportUnavailableError extends RealtimeError {
|
|
|
255
262
|
}
|
|
256
263
|
}
|
|
257
264
|
|
|
265
|
+
/**
|
|
266
|
+
* A `sync` node that can hear no change: a real database, the in-process bus, and no replicator in
|
|
267
|
+
* this process. A replicator in another process publishes into ITS in-process bus, so every live
|
|
268
|
+
* query and channel here is silent, with no error on either side. Refused at boot, where the
|
|
269
|
+
* topology is known, rather than discovered as a live feature that never updates.
|
|
270
|
+
*/
|
|
271
|
+
export class RealtimeTopologyError extends RealtimeError {
|
|
272
|
+
constructor() {
|
|
273
|
+
super({
|
|
274
|
+
code: 'X_REALTIME_TOPOLOGY',
|
|
275
|
+
cause:
|
|
276
|
+
'role sync runs on an external database over the in-process transport with no replicator in this process, so no committed change can reach it',
|
|
277
|
+
// Since 22.0.0 NATS_URL alone selects nothing: `realtime.transport` does, and a set NATS_URL
|
|
278
|
+
// under `'memory'` is refused, so the fix has to name both halves.
|
|
279
|
+
fix: "set realtime: { transport: 'nats', urlEnv: 'NATS_URL' } in app.config.ts and NATS_URL for every realtime role (web, sync, replicator), or run ROLE=sync with the replicator in one process: x dev --role sync,replicator",
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
258
284
|
/**
|
|
259
285
|
* The bytes on the bus socket are not the protocol we speak: an unknown NATS verb, a header block
|
|
260
286
|
* that is not `NATS/1.0`, a JetStream reply in a shape the API never produces. Always a version or
|
|
@@ -330,6 +356,22 @@ export class SocketUnauthenticatedError extends RealtimeError {
|
|
|
330
356
|
}
|
|
331
357
|
}
|
|
332
358
|
|
|
359
|
+
/**
|
|
360
|
+
* A browser page on another origin asked for a socket. No CORS applies to a websocket and the
|
|
361
|
+
* session cookie rides it, so admitting the upgrade would open a socket AS the visitor for a page
|
|
362
|
+
* that is not this app — cross-site websocket hijacking. Decided before `authenticate` and before
|
|
363
|
+
* the accept budget, so a hostile page costs neither.
|
|
364
|
+
*/
|
|
365
|
+
export class SocketOriginRefusedError extends RealtimeError {
|
|
366
|
+
constructor(args: { reason: string }) {
|
|
367
|
+
super({
|
|
368
|
+
code: 'X_SOCKET_ORIGIN_REFUSED',
|
|
369
|
+
cause: `the websocket upgrade was refused: ${args.reason}`,
|
|
370
|
+
fix: 'export APP_URL="https://www.example.com" # on the sync role: the origin the page is served on (or createSyncNode({ allowedOrigins }))',
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
333
375
|
/**
|
|
334
376
|
* `authenticate` raised instead of deciding. The same rule the row gate follows: a failure is not a
|
|
335
377
|
* denial, so the client is told to come back rather than told it may not connect — a token service
|
package/src/idb-fake.ts
CHANGED
|
@@ -13,6 +13,11 @@ type Tables = Map<string, Map<string, unknown>>;
|
|
|
13
13
|
export interface FakeIdbOptions {
|
|
14
14
|
/** `open` fails, as it does in a private window or with storage blocked. */
|
|
15
15
|
readonly blocked?: boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Every write ABORTS its transaction, as a quota refusal does: `abort` fires and nothing else —
|
|
18
|
+
* no `complete`, no transaction `error` event. Read at write time, so a test can flip it.
|
|
19
|
+
*/
|
|
20
|
+
quotaExceeded?: boolean;
|
|
16
21
|
}
|
|
17
22
|
|
|
18
23
|
/** A fresh, empty database server. Share one instance to simulate a reload over the same disk. */
|
|
@@ -28,7 +33,7 @@ export function fakeIndexedDb(options: FakeIdbOptions = {}): IdbFactoryLike {
|
|
|
28
33
|
}
|
|
29
34
|
const known = databases.get(name) ?? { version: 0, tables: new Map() };
|
|
30
35
|
databases.set(name, known);
|
|
31
|
-
const db = database(known.tables);
|
|
36
|
+
const db = database(known.tables, options);
|
|
32
37
|
request.result = db;
|
|
33
38
|
if (known.version < version) {
|
|
34
39
|
known.version = version;
|
|
@@ -41,7 +46,7 @@ export function fakeIndexedDb(options: FakeIdbOptions = {}): IdbFactoryLike {
|
|
|
41
46
|
};
|
|
42
47
|
}
|
|
43
48
|
|
|
44
|
-
function database(tables: Tables): IdbDatabaseLike {
|
|
49
|
+
function database(tables: Tables, options: FakeIdbOptions): IdbDatabaseLike {
|
|
45
50
|
return {
|
|
46
51
|
objectStoreNames: { contains: (name: string): boolean => tables.has(name) },
|
|
47
52
|
createObjectStore: (name: string): void => {
|
|
@@ -49,13 +54,20 @@ function database(tables: Tables): IdbDatabaseLike {
|
|
|
49
54
|
},
|
|
50
55
|
transaction(names: string | readonly string[]) {
|
|
51
56
|
let open = 0;
|
|
52
|
-
|
|
57
|
+
let failed = false;
|
|
53
58
|
const tx = {
|
|
54
59
|
oncomplete: null as (() => void) | null,
|
|
55
60
|
onerror: null as (() => void) | null,
|
|
61
|
+
onabort: null as (() => void) | null,
|
|
56
62
|
error: null as unknown,
|
|
57
63
|
objectStore: (name: string): IdbStoreLike => store(name),
|
|
58
64
|
};
|
|
65
|
+
const abort = (): void => {
|
|
66
|
+
if (failed) return;
|
|
67
|
+
failed = true;
|
|
68
|
+
tx.error = new DOMException('The quota has been exceeded.', 'QuotaExceededError');
|
|
69
|
+
queueMicrotask(() => tx.onabort?.());
|
|
70
|
+
};
|
|
59
71
|
const settleLater = (): void => {
|
|
60
72
|
queueMicrotask(() => {
|
|
61
73
|
if (open > 0 || failed) return;
|
|
@@ -82,7 +94,15 @@ function database(tables: Tables): IdbDatabaseLike {
|
|
|
82
94
|
const sorted = (): [string, unknown][] =>
|
|
83
95
|
[...table].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
84
96
|
return {
|
|
85
|
-
|
|
97
|
+
get: (key) => run(() => structuredClone(table.get(key))),
|
|
98
|
+
put: (value, key) =>
|
|
99
|
+
run(() => {
|
|
100
|
+
if (options.quotaExceeded === true) {
|
|
101
|
+
abort();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
table.set(key, structuredClone(value));
|
|
105
|
+
}),
|
|
86
106
|
delete: (key) => run(() => void table.delete(key)),
|
|
87
107
|
getAll: () => run(() => sorted().map(([, value]) => structuredClone(value))),
|
|
88
108
|
getAllKeys: () => run(() => sorted().map(([key]) => key)),
|
package/src/idb-types.ts
CHANGED
|
@@ -14,6 +14,7 @@ export interface IdbRequestLike<T> {
|
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
export interface IdbStoreLike {
|
|
17
|
+
get(key: string): IdbRequestLike<unknown>;
|
|
17
18
|
put(value: unknown, key: string): IdbRequestLike<unknown>;
|
|
18
19
|
delete(key: string): IdbRequestLike<unknown>;
|
|
19
20
|
getAll(): IdbRequestLike<unknown[]>;
|
|
@@ -23,6 +24,12 @@ export interface IdbStoreLike {
|
|
|
23
24
|
export interface IdbTransactionLike {
|
|
24
25
|
oncomplete: (() => void) | null;
|
|
25
26
|
onerror: (() => void) | null;
|
|
27
|
+
/**
|
|
28
|
+
* A transaction the browser ABORTED — a quota refusal is the ordinary one — fires `abort` and
|
|
29
|
+
* nothing else: no `complete`, and no `error` on the transaction. Unlistened, every write awaiting
|
|
30
|
+
* it hung forever.
|
|
31
|
+
*/
|
|
32
|
+
onabort: (() => void) | null;
|
|
26
33
|
error: unknown;
|
|
27
34
|
objectStore(name: string): IdbStoreLike;
|
|
28
35
|
}
|
package/src/index.ts
CHANGED
|
@@ -74,6 +74,7 @@ export {
|
|
|
74
74
|
ReplicaIdentityError,
|
|
75
75
|
ReplicationFailedError,
|
|
76
76
|
ReplicationProtocolError,
|
|
77
|
+
ReplicationTlsError,
|
|
77
78
|
ReplicatorSlotHeldError,
|
|
78
79
|
ServerRenderLiveError,
|
|
79
80
|
SubscriptionLimitError,
|
|
@@ -158,7 +159,6 @@ export {
|
|
|
158
159
|
export {
|
|
159
160
|
type BackoffPolicy,
|
|
160
161
|
BROWSER_RECONNECT_MAX_MS,
|
|
161
|
-
backoffDelay,
|
|
162
162
|
browserBackoff,
|
|
163
163
|
defaultBackoff,
|
|
164
164
|
type JitterMode,
|
package/src/live-definition.ts
CHANGED
|
@@ -142,7 +142,11 @@ export function liveQueryDefinition(
|
|
|
142
142
|
},
|
|
143
143
|
snapshot: async ({ input }): Promise<SnapshotResult> => {
|
|
144
144
|
const window = await resolve(input);
|
|
145
|
-
|
|
145
|
+
// The position is taken BEFORE the rows: the rows are then at least that new, so the claim
|
|
146
|
+
// is true. Taken after, a commit landing mid-read was claimed and missing — and every change
|
|
147
|
+
// up to it is dropped downstream as already folded.
|
|
148
|
+
const lsn = options.lsn?.() ?? '';
|
|
149
|
+
return { rows: await window.read(), lsn };
|
|
146
150
|
},
|
|
147
151
|
matcher: (input) => windows.get(queryHash(name, input))?.matcher ?? UNRESOLVED,
|
|
148
152
|
// Read off the same resolved window as the matcher, so the scope the client keys rows under and
|
package/src/live-fanout.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { Row, RowPatch } from './json';
|
|
|
10
10
|
import type { LiveSubscription } from './live-contract';
|
|
11
11
|
import { applyToWindow, bridgeChange } from './matcher-bridge';
|
|
12
12
|
import { type QueryEntry, refillWindowInLane } from './query-window';
|
|
13
|
-
import type
|
|
13
|
+
import { type Subscriber, type SubscriberGate, windowIndex } from './subscriber-gate';
|
|
14
14
|
import { type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
15
15
|
|
|
16
16
|
export interface FanoutDeps {
|
|
@@ -38,6 +38,15 @@ export async function fanoutChange(
|
|
|
38
38
|
): Promise<FanoutResult> {
|
|
39
39
|
// A window that missed a change must be replaced before it is patched again, and it can only be
|
|
40
40
|
// replaced here — a fanout holds this entry's lane, and `fillWindow` takes the same one.
|
|
41
|
+
// No read has landed in this window yet: there is nothing to patch, and a patch folded into the
|
|
42
|
+
// empty rows would move `entry.lsn` past the read in flight and get that read discarded as older
|
|
43
|
+
// than the window — every row but the patched one lost, permanently. Marked stale instead, so the
|
|
44
|
+
// read that lands is applied and followed by one that includes this change.
|
|
45
|
+
if (entry.applied === 0) {
|
|
46
|
+
entry.stale = true;
|
|
47
|
+
return { sent: 0, stale: 0 };
|
|
48
|
+
}
|
|
49
|
+
if (change.op === 'truncate') return await truncated(deps, entry, change);
|
|
41
50
|
if (entry.stale) await refillWindowInLane(entry);
|
|
42
51
|
// The consume-side twin of the replicator's own duplicate guard, which had none. `entry.lsn =
|
|
43
52
|
// change.lsn` was unconditional, so a change the window already holds — a redelivery, or one
|
|
@@ -57,6 +66,8 @@ export async function fanoutChange(
|
|
|
57
66
|
for (const patch of result.patches) deps.source.append(entry.qid, patch);
|
|
58
67
|
|
|
59
68
|
let sent = 0;
|
|
69
|
+
// Indexed once for every subscriber below, never searched per subscriber per patch.
|
|
70
|
+
const index = windowIndex(entry.rows);
|
|
60
71
|
for (const subscription of entry.subscribers.values()) {
|
|
61
72
|
if (result.refill) {
|
|
62
73
|
// The window lost its tail: guessing is how a sync engine silently diverges. Checked BEFORE
|
|
@@ -84,7 +95,8 @@ export async function fanoutChange(
|
|
|
84
95
|
entry,
|
|
85
96
|
who,
|
|
86
97
|
result.patches,
|
|
87
|
-
|
|
98
|
+
heldBy(subscription.cursor),
|
|
99
|
+
index,
|
|
88
100
|
);
|
|
89
101
|
} catch {
|
|
90
102
|
// Already counted and reported as a gate failure. Degrade this one subscriber the way a
|
|
@@ -112,6 +124,43 @@ export async function fanoutChange(
|
|
|
112
124
|
return { sent, stale: 0 };
|
|
113
125
|
}
|
|
114
126
|
|
|
127
|
+
/**
|
|
128
|
+
* Every row of a relation this window reads is gone. There is nothing to patch from — a truncate
|
|
129
|
+
* names no row — so the window is re-read now, in the lane, and every subscriber is re-snapshotted
|
|
130
|
+
* out of what came back: a window that kept the truncated rows until its next change would serve
|
|
131
|
+
* them to every new subscriber in the meantime.
|
|
132
|
+
*/
|
|
133
|
+
async function truncated(
|
|
134
|
+
deps: FanoutDeps,
|
|
135
|
+
entry: QueryEntry,
|
|
136
|
+
change: ChangeEvent,
|
|
137
|
+
): Promise<FanoutResult> {
|
|
138
|
+
if (!entry.shape.entities.includes(change.entity)) return { sent: 0, stale: 0 };
|
|
139
|
+
await refillWindowInLane(entry);
|
|
140
|
+
if (change.lsn > entry.lsn) entry.lsn = change.lsn;
|
|
141
|
+
let sent = 0;
|
|
142
|
+
for (const subscription of entry.subscribers.values()) {
|
|
143
|
+
subscription.socket.markDesynced(subscription.sid);
|
|
144
|
+
if (await resnapshot(deps, entry, subscription)) sent += 1;
|
|
145
|
+
}
|
|
146
|
+
return { sent, stale: 0 };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* A cursor's ids as a set, built once per ids ARRAY rather than once per subscriber per change.
|
|
151
|
+
* `advance` hands an update's cursor the same array it had (an update moves no id), so the cache
|
|
152
|
+
* holds across the common change and dies with the array; an insert or delete makes a new one.
|
|
153
|
+
*/
|
|
154
|
+
const heldSets = new WeakMap<readonly string[], ReadonlySet<string>>();
|
|
155
|
+
|
|
156
|
+
function heldBy(cursor: LiveCursor): ReadonlySet<string> {
|
|
157
|
+
const cached = heldSets.get(cursor.ids);
|
|
158
|
+
if (cached !== undefined) return cached;
|
|
159
|
+
const held = new Set(cursor.ids);
|
|
160
|
+
heldSets.set(cursor.ids, held);
|
|
161
|
+
return held;
|
|
162
|
+
}
|
|
163
|
+
|
|
115
164
|
/**
|
|
116
165
|
* The repair for one diverged subscriber, out of the window the lane is already holding — no DB
|
|
117
166
|
* read, one frame. Its cursor is rebuilt from what this subscriber may actually see, exactly as
|
package/src/live-query.ts
CHANGED
|
@@ -70,6 +70,7 @@ export class LiveQueryRegistry {
|
|
|
70
70
|
readonly #maxEntries: number;
|
|
71
71
|
/** What one lane needs, and nothing this class holds beyond it. */
|
|
72
72
|
readonly #fanout: FanoutDeps;
|
|
73
|
+
#lastLsn = '';
|
|
73
74
|
#staleChanges = 0;
|
|
74
75
|
|
|
75
76
|
constructor(options: LiveQueryRegistryOptions) {
|
|
@@ -363,7 +364,17 @@ export class LiveQueryRegistry {
|
|
|
363
364
|
* never per node — awaiting one entry before entering the next made one slow policy pass the
|
|
364
365
|
* whole node's pace, and let a lane that threw skip every entry behind it with nobody desynced.
|
|
365
366
|
*/
|
|
367
|
+
/**
|
|
368
|
+
* The newest change position this registry has been handed. What a node's snapshot may claim
|
|
369
|
+
* (`liveQueryDefinition`'s `lsn`): a read begun after it holds at least that change, and every
|
|
370
|
+
* later change is above it. `''` before the first.
|
|
371
|
+
*/
|
|
372
|
+
get lastLsn(): string {
|
|
373
|
+
return this.#lastLsn;
|
|
374
|
+
}
|
|
375
|
+
|
|
366
376
|
async deliver(change: ChangeEvent): Promise<number> {
|
|
377
|
+
if (change.lsn > this.#lastLsn) this.#lastLsn = change.lsn;
|
|
367
378
|
const lanes = [...this.#entries.values()].map(async (entry) => {
|
|
368
379
|
try {
|
|
369
380
|
const result = await entry.lock.run(() => fanoutChange(this.#fanout, entry, change));
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// The in-process replicator: committed row changes in, `ChangeEvent`s out, fanned into the node.
|
|
2
|
+
//
|
|
3
|
+
// Production decodes the write-ahead log. PGlite has no walsender and the memory driver has no log,
|
|
4
|
+
// so a test process had no change source at all — which is what left the `subscribe` fixture with
|
|
5
|
+
// no driver, and its five tests in `examples/dummy` asserting against a snapshot that never moved.
|
|
6
|
+
//
|
|
7
|
+
// WHAT IS REAL HERE, and it is everything downstream of the decoder: the matcher, the shared window,
|
|
8
|
+
// the per-subscriber `visible` gate, the cursor, the frames. This substitutes for the WAL DECODER
|
|
9
|
+
// and for nothing else — `@ultimat3/entity`'s `setRowObserver` reports what a repository wrote, in
|
|
10
|
+
// this process, and the events are shaped exactly as `PgLogicalReplicationFeed` shapes them.
|
|
11
|
+
//
|
|
12
|
+
// WHAT IS NOT: a write another process made is invisible, because nothing here reads a log. That is
|
|
13
|
+
// the honest bound, and it is why `selectChangeFeed` never picks it — the boot that installs it
|
|
14
|
+
// decides, and only under an embedded database.
|
|
15
|
+
//
|
|
16
|
+
// Lives in `@ultimat3/realtime/server` since 2026-09-23: it imports only `entity` and `realtime`,
|
|
17
|
+
// so it is a second change source beside the WAL decoder, and `x dev` booting it out of
|
|
18
|
+
// `@ultimat3/testing` put the test harness in every dev process's graph. `@ultimat3/testing`
|
|
19
|
+
// re-exports it until 22.0.0.
|
|
20
|
+
//
|
|
21
|
+
// `x dev` is the one boot that installs it outside a test, `As of 2026-09-05` (`@ultimat3/cli`'s
|
|
22
|
+
// `dev-live-feed.ts`), and only under the EMBEDDED database: PGlite has no walsender, every role
|
|
23
|
+
// runs in that one process, so the bound above holds by construction — and a real `DATABASE_URL`
|
|
24
|
+
// gets the decoder instead, never both.
|
|
25
|
+
|
|
26
|
+
import type { RowBulkChange, RowChange, RowObserver } from '@ultimat3/entity';
|
|
27
|
+
import type { ChangeEvent, ChangeOp } from './changefeed';
|
|
28
|
+
import type { Row } from './json';
|
|
29
|
+
import type { LiveQueryRegistry } from './live-query';
|
|
30
|
+
|
|
31
|
+
/** What a caller does with a change nobody could deliver. */
|
|
32
|
+
export interface LiveReplicatorOptions {
|
|
33
|
+
readonly registry: LiveQueryRegistry;
|
|
34
|
+
/**
|
|
35
|
+
* The node's declared channels, fed the same `ChangeEvent` — what a real node's change
|
|
36
|
+
* subscription does beside `registry.deliver` (`sync-node.ts`). Without it a channel's `records`
|
|
37
|
+
* frames never carried a write made under `x dev`, and every other tab stayed on the old row.
|
|
38
|
+
*/
|
|
39
|
+
readonly channels?: { deliverChange(change: ChangeEvent): unknown };
|
|
40
|
+
/** Tenant column, hoisted out of the row so fanout filters without parsing it. */
|
|
41
|
+
readonly tenantColumn?: string;
|
|
42
|
+
readonly onError?: (error: unknown) => void;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface LiveReplicator {
|
|
46
|
+
/** Resolves when every change observed so far has been fanned out. Never a sleep. */
|
|
47
|
+
settled(): Promise<void>;
|
|
48
|
+
/** Changes this replicator has delivered — the number a test asserts a patch count against. */
|
|
49
|
+
readonly delivered: number;
|
|
50
|
+
stop(): void;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* A `ChangeEvent` row is `Row` — a JSON object carrying an `id`. Every row a repository stores has
|
|
55
|
+
* one; the cast is what says so to a compiler that only sees `Record<string, unknown>`, and a row
|
|
56
|
+
* that genuinely has none fails downstream in `idOf`, with the entity named, exactly as a row off
|
|
57
|
+
* the wire would.
|
|
58
|
+
*/
|
|
59
|
+
const asRow = (value: Readonly<Record<string, unknown>> | null): Row | null =>
|
|
60
|
+
value === null ? null : (value as unknown as Row);
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Lexicographically comparable, which is the whole contract of an lsn — `formatLsn` in
|
|
64
|
+
* `@ultimat3/realtime` produces the same shape from a real WAL position. A counter is enough here
|
|
65
|
+
* because one process observes its own writes in the order it made them.
|
|
66
|
+
*/
|
|
67
|
+
const lsnOf = (position: number): string => position.toString(16).padStart(16, '0');
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Install the replicator for the length of one test. It takes over the process row observer and
|
|
71
|
+
* hands back whatever was installed before, because `bun test` shares one process across files and
|
|
72
|
+
* an unconditional clear would take an outer harness's observer with it.
|
|
73
|
+
*/
|
|
74
|
+
export async function startLiveReplicator(options: LiveReplicatorOptions): Promise<LiveReplicator> {
|
|
75
|
+
// Awaited BEFORE the observer exists, so installation is the last thing this function does and
|
|
76
|
+
// no write between the call and the install can slip past unobserved.
|
|
77
|
+
const entity = await import('@ultimat3/entity');
|
|
78
|
+
const { registry } = options;
|
|
79
|
+
const tenant = options.tenantColumn ?? 'orgId';
|
|
80
|
+
let position = 0;
|
|
81
|
+
let delivered = 0;
|
|
82
|
+
// One promise chain, because ORDERING is the guarantee the whole pipeline is built on — the same
|
|
83
|
+
// reason `InMemoryChangeFeed` serializes its deliveries rather than firing them concurrently.
|
|
84
|
+
let tail: Promise<void> = Promise.resolve();
|
|
85
|
+
let stopped = false;
|
|
86
|
+
|
|
87
|
+
const enqueue = (work: () => Promise<void>): void => {
|
|
88
|
+
tail = tail.then(work).catch((error: unknown) => {
|
|
89
|
+
// Never rethrown into the chain: one failed fanout must not silence every change behind it,
|
|
90
|
+
// and a rejection with nobody to hand it to ends the Bun process.
|
|
91
|
+
options.onError?.(error);
|
|
92
|
+
});
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const observer: RowObserver = {
|
|
96
|
+
onChange(change: RowChange): void {
|
|
97
|
+
if (stopped) return;
|
|
98
|
+
position += 1;
|
|
99
|
+
const at = position;
|
|
100
|
+
const row = change.after ?? change.before;
|
|
101
|
+
const orgId = typeof row?.[tenant] === 'string' ? (row[tenant] as string) : null;
|
|
102
|
+
const event: ChangeEvent = {
|
|
103
|
+
entity: change.entity,
|
|
104
|
+
// A repository's three row ops are three of the feed's four; a truncate never comes this way.
|
|
105
|
+
op: change.op satisfies ChangeOp,
|
|
106
|
+
before: asRow(change.before),
|
|
107
|
+
after: asRow(change.after),
|
|
108
|
+
lsn: lsnOf(at),
|
|
109
|
+
txid: String(at),
|
|
110
|
+
orgId,
|
|
111
|
+
// Deliberately not a clock read: the preload freezes `Date.now()`, and a change's commit
|
|
112
|
+
// time is not something any assertion in this repo reads. `at` keeps it monotonic anyway.
|
|
113
|
+
at,
|
|
114
|
+
// The keyed write it belongs to, read off the request scope — what the WAL decoder reads
|
|
115
|
+
// off the transaction's opening message — so a channel frame names it here as it would there.
|
|
116
|
+
...(change.write === undefined ? {} : { write: change.write }),
|
|
117
|
+
};
|
|
118
|
+
enqueue(async () => {
|
|
119
|
+
// Channels first, as the node does: a live query's fanout that throws must not also cost
|
|
120
|
+
// every declared channel the change.
|
|
121
|
+
options.channels?.deliverChange(event);
|
|
122
|
+
delivered += await registry.deliver(event);
|
|
123
|
+
});
|
|
124
|
+
},
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* A filtered write names rows this seam never saw, so there is no event to shape. Every window
|
|
128
|
+
* on the node is marked stale instead and re-read on the next change — `invalidate()` is the
|
|
129
|
+
* node's own answer to "the change stream skipped something", used here for the one write that
|
|
130
|
+
* genuinely does. Silence would be the alternative, and a subscriber told nothing happened
|
|
131
|
+
* diverges with nobody ever asking again.
|
|
132
|
+
*/
|
|
133
|
+
onBulk(_change: RowBulkChange): void {
|
|
134
|
+
if (stopped) return;
|
|
135
|
+
registry.invalidate();
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const previous = entity.setRowObserver(observer);
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
get delivered() {
|
|
143
|
+
return delivered;
|
|
144
|
+
},
|
|
145
|
+
settled: async () => {
|
|
146
|
+
// Twice: a fanout can enqueue nothing, but the writes that produced these changes may still
|
|
147
|
+
// be resolving their own promises when a test asks. Awaiting the chain, letting the
|
|
148
|
+
// microtask queue drain, then awaiting it again covers a change observed in between.
|
|
149
|
+
await tail;
|
|
150
|
+
for (let turn = 0; turn < 8; turn += 1) await Promise.resolve();
|
|
151
|
+
await tail;
|
|
152
|
+
},
|
|
153
|
+
stop: () => {
|
|
154
|
+
stopped = true;
|
|
155
|
+
// Restored, never cleared: one process runs every test file, and an outer harness's observer
|
|
156
|
+
// must survive an inner fixture finishing.
|
|
157
|
+
entity.setRowObserver(previous);
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
package/src/local-store-idb.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import type { ClientScope, RecordRows, Row } from '@ultimat3/core/page';
|
|
9
9
|
import { isJsonObject, renderThrowable, type UltimateError } from '@ultimat3/core/page';
|
|
10
10
|
import type { IdbDatabaseLike, IdbFactoryLike, IdbRequestLike } from './idb-types';
|
|
11
|
-
import type { QueueState } from './offline-queue';
|
|
11
|
+
import type { QueueChange, QueuedMutation, QueueState } from './offline-queue';
|
|
12
12
|
import { LocalStoreUnavailableError } from './page-errors';
|
|
13
13
|
|
|
14
14
|
/** One persisted row, by record type and record key. */
|
|
@@ -28,7 +28,12 @@ export interface LocalStore {
|
|
|
28
28
|
deletes: readonly Omit<PersistedRow, 'row'>[],
|
|
29
29
|
): Promise<void>;
|
|
30
30
|
queue(scope: string): Promise<QueueState | undefined>;
|
|
31
|
-
|
|
31
|
+
/**
|
|
32
|
+
* One change to one scope's outbox, BY KEY (`QueueChange`). It was `saveQueue(scope, state)`,
|
|
33
|
+
* a whole-queue save — and two tabs of one user each saved their own copy, so the last save won
|
|
34
|
+
* and the other tab's queued write was erased.
|
|
35
|
+
*/
|
|
36
|
+
writeQueue(scope: string, change: QueueChange): Promise<void>;
|
|
32
37
|
/** Everything of one scope — its rows AND its outbox. Sign-out, or any principal change. */
|
|
33
38
|
wipe(scope: string): Promise<void>;
|
|
34
39
|
/**
|
|
@@ -53,12 +58,26 @@ const OUTBOX = 'outbox';
|
|
|
53
58
|
/** JSON, never a joined string: a principal is opaque and may hold any separator. */
|
|
54
59
|
const rowKey = (scope: string, type: string, key: string): string =>
|
|
55
60
|
JSON.stringify([scope, type, key]);
|
|
61
|
+
/** One queued mutation, and one scope's sequence floor — the outbox's two record shapes. */
|
|
62
|
+
const mutationKey = (scope: string, key: string): string => JSON.stringify([scope, 'm', key]);
|
|
63
|
+
const seqSlot = (scope: string): string => JSON.stringify([scope, 'seq']);
|
|
64
|
+
|
|
65
|
+
/** The scope an outbox key belongs to: `[scope, …]`, or a pre-22.0.0 whole-queue record `scope`. */
|
|
66
|
+
function outboxScopeOf(raw: unknown): string | undefined {
|
|
67
|
+
if (typeof raw !== 'string') return undefined;
|
|
68
|
+
if (!raw.startsWith('[')) return raw;
|
|
69
|
+
const parts: unknown = JSON.parse(raw);
|
|
70
|
+
return Array.isArray(parts) && typeof parts[0] === 'string' ? parts[0] : undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const byQueueOrder = (a: QueuedMutation, b: QueuedMutation): number =>
|
|
74
|
+
a.seq - b.seq || a.enqueuedAt - b.enqueuedAt || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0);
|
|
56
75
|
|
|
57
76
|
/** Memory: tests, SSR, and the fallback when IndexedDB is unavailable. */
|
|
58
77
|
export class MemoryLocalStore implements LocalStore {
|
|
59
78
|
readonly kind = 'memory';
|
|
60
79
|
readonly #rows = new Map<string, PersistedRow & { readonly scope: string }>();
|
|
61
|
-
readonly #queues = new Map<string,
|
|
80
|
+
readonly #queues = new Map<string, { mutations: Map<string, QueuedMutation>; nextSeq: number }>();
|
|
62
81
|
|
|
63
82
|
async rows(scope: string): Promise<ReadonlyMap<string, RecordRows>> {
|
|
64
83
|
return group([...this.#rows.values()].filter((entry) => entry.scope === scope));
|
|
@@ -72,10 +91,19 @@ export class MemoryLocalStore implements LocalStore {
|
|
|
72
91
|
for (const put of puts) this.#rows.set(rowKey(scope, put.type, put.key), { ...put, scope });
|
|
73
92
|
}
|
|
74
93
|
async queue(scope: string): Promise<QueueState | undefined> {
|
|
75
|
-
|
|
94
|
+
const held = this.#queues.get(scope);
|
|
95
|
+
if (held === undefined) return undefined;
|
|
96
|
+
return {
|
|
97
|
+
mutations: structuredClone([...held.mutations.values()]).sort(byQueueOrder),
|
|
98
|
+
nextSeq: held.nextSeq,
|
|
99
|
+
};
|
|
76
100
|
}
|
|
77
|
-
async
|
|
78
|
-
this.#queues.
|
|
101
|
+
async writeQueue(scope: string, change: QueueChange): Promise<void> {
|
|
102
|
+
const held = this.#queues.get(scope) ?? { mutations: new Map(), nextSeq: 1 };
|
|
103
|
+
for (const key of change.deletes) held.mutations.delete(key);
|
|
104
|
+
for (const put of change.puts) held.mutations.set(put.key, structuredClone(put));
|
|
105
|
+
held.nextSeq = Math.max(held.nextSeq, change.nextSeq);
|
|
106
|
+
this.#queues.set(scope, held);
|
|
79
107
|
}
|
|
80
108
|
async wipe(scope: string): Promise<void> {
|
|
81
109
|
for (const [key, entry] of this.#rows) if (entry.scope === scope) this.#rows.delete(key);
|
|
@@ -116,15 +144,52 @@ class IdbLocalStore implements LocalStore {
|
|
|
116
144
|
await done(tx);
|
|
117
145
|
}
|
|
118
146
|
async queue(scope: string): Promise<QueueState | undefined> {
|
|
119
|
-
|
|
147
|
+
// Read-WRITE: a pre-22.0.0 whole-queue record for this scope is converted in the same
|
|
148
|
+
// transaction, so an upgrade keeps the writes a user queued on the previous version.
|
|
149
|
+
const tx = this.db.transaction(OUTBOX, 'readwrite');
|
|
120
150
|
const store = tx.objectStore(OUTBOX);
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
151
|
+
// Both issued before either is awaited, and awaited one by one: the conversion below writes in
|
|
152
|
+
// this transaction, and it must still be open when the reads land.
|
|
153
|
+
const asked = [answer(store.getAllKeys()), answer(store.getAll())] as const;
|
|
154
|
+
const keys = await asked[0];
|
|
155
|
+
const values = await asked[1];
|
|
156
|
+
const mutations = new Map<string, QueuedMutation>();
|
|
157
|
+
let nextSeq = 1;
|
|
158
|
+
let found = false;
|
|
159
|
+
let converted = false;
|
|
160
|
+
keys.forEach((raw, index) => {
|
|
161
|
+
if (outboxScopeOf(raw) !== scope) return;
|
|
162
|
+
found = true;
|
|
163
|
+
const value = values[index];
|
|
164
|
+
if (raw === scope) {
|
|
165
|
+
const legacy = value as QueueState;
|
|
166
|
+
for (const mutation of legacy.mutations) {
|
|
167
|
+
mutations.set(mutation.key, mutation);
|
|
168
|
+
store.put(mutation, mutationKey(scope, mutation.key));
|
|
169
|
+
}
|
|
170
|
+
nextSeq = Math.max(nextSeq, legacy.nextSeq);
|
|
171
|
+
store.put(nextSeq, seqSlot(scope));
|
|
172
|
+
store.delete(scope);
|
|
173
|
+
converted = true;
|
|
174
|
+
} else if (raw === seqSlot(scope)) {
|
|
175
|
+
nextSeq = Math.max(nextSeq, typeof value === 'number' ? value : 1);
|
|
176
|
+
} else {
|
|
177
|
+
const mutation = value as QueuedMutation;
|
|
178
|
+
mutations.set(mutation.key, mutation);
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
// Awaited only when something was written: a transaction that issued nothing after its reads
|
|
182
|
+
// has already completed, and a listener attached now would wait for an event that is gone.
|
|
183
|
+
if (converted) await done(tx);
|
|
184
|
+
return found ? { mutations: [...mutations.values()].sort(byQueueOrder), nextSeq } : undefined;
|
|
124
185
|
}
|
|
125
|
-
async
|
|
186
|
+
async writeQueue(scope: string, change: QueueChange): Promise<void> {
|
|
126
187
|
const tx = this.db.transaction(OUTBOX, 'readwrite');
|
|
127
|
-
tx.objectStore(OUTBOX)
|
|
188
|
+
const store = tx.objectStore(OUTBOX);
|
|
189
|
+
const floor = await answer(store.get(seqSlot(scope)));
|
|
190
|
+
for (const key of change.deletes) store.delete(mutationKey(scope, key));
|
|
191
|
+
for (const put of change.puts) store.put(put, mutationKey(scope, put.key));
|
|
192
|
+
store.put(Math.max(typeof floor === 'number' ? floor : 1, change.nextSeq), seqSlot(scope));
|
|
128
193
|
await done(tx);
|
|
129
194
|
}
|
|
130
195
|
async wipe(scope: string): Promise<void> {
|
|
@@ -136,7 +201,10 @@ class IdbLocalStore implements LocalStore {
|
|
|
136
201
|
const parts: unknown = JSON.parse(raw);
|
|
137
202
|
if (Array.isArray(parts) && parts[0] === scope) records.delete(raw);
|
|
138
203
|
}
|
|
139
|
-
tx.objectStore(OUTBOX)
|
|
204
|
+
const outbox = tx.objectStore(OUTBOX);
|
|
205
|
+
for (const raw of await answer(outbox.getAllKeys())) {
|
|
206
|
+
if (outboxScopeOf(raw) === scope && typeof raw === 'string') outbox.delete(raw);
|
|
207
|
+
}
|
|
140
208
|
await done(tx);
|
|
141
209
|
}
|
|
142
210
|
async wipeOthers(keep: string): Promise<void> {
|
|
@@ -152,8 +220,9 @@ class IdbLocalStore implements LocalStore {
|
|
|
152
220
|
const parts: unknown = JSON.parse(raw);
|
|
153
221
|
if (Array.isArray(parts) && parts[0] !== keep) records.delete(raw);
|
|
154
222
|
}
|
|
155
|
-
for (const
|
|
156
|
-
|
|
223
|
+
for (const raw of queueKeys) {
|
|
224
|
+
const scope = outboxScopeOf(raw);
|
|
225
|
+
if (typeof raw === 'string' && scope !== undefined && scope !== keep) outbox.delete(raw);
|
|
157
226
|
}
|
|
158
227
|
await done(tx);
|
|
159
228
|
}
|
|
@@ -214,11 +283,16 @@ function answer<T>(request: IdbRequestLike<T>): Promise<T> {
|
|
|
214
283
|
function done(tx: {
|
|
215
284
|
oncomplete: (() => void) | null;
|
|
216
285
|
onerror: (() => void) | null;
|
|
286
|
+
onabort: (() => void) | null;
|
|
217
287
|
error: unknown;
|
|
218
288
|
}): Promise<void> {
|
|
219
289
|
return new Promise((resolve, reject) => {
|
|
220
290
|
tx.oncomplete = (): void => resolve();
|
|
221
291
|
tx.onerror = (): void => reject(tx.error);
|
|
292
|
+
// A quota refusal ABORTS the transaction and fires nothing else, so a store that listened only
|
|
293
|
+
// for `complete` and `error` left `write`, `writeQueue`, `flush` and `enqueue` pending forever.
|
|
294
|
+
tx.onabort = (): void =>
|
|
295
|
+
reject(tx.error ?? new DOMException('the transaction was aborted', 'AbortError'));
|
|
222
296
|
});
|
|
223
297
|
}
|
|
224
298
|
|