@ultimat3/realtime 22.15.0 → 24.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +89 -102
- package/README.md +140 -20
- package/package.json +4 -4
- package/src/advisory-lock.ts +57 -0
- package/src/boot.ts +12 -7
- package/src/change-buffer.ts +96 -41
- package/src/changefeed-env.ts +2 -2
- package/src/changefeed.ts +39 -1
- package/src/channel-authz.ts +43 -12
- package/src/channel-latch.ts +52 -0
- package/src/channel-logs.ts +34 -4
- package/src/channel-records.ts +33 -2
- package/src/channel-sids.ts +37 -0
- package/src/channel.ts +143 -44
- package/src/client-channels.ts +10 -1
- package/src/client-contract.ts +4 -1
- package/src/client-frames.ts +34 -4
- package/src/client-heartbeat.ts +39 -9
- package/src/client.ts +1 -0
- package/src/cursor.ts +11 -0
- package/src/errors.ts +70 -19
- package/src/fanout.ts +13 -0
- package/src/index.ts +1 -1
- package/src/live-contract.ts +6 -2
- package/src/live-definition.ts +46 -11
- package/src/live-fanout.ts +70 -11
- package/src/live-floor.ts +35 -0
- package/src/live-query-options.ts +38 -0
- package/src/live-query.ts +119 -91
- package/src/live-refusal.ts +21 -0
- package/src/live-replicator.ts +6 -6
- package/src/live-resume.ts +74 -0
- package/src/live-tenant.ts +25 -0
- package/src/nats-client.ts +11 -2
- package/src/nats-open.ts +9 -0
- package/src/nats-transport.ts +23 -1
- package/src/offline-queue.ts +48 -5
- package/src/page-errors.ts +17 -0
- package/src/page-outbox.ts +121 -50
- package/src/pg-advisory-lock.ts +45 -1
- package/src/pg-auth.ts +9 -4
- package/src/pg-bytes.ts +4 -1
- package/src/pg-connection.ts +119 -21
- package/src/pg-entity-row.ts +27 -2
- package/src/pg-identity-tables.ts +25 -0
- package/src/pg-preflight.ts +50 -4
- package/src/pg-replication-row.ts +82 -0
- package/src/pg-replication.ts +44 -45
- package/src/pg-socket.ts +6 -4
- package/src/pg-tls.ts +18 -0
- package/src/pg-wire.ts +5 -3
- package/src/pgoutput.ts +19 -9
- package/src/presence.ts +85 -8
- package/src/query-window.ts +28 -7
- package/src/record-persister.ts +7 -2
- package/src/replication-errors.ts +8 -4
- package/src/replicator-envelope.ts +110 -0
- package/src/replicator-row.ts +75 -0
- package/src/replicator.ts +319 -186
- package/src/server.ts +11 -7
- package/src/socket-engine.ts +14 -2
- package/src/socket-idle.ts +3 -1
- package/src/socket-routes.ts +8 -3
- package/src/subscriber-gate.ts +11 -1
- package/src/sync-auth.ts +47 -2
- package/src/sync-frames.ts +73 -20
- package/src/sync-node-bounds.ts +15 -0
- package/src/sync-node-contract.ts +20 -2
- package/src/sync-node.ts +92 -17
- package/src/sync-origin.ts +26 -11
- package/src/sync-protocol.ts +13 -3
- package/src/sync-upgrade.ts +76 -8
- package/src/use-mutation.ts +33 -5
- package/src/wire-channel.ts +13 -4
package/src/pg-tls.ts
CHANGED
|
@@ -62,6 +62,24 @@ export function parseSsl(params: URLSearchParams): SslSettings {
|
|
|
62
62
|
/** Whether a mode asks the runtime for the verification at all. */
|
|
63
63
|
export const verifies = (ssl: SslMode): boolean => ssl === 'verify-ca' || ssl === 'verify-full';
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Whether a cleartext or md5 password request is answered. Cleartext IS the password; md5 is
|
|
67
|
+
* replayable. What each mode actually protects against:
|
|
68
|
+
*
|
|
69
|
+
* - `verify-ca` / `verify-full`: the session is encrypted to a server whose certificate was
|
|
70
|
+
* checked. The only modes that stop an ON-PATH attacker.
|
|
71
|
+
* - `require`: encrypted, to whoever answered. No certificate is verified, so this stops a PASSIVE
|
|
72
|
+
* listener and nothing else — an attacker on the path can terminate the TLS session itself and
|
|
73
|
+
* ask for the password. Answered anyway: it is libpq's behaviour, and managed databases whose
|
|
74
|
+
* CA an app does not hold depend on it. An operator who needs more names a `verify-*` mode.
|
|
75
|
+
* - `disable`: the operator SAYING the session is cleartext — the opt-in.
|
|
76
|
+
* - `allow` / `prefer` (the default, and what an unstated mode is): refused. They do not even
|
|
77
|
+
* promise encryption — a stripped upgrade falls back to cleartext without a word — so they stop
|
|
78
|
+
* neither kind of attacker, and nobody chose that.
|
|
79
|
+
*/
|
|
80
|
+
export const answersWeakAuth = (ssl: SslMode | undefined): boolean =>
|
|
81
|
+
ssl === 'disable' || ssl === 'require' || (ssl !== undefined && verifies(ssl));
|
|
82
|
+
|
|
65
83
|
/** OpenSSL's code for a certificate whose chain verified but whose names do not include the host. */
|
|
66
84
|
const HOST_MISMATCH = 'ERR_TLS_CERT_ALTNAME_INVALID';
|
|
67
85
|
|
package/src/pg-wire.ts
CHANGED
|
@@ -69,8 +69,10 @@ export class MessageReader {
|
|
|
69
69
|
if (held === 0) return undefined;
|
|
70
70
|
throw new ReplicationProtocolError({
|
|
71
71
|
stage: 'read',
|
|
72
|
-
detail:
|
|
73
|
-
|
|
72
|
+
detail:
|
|
73
|
+
`the connection closed with ${held} bytes of a partial message — the backend was ` +
|
|
74
|
+
'terminated mid-message, and the server log names the reason',
|
|
75
|
+
fix: 'x doctor --json',
|
|
74
76
|
});
|
|
75
77
|
}
|
|
76
78
|
this.#append(chunk);
|
|
@@ -245,7 +247,7 @@ export const FIXES: Readonly<Record<string, string>> = {
|
|
|
245
247
|
};
|
|
246
248
|
|
|
247
249
|
/** What a SQLSTATE this table has no entry for is answered with. */
|
|
248
|
-
const GENERIC_FIX = 'x doctor
|
|
250
|
+
const GENERIC_FIX = 'x doctor --json';
|
|
249
251
|
|
|
250
252
|
/**
|
|
251
253
|
* An `ErrorResponse` becomes the one error class whose `fix` names the command to run.
|
package/src/pgoutput.ts
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
// Decodes pgoutput logical-replication messages (protocol version 1
|
|
2
|
-
//
|
|
3
|
-
// per-connection relation cache
|
|
4
|
-
//
|
|
5
|
-
// What a tuple's TEXT means is `pg-values.ts`'s: this file frames messages, that one owns the type
|
|
6
|
-
// catalogue that turns postgres' text into the value a repository row holds.
|
|
1
|
+
// Decodes pgoutput logical-replication messages (protocol version 1) into typed PgOutputMessage
|
|
2
|
+
// values; the stream asks `messages 'true'`, which needs Postgres >= 14. Pure byte decoding, no
|
|
3
|
+
// I/O: a decoder owns the per-connection relation cache later tuples reference by oid. What a
|
|
4
|
+
// tuple's TEXT means is `pg-values.ts`'s — this file frames messages, that one owns the types.
|
|
7
5
|
|
|
8
6
|
import { isWriteDigest, WRITE_ORIGIN_WAL_PREFIX } from '@ultimat3/core';
|
|
9
7
|
import { ReplicationProtocolError } from './errors';
|
|
@@ -48,6 +46,12 @@ export type PgOutputMessage =
|
|
|
48
46
|
readonly relation: PgRelation;
|
|
49
47
|
readonly before: PhysicalRow | null;
|
|
50
48
|
readonly after: PhysicalRow;
|
|
49
|
+
/**
|
|
50
|
+
* Physical columns the new tuple sent as `'u'`: an out-of-line (TOAST) value the UPDATE did
|
|
51
|
+
* not touch, for which Postgres logs no bytes. They are absent from `after`, and this list is
|
|
52
|
+
* what tells "absent" apart from "the whole row".
|
|
53
|
+
*/
|
|
54
|
+
readonly unchanged: readonly string[];
|
|
51
55
|
}
|
|
52
56
|
| { readonly kind: 'delete'; readonly relation: PgRelation; readonly before: PhysicalRow }
|
|
53
57
|
| { readonly kind: 'truncate'; readonly relations: readonly PgRelation[] }
|
|
@@ -67,7 +71,11 @@ export type PgOutputMessage =
|
|
|
67
71
|
* `'u'` standing in for the columns a key-only tuple leaves out — so the column count always
|
|
68
72
|
* matches the relation, and only the per-column byte tells us whether a value is actually there.
|
|
69
73
|
*/
|
|
70
|
-
function decodeTupleData(
|
|
74
|
+
function decodeTupleData(
|
|
75
|
+
reader: ByteReader,
|
|
76
|
+
relation: PgRelation,
|
|
77
|
+
unchanged?: string[],
|
|
78
|
+
): PhysicalRow {
|
|
71
79
|
const count = reader.int16();
|
|
72
80
|
if (count !== relation.columns.length) {
|
|
73
81
|
throw new ReplicationProtocolError({
|
|
@@ -90,6 +98,7 @@ function decodeTupleData(reader: ByteReader, relation: PgRelation): PhysicalRow
|
|
|
90
98
|
if (kind === 'u') {
|
|
91
99
|
// Omitted, not nulled: "unchanged" and "set to null" are different facts about the row,
|
|
92
100
|
// and a missing key is the only encoding that keeps those two facts distinguishable.
|
|
101
|
+
unchanged?.push(column.name);
|
|
93
102
|
continue;
|
|
94
103
|
}
|
|
95
104
|
if (kind === 't') {
|
|
@@ -245,8 +254,9 @@ export class PgOutputDecoder {
|
|
|
245
254
|
detail: `expected the new-tuple marker "N" but got "${marker}"`,
|
|
246
255
|
});
|
|
247
256
|
}
|
|
248
|
-
const
|
|
249
|
-
|
|
257
|
+
const unchanged: string[] = [];
|
|
258
|
+
const after = decodeTupleData(reader, relation, unchanged);
|
|
259
|
+
return { kind: 'update', relation, before, after, unchanged };
|
|
250
260
|
}
|
|
251
261
|
|
|
252
262
|
#decodeDelete(reader: ByteReader): PgOutputMessage {
|
package/src/presence.ts
CHANGED
|
@@ -25,6 +25,13 @@ export const PRESENCE_SWEEP_PREFIX = 'presence.sweep';
|
|
|
25
25
|
*/
|
|
26
26
|
export const DEFAULT_MAX_PRESENCE_MEMBERS = 256;
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Member ids the sweep leader ships on its lease, for the other nodes to record. Bounded because
|
|
30
|
+
* the lease is one bus message: past it, a member is known to the leader alone, as every member
|
|
31
|
+
* was before the lease carried a roster.
|
|
32
|
+
*/
|
|
33
|
+
export const PRESENCE_LEASE_ROSTER_LIMIT = 4_096;
|
|
34
|
+
|
|
28
35
|
export interface PresenceOptions {
|
|
29
36
|
readonly transport: Transport;
|
|
30
37
|
/** Optional: without a hub, presence is queryable but silent (no join/leave frames). */
|
|
@@ -65,6 +72,10 @@ export class PresenceRegistry {
|
|
|
65
72
|
readonly #nodeId: string;
|
|
66
73
|
/** Diffing cache only — the truth is always `transport.shared`. Safe to lose. */
|
|
67
74
|
readonly #seen = new Map<string, Set<string>>();
|
|
75
|
+
/** Per topic this node LEADS: the roster on its lease, as written. */
|
|
76
|
+
readonly #shipped = new Map<string, string>();
|
|
77
|
+
/** Per topic this node FOLLOWS: the ids the leader's lease last named. */
|
|
78
|
+
readonly #heard = new Map<string, ReadonlySet<string>>();
|
|
68
79
|
|
|
69
80
|
constructor(options: PresenceOptions) {
|
|
70
81
|
this.#transport = options.transport;
|
|
@@ -100,7 +111,10 @@ export class PresenceRegistry {
|
|
|
100
111
|
return await this.roster(name);
|
|
101
112
|
}
|
|
102
113
|
|
|
103
|
-
/**
|
|
114
|
+
/**
|
|
115
|
+
* A member's beat: its TTL renewed, nothing announced and nothing read. `false` means the member
|
|
116
|
+
* had already expired: the caller must `join` again, not `heartbeat`.
|
|
117
|
+
*/
|
|
104
118
|
async heartbeat(name: Topic, id: string): Promise<boolean> {
|
|
105
119
|
const alive = await this.#transport.shared.touch(this.#key(name), id, this.#ttlMs);
|
|
106
120
|
if (!alive) this.#track(name).delete(id);
|
|
@@ -190,12 +204,18 @@ export class PresenceRegistry {
|
|
|
190
204
|
// A room nobody is in is not a room. Without this the cache keeps one entry per topic ever
|
|
191
205
|
// subscribed to, for the life of the process, and the sweep walks all of them forever.
|
|
192
206
|
if ((this.#seen.get(name)?.size ?? 0) === 0) {
|
|
193
|
-
this.#
|
|
207
|
+
this.#forget(name);
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
const lease = await this.#claimSweep(name);
|
|
211
|
+
if (!lease.leads) {
|
|
212
|
+
this.#record(name, lease.roster);
|
|
194
213
|
continue;
|
|
195
214
|
}
|
|
196
|
-
|
|
215
|
+
this.#heard.delete(name);
|
|
197
216
|
gone.push(...(await this.sweep(name)));
|
|
198
|
-
|
|
217
|
+
await this.#ship(name);
|
|
218
|
+
if ((this.#seen.get(name)?.size ?? 0) === 0) this.#forget(name);
|
|
199
219
|
}
|
|
200
220
|
return gone;
|
|
201
221
|
}
|
|
@@ -211,14 +231,59 @@ export class PresenceRegistry {
|
|
|
211
231
|
* different views is a duplicate `leave` for a member who has already gone — which is what a
|
|
212
232
|
* `leave` frame means anyway. What it never produces is nobody sweeping: a claim is re-put every
|
|
213
233
|
* pass, and a dead leader's expires within one TTL.
|
|
234
|
+
*
|
|
235
|
+
* The leader's claim CARRIES the ids it last saw live, and every follower reads it in the read
|
|
236
|
+
* the election already makes. That is what a takeover diffs against: a leader that dies takes
|
|
237
|
+
* its own members with it, and a successor that had skipped every sweep held nothing but its own
|
|
238
|
+
* — so their leaves were never announced, to anyone, until each client reconnected.
|
|
214
239
|
*/
|
|
215
|
-
async #claimSweep(
|
|
240
|
+
async #claimSweep(
|
|
241
|
+
name: Topic,
|
|
242
|
+
): Promise<{ readonly leads: boolean; readonly roster: readonly string[] | null }> {
|
|
216
243
|
const key = `${PRESENCE_SWEEP_PREFIX}.${name}`;
|
|
217
|
-
await this.#transport.shared.put(key, this.#nodeId, '', this.#ttlMs);
|
|
244
|
+
await this.#transport.shared.put(key, this.#nodeId, this.#shipped.get(name) ?? '', this.#ttlMs);
|
|
218
245
|
const claimants = await this.#transport.shared.entries(key);
|
|
219
246
|
let leader = this.#nodeId;
|
|
220
|
-
|
|
221
|
-
|
|
247
|
+
let roster: readonly string[] | null = null;
|
|
248
|
+
for (const claimant of claimants) {
|
|
249
|
+
if (claimant.member > leader) continue;
|
|
250
|
+
leader = claimant.member;
|
|
251
|
+
roster = parseRoster(claimant.value);
|
|
252
|
+
}
|
|
253
|
+
return { leads: leader === this.#nodeId, roster };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** A follower's pass: what the leader said is live is recorded, what it stopped saying is not. */
|
|
257
|
+
#record(name: Topic, roster: readonly string[] | null): void {
|
|
258
|
+
this.#shipped.delete(name);
|
|
259
|
+
// A leader that has not swept yet has said nothing: nothing is learned and nothing forgotten.
|
|
260
|
+
if (roster === null) return;
|
|
261
|
+
const tracked = this.#track(name);
|
|
262
|
+
const now = new Set(roster);
|
|
263
|
+
// Dropped by the leader while it is alive means the leader announced it.
|
|
264
|
+
for (const id of this.#heard.get(name) ?? []) if (!now.has(id)) tracked.delete(id);
|
|
265
|
+
for (const id of now) tracked.add(id);
|
|
266
|
+
this.#heard.set(name, now);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** The leader's roster, written onto its lease when it changed. */
|
|
270
|
+
async #ship(name: Topic): Promise<void> {
|
|
271
|
+
const ids = [...(this.#seen.get(name) ?? [])].sort().slice(0, PRESENCE_LEASE_ROSTER_LIMIT);
|
|
272
|
+
const value = JSON.stringify(ids);
|
|
273
|
+
if (this.#shipped.get(name) === value) return;
|
|
274
|
+
this.#shipped.set(name, value);
|
|
275
|
+
await this.#transport.shared.put(
|
|
276
|
+
`${PRESENCE_SWEEP_PREFIX}.${name}`,
|
|
277
|
+
this.#nodeId,
|
|
278
|
+
value,
|
|
279
|
+
this.#ttlMs,
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
#forget(name: Topic): void {
|
|
284
|
+
this.#seen.delete(name);
|
|
285
|
+
this.#shipped.delete(name);
|
|
286
|
+
this.#heard.delete(name);
|
|
222
287
|
}
|
|
223
288
|
|
|
224
289
|
/** Full-set frame for a client that just (re)connected — presence has no delta protocol. */
|
|
@@ -283,6 +348,18 @@ export function presenceFrame(
|
|
|
283
348
|
};
|
|
284
349
|
}
|
|
285
350
|
|
|
351
|
+
/** A lease value as the leader wrote it: a JSON list of member ids, or nothing yet. */
|
|
352
|
+
function parseRoster(value: string): readonly string[] | null {
|
|
353
|
+
if (value === '') return null;
|
|
354
|
+
try {
|
|
355
|
+
const parsed: unknown = JSON.parse(value);
|
|
356
|
+
if (!Array.isArray(parsed)) return null;
|
|
357
|
+
return parsed.filter((id): id is string => typeof id === 'string');
|
|
358
|
+
} catch {
|
|
359
|
+
return null;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
286
363
|
function parseMember(id: string, value: string): PresenceMember | null {
|
|
287
364
|
try {
|
|
288
365
|
const parsed: unknown = JSON.parse(value);
|
package/src/query-window.ts
CHANGED
|
@@ -30,6 +30,11 @@ export interface QueryEntry {
|
|
|
30
30
|
readonly qid: string;
|
|
31
31
|
readonly definition: LiveQueryDefinition;
|
|
32
32
|
readonly input: JsonValue;
|
|
33
|
+
/**
|
|
34
|
+
* The org this window is read for and patched for — its subscribers' own, `null` for subscribers
|
|
35
|
+
* that carry none. Part of `qid`, so two orgs on one `(query, input)` are two entries.
|
|
36
|
+
*/
|
|
37
|
+
readonly tenant: string | null;
|
|
33
38
|
/** Told to the client on every snapshot: the identity scope its rows belong under. */
|
|
34
39
|
readonly rowEntity: string | null;
|
|
35
40
|
/** The record key a row travels under; `null` = its `id` (a plain table). */
|
|
@@ -80,12 +85,14 @@ export function createEntry(
|
|
|
80
85
|
definition: LiveQueryDefinition,
|
|
81
86
|
input: JsonValue,
|
|
82
87
|
matcher: IncrementalMatcher,
|
|
83
|
-
options?: EntryOptions,
|
|
88
|
+
options?: EntryOptions & { readonly tenant?: string | null },
|
|
84
89
|
): QueryEntry {
|
|
90
|
+
const tenant = options?.tenant ?? null;
|
|
85
91
|
return {
|
|
86
92
|
qid,
|
|
87
93
|
definition,
|
|
88
94
|
input,
|
|
95
|
+
tenant,
|
|
89
96
|
// Resolved with the matcher, from the same build: `prepare` has already run, so a definition
|
|
90
97
|
// that compiles its shape per input can answer.
|
|
91
98
|
rowEntity: definition.rowEntity?.(input) ?? null,
|
|
@@ -96,7 +103,9 @@ export function createEntry(
|
|
|
96
103
|
// static declaration and can only be a superset of it. Preferring the matcher is what lets a
|
|
97
104
|
// definition built from a real query carry no static list at all.
|
|
98
105
|
entities: matcher.entities.length > 0 ? matcher.entities : definition.entities,
|
|
99
|
-
|
|
106
|
+
// The window's own tenant first: a change in another org is dropped before any predicate
|
|
107
|
+
// runs, whether or not the query's `where` happens to name one.
|
|
108
|
+
orgId: tenant ?? orgIdOf(input),
|
|
100
109
|
...(definition.columns ? { columns: definition.columns } : {}),
|
|
101
110
|
},
|
|
102
111
|
matcher,
|
|
@@ -122,9 +131,8 @@ export function createEntry(
|
|
|
122
131
|
* had already been fanned out would rewind every later subscriber to rows the window has moved
|
|
123
132
|
* past, so a stale read is discarded and its caller is served from the newer window instead.
|
|
124
133
|
*/
|
|
125
|
-
export async function fillWindow(
|
|
126
|
-
|
|
127
|
-
): Promise<{ rows: readonly Row[]; lsn: string }> {
|
|
134
|
+
export async function fillWindow(entry: QueryEntry): Promise<FilledWindow> {
|
|
135
|
+
let landed: FilledWindow['landed'] = null;
|
|
128
136
|
for (let attempt = 0; ; attempt += 1) {
|
|
129
137
|
// Read before `startRead` clears it: a second caller arriving during the read joins it and is
|
|
130
138
|
// not the one that forced it, which is what keeps one forced read from becoming N.
|
|
@@ -146,15 +154,28 @@ export async function fillWindow(
|
|
|
146
154
|
const first = entry.applied === 0;
|
|
147
155
|
if (isNewestRead(entry, pending) && (forced || first || result.lsn >= entry.lsn)) {
|
|
148
156
|
applyRead(entry, pending, result);
|
|
157
|
+
// A forced read outranks a first one: what it replaced was known to be wrong.
|
|
158
|
+
landed = forced || landed === 'forced' ? 'forced' : first ? 'first' : landed;
|
|
149
159
|
}
|
|
150
160
|
// A change reached this window while its first read was in flight and could not be folded,
|
|
151
161
|
// so what just landed may predate it: read once more before serving anyone a partial window.
|
|
152
162
|
return first && entry.stale && attempt < COLD_REREADS;
|
|
153
163
|
});
|
|
154
|
-
if (!again) return { rows: entry.rows, lsn: entry.lsn };
|
|
164
|
+
if (!again) return { rows: entry.rows, lsn: entry.lsn, landed };
|
|
155
165
|
}
|
|
156
166
|
}
|
|
157
167
|
|
|
168
|
+
/**
|
|
169
|
+
* The window a caller is served from, and whether THIS call put a read into it. `landed` is what
|
|
170
|
+
* decides the retained ring's floor (`change-buffer.ts`): `null` is a window that was already
|
|
171
|
+
* filled — its patch history is still true, and flooring it would evict every other subscriber's.
|
|
172
|
+
*/
|
|
173
|
+
export interface FilledWindow {
|
|
174
|
+
readonly rows: readonly Row[];
|
|
175
|
+
readonly lsn: string;
|
|
176
|
+
readonly landed: 'first' | 'forced' | null;
|
|
177
|
+
}
|
|
178
|
+
|
|
158
179
|
/**
|
|
159
180
|
* How many times a cold window re-reads because writes kept landing during its read. Bounded: a
|
|
160
181
|
* table written faster than it can be read would otherwise never serve a subscriber, and after the
|
|
@@ -253,7 +274,7 @@ function startRead(entry: QueryEntry): PendingRead {
|
|
|
253
274
|
*/
|
|
254
275
|
async function readSnapshot(entry: QueryEntry): Promise<SnapshotResult> {
|
|
255
276
|
try {
|
|
256
|
-
return await entry.definition.snapshot({ input: entry.input });
|
|
277
|
+
return await entry.definition.snapshot({ input: entry.input, tenant: entry.tenant });
|
|
257
278
|
} catch (error) {
|
|
258
279
|
entry.stale = true;
|
|
259
280
|
throw error;
|
package/src/record-persister.ts
CHANGED
|
@@ -88,10 +88,15 @@ export function recordPersister(options: RecordPersisterOptions): RecordPersiste
|
|
|
88
88
|
});
|
|
89
89
|
|
|
90
90
|
const restore = async (): Promise<number> => {
|
|
91
|
-
const
|
|
91
|
+
const owner = principal();
|
|
92
|
+
const scope = scopeKey(owner);
|
|
92
93
|
if (scope === undefined) return 0;
|
|
94
|
+
const stored = await options.local.rows(scope);
|
|
95
|
+
// The read is async and a principal change can land inside it: its rescope has already cleared
|
|
96
|
+
// the store, so restoring now would hand the previous principal's rows to the next one.
|
|
97
|
+
if (principal() !== owner) return 0;
|
|
93
98
|
let restored = 0;
|
|
94
|
-
for (const [type, rows] of
|
|
99
|
+
for (const [type, rows] of stored) {
|
|
95
100
|
if (!options.types.has(type)) continue;
|
|
96
101
|
options.store.restore(type, rows);
|
|
97
102
|
restored += Object.keys(rows).length;
|
|
@@ -17,10 +17,14 @@ export class ReplicationProtocolError extends RealtimeError {
|
|
|
17
17
|
constructor(args: { stage: string; detail: string; fix?: string }) {
|
|
18
18
|
super({
|
|
19
19
|
code: 'X_REPLICATION_PROTOCOL',
|
|
20
|
-
cause: `
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
// The requirement rides the cause when the caller names no fix of its own: a `fix:` is one
|
|
21
|
+
// command, and `x doctor` has no `db` subcommand for the old line to have been one.
|
|
22
|
+
cause:
|
|
23
|
+
`postgres replication ${args.stage}: ${args.detail}` +
|
|
24
|
+
(args.fix === undefined
|
|
25
|
+
? ' — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical'
|
|
26
|
+
: ''),
|
|
27
|
+
fix: args.fix ?? 'x doctor --json',
|
|
24
28
|
});
|
|
25
29
|
}
|
|
26
30
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// The bus envelope, read side: one published change decoded back into a `ChangeEvent` (rows revived
|
|
2
|
+
// through their entity's columns), with the producer and sequence a lost message is visible in —
|
|
3
|
+
// and the detector a `sync` node runs over them. The publisher is `replicator.ts`.
|
|
4
|
+
|
|
5
|
+
import { isWriteDigest } from '@ultimat3/core';
|
|
6
|
+
import type { ChangeEvent } from './changefeed';
|
|
7
|
+
import { reviveBusRow } from './replicator-row';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* What the bus actually carries: one change, plus who published it and where in that publisher's
|
|
11
|
+
* stream it sits.
|
|
12
|
+
*
|
|
13
|
+
* Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
|
|
14
|
+
* cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
|
|
15
|
+
* cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
|
|
16
|
+
* arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
|
|
17
|
+
*
|
|
18
|
+
* Both fields are optional on the wire, so a node reading a publisher that predates them simply
|
|
19
|
+
* detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
|
|
20
|
+
*/
|
|
21
|
+
export interface ChangeEnvelope {
|
|
22
|
+
readonly change: ChangeEvent;
|
|
23
|
+
/** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
|
|
24
|
+
readonly seq: number | null;
|
|
25
|
+
/** Identifies one replicator *run*. A new one restarts `seq` — and follows a stream that died. */
|
|
26
|
+
readonly producer: string | null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
|
|
30
|
+
export function parseChange(payload: string): ChangeEvent | null {
|
|
31
|
+
return parseEnvelope(payload)?.change ?? null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
|
|
35
|
+
export function parseEnvelope(payload: string): ChangeEnvelope | null {
|
|
36
|
+
try {
|
|
37
|
+
const parsed: unknown = JSON.parse(payload);
|
|
38
|
+
if (typeof parsed !== 'object' || parsed === null) return null;
|
|
39
|
+
const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
|
|
40
|
+
if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
|
|
41
|
+
if (
|
|
42
|
+
shape.op !== 'insert' &&
|
|
43
|
+
shape.op !== 'update' &&
|
|
44
|
+
shape.op !== 'delete' &&
|
|
45
|
+
shape.op !== 'truncate'
|
|
46
|
+
) {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
change: {
|
|
51
|
+
entity: shape.entity,
|
|
52
|
+
op: shape.op,
|
|
53
|
+
before: reviveBusRow(shape.entity, shape.before ?? null),
|
|
54
|
+
after: reviveBusRow(shape.entity, shape.after ?? null),
|
|
55
|
+
lsn: shape.lsn,
|
|
56
|
+
txid: typeof shape.txid === 'string' ? shape.txid : '',
|
|
57
|
+
orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
|
|
58
|
+
at: typeof shape.at === 'number' ? shape.at : 0,
|
|
59
|
+
// Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
|
|
60
|
+
// label here would cost every member the whole frame rather than one page its echo.
|
|
61
|
+
...(isWriteDigest(shape.write) ? { write: shape.write } : {}),
|
|
62
|
+
...(isNames(shape.omitted) ? { omitted: shape.omitted } : {}),
|
|
63
|
+
},
|
|
64
|
+
seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
|
|
65
|
+
producer: typeof shape.producer === 'string' ? shape.producer : null,
|
|
66
|
+
};
|
|
67
|
+
} catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** A non-empty list of property names — the only shape `ChangeEvent.omitted` crosses the bus in. */
|
|
73
|
+
const isNames = (value: unknown): value is readonly string[] =>
|
|
74
|
+
Array.isArray(value) && value.length > 0 && value.every((name) => typeof name === 'string');
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
|
|
78
|
+
* say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
|
|
79
|
+
* there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
|
|
80
|
+
* simply never happened as far as every subscriber on this node could tell.
|
|
81
|
+
*
|
|
82
|
+
* Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
|
|
83
|
+
* reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
|
|
84
|
+
* the first message this node ever reads: a node that joined late has missed everything by
|
|
85
|
+
* definition, and every subscription it holds started after it did.
|
|
86
|
+
*
|
|
87
|
+
* A NEW producer after a known one IS a gap. The run before it died — that is why there is a new
|
|
88
|
+
* one — and nothing says its last messages reached this node: the tail of a dead stream has no
|
|
89
|
+
* later sequence number to be missed against. One re-read per replicator restart is the price.
|
|
90
|
+
*/
|
|
91
|
+
export class SeqGapDetector {
|
|
92
|
+
readonly #next = new Map<string, number>();
|
|
93
|
+
|
|
94
|
+
/** `true` when at least one message between the last one and this one was never delivered. */
|
|
95
|
+
observe(envelope: ChangeEnvelope): boolean {
|
|
96
|
+
const { producer, seq } = envelope;
|
|
97
|
+
if (producer === null || seq === null) return false;
|
|
98
|
+
const expected = this.#next.get(producer);
|
|
99
|
+
const succeeds = expected === undefined && this.#next.size > 0;
|
|
100
|
+
// Never backwards: a redelivery must not lower the bar and turn the next legitimate message
|
|
101
|
+
// into a gap of its own.
|
|
102
|
+
this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
|
|
103
|
+
return succeeds || (expected !== undefined && seq > expected);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
|
|
107
|
+
forget(): void {
|
|
108
|
+
this.#next.clear();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Single responsibility: a change's rows across the replicator bus. The bus carries text, a row
|
|
2
|
+
// holds a `Date` and a `Uint8Array`, and `JSON.parse` returns neither — so the publisher writes a
|
|
3
|
+
// form JSON can carry and the sync node reads it back through the ENTITY's own columns.
|
|
4
|
+
|
|
5
|
+
import { entityForTable } from '@ultimat3/entity';
|
|
6
|
+
import type { Row } from './json';
|
|
7
|
+
|
|
8
|
+
const CHUNK = 0x8000;
|
|
9
|
+
|
|
10
|
+
/** Base64 of raw bytes. Chunked: one `String.fromCharCode(...bytes)` overflows the stack at ~100k. */
|
|
11
|
+
function toBase64(bytes: Uint8Array): string {
|
|
12
|
+
let binary = '';
|
|
13
|
+
for (let at = 0; at < bytes.length; at += CHUNK) {
|
|
14
|
+
binary += String.fromCharCode(...bytes.subarray(at, at + CHUNK));
|
|
15
|
+
}
|
|
16
|
+
return btoa(binary);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function fromBase64(text: string): Uint8Array {
|
|
20
|
+
const binary = atob(text);
|
|
21
|
+
const bytes = new Uint8Array(binary.length);
|
|
22
|
+
for (let at = 0; at < binary.length; at += 1) bytes[at] = binary.charCodeAt(at);
|
|
23
|
+
return bytes;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Publish side. A `Date` already serialises to the ISO text its column reads back; a `Uint8Array`
|
|
28
|
+
* serialises to `{"0":1,"1":2}`, which nothing reads back, so it crosses as base64. Only a
|
|
29
|
+
* top-level value can be one: `arrayOf()` refuses `bytes()` at declaration.
|
|
30
|
+
*/
|
|
31
|
+
export function busRow(row: Row | null): Row | null {
|
|
32
|
+
if (row === null) return null;
|
|
33
|
+
let out: Record<string, unknown> | undefined;
|
|
34
|
+
for (const [property, value] of Object.entries(row as Record<string, unknown>)) {
|
|
35
|
+
if (!(value instanceof Uint8Array)) continue;
|
|
36
|
+
out ??= { ...row };
|
|
37
|
+
out[property] = toBase64(value);
|
|
38
|
+
}
|
|
39
|
+
return out === undefined ? row : (out as unknown as Row);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Consume side: the row as the replicator held it before it was text. Without this every `Date`
|
|
44
|
+
* reached the matcher as an ISO string, `compareValues(Date, string)` fell to comparing their
|
|
45
|
+
* printed forms, and one edit to one column moved its row to the top of every window ordered by
|
|
46
|
+
* a timestamp — on every deployment with a bus, and in no in-process test, which hands the object
|
|
47
|
+
* over directly.
|
|
48
|
+
*
|
|
49
|
+
* Decided by the column that DECLARED the value, never by what the value looks like: a `text()`
|
|
50
|
+
* holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.entity`, which on the WAL
|
|
51
|
+
* path is the relation name. A table with no entity here, a property with no column, and a value
|
|
52
|
+
* its column refuses are all handed back as they arrived — a change that crosses unrevived is the
|
|
53
|
+
* old behaviour; a change dropped is a window that silently diverges.
|
|
54
|
+
*/
|
|
55
|
+
export function reviveBusRow(table: string, row: Row | null): Row | null {
|
|
56
|
+
if (row === null) return null;
|
|
57
|
+
const entity = entityForTable(table);
|
|
58
|
+
if (entity === undefined) return row;
|
|
59
|
+
const out: Record<string, unknown> = {};
|
|
60
|
+
for (const [property, value] of Object.entries(row as Record<string, unknown>)) {
|
|
61
|
+
out[property] = value;
|
|
62
|
+
if (value === null || !Object.hasOwn(entity.$columns, property)) continue;
|
|
63
|
+
const column = entity.$columns[property];
|
|
64
|
+
if (column === undefined || column.$meta.sealed !== undefined) continue;
|
|
65
|
+
try {
|
|
66
|
+
out[property] =
|
|
67
|
+
column.$meta.kind === 'bytea' && typeof value === 'string'
|
|
68
|
+
? fromBase64(value)
|
|
69
|
+
: column.$parse(value);
|
|
70
|
+
} catch {
|
|
71
|
+
// Kept as it arrived — see the function comment.
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return out as unknown as Row;
|
|
75
|
+
}
|