@ultimat3/realtime 24.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +61 -63
- package/README.md +131 -46
- package/package.json +9 -6
- package/src/advisory-lock.ts +10 -5
- package/src/boot.ts +14 -9
- package/src/changefeed-env.ts +8 -8
- package/src/changefeed.ts +42 -18
- package/src/channel-authz.ts +7 -7
- package/src/channel-describe.ts +1 -1
- package/src/channel-logs.ts +16 -25
- package/src/channel-presence.ts +3 -3
- package/src/channel-records.ts +3 -3
- package/src/channel-ref.ts +2 -2
- package/src/channel-registry.ts +2 -2
- package/src/channel.ts +8 -7
- package/src/client-channels.ts +2 -0
- package/src/client-frames.ts +9 -2
- package/src/client.ts +29 -2
- package/src/error-titles.ts +91 -0
- package/src/errors.ts +47 -90
- package/src/first-paint-hold.ts +49 -0
- package/src/idb-types.ts +2 -2
- package/src/index.ts +24 -6
- package/src/json.ts +16 -3
- package/src/live-contract.ts +14 -0
- package/src/live-definition.ts +12 -2
- package/src/live-deliver.ts +55 -0
- package/src/live-fanout.ts +81 -24
- package/src/live-query-options.ts +6 -0
- package/src/live-query.ts +66 -144
- package/src/live-reauth.ts +127 -0
- package/src/live-replicator.ts +28 -8
- package/src/live-resume.ts +44 -8
- package/src/live-rows.ts +32 -3
- package/src/live-spend.ts +50 -0
- package/src/local-store-idb.ts +5 -0
- package/src/matcher-bridge.ts +11 -5
- package/src/nats-client.ts +26 -4
- package/src/nats-lib-client.ts +36 -7
- package/src/nats-subscriptions.ts +63 -0
- package/src/nats-transport.ts +127 -15
- package/src/offline-queue.ts +5 -0
- package/src/page-errors.ts +8 -9
- package/src/page-outbox.ts +5 -5
- package/src/page-runtime-wait.ts +72 -0
- package/src/page-runtime.ts +32 -0
- package/src/page-socket.ts +7 -16
- package/src/page-store.ts +83 -16
- package/src/pg-advisory-lock.ts +9 -4
- package/src/pg-connection.ts +7 -1
- package/src/pg-replication.ts +6 -6
- package/src/pg-socket.ts +20 -3
- package/src/pg-values.ts +29 -14
- package/src/policy-gate.ts +7 -7
- package/src/presence.ts +2 -2
- package/src/principal-sockets.ts +75 -0
- package/src/query-window.ts +1 -1
- package/src/record-publisher.ts +178 -0
- package/src/record-store.ts +25 -4
- package/src/record-synced.ts +43 -1
- package/src/replication-errors.ts +3 -3
- package/src/replicator-envelope.ts +61 -11
- package/src/replicator-row.ts +1 -1
- package/src/replicator.ts +12 -23
- package/src/server.ts +36 -18
- package/src/socket-defaults.ts +20 -0
- package/src/socket.ts +14 -19
- package/src/subscription-book.ts +129 -71
- package/src/subscription-count.ts +151 -0
- package/src/sync-actor-change.ts +56 -0
- package/src/sync-bus-handlers.ts +131 -0
- package/src/sync-frames.ts +1 -1
- package/src/sync-node-bounds.ts +1 -1
- package/src/sync-node-contract.ts +16 -2
- package/src/sync-node.ts +115 -109
- package/src/sync-origin.ts +1 -1
- package/src/sync-protocol.ts +73 -13
- package/src/sync-upgrade.ts +39 -0
- package/src/type-pins.ts +9 -1
- package/src/use-channel.ts +19 -14
- package/src/use-connection.ts +3 -3
- package/src/use-mutation.ts +9 -16
- package/src/use-query.ts +11 -11
- package/src/use-record.ts +89 -10
- package/src/wire-channel.ts +5 -5
- package/src/wire-read.ts +2 -2
- package/src/idb-fake.ts +0 -133
- package/src/policy-fake.ts +0 -14
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// Committed rows an app hands over → change envelopes on the subject every sync node reads, for a
|
|
2
|
+
// deployment that runs no replicator. Channels only: a publisher has no commit position, so its
|
|
3
|
+
// changes feed the declared channels and never a live window (`sync-bus-handlers.ts`). One per
|
|
4
|
+
// process, built at boot: it is one producer, and its seq is what a node sees a lost message in.
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
assert,
|
|
8
|
+
type Clock,
|
|
9
|
+
currentWriteOrigin,
|
|
10
|
+
isWriteDigest,
|
|
11
|
+
systemClock,
|
|
12
|
+
uuidV7,
|
|
13
|
+
} from '@ultimat3/core';
|
|
14
|
+
import { type RecordProjection, recordProjection } from '@ultimat3/entity/record';
|
|
15
|
+
import { type ChangeEvent, formatLsn } from './changefeed';
|
|
16
|
+
import type { ChannelEntity } from './channel-decl';
|
|
17
|
+
import type { Transport } from './fanout';
|
|
18
|
+
import type { Row } from './json';
|
|
19
|
+
import { changeSubject } from './replicator';
|
|
20
|
+
import { encodeEnvelope } from './replicator-envelope';
|
|
21
|
+
|
|
22
|
+
export interface RecordPublisherOptions {
|
|
23
|
+
readonly transport: Transport;
|
|
24
|
+
/**
|
|
25
|
+
* Every entity this process publishes, claimed until `close()`. Claimed, `x dev`'s in-process
|
|
26
|
+
* bridge stops carrying that entity's repository writes to channels — the app's publish is then
|
|
27
|
+
* the one delivery, exactly as it is in production with no replicator.
|
|
28
|
+
*/
|
|
29
|
+
readonly entities: readonly ChannelEntity[];
|
|
30
|
+
/** Tenant column, hoisted into the subject as the replicator hoists it. Default `orgId`. */
|
|
31
|
+
readonly tenantColumn?: string;
|
|
32
|
+
readonly clock?: Clock;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface PublishOptions {
|
|
36
|
+
/** `upsert` (default): the rows are adopted. `delete`: the rows are gone, by their key. */
|
|
37
|
+
readonly op?: 'upsert' | 'delete';
|
|
38
|
+
/**
|
|
39
|
+
* The keyed write these rows belong to — `writeDigest(idempotencyKey)` — so the page that made
|
|
40
|
+
* it settles its own echo. Default: the keyed request this call runs inside, else `null`.
|
|
41
|
+
*/
|
|
42
|
+
readonly write?: string | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RecordPublisher extends Disposable {
|
|
46
|
+
/** This publisher's producer id: the stream its `seq` counts in. */
|
|
47
|
+
readonly producer: string;
|
|
48
|
+
/**
|
|
49
|
+
* Put COMMITTED rows on the bus, one change per row, in call order. Call it after the
|
|
50
|
+
* transaction commits: a row published from inside one is on every page before a rollback.
|
|
51
|
+
* Rejects with the first refusal once every row was tried; a refused row's seq stays spent, so
|
|
52
|
+
* every node reads the hole and its channel members re-run their catch-up read.
|
|
53
|
+
*/
|
|
54
|
+
publish(
|
|
55
|
+
entity: ChannelEntity,
|
|
56
|
+
rows: readonly Readonly<Record<string, unknown>>[],
|
|
57
|
+
options?: PublishOptions,
|
|
58
|
+
): Promise<void>;
|
|
59
|
+
/** Releases the claim. Idempotent; a closed publisher refuses to publish. */
|
|
60
|
+
close(): void;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Tables a live publisher claims, counted: two publishers may claim one. */
|
|
64
|
+
const claims = new Map<string, number>();
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Whether a `recordPublisher` in this process claims `table` — the relation, the vocabulary of
|
|
68
|
+
* `ChangeEvent.table` on every producer.
|
|
69
|
+
*/
|
|
70
|
+
export function isPublishedTable(table: string): boolean {
|
|
71
|
+
return claims.has(table);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function recordPublisher(options: RecordPublisherOptions): RecordPublisher {
|
|
75
|
+
const clock = options.clock ?? systemClock;
|
|
76
|
+
const tenant = options.tenantColumn ?? 'orgId';
|
|
77
|
+
const projections = new Map<string, RecordProjection>();
|
|
78
|
+
for (const declared of options.entities) {
|
|
79
|
+
const projection = recordProjection(declared);
|
|
80
|
+
projections.set(projection.type, projection);
|
|
81
|
+
}
|
|
82
|
+
const names = [...projections.values()].map((projection) => projection.table);
|
|
83
|
+
for (const name of names) claims.set(name, (claims.get(name) ?? 0) + 1);
|
|
84
|
+
const producer = uuidV7();
|
|
85
|
+
let seq = 0;
|
|
86
|
+
let open = true;
|
|
87
|
+
// One chain, so seq order is bus order whatever the caller awaits: a node reads a seq that
|
|
88
|
+
// arrives out of order as a hole that was never there.
|
|
89
|
+
let tail: Promise<unknown> = Promise.resolve();
|
|
90
|
+
|
|
91
|
+
const close = (): void => {
|
|
92
|
+
if (!open) return;
|
|
93
|
+
open = false;
|
|
94
|
+
for (const name of names) {
|
|
95
|
+
const held = (claims.get(name) ?? 1) - 1;
|
|
96
|
+
if (held > 0) claims.set(name, held);
|
|
97
|
+
else claims.delete(name);
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const changeOf = (
|
|
102
|
+
projection: RecordProjection,
|
|
103
|
+
row: Readonly<Record<string, unknown>>,
|
|
104
|
+
op: 'upsert' | 'delete',
|
|
105
|
+
write: string | null,
|
|
106
|
+
): ChangeEvent => {
|
|
107
|
+
// The record's own properties and nothing else: the entity's record schema omits every sealed
|
|
108
|
+
// column, so neither a secret nor a property the entity never declared can ride.
|
|
109
|
+
const columns = projection.schema.properties ?? {};
|
|
110
|
+
const image = Object.fromEntries(
|
|
111
|
+
Object.entries(row).filter(([property]) => Object.hasOwn(columns, property)),
|
|
112
|
+
) as Row;
|
|
113
|
+
// Refused here, before any row of the call is sequenced: a keyless row matches no record.
|
|
114
|
+
projection.key(image);
|
|
115
|
+
const orgId = image[tenant];
|
|
116
|
+
return {
|
|
117
|
+
table: projection.table,
|
|
118
|
+
op: op === 'delete' ? 'delete' : 'update',
|
|
119
|
+
before: op === 'delete' ? image : null,
|
|
120
|
+
after: op === 'delete' ? null : image,
|
|
121
|
+
// Stamped with the seq at send: it orders nothing outside this producer, which is why no
|
|
122
|
+
// live window is fed one.
|
|
123
|
+
lsn: '',
|
|
124
|
+
txid: producer,
|
|
125
|
+
orgId: typeof orgId === 'string' ? orgId : null,
|
|
126
|
+
at: clock.now().getTime(),
|
|
127
|
+
write,
|
|
128
|
+
};
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
const send = async (change: ChangeEvent): Promise<void> => {
|
|
132
|
+
seq += 1;
|
|
133
|
+
await options.transport.publish(
|
|
134
|
+
changeSubject(change),
|
|
135
|
+
encodeEnvelope({ ...change, lsn: formatLsn(seq) }, seq, producer, 'publisher'),
|
|
136
|
+
);
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const publish: RecordPublisher['publish'] = async (entity, rows, publishOptions = {}) => {
|
|
140
|
+
assert(
|
|
141
|
+
open,
|
|
142
|
+
`recordPublisher ${producer} was closed and published "${entity.$name}" after it`,
|
|
143
|
+
'build one recordPublisher per process at boot and close it at shutdown',
|
|
144
|
+
);
|
|
145
|
+
const projection = projections.get(entity.$name);
|
|
146
|
+
assert(
|
|
147
|
+
projection !== undefined,
|
|
148
|
+
`recordPublisher was asked to publish "${entity.$name}", which it did not claim`,
|
|
149
|
+
`add ${entity.$name} to recordPublisher({ entities: [...] }) where the publisher is built`,
|
|
150
|
+
);
|
|
151
|
+
const write =
|
|
152
|
+
publishOptions.write === undefined ? (currentWriteOrigin() ?? null) : publishOptions.write;
|
|
153
|
+
assert(
|
|
154
|
+
write === null || isWriteDigest(write),
|
|
155
|
+
`publish({ write }) for "${entity.$name}" is not a write digest`,
|
|
156
|
+
"pass writeDigest(idempotencyKey) from '@ultimat3/core', or null for a write no request made",
|
|
157
|
+
);
|
|
158
|
+
const op = publishOptions.op ?? 'upsert';
|
|
159
|
+
const changes = rows.map((row) => changeOf(projection, row, op, write));
|
|
160
|
+
const sent = tail.then(async () => {
|
|
161
|
+
// Every row the bus takes goes, a refused one in the middle included: its spent seq is the
|
|
162
|
+
// hole the nodes repair, and the caller hears the first refusal once the rest are out.
|
|
163
|
+
let refused: { readonly error: unknown } | undefined;
|
|
164
|
+
for (const change of changes) {
|
|
165
|
+
try {
|
|
166
|
+
await send(change);
|
|
167
|
+
} catch (error) {
|
|
168
|
+
refused ??= { error };
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
if (refused !== undefined) throw refused.error;
|
|
172
|
+
});
|
|
173
|
+
tail = sent.catch(() => undefined);
|
|
174
|
+
await sent;
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
return { producer, publish, close, [Symbol.dispose]: close };
|
|
178
|
+
}
|
package/src/record-store.ts
CHANGED
|
@@ -61,16 +61,21 @@ export class RecordStore implements RecordSink {
|
|
|
61
61
|
|
|
62
62
|
/** Every record of one type, as seen — what a mutator's `tx.<type>.all()` walks. */
|
|
63
63
|
all(type: string): readonly Row[] {
|
|
64
|
+
return this.keyed(type).map(([, row]) => row);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** `all`, with each record's key — what a whole-type reader orders by. */
|
|
68
|
+
keyed(type: string): readonly (readonly [string, Row])[] {
|
|
64
69
|
const prefix = `${type}:`;
|
|
65
|
-
const out: Row[] = [];
|
|
70
|
+
const out: (readonly [string, Row])[] = [];
|
|
66
71
|
const seen = new Set<RecordKey>();
|
|
67
72
|
for (const [rk, row] of this.#view) {
|
|
68
73
|
if (!rk.startsWith(prefix)) continue;
|
|
69
74
|
seen.add(rk);
|
|
70
|
-
if (row !== null) out.push(row);
|
|
75
|
+
if (row !== null) out.push([rk.slice(prefix.length), row]);
|
|
71
76
|
}
|
|
72
77
|
for (const [rk, row] of this.#synced.entries()) {
|
|
73
|
-
if (rk.startsWith(prefix) && !seen.has(rk)) out.push(row);
|
|
78
|
+
if (rk.startsWith(prefix) && !seen.has(rk)) out.push([rk.slice(prefix.length), row]);
|
|
74
79
|
}
|
|
75
80
|
return out;
|
|
76
81
|
}
|
|
@@ -153,6 +158,18 @@ export class RecordStore implements RecordSink {
|
|
|
153
158
|
if (this.#synced.release(rk)) this.#touchSynced(rk);
|
|
154
159
|
}
|
|
155
160
|
|
|
161
|
+
/** Every record of `type` is shown by one reader: none is evicted while it is held. */
|
|
162
|
+
retainType(type: string): void {
|
|
163
|
+
this.#synced.retainType(type);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** The last type hold leaving evicts the rows it alone kept (`SyncedLayer.releaseType`). */
|
|
167
|
+
releaseType(type: string): void {
|
|
168
|
+
this.batch(() => {
|
|
169
|
+
for (const rk of this.#synced.releaseType(type)) this.#touchSynced(rk);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
156
173
|
/**
|
|
157
174
|
* An optimistic write, visible to every holder before this returns. `apply` is the mutator's
|
|
158
175
|
* pure `local` half: it is REPLAYED over synced truth whenever synced truth moves, which is what
|
|
@@ -342,9 +359,13 @@ export class RecordStore implements RecordSink {
|
|
|
342
359
|
this.#touched = result.touched;
|
|
343
360
|
}
|
|
344
361
|
|
|
345
|
-
/**
|
|
362
|
+
/**
|
|
363
|
+
* The one way an overlay leaves: its entry, the digest that named it and its server wait go
|
|
364
|
+
* together. A wait left behind ran for its whole bound after a later settle removed the overlay.
|
|
365
|
+
*/
|
|
346
366
|
#forget(key: string): boolean {
|
|
347
367
|
this.#names.forget(key);
|
|
368
|
+
this.#wait.forget(key);
|
|
348
369
|
return this.#overlays.delete(key);
|
|
349
370
|
}
|
|
350
371
|
|
package/src/record-synced.ts
CHANGED
|
@@ -8,6 +8,10 @@ import type { RecordKey } from './record-key';
|
|
|
8
8
|
export class SyncedLayer {
|
|
9
9
|
readonly #rows = new Map<RecordKey, Row>();
|
|
10
10
|
readonly #holds = new Map<RecordKey, number>();
|
|
11
|
+
/** Types a whole-type reader holds (`useRecords(type, selection)`): none of their rows is evicted. */
|
|
12
|
+
readonly #typeHolds = new Map<string, number>();
|
|
13
|
+
/** Per held type, the rows whose last key holder left while it was held: the hold's to evict. */
|
|
14
|
+
readonly #spared = new Map<string, Set<RecordKey>>();
|
|
11
15
|
/** Keys restored from disk and not yet confirmed: the first server row REPLACES, never merges. */
|
|
12
16
|
readonly #provisional = new Set<RecordKey>();
|
|
13
17
|
|
|
@@ -74,6 +78,8 @@ export class SyncedLayer {
|
|
|
74
78
|
/** Answers whether a row was there to remove. */
|
|
75
79
|
remove(rk: RecordKey): boolean {
|
|
76
80
|
this.#provisional.delete(rk);
|
|
81
|
+
// Gone is no longer spared: a row the server sends again later is a new arrival, not this one.
|
|
82
|
+
this.#spared.get(rk.slice(0, rk.indexOf(':')))?.delete(rk);
|
|
77
83
|
return this.#rows.delete(rk);
|
|
78
84
|
}
|
|
79
85
|
|
|
@@ -81,7 +87,10 @@ export class SyncedLayer {
|
|
|
81
87
|
this.#holds.set(rk, (this.#holds.get(rk) ?? 0) + 1);
|
|
82
88
|
}
|
|
83
89
|
|
|
84
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* The last holder leaving evicts the row — unless a reader holds its whole type, which shows
|
|
92
|
+
* every row of it and would lose this one to another holder's unmount. Answers whether a row went.
|
|
93
|
+
*/
|
|
85
94
|
release(rk: RecordKey): boolean {
|
|
86
95
|
const holds = this.#holds.get(rk);
|
|
87
96
|
if (holds === undefined) return false;
|
|
@@ -90,11 +99,44 @@ export class SyncedLayer {
|
|
|
90
99
|
return false;
|
|
91
100
|
}
|
|
92
101
|
this.#holds.delete(rk);
|
|
102
|
+
const type = rk.slice(0, rk.indexOf(':'));
|
|
103
|
+
if (this.#typeHolds.has(type)) {
|
|
104
|
+
const spared = this.#spared.get(type) ?? new Set<RecordKey>();
|
|
105
|
+
spared.add(rk);
|
|
106
|
+
this.#spared.set(type, spared);
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
93
109
|
return this.remove(rk);
|
|
94
110
|
}
|
|
95
111
|
|
|
112
|
+
retainType(type: string): void {
|
|
113
|
+
this.#typeHolds.set(type, (this.#typeHolds.get(type) ?? 0) + 1);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The last type hold leaving evicts what it spared — each row whose last key holder left while
|
|
118
|
+
* the type was held and that no key holder has taken since — exactly as that release would have.
|
|
119
|
+
* A row no key holder ever held is not the hold's to evict. Answers the rows that went.
|
|
120
|
+
*/
|
|
121
|
+
releaseType(type: string): readonly RecordKey[] {
|
|
122
|
+
const holds = this.#typeHolds.get(type) ?? 0;
|
|
123
|
+
if (holds > 1) {
|
|
124
|
+
this.#typeHolds.set(type, holds - 1);
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
this.#typeHolds.delete(type);
|
|
128
|
+
const spared = this.#spared.get(type);
|
|
129
|
+
this.#spared.delete(type);
|
|
130
|
+
const evicted: RecordKey[] = [];
|
|
131
|
+
for (const rk of spared ?? []) {
|
|
132
|
+
if (!this.#holds.has(rk) && this.remove(rk)) evicted.push(rk);
|
|
133
|
+
}
|
|
134
|
+
return evicted;
|
|
135
|
+
}
|
|
136
|
+
|
|
96
137
|
clear(): void {
|
|
97
138
|
this.#rows.clear();
|
|
98
139
|
this.#provisional.clear();
|
|
140
|
+
this.#spared.clear();
|
|
99
141
|
}
|
|
100
142
|
}
|
|
@@ -63,8 +63,8 @@ export class ReplicationTlsError extends RealtimeError {
|
|
|
63
63
|
/**
|
|
64
64
|
* A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
|
|
65
65
|
* nothing is wrong with this process: the database already has its one replicator, and a second
|
|
66
|
-
* one that started anyway would publish every change twice.
|
|
67
|
-
*
|
|
66
|
+
* one that started anyway would publish every change twice. Thrown only under `x dev`: a container
|
|
67
|
+
* that loses the lock stays up unready and takes over when it frees (`cli/src/role-replicator.ts`).
|
|
68
68
|
*/
|
|
69
69
|
export class ReplicatorSlotHeldError extends RealtimeError {
|
|
70
70
|
constructor(args: { key: string; holder?: string | undefined }) {
|
|
@@ -73,7 +73,7 @@ export class ReplicatorSlotHeldError extends RealtimeError {
|
|
|
73
73
|
cause:
|
|
74
74
|
`advisory lock ${args.key} is held${args.holder === undefined ? '' : ` by ${args.holder}`}` +
|
|
75
75
|
' — one database has exactly one replicator',
|
|
76
|
-
fix: '
|
|
76
|
+
fix: 'x dev --role replicator # rerun after the replicator already on this DATABASE_URL stops — one database has one replicator',
|
|
77
77
|
});
|
|
78
78
|
}
|
|
79
79
|
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
// The bus envelope,
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// The bus envelope, both sides: the one encoder every producer writes with (`replicator.ts`,
|
|
2
|
+
// `record-publisher.ts`), its decode back into a `ChangeEvent` (rows revived through their entity's
|
|
3
|
+
// columns) with the producer and sequence a lost message is visible in, and the detector a `sync`
|
|
4
|
+
// node runs over them.
|
|
4
5
|
|
|
5
6
|
import { isWriteDigest } from '@ultimat3/core';
|
|
6
7
|
import type { ChangeEvent } from './changefeed';
|
|
7
|
-
import { reviveBusRow } from './replicator-row';
|
|
8
|
+
import { busRow, reviveBusRow } from './replicator-row';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* What the bus actually carries: one change, plus who published it and where in that publisher's
|
|
@@ -24,6 +25,36 @@ export interface ChangeEnvelope {
|
|
|
24
25
|
readonly seq: number | null;
|
|
25
26
|
/** Identifies one replicator *run*. A new one restarts `seq` — and follows a stream that died. */
|
|
26
27
|
readonly producer: string | null;
|
|
28
|
+
/**
|
|
29
|
+
* Who sequenced it. Absent is the replicator (every publisher that predates the field). A
|
|
30
|
+
* `recordPublisher` says `'publisher'`: its changes carry no commit position, so they feed the
|
|
31
|
+
* declared channels and never a live window, and several publishers run side by side.
|
|
32
|
+
*/
|
|
33
|
+
readonly source?: EnvelopeSource;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** The two producers of the change bus: the one WAL replicator, and an app's record publishers. */
|
|
37
|
+
export type EnvelopeSource = 'replicator' | 'publisher';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Producer side: one change as the bus carries it. Flat — the change plus `seq`, `producer` and,
|
|
41
|
+
* for a record publisher, `source` — so a consumer that only knows the change reads it unchanged
|
|
42
|
+
* (`parseChange`). Rows cross as `busRow` writes them: bytes as base64, revived by the entity.
|
|
43
|
+
*/
|
|
44
|
+
export function encodeEnvelope(
|
|
45
|
+
change: ChangeEvent,
|
|
46
|
+
seq: number,
|
|
47
|
+
producer: string,
|
|
48
|
+
source?: EnvelopeSource,
|
|
49
|
+
): string {
|
|
50
|
+
return JSON.stringify({
|
|
51
|
+
...change,
|
|
52
|
+
before: busRow(change.before),
|
|
53
|
+
after: busRow(change.after),
|
|
54
|
+
seq,
|
|
55
|
+
producer,
|
|
56
|
+
...(source === 'publisher' ? { source } : {}),
|
|
57
|
+
});
|
|
27
58
|
}
|
|
28
59
|
|
|
29
60
|
/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
|
|
@@ -36,8 +67,17 @@ export function parseEnvelope(payload: string): ChangeEnvelope | null {
|
|
|
36
67
|
try {
|
|
37
68
|
const parsed: unknown = JSON.parse(payload);
|
|
38
69
|
if (typeof parsed !== 'object' || parsed === null) return null;
|
|
39
|
-
const shape = parsed as Partial<ChangeEvent> & {
|
|
40
|
-
|
|
70
|
+
const shape = parsed as Partial<ChangeEvent> & {
|
|
71
|
+
seq?: unknown;
|
|
72
|
+
producer?: unknown;
|
|
73
|
+
source?: unknown;
|
|
74
|
+
entity?: unknown;
|
|
75
|
+
};
|
|
76
|
+
// `entity` is the field a 24.x replicator wrote the table under. Read, never written: across a
|
|
77
|
+
// rolling deploy to 25.0.0 the replicator and the sync nodes run different builds for minutes,
|
|
78
|
+
// and a node refusing every change of the old one is every live view frozen for that window.
|
|
79
|
+
const table = typeof shape.table === 'string' ? shape.table : shape.entity;
|
|
80
|
+
if (typeof table !== 'string' || typeof shape.lsn !== 'string') return null;
|
|
41
81
|
if (
|
|
42
82
|
shape.op !== 'insert' &&
|
|
43
83
|
shape.op !== 'update' &&
|
|
@@ -48,21 +88,22 @@ export function parseEnvelope(payload: string): ChangeEnvelope | null {
|
|
|
48
88
|
}
|
|
49
89
|
return {
|
|
50
90
|
change: {
|
|
51
|
-
|
|
91
|
+
table,
|
|
52
92
|
op: shape.op,
|
|
53
|
-
before: reviveBusRow(
|
|
54
|
-
after: reviveBusRow(
|
|
93
|
+
before: reviveBusRow(table, shape.before ?? null),
|
|
94
|
+
after: reviveBusRow(table, shape.after ?? null),
|
|
55
95
|
lsn: shape.lsn,
|
|
56
96
|
txid: typeof shape.txid === 'string' ? shape.txid : '',
|
|
57
97
|
orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
|
|
58
98
|
at: typeof shape.at === 'number' ? shape.at : 0,
|
|
59
99
|
// Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
|
|
60
100
|
// label here would cost every member the whole frame rather than one page its echo.
|
|
61
|
-
|
|
101
|
+
write: isWriteDigest(shape.write) ? shape.write : null,
|
|
62
102
|
...(isNames(shape.omitted) ? { omitted: shape.omitted } : {}),
|
|
63
103
|
},
|
|
64
104
|
seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
|
|
65
105
|
producer: typeof shape.producer === 'string' ? shape.producer : null,
|
|
106
|
+
...(shape.source === 'publisher' ? { source: 'publisher' as const } : {}),
|
|
66
107
|
};
|
|
67
108
|
} catch {
|
|
68
109
|
return null;
|
|
@@ -87,16 +128,24 @@ const isNames = (value: unknown): value is readonly string[] =>
|
|
|
87
128
|
* A NEW producer after a known one IS a gap. The run before it died — that is why there is a new
|
|
88
129
|
* one — and nothing says its last messages reached this node: the tail of a dead stream has no
|
|
89
130
|
* later sequence number to be missed against. One re-read per replicator restart is the price.
|
|
131
|
+
*
|
|
132
|
+
* Only a REPLICATOR run succeeds another. Record publishers (`source: 'publisher'`) run side by
|
|
133
|
+
* side, one per app process, so a new one says nothing about the others; and a replicator run
|
|
134
|
+
* after a publisher is no successor either. A hole inside any one producer's sequence is a gap.
|
|
90
135
|
*/
|
|
91
136
|
export class SeqGapDetector {
|
|
92
137
|
readonly #next = new Map<string, number>();
|
|
138
|
+
/** Replicator runs read — the producers a NEW replicator run succeeds. */
|
|
139
|
+
readonly #runs = new Set<string>();
|
|
93
140
|
|
|
94
141
|
/** `true` when at least one message between the last one and this one was never delivered. */
|
|
95
142
|
observe(envelope: ChangeEnvelope): boolean {
|
|
96
143
|
const { producer, seq } = envelope;
|
|
97
144
|
if (producer === null || seq === null) return false;
|
|
98
145
|
const expected = this.#next.get(producer);
|
|
99
|
-
const
|
|
146
|
+
const replicator = envelope.source !== 'publisher';
|
|
147
|
+
const succeeds = replicator && expected === undefined && this.#runs.size > 0;
|
|
148
|
+
if (replicator) this.#runs.add(producer);
|
|
100
149
|
// Never backwards: a redelivery must not lower the bar and turn the next legitimate message
|
|
101
150
|
// into a gap of its own.
|
|
102
151
|
this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
|
|
@@ -106,5 +155,6 @@ export class SeqGapDetector {
|
|
|
106
155
|
/** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
|
|
107
156
|
forget(): void {
|
|
108
157
|
this.#next.clear();
|
|
158
|
+
this.#runs.clear();
|
|
109
159
|
}
|
|
110
160
|
}
|
package/src/replicator-row.ts
CHANGED
|
@@ -47,7 +47,7 @@ export function busRow(row: Row | null): Row | null {
|
|
|
47
47
|
* over directly.
|
|
48
48
|
*
|
|
49
49
|
* Decided by the column that DECLARED the value, never by what the value looks like: a `text()`
|
|
50
|
-
* holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.
|
|
50
|
+
* holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.table`, which on the WAL
|
|
51
51
|
* path is the relation name. A table with no entity here, a property with no column, and a value
|
|
52
52
|
* its column refuses are all handed back as they arrived — a change that crosses unrevived is the
|
|
53
53
|
* old behaviour; a change dropped is a window that silently diverges.
|
package/src/replicator.ts
CHANGED
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
// its feed and reports `/readyz` false. A stream that ENDS — or a lock that is LOST — is this file's
|
|
12
12
|
// to answer: `running` goes false, everything is let go, and the process asks again on a backoff.
|
|
13
13
|
|
|
14
|
-
import { logger, renderThrowable,
|
|
14
|
+
import { logger, renderThrowable, uuidV7, withSpan } from '@ultimat3/core';
|
|
15
15
|
import type { AdvisoryLock } from './advisory-lock';
|
|
16
16
|
import type { ChangeEvent, ChangeFeed } from './changefeed';
|
|
17
17
|
import { ReplicationFailedError } from './errors';
|
|
18
18
|
import type { Transport } from './fanout';
|
|
19
|
-
import {
|
|
19
|
+
import { encodeEnvelope } from './replicator-envelope';
|
|
20
20
|
import {
|
|
21
21
|
type BackoffPolicy,
|
|
22
22
|
defaultBackoff,
|
|
@@ -30,9 +30,9 @@ export const CHANGE_SUBJECT_PREFIX = 'x.change';
|
|
|
30
30
|
/** Every change subject: what a `sync` node subscribes to. A changefeed subject, never a channel. */
|
|
31
31
|
export const CHANGE_SUBJECT_ALL = `${CHANGE_SUBJECT_PREFIX}.>`;
|
|
32
32
|
|
|
33
|
-
/** `x.change.<
|
|
33
|
+
/** `x.change.<table>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
|
|
34
34
|
export function changeSubject(change: ChangeEvent): string {
|
|
35
|
-
return `${CHANGE_SUBJECT_PREFIX}.${change.
|
|
35
|
+
return `${CHANGE_SUBJECT_PREFIX}.${change.table}.${change.orgId ?? '_'}`;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export interface ReplicatorOptions {
|
|
@@ -56,7 +56,7 @@ export interface ReplicatorStats {
|
|
|
56
56
|
readonly restarts: number;
|
|
57
57
|
/**
|
|
58
58
|
* Why this replicator is not running although nothing stopped it — the stream's own reason, the
|
|
59
|
-
* last restart's, or the change (
|
|
59
|
+
* last restart's, or the change (table, lsn, how many times) the bus keeps refusing. `null`
|
|
60
60
|
* while it runs, and after a `stop()`.
|
|
61
61
|
*/
|
|
62
62
|
readonly failure: string | null;
|
|
@@ -87,7 +87,7 @@ interface Run {
|
|
|
87
87
|
|
|
88
88
|
/** The change a run could not publish, kept until one goes out. */
|
|
89
89
|
interface Refusal {
|
|
90
|
-
readonly
|
|
90
|
+
readonly table: string;
|
|
91
91
|
readonly lsn: string;
|
|
92
92
|
readonly error: string;
|
|
93
93
|
times: number;
|
|
@@ -107,7 +107,7 @@ const fencedOut = (reason: string): ReplicationFailedError =>
|
|
|
107
107
|
fix: 'x doctor --json',
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
-
export function
|
|
110
|
+
export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
|
|
111
111
|
const subjectOf = options.subjectOf ?? changeSubject;
|
|
112
112
|
const backoff = options.backoff ?? defaultBackoff;
|
|
113
113
|
/** A stream is pumping under a lock this process holds. Internal: `running` is stricter. */
|
|
@@ -119,7 +119,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
|
|
|
119
119
|
// One id per run, not per process: a replicator that took the lock back after a crash publishes
|
|
120
120
|
// from its persisted lsn, and a consumer must read that as a new stream rather than as a gap in
|
|
121
121
|
// the old one.
|
|
122
|
-
let producer =
|
|
122
|
+
let producer = uuidV7();
|
|
123
123
|
let seq = 0;
|
|
124
124
|
/** The start in flight, if any — see `start()` on why `pumping` cannot answer that question. */
|
|
125
125
|
let starting: Promise<boolean> | undefined;
|
|
@@ -200,23 +200,12 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
|
|
|
200
200
|
await withSpan('realtime.replicate', async () => {
|
|
201
201
|
seq += 1;
|
|
202
202
|
try {
|
|
203
|
-
|
|
204
|
-
// the change reads it unchanged — `parseChange` still answers on the same payload.
|
|
205
|
-
await options.transport.publish(
|
|
206
|
-
subjectOf(change),
|
|
207
|
-
JSON.stringify({
|
|
208
|
-
...change,
|
|
209
|
-
before: busRow(change.before),
|
|
210
|
-
after: busRow(change.after),
|
|
211
|
-
seq,
|
|
212
|
-
producer,
|
|
213
|
-
}),
|
|
214
|
-
);
|
|
203
|
+
await options.transport.publish(subjectOf(change), encodeEnvelope(change, seq, producer));
|
|
215
204
|
} catch (thrown) {
|
|
216
205
|
const error = renderThrowable(thrown);
|
|
217
206
|
const same = refusal !== null && refusal.lsn === change.lsn;
|
|
218
207
|
refusal = {
|
|
219
|
-
|
|
208
|
+
table: change.table,
|
|
220
209
|
lsn: change.lsn,
|
|
221
210
|
error,
|
|
222
211
|
times: same ? (refusal?.times ?? 0) + 1 : 1,
|
|
@@ -238,7 +227,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
|
|
|
238
227
|
const describeFailure = (reason: string): string =>
|
|
239
228
|
refusal === null
|
|
240
229
|
? reason
|
|
241
|
-
: `${refusal.
|
|
230
|
+
: `${refusal.table} at lsn ${refusal.lsn} could not be published ` +
|
|
242
231
|
`(${refusal.times} time(s) in a row): ${refusal.error}`;
|
|
243
232
|
|
|
244
233
|
const begin = async (): Promise<boolean> => {
|
|
@@ -246,7 +235,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
|
|
|
246
235
|
logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
|
|
247
236
|
return false;
|
|
248
237
|
}
|
|
249
|
-
producer =
|
|
238
|
+
producer = uuidV7();
|
|
250
239
|
seq = 0;
|
|
251
240
|
const run: Run = { over: null };
|
|
252
241
|
current = run;
|