@ultimat3/realtime 24.0.0 → 25.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CLAUDE.md +61 -63
  2. package/README.md +131 -46
  3. package/package.json +9 -6
  4. package/src/advisory-lock.ts +10 -5
  5. package/src/boot.ts +14 -9
  6. package/src/changefeed-env.ts +8 -8
  7. package/src/changefeed.ts +42 -18
  8. package/src/channel-authz.ts +7 -7
  9. package/src/channel-describe.ts +1 -1
  10. package/src/channel-logs.ts +16 -25
  11. package/src/channel-presence.ts +3 -3
  12. package/src/channel-records.ts +3 -3
  13. package/src/channel-ref.ts +2 -2
  14. package/src/channel-registry.ts +2 -2
  15. package/src/channel.ts +8 -7
  16. package/src/client-channels.ts +2 -0
  17. package/src/client-frames.ts +9 -2
  18. package/src/client.ts +29 -2
  19. package/src/error-titles.ts +91 -0
  20. package/src/errors.ts +47 -90
  21. package/src/first-paint-hold.ts +49 -0
  22. package/src/idb-types.ts +2 -2
  23. package/src/index.ts +24 -6
  24. package/src/json.ts +16 -3
  25. package/src/live-contract.ts +14 -0
  26. package/src/live-definition.ts +12 -2
  27. package/src/live-deliver.ts +55 -0
  28. package/src/live-fanout.ts +81 -24
  29. package/src/live-query-options.ts +6 -0
  30. package/src/live-query.ts +66 -144
  31. package/src/live-reauth.ts +127 -0
  32. package/src/live-replicator.ts +28 -8
  33. package/src/live-resume.ts +44 -8
  34. package/src/live-rows.ts +32 -3
  35. package/src/live-spend.ts +50 -0
  36. package/src/local-store-idb.ts +5 -0
  37. package/src/matcher-bridge.ts +11 -5
  38. package/src/nats-client.ts +26 -4
  39. package/src/nats-lib-client.ts +36 -7
  40. package/src/nats-subscriptions.ts +63 -0
  41. package/src/nats-transport.ts +127 -15
  42. package/src/offline-queue.ts +5 -0
  43. package/src/page-errors.ts +8 -9
  44. package/src/page-outbox.ts +5 -5
  45. package/src/page-runtime-wait.ts +72 -0
  46. package/src/page-runtime.ts +32 -0
  47. package/src/page-socket.ts +7 -16
  48. package/src/page-store.ts +83 -16
  49. package/src/pg-advisory-lock.ts +9 -4
  50. package/src/pg-connection.ts +7 -1
  51. package/src/pg-replication.ts +6 -6
  52. package/src/pg-socket.ts +20 -3
  53. package/src/pg-values.ts +29 -14
  54. package/src/policy-gate.ts +7 -7
  55. package/src/presence.ts +2 -2
  56. package/src/principal-sockets.ts +75 -0
  57. package/src/query-window.ts +1 -1
  58. package/src/record-publisher.ts +178 -0
  59. package/src/record-store.ts +25 -4
  60. package/src/record-synced.ts +43 -1
  61. package/src/replication-errors.ts +3 -3
  62. package/src/replicator-envelope.ts +61 -11
  63. package/src/replicator-row.ts +1 -1
  64. package/src/replicator.ts +12 -23
  65. package/src/server.ts +36 -18
  66. package/src/socket-defaults.ts +20 -0
  67. package/src/socket.ts +14 -19
  68. package/src/subscription-book.ts +129 -71
  69. package/src/subscription-count.ts +151 -0
  70. package/src/sync-actor-change.ts +56 -0
  71. package/src/sync-bus-handlers.ts +131 -0
  72. package/src/sync-frames.ts +1 -1
  73. package/src/sync-node-bounds.ts +1 -1
  74. package/src/sync-node-contract.ts +16 -2
  75. package/src/sync-node.ts +115 -109
  76. package/src/sync-origin.ts +1 -1
  77. package/src/sync-protocol.ts +73 -13
  78. package/src/sync-upgrade.ts +39 -0
  79. package/src/type-pins.ts +9 -1
  80. package/src/use-channel.ts +19 -14
  81. package/src/use-connection.ts +3 -3
  82. package/src/use-mutation.ts +9 -16
  83. package/src/use-query.ts +11 -11
  84. package/src/use-record.ts +89 -10
  85. package/src/wire-channel.ts +5 -5
  86. package/src/wire-read.ts +2 -2
  87. package/src/idb-fake.ts +0 -133
  88. package/src/policy-fake.ts +0 -14
@@ -0,0 +1,178 @@
1
+ // Committed rows an app hands over → change envelopes on the subject every sync node reads, for a
2
+ // deployment that runs no replicator. Channels only: a publisher has no commit position, so its
3
+ // changes feed the declared channels and never a live window (`sync-bus-handlers.ts`). One per
4
+ // process, built at boot: it is one producer, and its seq is what a node sees a lost message in.
5
+
6
+ import {
7
+ assert,
8
+ type Clock,
9
+ currentWriteOrigin,
10
+ isWriteDigest,
11
+ systemClock,
12
+ uuidV7,
13
+ } from '@ultimat3/core';
14
+ import { type RecordProjection, recordProjection } from '@ultimat3/entity/record';
15
+ import { type ChangeEvent, formatLsn } from './changefeed';
16
+ import type { ChannelEntity } from './channel-decl';
17
+ import type { Transport } from './fanout';
18
+ import type { Row } from './json';
19
+ import { changeSubject } from './replicator';
20
+ import { encodeEnvelope } from './replicator-envelope';
21
+
22
+ export interface RecordPublisherOptions {
23
+ readonly transport: Transport;
24
+ /**
25
+ * Every entity this process publishes, claimed until `close()`. Claimed, `x dev`'s in-process
26
+ * bridge stops carrying that entity's repository writes to channels — the app's publish is then
27
+ * the one delivery, exactly as it is in production with no replicator.
28
+ */
29
+ readonly entities: readonly ChannelEntity[];
30
+ /** Tenant column, hoisted into the subject as the replicator hoists it. Default `orgId`. */
31
+ readonly tenantColumn?: string;
32
+ readonly clock?: Clock;
33
+ }
34
+
35
+ export interface PublishOptions {
36
+ /** `upsert` (default): the rows are adopted. `delete`: the rows are gone, by their key. */
37
+ readonly op?: 'upsert' | 'delete';
38
+ /**
39
+ * The keyed write these rows belong to — `writeDigest(idempotencyKey)` — so the page that made
40
+ * it settles its own echo. Default: the keyed request this call runs inside, else `null`.
41
+ */
42
+ readonly write?: string | null;
43
+ }
44
+
45
+ export interface RecordPublisher extends Disposable {
46
+ /** This publisher's producer id: the stream its `seq` counts in. */
47
+ readonly producer: string;
48
+ /**
49
+ * Put COMMITTED rows on the bus, one change per row, in call order. Call it after the
50
+ * transaction commits: a row published from inside one is on every page before a rollback.
51
+ * Rejects with the first refusal once every row was tried; a refused row's seq stays spent, so
52
+ * every node reads the hole and its channel members re-run their catch-up read.
53
+ */
54
+ publish(
55
+ entity: ChannelEntity,
56
+ rows: readonly Readonly<Record<string, unknown>>[],
57
+ options?: PublishOptions,
58
+ ): Promise<void>;
59
+ /** Releases the claim. Idempotent; a closed publisher refuses to publish. */
60
+ close(): void;
61
+ }
62
+
63
+ /** Tables a live publisher claims, counted: two publishers may claim one. */
64
+ const claims = new Map<string, number>();
65
+
66
+ /**
67
+ * Whether a `recordPublisher` in this process claims `table` — the relation, the vocabulary of
68
+ * `ChangeEvent.table` on every producer.
69
+ */
70
+ export function isPublishedTable(table: string): boolean {
71
+ return claims.has(table);
72
+ }
73
+
74
+ export function recordPublisher(options: RecordPublisherOptions): RecordPublisher {
75
+ const clock = options.clock ?? systemClock;
76
+ const tenant = options.tenantColumn ?? 'orgId';
77
+ const projections = new Map<string, RecordProjection>();
78
+ for (const declared of options.entities) {
79
+ const projection = recordProjection(declared);
80
+ projections.set(projection.type, projection);
81
+ }
82
+ const names = [...projections.values()].map((projection) => projection.table);
83
+ for (const name of names) claims.set(name, (claims.get(name) ?? 0) + 1);
84
+ const producer = uuidV7();
85
+ let seq = 0;
86
+ let open = true;
87
+ // One chain, so seq order is bus order whatever the caller awaits: a node reads a seq that
88
+ // arrives out of order as a hole that was never there.
89
+ let tail: Promise<unknown> = Promise.resolve();
90
+
91
+ const close = (): void => {
92
+ if (!open) return;
93
+ open = false;
94
+ for (const name of names) {
95
+ const held = (claims.get(name) ?? 1) - 1;
96
+ if (held > 0) claims.set(name, held);
97
+ else claims.delete(name);
98
+ }
99
+ };
100
+
101
+ const changeOf = (
102
+ projection: RecordProjection,
103
+ row: Readonly<Record<string, unknown>>,
104
+ op: 'upsert' | 'delete',
105
+ write: string | null,
106
+ ): ChangeEvent => {
107
+ // The record's own properties and nothing else: the entity's record schema omits every sealed
108
+ // column, so neither a secret nor a property the entity never declared can ride.
109
+ const columns = projection.schema.properties ?? {};
110
+ const image = Object.fromEntries(
111
+ Object.entries(row).filter(([property]) => Object.hasOwn(columns, property)),
112
+ ) as Row;
113
+ // Refused here, before any row of the call is sequenced: a keyless row matches no record.
114
+ projection.key(image);
115
+ const orgId = image[tenant];
116
+ return {
117
+ table: projection.table,
118
+ op: op === 'delete' ? 'delete' : 'update',
119
+ before: op === 'delete' ? image : null,
120
+ after: op === 'delete' ? null : image,
121
+ // Stamped with the seq at send: it orders nothing outside this producer, which is why no
122
+ // live window is fed one.
123
+ lsn: '',
124
+ txid: producer,
125
+ orgId: typeof orgId === 'string' ? orgId : null,
126
+ at: clock.now().getTime(),
127
+ write,
128
+ };
129
+ };
130
+
131
+ const send = async (change: ChangeEvent): Promise<void> => {
132
+ seq += 1;
133
+ await options.transport.publish(
134
+ changeSubject(change),
135
+ encodeEnvelope({ ...change, lsn: formatLsn(seq) }, seq, producer, 'publisher'),
136
+ );
137
+ };
138
+
139
+ const publish: RecordPublisher['publish'] = async (entity, rows, publishOptions = {}) => {
140
+ assert(
141
+ open,
142
+ `recordPublisher ${producer} was closed and published "${entity.$name}" after it`,
143
+ 'build one recordPublisher per process at boot and close it at shutdown',
144
+ );
145
+ const projection = projections.get(entity.$name);
146
+ assert(
147
+ projection !== undefined,
148
+ `recordPublisher was asked to publish "${entity.$name}", which it did not claim`,
149
+ `add ${entity.$name} to recordPublisher({ entities: [...] }) where the publisher is built`,
150
+ );
151
+ const write =
152
+ publishOptions.write === undefined ? (currentWriteOrigin() ?? null) : publishOptions.write;
153
+ assert(
154
+ write === null || isWriteDigest(write),
155
+ `publish({ write }) for "${entity.$name}" is not a write digest`,
156
+ "pass writeDigest(idempotencyKey) from '@ultimat3/core', or null for a write no request made",
157
+ );
158
+ const op = publishOptions.op ?? 'upsert';
159
+ const changes = rows.map((row) => changeOf(projection, row, op, write));
160
+ const sent = tail.then(async () => {
161
+ // Every row the bus takes goes, a refused one in the middle included: its spent seq is the
162
+ // hole the nodes repair, and the caller hears the first refusal once the rest are out.
163
+ let refused: { readonly error: unknown } | undefined;
164
+ for (const change of changes) {
165
+ try {
166
+ await send(change);
167
+ } catch (error) {
168
+ refused ??= { error };
169
+ }
170
+ }
171
+ if (refused !== undefined) throw refused.error;
172
+ });
173
+ tail = sent.catch(() => undefined);
174
+ await sent;
175
+ };
176
+
177
+ return { producer, publish, close, [Symbol.dispose]: close };
178
+ }
@@ -61,16 +61,21 @@ export class RecordStore implements RecordSink {
61
61
 
62
62
  /** Every record of one type, as seen — what a mutator's `tx.<type>.all()` walks. */
63
63
  all(type: string): readonly Row[] {
64
+ return this.keyed(type).map(([, row]) => row);
65
+ }
66
+
67
+ /** `all`, with each record's key — what a whole-type reader orders by. */
68
+ keyed(type: string): readonly (readonly [string, Row])[] {
64
69
  const prefix = `${type}:`;
65
- const out: Row[] = [];
70
+ const out: (readonly [string, Row])[] = [];
66
71
  const seen = new Set<RecordKey>();
67
72
  for (const [rk, row] of this.#view) {
68
73
  if (!rk.startsWith(prefix)) continue;
69
74
  seen.add(rk);
70
- if (row !== null) out.push(row);
75
+ if (row !== null) out.push([rk.slice(prefix.length), row]);
71
76
  }
72
77
  for (const [rk, row] of this.#synced.entries()) {
73
- if (rk.startsWith(prefix) && !seen.has(rk)) out.push(row);
78
+ if (rk.startsWith(prefix) && !seen.has(rk)) out.push([rk.slice(prefix.length), row]);
74
79
  }
75
80
  return out;
76
81
  }
@@ -153,6 +158,18 @@ export class RecordStore implements RecordSink {
153
158
  if (this.#synced.release(rk)) this.#touchSynced(rk);
154
159
  }
155
160
 
161
+ /** Every record of `type` is shown by one reader: none is evicted while it is held. */
162
+ retainType(type: string): void {
163
+ this.#synced.retainType(type);
164
+ }
165
+
166
+ /** The last type hold leaving evicts the rows it alone kept (`SyncedLayer.releaseType`). */
167
+ releaseType(type: string): void {
168
+ this.batch(() => {
169
+ for (const rk of this.#synced.releaseType(type)) this.#touchSynced(rk);
170
+ });
171
+ }
172
+
156
173
  /**
157
174
  * An optimistic write, visible to every holder before this returns. `apply` is the mutator's
158
175
  * pure `local` half: it is REPLAYED over synced truth whenever synced truth moves, which is what
@@ -342,9 +359,13 @@ export class RecordStore implements RecordSink {
342
359
  this.#touched = result.touched;
343
360
  }
344
361
 
345
- /** The one way an overlay leaves: its entry and the digest that named it go together. */
362
+ /**
363
+ * The one way an overlay leaves: its entry, the digest that named it and its server wait go
364
+ * together. A wait left behind ran for its whole bound after a later settle removed the overlay.
365
+ */
346
366
  #forget(key: string): boolean {
347
367
  this.#names.forget(key);
368
+ this.#wait.forget(key);
348
369
  return this.#overlays.delete(key);
349
370
  }
350
371
 
@@ -8,6 +8,10 @@ import type { RecordKey } from './record-key';
8
8
  export class SyncedLayer {
9
9
  readonly #rows = new Map<RecordKey, Row>();
10
10
  readonly #holds = new Map<RecordKey, number>();
11
+ /** Types a whole-type reader holds (`useRecords(type, selection)`): none of their rows is evicted. */
12
+ readonly #typeHolds = new Map<string, number>();
13
+ /** Per held type, the rows whose last key holder left while it was held: the hold's to evict. */
14
+ readonly #spared = new Map<string, Set<RecordKey>>();
11
15
  /** Keys restored from disk and not yet confirmed: the first server row REPLACES, never merges. */
12
16
  readonly #provisional = new Set<RecordKey>();
13
17
 
@@ -74,6 +78,8 @@ export class SyncedLayer {
74
78
  /** Answers whether a row was there to remove. */
75
79
  remove(rk: RecordKey): boolean {
76
80
  this.#provisional.delete(rk);
81
+ // Gone is no longer spared: a row the server sends again later is a new arrival, not this one.
82
+ this.#spared.get(rk.slice(0, rk.indexOf(':')))?.delete(rk);
77
83
  return this.#rows.delete(rk);
78
84
  }
79
85
 
@@ -81,7 +87,10 @@ export class SyncedLayer {
81
87
  this.#holds.set(rk, (this.#holds.get(rk) ?? 0) + 1);
82
88
  }
83
89
 
84
- /** The last holder leaving evicts the row. Answers whether a row went. */
90
+ /**
91
+ * The last holder leaving evicts the row — unless a reader holds its whole type, which shows
92
+ * every row of it and would lose this one to another holder's unmount. Answers whether a row went.
93
+ */
85
94
  release(rk: RecordKey): boolean {
86
95
  const holds = this.#holds.get(rk);
87
96
  if (holds === undefined) return false;
@@ -90,11 +99,44 @@ export class SyncedLayer {
90
99
  return false;
91
100
  }
92
101
  this.#holds.delete(rk);
102
+ const type = rk.slice(0, rk.indexOf(':'));
103
+ if (this.#typeHolds.has(type)) {
104
+ const spared = this.#spared.get(type) ?? new Set<RecordKey>();
105
+ spared.add(rk);
106
+ this.#spared.set(type, spared);
107
+ return false;
108
+ }
93
109
  return this.remove(rk);
94
110
  }
95
111
 
112
+ retainType(type: string): void {
113
+ this.#typeHolds.set(type, (this.#typeHolds.get(type) ?? 0) + 1);
114
+ }
115
+
116
+ /**
117
+ * The last type hold leaving evicts what it spared — each row whose last key holder left while
118
+ * the type was held and that no key holder has taken since — exactly as that release would have.
119
+ * A row no key holder ever held is not the hold's to evict. Answers the rows that went.
120
+ */
121
+ releaseType(type: string): readonly RecordKey[] {
122
+ const holds = this.#typeHolds.get(type) ?? 0;
123
+ if (holds > 1) {
124
+ this.#typeHolds.set(type, holds - 1);
125
+ return [];
126
+ }
127
+ this.#typeHolds.delete(type);
128
+ const spared = this.#spared.get(type);
129
+ this.#spared.delete(type);
130
+ const evicted: RecordKey[] = [];
131
+ for (const rk of spared ?? []) {
132
+ if (!this.#holds.has(rk) && this.remove(rk)) evicted.push(rk);
133
+ }
134
+ return evicted;
135
+ }
136
+
96
137
  clear(): void {
97
138
  this.#rows.clear();
98
139
  this.#provisional.clear();
140
+ this.#spared.clear();
99
141
  }
100
142
  }
@@ -63,8 +63,8 @@ export class ReplicationTlsError extends RealtimeError {
63
63
  /**
64
64
  * A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
65
65
  * nothing is wrong with this process: the database already has its one replicator, and a second
66
- * one that started anyway would publish every change twice. Terminal for a container whose whole
67
- * job is that role — the scheduler is the thing that has to change, not the connection.
66
+ * one that started anyway would publish every change twice. Thrown only under `x dev`: a container
67
+ * that loses the lock stays up unready and takes over when it frees (`cli/src/role-replicator.ts`).
68
68
  */
69
69
  export class ReplicatorSlotHeldError extends RealtimeError {
70
70
  constructor(args: { key: string; holder?: string | undefined }) {
@@ -73,7 +73,7 @@ export class ReplicatorSlotHeldError extends RealtimeError {
73
73
  cause:
74
74
  `advisory lock ${args.key} is held${args.holder === undefined ? '' : ` by ${args.holder}`}` +
75
75
  ' — one database has exactly one replicator',
76
- fix: 'scale the replicator to 1 per database: kubectl scale deploy/replicator --replicas=1',
76
+ fix: 'x dev --role replicator # rerun after the replicator already on this DATABASE_URL stops — one database has one replicator',
77
77
  });
78
78
  }
79
79
  }
@@ -1,10 +1,11 @@
1
- // The bus envelope, read side: one published change decoded back into a `ChangeEvent` (rows revived
2
- // through their entity's columns), with the producer and sequence a lost message is visible in —
3
- // and the detector a `sync` node runs over them. The publisher is `replicator.ts`.
1
+ // The bus envelope, both sides: the one encoder every producer writes with (`replicator.ts`,
2
+ // `record-publisher.ts`), its decode back into a `ChangeEvent` (rows revived through their entity's
3
+ // columns) with the producer and sequence a lost message is visible in, and the detector a `sync`
4
+ // node runs over them.
4
5
 
5
6
  import { isWriteDigest } from '@ultimat3/core';
6
7
  import type { ChangeEvent } from './changefeed';
7
- import { reviveBusRow } from './replicator-row';
8
+ import { busRow, reviveBusRow } from './replicator-row';
8
9
 
9
10
  /**
10
11
  * What the bus actually carries: one change, plus who published it and where in that publisher's
@@ -24,6 +25,36 @@ export interface ChangeEnvelope {
24
25
  readonly seq: number | null;
25
26
  /** Identifies one replicator *run*. A new one restarts `seq` — and follows a stream that died. */
26
27
  readonly producer: string | null;
28
+ /**
29
+ * Who sequenced it. Absent is the replicator (every publisher that predates the field). A
30
+ * `recordPublisher` says `'publisher'`: its changes carry no commit position, so they feed the
31
+ * declared channels and never a live window, and several publishers run side by side.
32
+ */
33
+ readonly source?: EnvelopeSource;
34
+ }
35
+
36
+ /** The two producers of the change bus: the one WAL replicator, and an app's record publishers. */
37
+ export type EnvelopeSource = 'replicator' | 'publisher';
38
+
39
+ /**
40
+ * Producer side: one change as the bus carries it. Flat — the change plus `seq`, `producer` and,
41
+ * for a record publisher, `source` — so a consumer that only knows the change reads it unchanged
42
+ * (`parseChange`). Rows cross as `busRow` writes them: bytes as base64, revived by the entity.
43
+ */
44
+ export function encodeEnvelope(
45
+ change: ChangeEvent,
46
+ seq: number,
47
+ producer: string,
48
+ source?: EnvelopeSource,
49
+ ): string {
50
+ return JSON.stringify({
51
+ ...change,
52
+ before: busRow(change.before),
53
+ after: busRow(change.after),
54
+ seq,
55
+ producer,
56
+ ...(source === 'publisher' ? { source } : {}),
57
+ });
27
58
  }
28
59
 
29
60
  /** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
@@ -36,8 +67,17 @@ export function parseEnvelope(payload: string): ChangeEnvelope | null {
36
67
  try {
37
68
  const parsed: unknown = JSON.parse(payload);
38
69
  if (typeof parsed !== 'object' || parsed === null) return null;
39
- const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
40
- if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
70
+ const shape = parsed as Partial<ChangeEvent> & {
71
+ seq?: unknown;
72
+ producer?: unknown;
73
+ source?: unknown;
74
+ entity?: unknown;
75
+ };
76
+ // `entity` is the field a 24.x replicator wrote the table under. Read, never written: across a
77
+ // rolling deploy to 25.0.0 the replicator and the sync nodes run different builds for minutes,
78
+ // and a node refusing every change of the old one is every live view frozen for that window.
79
+ const table = typeof shape.table === 'string' ? shape.table : shape.entity;
80
+ if (typeof table !== 'string' || typeof shape.lsn !== 'string') return null;
41
81
  if (
42
82
  shape.op !== 'insert' &&
43
83
  shape.op !== 'update' &&
@@ -48,21 +88,22 @@ export function parseEnvelope(payload: string): ChangeEnvelope | null {
48
88
  }
49
89
  return {
50
90
  change: {
51
- entity: shape.entity,
91
+ table,
52
92
  op: shape.op,
53
- before: reviveBusRow(shape.entity, shape.before ?? null),
54
- after: reviveBusRow(shape.entity, shape.after ?? null),
93
+ before: reviveBusRow(table, shape.before ?? null),
94
+ after: reviveBusRow(table, shape.after ?? null),
55
95
  lsn: shape.lsn,
56
96
  txid: typeof shape.txid === 'string' ? shape.txid : '',
57
97
  orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
58
98
  at: typeof shape.at === 'number' ? shape.at : 0,
59
99
  // Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
60
100
  // label here would cost every member the whole frame rather than one page its echo.
61
- ...(isWriteDigest(shape.write) ? { write: shape.write } : {}),
101
+ write: isWriteDigest(shape.write) ? shape.write : null,
62
102
  ...(isNames(shape.omitted) ? { omitted: shape.omitted } : {}),
63
103
  },
64
104
  seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
65
105
  producer: typeof shape.producer === 'string' ? shape.producer : null,
106
+ ...(shape.source === 'publisher' ? { source: 'publisher' as const } : {}),
66
107
  };
67
108
  } catch {
68
109
  return null;
@@ -87,16 +128,24 @@ const isNames = (value: unknown): value is readonly string[] =>
87
128
  * A NEW producer after a known one IS a gap. The run before it died — that is why there is a new
88
129
  * one — and nothing says its last messages reached this node: the tail of a dead stream has no
89
130
  * later sequence number to be missed against. One re-read per replicator restart is the price.
131
+ *
132
+ * Only a REPLICATOR run succeeds another. Record publishers (`source: 'publisher'`) run side by
133
+ * side, one per app process, so a new one says nothing about the others; and a replicator run
134
+ * after a publisher is no successor either. A hole inside any one producer's sequence is a gap.
90
135
  */
91
136
  export class SeqGapDetector {
92
137
  readonly #next = new Map<string, number>();
138
+ /** Replicator runs read — the producers a NEW replicator run succeeds. */
139
+ readonly #runs = new Set<string>();
93
140
 
94
141
  /** `true` when at least one message between the last one and this one was never delivered. */
95
142
  observe(envelope: ChangeEnvelope): boolean {
96
143
  const { producer, seq } = envelope;
97
144
  if (producer === null || seq === null) return false;
98
145
  const expected = this.#next.get(producer);
99
- const succeeds = expected === undefined && this.#next.size > 0;
146
+ const replicator = envelope.source !== 'publisher';
147
+ const succeeds = replicator && expected === undefined && this.#runs.size > 0;
148
+ if (replicator) this.#runs.add(producer);
100
149
  // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
101
150
  // into a gap of its own.
102
151
  this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
@@ -106,5 +155,6 @@ export class SeqGapDetector {
106
155
  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
107
156
  forget(): void {
108
157
  this.#next.clear();
158
+ this.#runs.clear();
109
159
  }
110
160
  }
@@ -47,7 +47,7 @@ export function busRow(row: Row | null): Row | null {
47
47
  * over directly.
48
48
  *
49
49
  * Decided by the column that DECLARED the value, never by what the value looks like: a `text()`
50
- * holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.entity`, which on the WAL
50
+ * holding `2026-08-09T12:00:00.000Z` stays text. `table` is `ChangeEvent.table`, which on the WAL
51
51
  * path is the relation name. A table with no entity here, a property with no column, and a value
52
52
  * its column refuses are all handed back as they arrived — a change that crosses unrevived is the
53
53
  * old behaviour; a change dropped is a window that silently diverges.
package/src/replicator.ts CHANGED
@@ -11,12 +11,12 @@
11
11
  // its feed and reports `/readyz` false. A stream that ENDS — or a lock that is LOST — is this file's
12
12
  // to answer: `running` goes false, everything is let go, and the process asks again on a backoff.
13
13
 
14
- import { logger, renderThrowable, uuid, withSpan } from '@ultimat3/core';
14
+ import { logger, renderThrowable, uuidV7, withSpan } from '@ultimat3/core';
15
15
  import type { AdvisoryLock } from './advisory-lock';
16
16
  import type { ChangeEvent, ChangeFeed } from './changefeed';
17
17
  import { ReplicationFailedError } from './errors';
18
18
  import type { Transport } from './fanout';
19
- import { busRow } from './replicator-row';
19
+ import { encodeEnvelope } from './replicator-envelope';
20
20
  import {
21
21
  type BackoffPolicy,
22
22
  defaultBackoff,
@@ -30,9 +30,9 @@ export const CHANGE_SUBJECT_PREFIX = 'x.change';
30
30
  /** Every change subject: what a `sync` node subscribes to. A changefeed subject, never a channel. */
31
31
  export const CHANGE_SUBJECT_ALL = `${CHANGE_SUBJECT_PREFIX}.>`;
32
32
 
33
- /** `x.change.<entity>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
33
+ /** `x.change.<table>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
34
34
  export function changeSubject(change: ChangeEvent): string {
35
- return `${CHANGE_SUBJECT_PREFIX}.${change.entity}.${change.orgId ?? '_'}`;
35
+ return `${CHANGE_SUBJECT_PREFIX}.${change.table}.${change.orgId ?? '_'}`;
36
36
  }
37
37
 
38
38
  export interface ReplicatorOptions {
@@ -56,7 +56,7 @@ export interface ReplicatorStats {
56
56
  readonly restarts: number;
57
57
  /**
58
58
  * Why this replicator is not running although nothing stopped it — the stream's own reason, the
59
- * last restart's, or the change (entity, lsn, how many times) the bus keeps refusing. `null`
59
+ * last restart's, or the change (table, lsn, how many times) the bus keeps refusing. `null`
60
60
  * while it runs, and after a `stop()`.
61
61
  */
62
62
  readonly failure: string | null;
@@ -87,7 +87,7 @@ interface Run {
87
87
 
88
88
  /** The change a run could not publish, kept until one goes out. */
89
89
  interface Refusal {
90
- readonly entity: string;
90
+ readonly table: string;
91
91
  readonly lsn: string;
92
92
  readonly error: string;
93
93
  times: number;
@@ -107,7 +107,7 @@ const fencedOut = (reason: string): ReplicationFailedError =>
107
107
  fix: 'x doctor --json',
108
108
  });
109
109
 
110
- export function createReplicator(options: ReplicatorOptions): Replicator {
110
+ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
111
111
  const subjectOf = options.subjectOf ?? changeSubject;
112
112
  const backoff = options.backoff ?? defaultBackoff;
113
113
  /** A stream is pumping under a lock this process holds. Internal: `running` is stricter. */
@@ -119,7 +119,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
119
119
  // One id per run, not per process: a replicator that took the lock back after a crash publishes
120
120
  // from its persisted lsn, and a consumer must read that as a new stream rather than as a gap in
121
121
  // the old one.
122
- let producer = uuid();
122
+ let producer = uuidV7();
123
123
  let seq = 0;
124
124
  /** The start in flight, if any — see `start()` on why `pumping` cannot answer that question. */
125
125
  let starting: Promise<boolean> | undefined;
@@ -200,23 +200,12 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
200
200
  await withSpan('realtime.replicate', async () => {
201
201
  seq += 1;
202
202
  try {
203
- // The envelope is the change plus two fields, flat, so a consumer that only knows about
204
- // the change reads it unchanged — `parseChange` still answers on the same payload.
205
- await options.transport.publish(
206
- subjectOf(change),
207
- JSON.stringify({
208
- ...change,
209
- before: busRow(change.before),
210
- after: busRow(change.after),
211
- seq,
212
- producer,
213
- }),
214
- );
203
+ await options.transport.publish(subjectOf(change), encodeEnvelope(change, seq, producer));
215
204
  } catch (thrown) {
216
205
  const error = renderThrowable(thrown);
217
206
  const same = refusal !== null && refusal.lsn === change.lsn;
218
207
  refusal = {
219
- entity: change.entity,
208
+ table: change.table,
220
209
  lsn: change.lsn,
221
210
  error,
222
211
  times: same ? (refusal?.times ?? 0) + 1 : 1,
@@ -238,7 +227,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
238
227
  const describeFailure = (reason: string): string =>
239
228
  refusal === null
240
229
  ? reason
241
- : `${refusal.entity} at lsn ${refusal.lsn} could not be published ` +
230
+ : `${refusal.table} at lsn ${refusal.lsn} could not be published ` +
242
231
  `(${refusal.times} time(s) in a row): ${refusal.error}`;
243
232
 
244
233
  const begin = async (): Promise<boolean> => {
@@ -246,7 +235,7 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
246
235
  logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
247
236
  return false;
248
237
  }
249
- producer = uuid();
238
+ producer = uuidV7();
250
239
  seq = 0;
251
240
  const run: Run = { over: null };
252
241
  current = run;