@ultimat3/realtime 23.0.0 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CLAUDE.md +118 -126
  2. package/README.md +229 -54
  3. package/package.json +9 -6
  4. package/src/advisory-lock.ts +62 -0
  5. package/src/boot.ts +25 -15
  6. package/src/change-buffer.ts +96 -41
  7. package/src/changefeed-env.ts +9 -9
  8. package/src/changefeed.ts +81 -19
  9. package/src/channel-authz.ts +48 -17
  10. package/src/channel-describe.ts +1 -1
  11. package/src/channel-latch.ts +52 -0
  12. package/src/channel-logs.ts +36 -15
  13. package/src/channel-presence.ts +3 -3
  14. package/src/channel-records.ts +35 -4
  15. package/src/channel-ref.ts +2 -2
  16. package/src/channel-registry.ts +2 -2
  17. package/src/channel-sids.ts +37 -0
  18. package/src/channel.ts +148 -48
  19. package/src/client-channels.ts +12 -1
  20. package/src/client-contract.ts +4 -1
  21. package/src/client-frames.ts +16 -3
  22. package/src/client-heartbeat.ts +39 -9
  23. package/src/client.ts +30 -2
  24. package/src/cursor.ts +11 -0
  25. package/src/error-titles.ts +91 -0
  26. package/src/errors.ts +113 -105
  27. package/src/fanout.ts +13 -0
  28. package/src/first-paint-hold.ts +49 -0
  29. package/src/idb-types.ts +2 -2
  30. package/src/index.ts +25 -7
  31. package/src/json.ts +16 -3
  32. package/src/live-contract.ts +14 -0
  33. package/src/live-definition.ts +34 -9
  34. package/src/live-deliver.ts +55 -0
  35. package/src/live-fanout.ts +122 -29
  36. package/src/live-floor.ts +35 -0
  37. package/src/live-query-options.ts +44 -0
  38. package/src/live-query.ts +96 -175
  39. package/src/live-reauth.ts +127 -0
  40. package/src/live-replicator.ts +28 -8
  41. package/src/live-resume.ts +44 -8
  42. package/src/live-rows.ts +32 -3
  43. package/src/live-spend.ts +50 -0
  44. package/src/local-store-idb.ts +5 -0
  45. package/src/matcher-bridge.ts +11 -5
  46. package/src/nats-client.ts +37 -6
  47. package/src/nats-lib-client.ts +36 -7
  48. package/src/nats-subscriptions.ts +63 -0
  49. package/src/nats-transport.ts +149 -15
  50. package/src/offline-queue.ts +53 -5
  51. package/src/page-errors.ts +25 -9
  52. package/src/page-outbox.ts +125 -54
  53. package/src/page-runtime-wait.ts +72 -0
  54. package/src/page-runtime.ts +32 -0
  55. package/src/page-socket.ts +7 -16
  56. package/src/page-store.ts +83 -16
  57. package/src/pg-advisory-lock.ts +54 -5
  58. package/src/pg-auth.ts +9 -4
  59. package/src/pg-bytes.ts +4 -1
  60. package/src/pg-connection.ts +126 -22
  61. package/src/pg-entity-row.ts +16 -0
  62. package/src/pg-identity-tables.ts +25 -0
  63. package/src/pg-preflight.ts +50 -4
  64. package/src/pg-replication-row.ts +82 -0
  65. package/src/pg-replication.ts +50 -51
  66. package/src/pg-socket.ts +26 -7
  67. package/src/pg-tls.ts +18 -0
  68. package/src/pg-values.ts +29 -14
  69. package/src/pg-wire.ts +5 -3
  70. package/src/pgoutput.ts +19 -9
  71. package/src/policy-gate.ts +7 -7
  72. package/src/presence.ts +87 -10
  73. package/src/principal-sockets.ts +75 -0
  74. package/src/query-window.ts +17 -5
  75. package/src/record-persister.ts +7 -2
  76. package/src/record-publisher.ts +178 -0
  77. package/src/record-store.ts +25 -4
  78. package/src/record-synced.ts +43 -1
  79. package/src/replication-errors.ts +11 -7
  80. package/src/replicator-envelope.ts +160 -0
  81. package/src/replicator-row.ts +75 -0
  82. package/src/replicator.ts +313 -191
  83. package/src/server.ts +45 -23
  84. package/src/socket-defaults.ts +20 -0
  85. package/src/socket-engine.ts +14 -2
  86. package/src/socket-idle.ts +3 -1
  87. package/src/socket-routes.ts +8 -3
  88. package/src/socket.ts +14 -19
  89. package/src/subscriber-gate.ts +11 -1
  90. package/src/subscription-book.ts +129 -71
  91. package/src/subscription-count.ts +151 -0
  92. package/src/sync-actor-change.ts +56 -0
  93. package/src/sync-auth.ts +47 -2
  94. package/src/sync-bus-handlers.ts +131 -0
  95. package/src/sync-frames.ts +74 -21
  96. package/src/sync-node-bounds.ts +16 -1
  97. package/src/sync-node-contract.ts +36 -4
  98. package/src/sync-node.ts +163 -82
  99. package/src/sync-origin.ts +27 -12
  100. package/src/sync-protocol.ts +86 -16
  101. package/src/sync-upgrade.ts +115 -8
  102. package/src/type-pins.ts +9 -1
  103. package/src/use-channel.ts +19 -14
  104. package/src/use-connection.ts +3 -3
  105. package/src/use-mutation.ts +41 -20
  106. package/src/use-query.ts +11 -11
  107. package/src/use-record.ts +89 -10
  108. package/src/wire-channel.ts +18 -9
  109. package/src/wire-read.ts +2 -2
  110. package/src/idb-fake.ts +0 -133
  111. package/src/policy-fake.ts +0 -14
@@ -0,0 +1,62 @@
1
+ // Single responsibility: the mutual-exclusion contract behind "exactly one replicator per database",
2
+ // and its single-process implementation. The Postgres one is `pg-advisory-lock.ts`.
3
+
4
+ /** Session-scoped mutual exclusion. Postgres-backed in production, in-memory for `x dev`. */
5
+ export interface AdvisoryLock {
6
+ readonly key: string;
7
+ /** `false` means another process holds it — never block, never steal. */
8
+ tryAcquire(): Promise<boolean>;
9
+ release(): Promise<void>;
10
+ /**
11
+ * Let go WITHOUT asking: drop the session, synchronously, and wait for nothing. Closing the
12
+ * session releases the lock, so this is the release for a session that may be dead — an unlock
13
+ * statement on one is a wait nothing ends. Never calls `onLost`. A no-op when nothing is held.
14
+ */
15
+ abandon(): void;
16
+ /**
17
+ * Told when a lock this object HELD is gone without `release()` — its session died. Never for a
18
+ * release, never for a `tryAcquire` that answered `false`. Returns the unsubscribe. The holder
19
+ * must stop doing what the lock allowed: the database will give it to the next process that asks.
20
+ */
21
+ onLost(listener: (reason: string) => void): () => void;
22
+ }
23
+
24
+ /** Single-process default: correct for `x dev` and tests, useless across containers (by design). */
25
+ export class MemoryAdvisoryLock implements AdvisoryLock {
26
+ static readonly #held = new Set<string>();
27
+ readonly key: string;
28
+ #mine = false;
29
+
30
+ constructor(key: string) {
31
+ this.key = key;
32
+ }
33
+
34
+ async tryAcquire(): Promise<boolean> {
35
+ if (MemoryAdvisoryLock.#held.has(this.key)) return false;
36
+ MemoryAdvisoryLock.#held.add(this.key);
37
+ this.#mine = true;
38
+ return true;
39
+ }
40
+
41
+ async release(): Promise<void> {
42
+ if (!this.#mine) return;
43
+ MemoryAdvisoryLock.#held.delete(this.key);
44
+ this.#mine = false;
45
+ }
46
+
47
+ abandon(): void {
48
+ if (!this.#mine) return;
49
+ MemoryAdvisoryLock.#held.delete(this.key);
50
+ this.#mine = false;
51
+ }
52
+
53
+ /** A set in this process's own heap has no session to lose: the listener is never called. */
54
+ onLost(): () => void {
55
+ return () => undefined;
56
+ }
57
+ }
58
+
59
+ /** The one way to build the in-process lock — the twin of `postgresAdvisoryLock()`; the class is a type in the barrel only (`X_FACTORY_NAME_SPELLING`). */
60
+ export function memoryAdvisoryLock(key: string): MemoryAdvisoryLock {
61
+ return new MemoryAdvisoryLock(key);
62
+ }
package/src/boot.ts CHANGED
@@ -1,13 +1,14 @@
1
- // `@ultimat3/realtime/boot` — the page's boot, ONE classic script per document (plan 101), built
2
- // and served by the CLI the way the sync worker is. It restores the principal's persisted records
3
- // and opens the outbox, so a reload replays what the previous load queued even on a page whose
4
- // islands only read. Here and not in `page-store.ts`, so no island bundle carries it.
1
+ // `@ultimat3/realtime/boot` — the page's boot, ONE classic script per document (plan 101), served
2
+ // by the CLI like the sync worker. It installs the page runtime (store, socket, host, channel book;
3
+ // islands await and read it, never carry it — #505), restores the principal's persisted records
4
+ // and opens the outbox, so a reload replays what the last load queued even if islands only read.
5
5
 
6
6
  import type { ClientScope } from '@ultimat3/core/page';
7
7
  import { pageClient } from '@ultimat3/core/page';
8
8
  import { pageLocalStore, scopeKey } from './local-store-idb';
9
9
  import { pageOutbox } from './page-outbox';
10
- import { BOOT_KEY, BOOT_RELEASE_KEY, type BootHost, pageRealtime } from './page-store';
10
+ import { installPageRuntime } from './page-runtime';
11
+ import { BOOT_KEY, BOOT_RELEASE_KEY, type BootHost } from './page-store';
11
12
  import { persistedTypes, type RecordPersister, recordPersister } from './record-persister';
12
13
  import type { RecordStore } from './record-store';
13
14
 
@@ -15,9 +16,8 @@ import type { RecordStore } from './record-store';
15
16
  * The types the document marks persisted, read back off disk, then the page's outbox opened. None
16
17
  * marked is no record disk.
17
18
  */
18
- async function restoreFromDisk(store: RecordStore, principal: Principal): Promise<void> {
19
+ async function restoreFromDisk(store: RecordStore, principal: () => Principal): Promise<void> {
19
20
  const types = persistedTypes();
20
- const keep = scopeKey(principal);
21
21
  let persister: RecordPersister | undefined;
22
22
  try {
23
23
  // A sign-out by full navigation (a form post and a redirect) never calls `rescope()`, so the
@@ -26,7 +26,12 @@ async function restoreFromDisk(store: RecordStore, principal: Principal): Promis
26
26
  // restored. An unscoped page (rendered for nobody) wipes nothing — it knows no principal.
27
27
  // Trade-off: two principals in two tabs of one browser — the newer boot wipes the other's
28
28
  // disk; that tab keeps its records in memory and re-persists them on its next write.
29
- if (keep !== undefined) await (await pageLocalStore()).wipeOthers(keep);
29
+ //
30
+ // WHOSE disk is kept is decided AT the wipe, after the store has opened: decided before that
31
+ // await, a `rescope()` during it kept the principal that left and wiped the one that arrived.
32
+ const disk = await pageLocalStore();
33
+ const keep = scopeKey(principal());
34
+ if (keep !== undefined) await disk.wipeOthers(keep);
30
35
  if (types.size > 0) {
31
36
  persister = recordPersister({ store, local: await pageLocalStore(), types });
32
37
  await persister.restore();
@@ -46,20 +51,25 @@ async function restoreFromDisk(store: RecordStore, principal: Principal): Promis
46
51
  type Principal = ClientScope['principal'];
47
52
 
48
53
  /**
49
- * Once per page, whoever calls first; the promise is what `pageRealtime().booted` answers. The
50
- * scope is the page's own — a parameter only so a test can boot an unscoped page (an OBJECT, so an
51
- * explicit `undefined` principal is not mistaken for "use the default").
54
+ * Once per page, whoever calls first; the promise is what `pageRealtime().booted` answers — the
55
+ * one moment an island may treat the page's records as restored (#506 holds first paint on it).
56
+ * The runtime is installed synchronously, before the first await, so an island waiting on this
57
+ * script (`awaitPageRuntime`) is released by the script's `load` with the store already there.
58
+ * The scope is the page's own — a parameter only so a test can boot an unscoped page (an OBJECT,
59
+ * so an explicit `undefined` principal is not mistaken for "use the default").
52
60
  */
53
- export function bootPage(
54
- scope: { readonly principal: Principal } = pageClient().scope,
55
- ): Promise<void> {
61
+ export function bootPage(scope?: { readonly principal: Principal }): Promise<void> {
56
62
  const host = globalThis as BootHost;
57
63
  const started = host[BOOT_KEY];
58
64
  const waiting = host[BOOT_RELEASE_KEY];
59
65
  // Already booting — unless what sits there is an island's placeholder, which this boot claims.
60
66
  if (started !== undefined && waiting === undefined) return started;
61
67
  Reflect.deleteProperty(host, BOOT_RELEASE_KEY);
62
- const booted = restoreFromDisk(pageRealtime().store, scope.principal);
68
+ const page = installPageRuntime();
69
+ // Asked each time, never captured: the page's principal is whatever `rescope()` last said.
70
+ const principal = (): Principal =>
71
+ scope === undefined ? pageClient().scope.principal : scope.principal;
72
+ const booted = restoreFromDisk(page.store, principal);
63
73
  if (started !== undefined && waiting !== undefined) {
64
74
  void booted.then(waiting);
65
75
  return started;
@@ -36,11 +36,41 @@ export interface ChangeBufferOptions {
36
36
  export const DEFAULT_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
37
37
  export const DEFAULT_MAX_BUFFER_BYTES_PER_QUERY = 1024 * 1024;
38
38
 
39
+ /**
40
+ * THE FLOOR RULE, stated once. A ring answers a resume only for a cursor it can PROVE it holds
41
+ * every later change for; anything else is `null`, and the caller takes one bounded snapshot.
42
+ *
43
+ * | The floor is set by | to | a cursor AT the floor |
44
+ * |---|---|---|
45
+ * | the ring's first patch (no read floored it) | that patch's lsn | resumes — it has that patch |
46
+ * | an eviction | the evicted patch's lsn | resumes — everything after it is retained |
47
+ * | a window's FIRST read, on a node that holds a position | the read's lsn | resumes — the cursor may be that read's own |
48
+ * | a window's FIRST read with NO position behind it (`sole`) | the read's own mark | resumes, and it is the ONLY cursor below the first patch that does |
49
+ * | a FORCED re-read (the window missed a change), a truncate, a partial row | the read's lsn, EXCLUSIVE | is refused |
50
+ *
51
+ * Why `sole`: a node that has received no change holds no position, so its read is marked with
52
+ * the node's own origin (`LiveQueryRegistry`), which sorts below every real lsn. A plain floor
53
+ * there admits every foreign cursor — each is "above" it — for a history this node never held. So
54
+ * the ring answers that one mark, and otherwise only a cursor at or after its FIRST patch.
55
+ *
56
+ * Why the last row is exclusive: a re-read's lsn is the node's last SEEN position, which does not
57
+ * move for the changes it missed — so "the window at L before the gap" and "the window re-read at
58
+ * L with the missed rows in it" carry the same cursor, and only refusing L tells them apart.
59
+ *
60
+ * What never moves the floor: a read served out of a window that is already filled. The retained
61
+ * patches are still its true history, and flooring it evicts every other subscriber's resume.
62
+ */
39
63
  interface Ring {
40
64
  patches: RowPatch[];
41
65
  bytes: number;
42
- /** Highest lsn already dropped. A cursor at or after this is still resumable. */
43
- evictedThrough: string | null;
66
+ /** The lsn this ring is complete from. Set at birth and only ever raised. */
67
+ evictedThrough: string;
68
+ /** Whether a cursor exactly AT the floor is refused — see the rule above. */
69
+ exclusive: boolean;
70
+ /** The one cursor a position-less birth vouches for, until an eviction or a re-read ends it. */
71
+ sole: string | null;
72
+ /** The first patch appended after a `sole` birth: where foreign cursors become answerable. */
73
+ anchor: string | null;
44
74
  }
45
75
 
46
76
  const encoder = new TextEncoder();
@@ -52,12 +82,6 @@ function patchBytes(patch: RowPatch): number {
52
82
 
53
83
  export class RingChangeBuffer implements ResumeSource {
54
84
  readonly #rings = new Map<string, Ring>();
55
- /**
56
- * Query hashes whose ring was dropped, so the ring the NEXT change builds knows it does not
57
- * carry the history before it. A `Set` of ids and not a `Map` of lsns: what the next ring is
58
- * complete from is its own first patch, which only that patch can name.
59
- */
60
- readonly #forgotten = new Set<string>();
61
85
  readonly #capacity: number;
62
86
  readonly #maxQueries: number;
63
87
  readonly #maxBytesPerQuery: number;
@@ -86,17 +110,20 @@ export class RingChangeBuffer implements ResumeSource {
86
110
 
87
111
  append(qid: string, patch: RowPatch): void {
88
112
  const existing = this.#rings.get(qid);
89
- // A ring RE-CREATED after a `forget` is complete only from this patch onward: everything before
90
- // it went with the ring, and — on the `unsubscribe` path — the entry went too, so the changes
91
- // in between were never appended at all. `evictedThrough` at its own first lsn is what makes
92
- // `since` refuse a cursor from before that, which it could not do while the field came back
93
- // `null` and every earlier cursor read as in-window on a ring that had held none of it.
94
- const reborn = this.#forgotten.delete(qid);
113
+ // A ring is complete only from where it was born. Born HERE, from a patch, that is the patch
114
+ // itself: this node held no entry for whatever came before, so nothing before it was ever
115
+ // appended. It used to be born with no floor at all, and a cursor minted on another node —
116
+ // the rolling-deploy shape — read as in-window on a ring that had held none of its history.
117
+ // `floorAt` is the earlier, cheaper birth: the lsn the window was read at.
95
118
  const ring: Ring = existing ?? {
96
119
  patches: [],
97
120
  bytes: 0,
98
- evictedThrough: reborn ? patch.lsn : null,
121
+ evictedThrough: patch.lsn,
122
+ exclusive: false,
123
+ sole: null,
124
+ anchor: null,
99
125
  };
126
+ if (ring.sole !== null && ring.anchor === null) ring.anchor = patch.lsn;
100
127
  ring.patches.push(patch);
101
128
  const cost = patchBytes(patch);
102
129
  ring.bytes += cost;
@@ -106,8 +133,48 @@ export class RingChangeBuffer implements ResumeSource {
106
133
  while (ring.patches.length > this.#capacity || ring.bytes > this.#maxBytesPerQuery) {
107
134
  if (!this.#shift(ring)) break;
108
135
  }
109
- // Re-insert to move this qid to the tail of the LRU order.
110
- if (existing) this.#rings.delete(qid);
136
+ this.#touch(qid, ring);
137
+ }
138
+
139
+ /**
140
+ * The window behind `qid` was read at `lsn`: the ring is complete from there, and from nowhere
141
+ * earlier. The caller decides WHEN (a read that landed, never one served from a filled window)
142
+ * and whether the floor is `exclusive` (a forced re-read) — the rule is on `Ring`.
143
+ *
144
+ * Only ever raises the floor, or tightens it at the same lsn. `''` is no position and says nothing.
145
+ */
146
+ floorAt(
147
+ qid: string,
148
+ lsn: string,
149
+ options: { readonly exclusive?: boolean; readonly sole?: boolean } = {},
150
+ ): void {
151
+ if (lsn === '') return;
152
+ const exclusive = options.exclusive === true;
153
+ const existing = this.#rings.get(qid);
154
+ const ring: Ring = existing ?? {
155
+ patches: [],
156
+ bytes: 0,
157
+ evictedThrough: lsn,
158
+ exclusive,
159
+ sole: options.sole === true ? lsn : null,
160
+ anchor: null,
161
+ };
162
+ if (lsn > ring.evictedThrough || (lsn === ring.evictedThrough && exclusive)) {
163
+ // Superseded, not evicted: what the read replaced is history with a gap in it.
164
+ for (let first = ring.patches[0]; first !== undefined && first.lsn <= lsn; ) {
165
+ this.#shift(ring);
166
+ first = ring.patches[0];
167
+ }
168
+ ring.evictedThrough = lsn;
169
+ ring.exclusive = exclusive;
170
+ ring.sole = null;
171
+ }
172
+ this.#touch(qid, ring);
173
+ }
174
+
175
+ /** Move `qid` to the tail of the LRU order, then hold both node-wide ceilings. */
176
+ #touch(qid: string, ring: Ring): void {
177
+ this.#rings.delete(qid);
111
178
  this.#rings.set(qid, ring);
112
179
  while (this.#rings.size > this.#maxQueries || this.#bytes > this.#maxBytes) {
113
180
  const oldest = this.#rings.keys().next();
@@ -126,13 +193,19 @@ export class RingChangeBuffer implements ResumeSource {
126
193
  ring.bytes -= cost;
127
194
  this.#bytes -= cost;
128
195
  ring.evictedThrough = dropped.lsn;
196
+ ring.exclusive = false;
197
+ ring.sole = null;
129
198
  return true;
130
199
  }
131
200
 
132
201
  since(qid: string, lsn: string): RowPatch[] | null {
133
202
  const ring = this.#rings.get(qid);
134
203
  if (!ring) return null;
135
- if (ring.evictedThrough !== null && lsn < ring.evictedThrough) return null;
204
+ if (lsn < ring.evictedThrough) return null;
205
+ if (ring.exclusive && lsn === ring.evictedThrough) return null;
206
+ if (ring.sole !== null && lsn !== ring.sole && (ring.anchor === null || lsn < ring.anchor)) {
207
+ return null;
208
+ }
136
209
  return ring.patches.filter((patch) => patch.lsn > lsn);
137
210
  }
138
211
 
@@ -149,29 +222,11 @@ export class RingChangeBuffer implements ResumeSource {
149
222
  */
150
223
  forget(qid: string): void {
151
224
  const ring = this.#rings.get(qid);
152
- if (ring !== undefined) {
153
- this.#bytes -= ring.bytes;
154
- this.#rings.delete(qid);
155
- }
156
- // The qid is remembered, the patches are not — and unconditionally, because the ring being
157
- // absent is not the history being intact. Both callers lose history here and neither could say
158
- // so: `LiveQueryRegistry.unsubscribe` drops the ENTRY, so every change until the next
159
- // subscriber is never appended at all, and the LRU fires on a query that still has LIVE
160
- // subscribers. Either way the next `append` was building a ring that reported itself complete
161
- // from the beginning of time, so a client reconnecting inside `maxLagMs` folded a partial patch
162
- // list onto a stale window with `shouldResnapshot` answering `in-window` and nothing marked
163
- // desynced — permanently divergent on a healthy socket. What the tombstone costs in exchange is
164
- // a resume that could have been a delta taking the snapshot path; that is one bounded read, and
165
- // it is the direction this package errs in everywhere else.
166
- this.#forgotten.add(qid);
167
- // Bounded like everything else here: insertion-ordered, so the oldest tombstone goes first.
168
- // Losing one costs a resume that could have been a delta; keeping them unbounded costs memory
169
- // a client-chosen input mints at will.
170
- while (this.#forgotten.size > this.#maxQueries) {
171
- const oldest = this.#forgotten.values().next();
172
- if (oldest.done === true) break;
173
- this.#forgotten.delete(oldest.value);
174
- }
225
+ if (ring === undefined) return;
226
+ this.#bytes -= ring.bytes;
227
+ this.#rings.delete(qid);
228
+ // Nothing is remembered about it: the next ring for this qid is born with its own floor, so
229
+ // it cannot claim the history that went with this one.
175
230
  }
176
231
 
177
232
  get queryCount(): number {
@@ -1,18 +1,18 @@
1
1
  // Single responsibility: environment → change feed. The one place that decides which `ChangeFeed`
2
2
  // a boot installs, so `x dev`, a replicator container and any custom host resolve it identically.
3
- // `PgLogicalReplicationFeed` is useless until something constructs it from a connection string;
3
+ // `PostgresChangeFeed` is useless until something constructs it from a connection string;
4
4
  // this is that something, keyed on env rather than a config field so one image deploys everywhere.
5
5
 
6
6
  import type { Clock } from '@ultimat3/core';
7
7
  import { ConfigInvalidError } from '@ultimat3/core';
8
+ import type { AdvisoryLock } from './advisory-lock';
9
+ import { memoryAdvisoryLock } from './advisory-lock';
8
10
  import type { ChangeFeed } from './changefeed';
9
- import { InMemoryChangeFeed, PgLogicalReplicationFeed } from './changefeed';
10
- import { PgAdvisoryLock } from './pg-advisory-lock';
11
+ import { memoryChangeFeed, postgresChangeFeed } from './changefeed';
12
+ import { postgresAdvisoryLock } from './pg-advisory-lock';
11
13
  import type { PgTarget } from './pg-socket';
12
14
  import { parsePgUrl } from './pg-socket';
13
15
  import type { PgStream } from './pg-wire';
14
- import type { AdvisoryLock } from './replicator';
15
- import { InMemoryAdvisoryLock } from './replicator';
16
16
  import type { Rng } from './thundering-herd';
17
17
 
18
18
  /** The keys read here, and nothing else. Named once so docs and tests cannot drift from the code. */
@@ -105,12 +105,12 @@ export function selectChangeFeed(
105
105
 
106
106
  if (url === undefined) {
107
107
  return {
108
- feed: new InMemoryChangeFeed(options.retain === undefined ? {} : { retain: options.retain }),
108
+ feed: memoryChangeFeed(options.retain === undefined ? {} : { retain: options.retain }),
109
109
  mode: 'embedded',
110
110
  detail: 'in-process change feed — set DATABASE_URL to decode a real WAL',
111
111
  slot: null,
112
112
  // Single-process mutual exclusion, which is all one process needs and all it can enforce.
113
- lock: new InMemoryAdvisoryLock(replicatorLockKey(DEFAULT_REPLICATION_SLOT)),
113
+ lock: memoryAdvisoryLock(replicatorLockKey(DEFAULT_REPLICATION_SLOT)),
114
114
  };
115
115
  }
116
116
 
@@ -122,7 +122,7 @@ export function selectChangeFeed(
122
122
  return {
123
123
  // Slot and publication are validated inside the feed, against the same identifier rule the
124
124
  // replication command needs — a second copy of that regex here is a second thing to keep true.
125
- feed: new PgLogicalReplicationFeed({
125
+ feed: postgresChangeFeed({
126
126
  url,
127
127
  slot,
128
128
  publication: nonEmpty(env['REPLICATION_PUBLICATION']) ?? DEFAULT_REPLICATION_PUBLICATION,
@@ -136,7 +136,7 @@ export function selectChangeFeed(
136
136
  slot,
137
137
  // Taken on the same database the feed reads, so the lock and the slot cannot end up in
138
138
  // different places — which is the only way "exactly one replicator" could quietly become two.
139
- lock: new PgAdvisoryLock({
139
+ lock: postgresAdvisoryLock({
140
140
  url,
141
141
  key: replicatorLockKey(slot),
142
142
  stream: options.stream,
package/src/changefeed.ts CHANGED
@@ -20,8 +20,13 @@ import type { Rng } from './thundering-herd';
20
20
  export type ChangeOp = 'insert' | 'update' | 'delete' | 'truncate';
21
21
 
22
22
  export interface ChangeEvent<R extends Row = Row> {
23
- /** Entity name, not table name — the matcher's dependency sets are declared in entity terms. */
24
- readonly entity: string;
23
+ /**
24
+ * The RELATION (table) name, on every producer: the WAL decoder reads it off the Relation
25
+ * message, the in-process bridge maps the row observer's entity name onto it, and a
26
+ * `recordPublisher` sends `projection.table`. A channel matches `projection.table`, a live shape
27
+ * is `from('<table>', …)`; the record TYPE (entity name) is derived from it (`recordTypeForTable`).
28
+ */
29
+ readonly table: string;
25
30
  readonly op: ChangeOp;
26
31
  readonly before: R | null;
27
32
  readonly after: R | null;
@@ -34,23 +39,45 @@ export interface ChangeEvent<R extends Row = Row> {
34
39
  readonly at: number;
35
40
  /**
36
41
  * The write that made this change: `writeDigest` of the idempotency key its request carried
37
- * (`@ultimat3/core`). Absent for a change no keyed request made. Read off the WAL message the
38
- * Postgres driver writes first in the transaction, or off the request scope in-process; a
39
- * channel stamps it on the `records` frame so the writing page recognises its own echo.
42
+ * (`@ultimat3/core`). `null` for a change no keyed request made — a job, a script, raw SQL. Read
43
+ * off the WAL message the Postgres driver writes first in the transaction, or off the request
44
+ * scope in-process; a channel stamps it on its `records` frame and a live query on its `patch`
45
+ * frame, so the writing page settles its own echo in the same batch. REQUIRED, never optional:
46
+ * a producer that forgets it is a type error, not a page that paints truth plus overlay.
40
47
  */
41
- readonly write?: string;
48
+ readonly write: string | null;
49
+ /**
50
+ * Row properties `after` does NOT carry because Postgres did not log them: an UPDATE that left
51
+ * an out-of-line (TOAST) value untouched sends no bytes for it, and under any replica identity
52
+ * but FULL there is no old image to read it from. Absent when `after` is the whole row. A
53
+ * consumer must not adopt `after` as the row while this is set — it re-reads instead.
54
+ */
55
+ readonly omitted?: readonly string[];
42
56
  }
43
57
 
44
58
  export interface ChangeFeedStartOptions {
45
59
  /** Resume position. Omitted means "from now". */
46
60
  readonly from?: string;
47
61
  readonly onChange: (event: ChangeEvent) => void | Promise<void>;
62
+ /**
63
+ * The feed stopped delivering ON ITS OWN — the walsender ended the copy, a decode failed, the
64
+ * handler rejected — and will not resume until `start()` is called again. Never called for a
65
+ * `stop()`. This is the only way out a pump has: nothing awaits its read loop, so a death that
66
+ * was only recorded was a death nobody saw.
67
+ */
68
+ readonly onEnd?: (reason: string) => void;
48
69
  }
49
70
 
50
71
  export interface ChangeFeed {
51
72
  readonly source: string;
52
73
  start(options: ChangeFeedStartOptions): Promise<void>;
53
74
  stop(): Promise<void>;
75
+ /**
76
+ * Go silent NOW, without asking: synchronous, nothing written, nothing awaited. For a stream
77
+ * that may already be dead — `stop()` says goodbye first, and a goodbye to a black-holed socket
78
+ * is never answered. After it returns no further change is delivered and `onEnd` is not called.
79
+ */
80
+ abandon(): void;
54
81
  /** Highest lsn delivered to the handler; the replicator persists this to survive a restart. */
55
82
  lastLsn(): string | null;
56
83
  }
@@ -66,7 +93,7 @@ export function parseLsn(pgLsn: string): string {
66
93
  return formatLsn((BigInt(`0x${high}`) << 32n) | BigInt(`0x${low}`));
67
94
  }
68
95
 
69
- export interface InMemoryChangeFeedOptions {
96
+ export interface MemoryChangeFeedOptions {
70
97
  /** Retained events, so a `start({ from })` inside the window replays instead of skipping. */
71
98
  readonly retain?: number;
72
99
  }
@@ -76,7 +103,7 @@ export interface InMemoryChangeFeedOptions {
76
103
  * ordering is the guarantee the whole pipeline is built on, so it is enforced here rather than
77
104
  * assumed downstream.
78
105
  */
79
- export class InMemoryChangeFeed implements ChangeFeed {
106
+ export class MemoryChangeFeed implements ChangeFeed {
80
107
  readonly source = 'in-memory';
81
108
  readonly #retained: ChangeEvent[] = [];
82
109
  readonly #retain: number;
@@ -85,7 +112,7 @@ export class InMemoryChangeFeed implements ChangeFeed {
85
112
  #position = 0n;
86
113
  #lastLsn: string | null = null;
87
114
 
88
- constructor(options: InMemoryChangeFeedOptions = {}) {
115
+ constructor(options: MemoryChangeFeedOptions = {}) {
89
116
  this.#retain = finiteOption('the change feed', 'retain', options.retain ?? 1024);
90
117
  }
91
118
 
@@ -103,6 +130,10 @@ export class InMemoryChangeFeed implements ChangeFeed {
103
130
  await this.#tail;
104
131
  }
105
132
 
133
+ abandon(): void {
134
+ this.#handler = null;
135
+ }
136
+
106
137
  lastLsn(): string | null {
107
138
  return this.#lastLsn;
108
139
  }
@@ -118,13 +149,19 @@ export class InMemoryChangeFeed implements ChangeFeed {
118
149
 
119
150
  /** Ergonomic emit: assigns the next lsn and txid so tests read as domain events. */
120
151
  async push(
121
- entity: string,
152
+ table: string,
122
153
  op: ChangeOp,
123
- rows: { before?: Row | null; after?: Row | null; orgId?: string | null; at?: number },
154
+ rows: {
155
+ before?: Row | null;
156
+ after?: Row | null;
157
+ orgId?: string | null;
158
+ at?: number;
159
+ write?: string | null;
160
+ },
124
161
  ): Promise<ChangeEvent> {
125
162
  this.#position += 1n;
126
163
  const event: ChangeEvent = {
127
- entity,
164
+ table,
128
165
  op,
129
166
  before: rows.before ?? null,
130
167
  after: rows.after ?? null,
@@ -132,6 +169,7 @@ export class InMemoryChangeFeed implements ChangeFeed {
132
169
  txid: this.#position.toString(10),
133
170
  orgId: rows.orgId ?? null,
134
171
  at: rows.at ?? 0,
172
+ write: rows.write ?? null,
135
173
  };
136
174
  await this.emit(event);
137
175
  return event;
@@ -147,7 +185,7 @@ export class InMemoryChangeFeed implements ChangeFeed {
147
185
  // The lane chains on a SETTLED shadow, never on `result` — `window-lock.ts` solves the same
148
186
  // problem the same way. Chained on the live tail, one rejected link poisoned every link behind
149
187
  // it: later changes rejected with the FIRST error, the handler was never called again and
150
- // `lastLsn()` froze. Reachable on any single-node deployment, because `createReplicator`'s
188
+ // `lastLsn()` froze. Reachable on any single-node deployment, because `changeFeedReplicator`'s
151
189
  // `onChange` awaits `transport.publish(...)` and a closed `InProcessTransport` refuses — one
152
190
  // transient publish failure ended change delivery for the life of the process. The rejection
153
191
  // still reaches the caller that pushed THAT event, and only it; `stop()` awaits the shadow, so
@@ -157,10 +195,15 @@ export class InMemoryChangeFeed implements ChangeFeed {
157
195
  }
158
196
  }
159
197
 
198
+ /** The one way to build the in-process feed — the twin of `postgresChangeFeed()`; the class is a type in the barrel only (`X_FACTORY_NAME_SPELLING`). */
199
+ export function memoryChangeFeed(options: MemoryChangeFeedOptions = {}): MemoryChangeFeed {
200
+ return new MemoryChangeFeed(options);
201
+ }
202
+
160
203
  /** Settles the shadow lane whichever way the delivery went. Nothing observes the value. */
161
204
  const ignore = (): void => undefined;
162
205
 
163
- export interface PgLogicalReplicationOptions {
206
+ export interface PostgresChangeFeedOptions {
164
207
  /** Connection string for a role with REPLICATION: `postgres://user:pass@host:5432/db`. */
165
208
  readonly url: string;
166
209
  /** Replication slot name. Exactly one `replicator` process may hold it. */
@@ -175,37 +218,51 @@ export interface PgLogicalReplicationOptions {
175
218
  readonly rng?: Rng | undefined;
176
219
  /** The byte pipe, injected. Defaults to `Bun.connect`; a test drives a scripted server instead. */
177
220
  readonly stream?: ((target: PgTarget) => Promise<PgStream>) | undefined;
221
+ /**
222
+ * Tables whose DELETE must carry the whole old row — a channel with params routes a removal by
223
+ * columns only `REPLICA IDENTITY FULL` logs. Preflight warns for each that is not FULL. Defaults
224
+ * to what the declared channels need (`pg-identity-tables.ts`).
225
+ */
226
+ readonly fullIdentityTables?: readonly string[] | undefined;
178
227
  }
179
228
 
180
229
  /**
181
230
  * The production feed: `pgoutput` decoding off a logical replication slot. Everything about *how*
182
231
  * lives in `pg-replication.ts`; what this class adds is the `ChangeFeed` contract the matcher, the
183
- * replicator and the fanout are written against — so swapping it for `InMemoryChangeFeed` in `x dev`
232
+ * replicator and the fanout are written against — so swapping it for `MemoryChangeFeed` in `x dev`
184
233
  * changes nothing downstream.
185
234
  */
186
- export class PgLogicalReplicationFeed implements ChangeFeed {
235
+ export class PostgresChangeFeed implements ChangeFeed {
187
236
  readonly source = 'pg-logical-replication';
188
237
  readonly #stream: PgReplicationStream;
189
238
 
190
- constructor(options: PgLogicalReplicationOptions) {
239
+ constructor(options: PostgresChangeFeedOptions) {
191
240
  if (options.entities.length === 0) {
192
241
  throw new ReplicationFailedError({
193
242
  stage: 'preflight',
194
243
  detail: 'the feed was given an empty entity list, so no change could ever match',
195
- fix: 'pass the entities the publication covers: new PgLogicalReplicationFeed({ entities: [...] })',
244
+ fix: 'pass the entities the publication covers: postgresChangeFeed({ entities: [...] })',
196
245
  });
197
246
  }
198
247
  this.#stream = new PgReplicationStream(options);
199
248
  }
200
249
 
201
250
  async start(options: ChangeFeedStartOptions): Promise<void> {
202
- await this.#stream.start({ from: options.from, onChange: options.onChange });
251
+ await this.#stream.start({
252
+ from: options.from,
253
+ onChange: options.onChange,
254
+ onEnd: options.onEnd,
255
+ });
203
256
  }
204
257
 
205
258
  async stop(): Promise<void> {
206
259
  await this.#stream.stop();
207
260
  }
208
261
 
262
+ abandon(): void {
263
+ this.#stream.abandon();
264
+ }
265
+
209
266
  lastLsn(): string | null {
210
267
  return this.#stream.lastLsn();
211
268
  }
@@ -215,3 +272,8 @@ export class PgLogicalReplicationFeed implements ChangeFeed {
215
272
  return this.#stream.stats();
216
273
  }
217
274
  }
275
+
276
+ /** The one way to build the WAL-backed feed — the twin of `memoryChangeFeed()`; the class is a type in the barrel only (`X_FACTORY_NAME_SPELLING`). */
277
+ export function postgresChangeFeed(options: PostgresChangeFeedOptions): PostgresChangeFeed {
278
+ return new PostgresChangeFeed(options);
279
+ }