@ultimat3/realtime 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +180 -0
- package/package.json +36 -0
- package/src/change-buffer.ts +69 -0
- package/src/changefeed-env.ts +146 -0
- package/src/changefeed.ts +191 -0
- package/src/channel.ts +181 -0
- package/src/client.ts +439 -0
- package/src/cursor.ts +188 -0
- package/src/errors.ts +253 -0
- package/src/fanout.ts +159 -0
- package/src/hooks.ts +230 -0
- package/src/index.ts +328 -0
- package/src/json.ts +76 -0
- package/src/live-definition.ts +144 -0
- package/src/live-query.ts +449 -0
- package/src/local-store.ts +188 -0
- package/src/matcher-bridge.ts +169 -0
- package/src/nats-commands.ts +97 -0
- package/src/nats-connection-fixture.ts +105 -0
- package/src/nats-connection.ts +464 -0
- package/src/nats-fake.ts +431 -0
- package/src/nats-jetstream.ts +226 -0
- package/src/nats-kv.ts +157 -0
- package/src/nats-protocol.ts +222 -0
- package/src/nats-socket.ts +236 -0
- package/src/nats-transport.ts +257 -0
- package/src/offline-queue.ts +206 -0
- package/src/pg-advisory-lock.ts +98 -0
- package/src/pg-auth.ts +300 -0
- package/src/pg-bytes.ts +185 -0
- package/src/pg-connection-fixture.ts +215 -0
- package/src/pg-connection.ts +337 -0
- package/src/pg-entity-row.ts +130 -0
- package/src/pg-replication-fixture.ts +261 -0
- package/src/pg-replication.ts +396 -0
- package/src/pg-socket.ts +265 -0
- package/src/pg-wire.ts +192 -0
- package/src/pgoutput.ts +297 -0
- package/src/policy-gate.ts +56 -0
- package/src/presence.ts +219 -0
- package/src/rebase.ts +198 -0
- package/src/replicator.ts +185 -0
- package/src/socket.ts +208 -0
- package/src/sync-node.ts +400 -0
- package/src/sync-protocol.ts +376 -0
- package/src/thundering-herd.ts +141 -0
- package/src/transport-env.ts +104 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 developerz.ai
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# ⚡ @ultimat3/realtime
|
|
2
|
+
|
|
3
|
+
Three tiers, one ladder, one protocol. Climbing a rung is a config change, never a rewrite.
|
|
4
|
+
|
|
5
|
+
## The ladder
|
|
6
|
+
|
|
7
|
+
| Tier | What it gives you | What it costs you |
|
|
8
|
+
|---|---|---|
|
|
9
|
+
| **1 — channels** | `publish`/`subscribe` on typed topics, presence, cursors, typing indicators | ~0. Pub/sub over Bun's native WS. No DB, no replication slot |
|
|
10
|
+
| **2 — live queries** | the list updates when someone else edits; your own click feels instant | one change feed + a matcher per query id + a bounded change window |
|
|
11
|
+
| **3 — local-first** | writes that survive being offline | a durable local store, a rebase log, client-side migrations, a conflict story per mutator |
|
|
12
|
+
|
|
13
|
+
Tier 2 covers ~90% of "make it realtime". Tier 3 buys exactly one extra property — offline writes — and charges a client database for it. Do not buy it by accident.
|
|
14
|
+
|
|
15
|
+
## Same mutator at every rung
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
// query
|
|
19
|
+
export const liveFeed = query({
|
|
20
|
+
input: t.object({ orgId: t.uuid }),
|
|
21
|
+
policy: can('feed:read'),
|
|
22
|
+
live: true,
|
|
23
|
+
sql: ({ orgId }) => db.posts.where({ orgId }).orderBy('createdAt').limit(50),
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
// mutator (action + optimistic local twin)
|
|
27
|
+
export const likePost = mutator({
|
|
28
|
+
// Convergent, not incremental: `local` replays on every rebase, so applying it N times has to
|
|
29
|
+
// equal applying it once — `likedByMe` is what makes the second application a no-op.
|
|
30
|
+
local(tx, { postId }) {
|
|
31
|
+
tx.posts.update(postId, (p) =>
|
|
32
|
+
p.likedByMe ? {} : { likedByMe: true, likeCount: p.likeCount + 1 });
|
|
33
|
+
},
|
|
34
|
+
async server(ctx, { postId }) { return ctx.posts.like(postId); },
|
|
35
|
+
conflict: 'server-wins', // | 'last-write-wins' | custom(merge)
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`persist: true` on the query moves that route from tier 2 to tier 3. Same mutator, same authz, same
|
|
40
|
+
frames — `local` starts writing to a durable store and the mutation queue starts surviving reloads.
|
|
41
|
+
**One protocol serves all three tiers**: a channel message, a live-query patch and an offline
|
|
42
|
+
mutation drain are frames in the same discriminated union (`src/sync-protocol.ts`), so the client's
|
|
43
|
+
frame handler is unchanged between rungs.
|
|
44
|
+
|
|
45
|
+
`local` must be pure — no I/O, no `Date.now()`, no `Math.random()` — because rebase replays it.
|
|
46
|
+
|
|
47
|
+
## Public API
|
|
48
|
+
|
|
49
|
+
| Concern | Export |
|
|
50
|
+
|---|---|
|
|
51
|
+
| tier 1 | `topic`, `ChannelHub`, `PresenceRegistry`, `SyncSocket`, `SocketRegistry` |
|
|
52
|
+
| tier 2 | `LiveQueryRegistry`, `InMemoryChangeFeed`, `PgLogicalReplicationFeed`, `selectChangeFeed`, `createReplicator`, `PgAdvisoryLock`, `matcherFor` |
|
|
53
|
+
| replication | `parsePgUrl`, `bunPgStream`, `PgOutputDecoder`, `entityRow`, `changeLsn`, `commitPositionOf` |
|
|
54
|
+
| fanout | `Transport`, `InProcessTransport`, `NatsTransport`, `selectTransport`, `subjectMatches` |
|
|
55
|
+
| the bus client | `NatsConnection`, `NatsProtocolParser`, `NatsKvSet`, `ensureKvBucket`, `parseNatsUrl`, `bunNatsStream`, `FakeNatsServer` |
|
|
56
|
+
| reconnect | `LiveCursor`, `resumeFrom`, `shouldResnapshot`, `defaultReconnectBudget`, `RingChangeBuffer`, `backoffDelay`, `drainPlan`, `AcceptBudget` |
|
|
57
|
+
| tier 3 | `MemoryLocalStore`, `createOpfsLocalStore`, `OfflineQueue`, `RebaseLog`, `reconcile`, `custom` |
|
|
58
|
+
| wire | `PROTOCOL_VERSION`, `encode`, `decode`, `Frame` |
|
|
59
|
+
| halves | `LiveClient` (client), `createSyncNode` / `listenSyncNode` (`sync` role) |
|
|
60
|
+
| hooks | `setLiveClient`, `useLive`, `useConnection`, `useMutation`, `useMutationQueue` |
|
|
61
|
+
|
|
62
|
+
## The four hooks
|
|
63
|
+
|
|
64
|
+
Register the client once, in the app entry. Every hook reads it from there — no hook takes a client
|
|
65
|
+
argument, and one that runs before the registration is `X_LIVE_CLIENT_MISSING`, never a default.
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
setLiveClient(new LiveClient({ signal: createSignal, connect, buildId, store, queue }));
|
|
69
|
+
|
|
70
|
+
const feed = useLive(liveFeed, () => ({ orgId: actor.orgId })); // feed(), feed.state(), feed.unsubscribe()
|
|
71
|
+
const connection = useConnection(); // .offline .online .reconnectAt .updateAvailable
|
|
72
|
+
const like = useMutation(likePost); // await like(input); like.pending
|
|
73
|
+
const queue = useMutationQueue(); // .pending .failed .drain()
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
| Rule | Why |
|
|
77
|
+
|---|---|
|
|
78
|
+
| **No `solid-js` import.** Reactivity is the `SignalFactory` the client was built with | one reactive runtime per app, and a tier-3 package that installs and tests with none |
|
|
79
|
+
| Every member is a **getter**, every result set an **accessor** | a value snapshotted at hook time never re-renders |
|
|
80
|
+
| A thunk `input` is read **once**, at subscribe time | nothing here re-runs it; changing input is a new subscription |
|
|
81
|
+
| The caller owns `unsubscribe` | this layer does not know what a mount is |
|
|
82
|
+
| `pending` / `failed` are read off the queue, through an invalidation signal refreshed on each `mutate` and `drain` | the count is never a second copy of the queue, and `OfflineQueue` holds arrays, not signals |
|
|
83
|
+
|
|
84
|
+
Tier 2 has no queue, so `pending` is `0` there — stated, not guessed.
|
|
85
|
+
|
|
86
|
+
Authz goes through `@ultimat3/query`'s `guard`, which is the only contact with `@ultimat3/policy`.
|
|
87
|
+
One authz system, never two: `policy` is evaluated **once per subscriber**, never once per query.
|
|
88
|
+
Two actors on one live query get two different result sets, and a row that leaves an actor's policy
|
|
89
|
+
is delivered to them as a `delete` — never as silence.
|
|
90
|
+
|
|
91
|
+
## Reconnect is the hard part
|
|
92
|
+
|
|
93
|
+
A deploy drops N sockets at once and every one asks "what changed since X?". If that answer needs
|
|
94
|
+
arbitrary WAL replay or a re-run of every query, a rolling restart becomes a self-inflicted outage
|
|
95
|
+
that outlasts the deploy. The design confronts it with exactly two paths and no third:
|
|
96
|
+
|
|
97
|
+
| Path | When | Cost |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| **delta** | the cursor's gap is inside the retained window and inside the budget | one buffer read, zero DB work |
|
|
100
|
+
| **snapshot** | out of window, past `maxLagMs`, or past `reconnectBudget` | one bounded indexed query |
|
|
101
|
+
|
|
102
|
+
A `LiveCursor` is `lsn` + result-set `digest` + last-seen `ids` + `count`. The digest is
|
|
103
|
+
order-sensitive, so a re-sort is detected; the ids let a delta be re-filtered per subscriber, because
|
|
104
|
+
the retained window stores **pre-policy** patches. `resumeFrom()` picks the path,
|
|
105
|
+
`shouldResnapshot()` explains it, and the budget is a cost model in patch-equivalents
|
|
106
|
+
(`snapshotCost: 250` = "replaying 250 patches costs a snapshot") so the expensive path is *chosen*,
|
|
107
|
+
never stumbled into.
|
|
108
|
+
|
|
109
|
+
On drain, `drainPlan()` gives every client its own jittered slot in a spread window and the node
|
|
110
|
+
sends a `reconnect` frame carrying that delay — clients redistribute instead of stampeding.
|
|
111
|
+
`AcceptBudget` is the receiving node's token bucket, and a refusal always carries a retry delay,
|
|
112
|
+
because refusing without one just moves the herd next door.
|
|
113
|
+
|
|
114
|
+
### Limits, stated plainly
|
|
115
|
+
|
|
116
|
+
- **The change window is per node.** A client that reconnects to a *different* `sync` node has no
|
|
117
|
+
window there and takes a snapshot. Making the window shared (replicator-side, request/reply) is
|
|
118
|
+
gated on the milestone-6 reconnect benchmark — 50k sockets, forced restart, measure
|
|
119
|
+
time-to-consistent — because that number decides the topology.
|
|
120
|
+
- **A delta resume leaves the digest unverified** (`DIGEST_UNVERIFIED`). Only a snapshot re-establishes
|
|
121
|
+
it. `verifyDigest()` is how a client detects drift and asks for a fresh one.
|
|
122
|
+
- **Backpressure drops patch frames.** That is safe *only* because a re-snapshot is cheap: the drop
|
|
123
|
+
is recorded on the socket (`desynced`) and the next flush re-snapshots rather than diverging.
|
|
124
|
+
- **`PgLogicalReplicationFeed` decodes `pgoutput` off a real slot** — its own Postgres v3 client
|
|
125
|
+
(SCRAM-SHA-256, in-band TLS, CopyBoth), no driver dependency. It preflights `wal_level`, the
|
|
126
|
+
publication and the slot, creates the slot when there is none, and confirms the slot as it goes so
|
|
127
|
+
the WAL does not grow without bound. `InMemoryChangeFeed` + `InProcessTransport` remain the
|
|
128
|
+
defaults for `x dev` and every test.
|
|
129
|
+
- **`selectChangeFeed(env, { entities })` decides which feed a boot installs** — same law
|
|
130
|
+
`selectMailDriver` follows: an unset variable means the embedded default. It returns `{ feed,
|
|
131
|
+
mode, detail, slot, lock }`: `mode` is `'embedded' | 'external'`, `detail` is the env key that
|
|
132
|
+
selected it and never a credential, and `lock` is the `AdvisoryLock` for that feed — built here
|
|
133
|
+
rather than by the caller, because constructing one needs the URL and the URL carries a password.
|
|
134
|
+
Neither `DATABASE_URL` nor `REPLICATION_URL` set → `InMemoryChangeFeed`,
|
|
135
|
+
`mode: 'embedded'`. `REPLICATION_URL` wins when both are set, but naming a different host, port or
|
|
136
|
+
database than `DATABASE_URL` is refused at boot with `X_CONFIG_INVALID` — a feed streaming the
|
|
137
|
+
wrong database's WAL would be silently wrong forever. `REPLICATION_SLOT` (default `x_replicator`)
|
|
138
|
+
and `REPLICATION_PUBLICATION` (default `x_changes`) name the slot and publication, both checked
|
|
139
|
+
against `[a-z_][a-z0-9_]*` before they reach a replication command.
|
|
140
|
+
- **`PgAdvisoryLock` is the production `AdvisoryLock`** — `SELECT
|
|
141
|
+
pg_try_advisory_lock(hashtext('x:replicator:<slot>'))` on its own session. Session-scoped, so a
|
|
142
|
+
crashed replicator releases it automatically: no lease renewal, no fencing token, no split brain.
|
|
143
|
+
`InMemoryAdvisoryLock` remains the single-process default for `x dev` and tests.
|
|
144
|
+
- **`selectTransport(env)` decides which transport a boot fans out on** — the same law again, and
|
|
145
|
+
the only place that reads `NATS_URL`. It returns `{ transport, mode, detail, bucket,
|
|
146
|
+
presenceTtlMs, connect }`: unset → `InProcessTransport` and `mode: 'embedded'`, set → a
|
|
147
|
+
`NatsTransport` on the KV bucket `NATS_KV_BUCKET` names (default `x_presence`, so two apps on one
|
|
148
|
+
cluster do not share one presence namespace), validated here rather than on first connect.
|
|
149
|
+
`presenceTtlMs` comes back with it because the bucket's whole-stream age limit was derived from
|
|
150
|
+
it — a `PresenceRegistry` given a different number would report members leaving that never left.
|
|
151
|
+
Selection is pure; `connect()` is the dial, so an unreachable bus fails at boot.
|
|
152
|
+
- **`NatsTransport` speaks NATS itself** — its own protocol codec and session over `Bun.connect`,
|
|
153
|
+
no client dependency. Fanout is core NATS; `shared` is a JetStream KV bucket the transport
|
|
154
|
+
creates on first connect, one key per presence member, expired by the **server's** per-message
|
|
155
|
+
TTL so a node that dies needs nobody to notice. Subscriptions are held as *intent*, so a lost
|
|
156
|
+
connection re-dials and re-subscribes underneath the caller — which is what makes `sync`
|
|
157
|
+
stateless. That bucket needs nats-server ≥ 2.11 (batch direct get, per-message TTL); an older
|
|
158
|
+
one is `X_TRANSPORT_PROTOCOL` on the first dial, never a retry loop, because no amount of
|
|
159
|
+
reconnecting makes a server newer.
|
|
160
|
+
- **The lsn is `<commit position><row position in the transaction>`, 24 hex characters.** Neither
|
|
161
|
+
half works alone: every row of one transaction shares a commit lsn, and logical decoding emits
|
|
162
|
+
*transactions* in commit order, so per-record WAL positions are not monotonic across them. The
|
|
163
|
+
pair sorts in delivery order and is byte-identical on replay, which is what turns at-least-once
|
|
164
|
+
redelivery into a drop instead of a duplicate.
|
|
165
|
+
- **A live query needs `REPLICA IDENTITY FULL`.** Deciding whether a row *left* a result set needs
|
|
166
|
+
the old values; with the default identity a delete replicates only the key columns.
|
|
167
|
+
- Tier 3's OPFS SQLite store is browser-only and throws until the browser entry ships; `MemoryLocalStore`
|
|
168
|
+
implements the full journal/rollback/replay semantics today.
|
|
169
|
+
|
|
170
|
+
## Errors
|
|
171
|
+
|
|
172
|
+
`X_TOPIC_FORBIDDEN` · `X_SUBSCRIPTION_LIMIT` · `X_PROTOCOL_VERSION` · `X_CURSOR_STALE` ·
|
|
173
|
+
`X_REBASE_CONFLICT` · `X_TRANSPORT_UNAVAILABLE` · `X_TRANSPORT_PROTOCOL` ·
|
|
174
|
+
`X_REPLICATION_FAILED` · `X_REPLICATION_PROTOCOL` · `X_REPLICATOR_SLOT_HELD` ·
|
|
175
|
+
`X_LIVE_CLIENT_MISSING` · `X_NOT_IMPLEMENTED`
|
|
176
|
+
|
|
177
|
+
Topics deny by default: a topic with no matching guard is forbidden. An authz hole is not a config
|
|
178
|
+
option someone forgot to set.
|
|
179
|
+
|
|
180
|
+
`As of 2026-07`: tiers 1–2 target v1, tier 3 targets v2.
|
package/package.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@ultimat3/realtime",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/developerz-ai/ultimate.git",
|
|
10
|
+
"directory": "packages/realtime"
|
|
11
|
+
},
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public",
|
|
14
|
+
"provenance": true
|
|
15
|
+
},
|
|
16
|
+
"exports": {
|
|
17
|
+
".": "./src/index.ts"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"src",
|
|
21
|
+
"!src/**/*.test.ts",
|
|
22
|
+
"README.md",
|
|
23
|
+
"LICENSE"
|
|
24
|
+
],
|
|
25
|
+
"engines": {
|
|
26
|
+
"bun": ">=1.3.0"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"typecheck": "tsc --noEmit -p tsconfig.json",
|
|
30
|
+
"test": "bun test"
|
|
31
|
+
},
|
|
32
|
+
"dependencies": {
|
|
33
|
+
"@ultimat3/core": "1.0.0",
|
|
34
|
+
"@ultimat3/query": "1.0.0"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// The bounded per-query change window that makes reconnect a delta instead of a refetch.
|
|
2
|
+
// Lives on the `replicator` (one per DB), so a reconnecting client costs zero DB work while its
|
|
3
|
+
// gap is inside the window. Outside it, `resumeFrom` takes one snapshot — never WAL traversal.
|
|
4
|
+
|
|
5
|
+
import type { ResumeSource } from './cursor';
|
|
6
|
+
import type { RowPatch } from './json';
|
|
7
|
+
|
|
8
|
+
export interface ChangeBufferOptions {
|
|
9
|
+
/** Retained patches per query hash. */
|
|
10
|
+
readonly capacity?: number;
|
|
11
|
+
/** Retained query hashes; the least-recently-written is dropped first. */
|
|
12
|
+
readonly maxQueries?: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
interface Ring {
|
|
16
|
+
patches: RowPatch[];
|
|
17
|
+
/** Highest lsn already dropped. A cursor at or after this is still resumable. */
|
|
18
|
+
evictedThrough: string | null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class RingChangeBuffer implements ResumeSource {
|
|
22
|
+
readonly #rings = new Map<string, Ring>();
|
|
23
|
+
readonly #capacity: number;
|
|
24
|
+
readonly #maxQueries: number;
|
|
25
|
+
|
|
26
|
+
constructor(options: ChangeBufferOptions = {}) {
|
|
27
|
+
this.#capacity = options.capacity ?? 1024;
|
|
28
|
+
this.#maxQueries = options.maxQueries ?? 4096;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
append(qid: string, patch: RowPatch): void {
|
|
32
|
+
const existing = this.#rings.get(qid);
|
|
33
|
+
const ring: Ring = existing ?? { patches: [], evictedThrough: null };
|
|
34
|
+
ring.patches.push(patch);
|
|
35
|
+
while (ring.patches.length > this.#capacity) {
|
|
36
|
+
const dropped = ring.patches.shift();
|
|
37
|
+
if (dropped) ring.evictedThrough = dropped.lsn;
|
|
38
|
+
}
|
|
39
|
+
// Re-insert to move this qid to the tail of the LRU order.
|
|
40
|
+
if (existing) this.#rings.delete(qid);
|
|
41
|
+
this.#rings.set(qid, ring);
|
|
42
|
+
if (this.#rings.size > this.#maxQueries) {
|
|
43
|
+
const oldest = this.#rings.keys().next();
|
|
44
|
+
if (!oldest.done) this.#rings.delete(oldest.value);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
since(qid: string, lsn: string): RowPatch[] | null {
|
|
49
|
+
const ring = this.#rings.get(qid);
|
|
50
|
+
if (!ring) return null;
|
|
51
|
+
if (ring.evictedThrough !== null && lsn < ring.evictedThrough) return null;
|
|
52
|
+
return ring.patches.filter((patch) => patch.lsn > lsn);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
headLsn(qid: string): string | null {
|
|
56
|
+
const ring = this.#rings.get(qid);
|
|
57
|
+
const last = ring?.patches.at(-1);
|
|
58
|
+
return last ? last.lsn : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Called when the last subscriber of a query goes away, so an idle query stops costing memory. */
|
|
62
|
+
forget(qid: string): void {
|
|
63
|
+
this.#rings.delete(qid);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
get queryCount(): number {
|
|
67
|
+
return this.#rings.size;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// Single responsibility: environment → change feed. The one place that decides which `ChangeFeed`
|
|
2
|
+
// a boot installs, so `x dev`, a replicator container and any custom host resolve it identically.
|
|
3
|
+
// `PgLogicalReplicationFeed` is useless until something constructs it from a connection string;
|
|
4
|
+
// this is that something, keyed on env rather than a config field so one image deploys everywhere.
|
|
5
|
+
|
|
6
|
+
import type { Clock } from '@ultimat3/core';
|
|
7
|
+
import { ConfigInvalidError } from '@ultimat3/core';
|
|
8
|
+
import type { ChangeFeed } from './changefeed';
|
|
9
|
+
import { InMemoryChangeFeed, PgLogicalReplicationFeed } from './changefeed';
|
|
10
|
+
import { PgAdvisoryLock } from './pg-advisory-lock';
|
|
11
|
+
import type { PgTarget } from './pg-socket';
|
|
12
|
+
import { parsePgUrl } from './pg-socket';
|
|
13
|
+
import type { PgStream } from './pg-wire';
|
|
14
|
+
import type { AdvisoryLock } from './replicator';
|
|
15
|
+
import { InMemoryAdvisoryLock } from './replicator';
|
|
16
|
+
import type { Rng } from './thundering-herd';
|
|
17
|
+
|
|
18
|
+
/** The keys read here, and nothing else. Named once so docs and tests cannot drift from the code. */
|
|
19
|
+
export const REPLICATION_ENV_KEYS = [
|
|
20
|
+
'DATABASE_URL',
|
|
21
|
+
'REPLICATION_URL',
|
|
22
|
+
'REPLICATION_SLOT',
|
|
23
|
+
'REPLICATION_PUBLICATION',
|
|
24
|
+
] as const;
|
|
25
|
+
|
|
26
|
+
/** The slot one replicator holds, and the publication it decodes, when env names neither. */
|
|
27
|
+
export const DEFAULT_REPLICATION_SLOT = 'x_replicator';
|
|
28
|
+
export const DEFAULT_REPLICATION_PUBLICATION = 'x_changes';
|
|
29
|
+
|
|
30
|
+
/** The advisory-lock key for a slot. One database, one replicator, one key — derived, never typed. */
|
|
31
|
+
export const replicatorLockKey = (slot: string): string => `x:replicator:${slot}`;
|
|
32
|
+
|
|
33
|
+
export type ReplicationEnvironment = Readonly<Record<string, string | undefined>>;
|
|
34
|
+
|
|
35
|
+
export interface ChangeFeedSelection {
|
|
36
|
+
readonly feed: ChangeFeed;
|
|
37
|
+
/** `embedded` is the in-process feed; `external` decodes a real WAL. */
|
|
38
|
+
readonly mode: 'embedded' | 'external';
|
|
39
|
+
/**
|
|
40
|
+
* Why this feed, in one line: the env key that selected it, or what to set to change it. A boot
|
|
41
|
+
* prints it, so "which WAL is this process reading" is never a guess — and it is the env key
|
|
42
|
+
* rather than the URL, because a replication URL carries a password.
|
|
43
|
+
*/
|
|
44
|
+
readonly detail: string;
|
|
45
|
+
/** Null in embedded mode: there is no slot to lock, and nothing to be the second replicator of. */
|
|
46
|
+
readonly slot: string | null;
|
|
47
|
+
/**
|
|
48
|
+
* The lock that keeps "one replicator per database" true for this feed. Returned here rather
|
|
49
|
+
* than built by the caller so the connection string stays inside this module — a caller that
|
|
50
|
+
* had to construct the lock itself would need the URL, and the URL carries a password.
|
|
51
|
+
*/
|
|
52
|
+
readonly lock: AdvisoryLock;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface SelectChangeFeedOptions {
|
|
56
|
+
/** Entities to decode. Anything else is skipped before it reaches the matcher. */
|
|
57
|
+
readonly entities: readonly string[];
|
|
58
|
+
/** Retained events in the embedded feed, so a `start({ from })` replays instead of skipping. */
|
|
59
|
+
readonly retain?: number | undefined;
|
|
60
|
+
readonly clock?: Clock | undefined;
|
|
61
|
+
readonly rng?: Rng | undefined;
|
|
62
|
+
readonly stream?: ((target: PgTarget) => Promise<PgStream>) | undefined;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const nonEmpty = (value: string | undefined): string | undefined =>
|
|
66
|
+
value === undefined || value.trim().length === 0 ? undefined : value.trim();
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* A feed pointed at a *different* database than the app writes to decodes a WAL in which the app's
|
|
70
|
+
* own transactions never appear: every live query stays on its first snapshot forever and nothing
|
|
71
|
+
* downstream can tell. There is no runtime symptom to debug, so the two URLs are compared at the
|
|
72
|
+
* boundary — the only place both are still in hand.
|
|
73
|
+
*/
|
|
74
|
+
function assertSameDatabase(replicationUrl: string, databaseUrl: string): void {
|
|
75
|
+
const replication = parsePgUrl(replicationUrl);
|
|
76
|
+
const application = parsePgUrl(databaseUrl);
|
|
77
|
+
const differs =
|
|
78
|
+
replication.host !== application.host ||
|
|
79
|
+
replication.port !== application.port ||
|
|
80
|
+
replication.database !== application.database;
|
|
81
|
+
if (!differs) return;
|
|
82
|
+
throw new ConfigInvalidError({
|
|
83
|
+
cause:
|
|
84
|
+
`REPLICATION_URL names ${replication.host}:${replication.port}/${replication.database} but ` +
|
|
85
|
+
`DATABASE_URL names ${application.host}:${application.port}/${application.database} — the ` +
|
|
86
|
+
'feed would decode a WAL the app never writes to',
|
|
87
|
+
fix: 'point REPLICATION_URL at the same host, port and database as DATABASE_URL, changing only the role',
|
|
88
|
+
meta: { REPLICATION_URL: replication.database, DATABASE_URL: application.database },
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* No connection string means the in-process feed — the same "an unset variable means the embedded
|
|
94
|
+
* default" law the db, events and storage bindings follow. `REPLICATION_URL` exists because the
|
|
95
|
+
* app's own role usually lacks `REPLICATION`; it overrides which credentials are used, never which
|
|
96
|
+
* database is read, which is what `assertSameDatabase` holds to.
|
|
97
|
+
*/
|
|
98
|
+
export function selectChangeFeed(
|
|
99
|
+
env: ReplicationEnvironment,
|
|
100
|
+
options: SelectChangeFeedOptions,
|
|
101
|
+
): ChangeFeedSelection {
|
|
102
|
+
const databaseUrl = nonEmpty(env['DATABASE_URL']);
|
|
103
|
+
const replicationUrl = nonEmpty(env['REPLICATION_URL']);
|
|
104
|
+
const url = replicationUrl ?? databaseUrl;
|
|
105
|
+
|
|
106
|
+
if (url === undefined) {
|
|
107
|
+
return {
|
|
108
|
+
feed: new InMemoryChangeFeed(options.retain === undefined ? {} : { retain: options.retain }),
|
|
109
|
+
mode: 'embedded',
|
|
110
|
+
detail: 'in-process change feed — set DATABASE_URL to decode a real WAL',
|
|
111
|
+
slot: null,
|
|
112
|
+
// Single-process mutual exclusion, which is all one process needs and all it can enforce.
|
|
113
|
+
lock: new InMemoryAdvisoryLock(replicatorLockKey(DEFAULT_REPLICATION_SLOT)),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (replicationUrl !== undefined && databaseUrl !== undefined) {
|
|
118
|
+
assertSameDatabase(replicationUrl, databaseUrl);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const slot = nonEmpty(env['REPLICATION_SLOT']) ?? DEFAULT_REPLICATION_SLOT;
|
|
122
|
+
return {
|
|
123
|
+
// Slot and publication are validated inside the feed, against the same identifier rule the
|
|
124
|
+
// replication command needs — a second copy of that regex here is a second thing to keep true.
|
|
125
|
+
feed: new PgLogicalReplicationFeed({
|
|
126
|
+
url,
|
|
127
|
+
slot,
|
|
128
|
+
publication: nonEmpty(env['REPLICATION_PUBLICATION']) ?? DEFAULT_REPLICATION_PUBLICATION,
|
|
129
|
+
entities: options.entities,
|
|
130
|
+
clock: options.clock,
|
|
131
|
+
rng: options.rng,
|
|
132
|
+
stream: options.stream,
|
|
133
|
+
}),
|
|
134
|
+
mode: 'external',
|
|
135
|
+
detail: replicationUrl === undefined ? 'DATABASE_URL' : 'REPLICATION_URL',
|
|
136
|
+
slot,
|
|
137
|
+
// Taken on the same database the feed reads, so the lock and the slot cannot end up in
|
|
138
|
+
// different places — which is the only way "exactly one replicator" could quietly become two.
|
|
139
|
+
lock: new PgAdvisoryLock({
|
|
140
|
+
url,
|
|
141
|
+
key: replicatorLockKey(slot),
|
|
142
|
+
stream: options.stream,
|
|
143
|
+
rng: options.rng,
|
|
144
|
+
}),
|
|
145
|
+
};
|
|
146
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// The ordered stream of committed row changes. Production source is Postgres logical replication;
|
|
2
|
+
// `x dev` and every test use the in-memory feed. Both satisfy one interface, so the matcher, the
|
|
3
|
+
// replicator, and the fanout never learn which one they are attached to.
|
|
4
|
+
|
|
5
|
+
import type { Clock } from '@ultimat3/core';
|
|
6
|
+
import { ReplicationFailedError } from './errors';
|
|
7
|
+
import type { Row } from './json';
|
|
8
|
+
import { PgReplicationStream, type ReplicationStreamStats } from './pg-replication';
|
|
9
|
+
import type { PgTarget } from './pg-socket';
|
|
10
|
+
import type { PgStream } from './pg-wire';
|
|
11
|
+
import type { Rng } from './thundering-herd';
|
|
12
|
+
|
|
13
|
+
export type ChangeOp = 'insert' | 'update' | 'delete';
|
|
14
|
+
|
|
15
|
+
export interface ChangeEvent<R extends Row = Row> {
|
|
16
|
+
/** Entity name, not table name — the matcher's dependency sets are declared in entity terms. */
|
|
17
|
+
readonly entity: string;
|
|
18
|
+
readonly op: ChangeOp;
|
|
19
|
+
readonly before: R | null;
|
|
20
|
+
readonly after: R | null;
|
|
21
|
+
/** Lexicographically comparable position. Use `formatLsn` / `parseLsn`, never a raw pg string. */
|
|
22
|
+
readonly lsn: string;
|
|
23
|
+
readonly txid: string;
|
|
24
|
+
/** Tenant column, hoisted out of the row so fanout can filter without parsing it. */
|
|
25
|
+
readonly orgId: string | null;
|
|
26
|
+
/** Commit time, epoch ms. */
|
|
27
|
+
readonly at: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface ChangeFeedStartOptions {
|
|
31
|
+
/** Resume position. Omitted means "from now". */
|
|
32
|
+
readonly from?: string;
|
|
33
|
+
readonly onChange: (event: ChangeEvent) => void | Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ChangeFeed {
|
|
37
|
+
readonly source: string;
|
|
38
|
+
start(options: ChangeFeedStartOptions): Promise<void>;
|
|
39
|
+
stop(): Promise<void>;
|
|
40
|
+
/** Highest lsn delivered to the handler; the replicator persists this to survive a restart. */
|
|
41
|
+
lastLsn(): string | null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** 16-hex zero-padded so string comparison equals numeric comparison. */
|
|
45
|
+
export function formatLsn(position: bigint | number): string {
|
|
46
|
+
return BigInt(position).toString(16).padStart(16, '0');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Postgres prints LSNs as `0/16B3748`. Both halves are hex; join them into one sortable value. */
|
|
50
|
+
export function parseLsn(pgLsn: string): string {
|
|
51
|
+
const [high = '0', low = '0'] = pgLsn.split('/');
|
|
52
|
+
return formatLsn((BigInt(`0x${high}`) << 32n) | BigInt(`0x${low}`));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface InMemoryChangeFeedOptions {
|
|
56
|
+
/** Retained events, so a `start({ from })` inside the window replays instead of skipping. */
|
|
57
|
+
readonly retain?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The blessed development and test feed. Deliveries are serialized through one promise chain:
|
|
62
|
+
* ordering is the guarantee the whole pipeline is built on, so it is enforced here rather than
|
|
63
|
+
* assumed downstream.
|
|
64
|
+
*/
|
|
65
|
+
export class InMemoryChangeFeed implements ChangeFeed {
|
|
66
|
+
readonly source = 'in-memory';
|
|
67
|
+
readonly #retained: ChangeEvent[] = [];
|
|
68
|
+
readonly #retain: number;
|
|
69
|
+
#handler: ChangeFeedStartOptions['onChange'] | null = null;
|
|
70
|
+
#tail: Promise<void> = Promise.resolve();
|
|
71
|
+
#position = 0n;
|
|
72
|
+
#lastLsn: string | null = null;
|
|
73
|
+
|
|
74
|
+
constructor(options: InMemoryChangeFeedOptions = {}) {
|
|
75
|
+
this.#retain = options.retain ?? 1024;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async start(options: ChangeFeedStartOptions): Promise<void> {
|
|
79
|
+
this.#handler = options.onChange;
|
|
80
|
+
const from = options.from;
|
|
81
|
+
if (from === undefined) return;
|
|
82
|
+
for (const event of this.#retained) {
|
|
83
|
+
if (event.lsn > from) await this.#deliver(event);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async stop(): Promise<void> {
|
|
88
|
+
this.#handler = null;
|
|
89
|
+
await this.#tail;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
lastLsn(): string | null {
|
|
93
|
+
return this.#lastLsn;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Append a fully-formed event. Used by tests that need an exact lsn or txid. */
|
|
97
|
+
async emit(event: ChangeEvent): Promise<void> {
|
|
98
|
+
if (this.#retain > 0) {
|
|
99
|
+
this.#retained.push(event);
|
|
100
|
+
while (this.#retained.length > this.#retain) this.#retained.shift();
|
|
101
|
+
}
|
|
102
|
+
await this.#deliver(event);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Ergonomic emit: assigns the next lsn and txid so tests read as domain events. */
|
|
106
|
+
async push(
|
|
107
|
+
entity: string,
|
|
108
|
+
op: ChangeOp,
|
|
109
|
+
rows: { before?: Row | null; after?: Row | null; orgId?: string | null; at?: number },
|
|
110
|
+
): Promise<ChangeEvent> {
|
|
111
|
+
this.#position += 1n;
|
|
112
|
+
const event: ChangeEvent = {
|
|
113
|
+
entity,
|
|
114
|
+
op,
|
|
115
|
+
before: rows.before ?? null,
|
|
116
|
+
after: rows.after ?? null,
|
|
117
|
+
lsn: formatLsn(this.#position),
|
|
118
|
+
txid: this.#position.toString(10),
|
|
119
|
+
orgId: rows.orgId ?? null,
|
|
120
|
+
at: rows.at ?? 0,
|
|
121
|
+
};
|
|
122
|
+
await this.emit(event);
|
|
123
|
+
return event;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async #deliver(event: ChangeEvent): Promise<void> {
|
|
127
|
+
const handler = this.#handler;
|
|
128
|
+
if (!handler) return;
|
|
129
|
+
this.#tail = this.#tail.then(async () => {
|
|
130
|
+
await handler(event);
|
|
131
|
+
this.#lastLsn = event.lsn;
|
|
132
|
+
});
|
|
133
|
+
await this.#tail;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface PgLogicalReplicationOptions {
|
|
138
|
+
/** Connection string for a role with REPLICATION: `postgres://user:pass@host:5432/db`. */
|
|
139
|
+
readonly url: string;
|
|
140
|
+
/** Replication slot name. Exactly one `replicator` process may hold it. */
|
|
141
|
+
readonly slot: string;
|
|
142
|
+
readonly publication: string;
|
|
143
|
+
/** Entities to decode; anything else is skipped before it reaches the matcher. */
|
|
144
|
+
readonly entities: readonly string[];
|
|
145
|
+
/** How often the slot is confirmed. Longer means more WAL retained after a crash. */
|
|
146
|
+
readonly statusIntervalMs?: number | undefined;
|
|
147
|
+
readonly clock?: Clock | undefined;
|
|
148
|
+
/** Injected so the SCRAM nonce is deterministic under a seeded test. */
|
|
149
|
+
readonly rng?: Rng | undefined;
|
|
150
|
+
/** The byte pipe, injected. Defaults to `Bun.connect`; a test drives a scripted server instead. */
|
|
151
|
+
readonly stream?: ((target: PgTarget) => Promise<PgStream>) | undefined;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* The production feed: `pgoutput` decoding off a logical replication slot. Everything about *how*
|
|
156
|
+
* lives in `pg-replication.ts`; what this class adds is the `ChangeFeed` contract the matcher, the
|
|
157
|
+
* replicator and the fanout are written against — so swapping it for `InMemoryChangeFeed` in `x dev`
|
|
158
|
+
* changes nothing downstream.
|
|
159
|
+
*/
|
|
160
|
+
export class PgLogicalReplicationFeed implements ChangeFeed {
|
|
161
|
+
readonly source = 'pg-logical-replication';
|
|
162
|
+
readonly #stream: PgReplicationStream;
|
|
163
|
+
|
|
164
|
+
constructor(options: PgLogicalReplicationOptions) {
|
|
165
|
+
if (options.entities.length === 0) {
|
|
166
|
+
throw new ReplicationFailedError({
|
|
167
|
+
stage: 'preflight',
|
|
168
|
+
detail: 'the feed was given an empty entity list, so no change could ever match',
|
|
169
|
+
fix: 'pass the entities the publication covers: new PgLogicalReplicationFeed({ entities: [...] })',
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
this.#stream = new PgReplicationStream(options);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async start(options: ChangeFeedStartOptions): Promise<void> {
|
|
176
|
+
await this.#stream.start({ from: options.from, onChange: options.onChange });
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async stop(): Promise<void> {
|
|
180
|
+
await this.#stream.stop();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
lastLsn(): string | null {
|
|
184
|
+
return this.#stream.lastLsn();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Delivered / skipped / replayed counts, for `/readyz` and the `x dev` dashboard. */
|
|
188
|
+
stats(): ReplicationStreamStats {
|
|
189
|
+
return this.#stream.stats();
|
|
190
|
+
}
|
|
191
|
+
}
|