@ultimat3/realtime 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +79 -20
- package/README.md +39 -15
- package/package.json +3 -3
- package/src/changefeed.ts +14 -2
- package/src/client-contract.ts +3 -2
- package/src/client-heartbeat.ts +4 -3
- package/src/cursor.ts +2 -2
- package/src/errors.ts +30 -0
- package/src/index.ts +6 -6
- package/src/json.ts +8 -48
- package/src/live-contract.ts +11 -18
- package/src/live-definition.ts +5 -5
- package/src/live-query.ts +3 -7
- package/src/pg-preflight.ts +117 -0
- package/src/pg-replication.ts +20 -66
- package/src/rebase.ts +7 -2
- package/src/subscriber-gate.ts +18 -1
- package/src/sync-frames.ts +49 -12
- package/src/sync-node.ts +30 -8
- package/src/sync-protocol.ts +2 -2
- package/src/sync-upgrade.ts +20 -2
- package/src/thundering-herd.ts +10 -0
package/CLAUDE.md
CHANGED
|
@@ -214,6 +214,22 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
214
214
|
threw skipped the close and the pump await, leaking the socket and telling a supervisor the
|
|
215
215
|
teardown was over before it had begun. Every step runs whatever the step before it did, and the
|
|
216
216
|
first failure is rethrown only once the connection is closed and the pump has ended.
|
|
217
|
+
- **`REPLICA IDENTITY FULL` is asked about at preflight, warned about, and counted — never thrown**
|
|
218
|
+
(added 2026-08-19). `pg-replication.ts`'s `#deliver` hands a `delete` its `message.before`,
|
|
219
|
+
and under any identity but FULL that tuple is the KEY COLUMNS ALONE — which `toRow` accepts,
|
|
220
|
+
because it only requires a text `id`. So `bridgeChange` decided "did this row leave the result
|
|
221
|
+
set" from a one-column row, `visible` read `undefined` for every column the identity did not
|
|
222
|
+
carry, and nothing anywhere recorded that it had happened: no emit, no check, and
|
|
223
|
+
`X_LIVE_REPLICA_IDENTITY` existed in neither the source nor the manifest. The check is a fourth
|
|
224
|
+
`connection.query` in `preflight`, and it **must** stay ahead of
|
|
225
|
+
`pg_create_logical_replication_slot` — changing the identity after a slot exists does not reach
|
|
226
|
+
what that slot decodes. It WARNS (`logger.warn(code, { cause, fix, tables })`, the message being
|
|
227
|
+
the code alone) because a throw would stop every app on the default identity from booting, which
|
|
228
|
+
is a worse outcome than the partial rows. `ReplicationStreamStats.partialBefore` is the runtime
|
|
229
|
+
half, read off `PgRelation.replicaIdentity` and never off the tuple: a DEFAULT-identity table
|
|
230
|
+
whose non-key columns happen to be NULL sends the bytes a FULL one does, so counting absent keys
|
|
231
|
+
would undercount exactly the rows a policy is most likely to misjudge. A hard refusal in the
|
|
232
|
+
`x verify` step is the follow-up and lives in `@ultimat3/cli`.
|
|
217
233
|
- A change lsn is `<16 hex commit position><8 hex row position in that transaction>`. Never order by
|
|
218
234
|
either half alone: the commit lsn repeats within a transaction, and per-record WAL positions are
|
|
219
235
|
not monotonic across transactions. Never make it depend on wall time, the entity list or a process
|
|
@@ -311,6 +327,33 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
311
327
|
double presence membership, double fanout — until the tab closed. `#socket` is nulled before the
|
|
312
328
|
close so the corpse's `onClose` takes its early return, and `onMessage` carries the same identity
|
|
313
329
|
guard `onClose` already had.
|
|
330
|
+
- **The grant is recorded BEFORE `server.upgrade`, and released on the path that never opens.**
|
|
331
|
+
Bun runs `websocket.open` SYNCHRONOUSLY inside `server.upgrade` and does not return until it has
|
|
332
|
+
(bun 1.3.14), and `open` is where `sync-node` reads the `GrantBook` for the socket's actor.
|
|
333
|
+
Recorded on the line after the upgrade — which it was — every authenticated socket on a real node
|
|
334
|
+
carried `actor: null`: `ChannelHub.#authorize` denied every topic with `X_TOPIC_FORBIDDEN`,
|
|
335
|
+
`authorize`/`visible` decided about nobody, `maxPerTenant` never applied and `hello.actorId` was
|
|
336
|
+
null. It did not self-repair, because `GrantBook.expired()` skips a grant with no `expiresAt` —
|
|
337
|
+
the shape `authenticate: async () => ({ actor })` produces. `onUngranted` is what makes the
|
|
338
|
+
correct order safe (only a `close` callback deletes a grant, and an upgrade that never took gets
|
|
339
|
+
no callback); it is REQUIRED on `UpgradeDeps`, so a second host of `handleUpgrade` cannot forget
|
|
340
|
+
it. The harness is half the rule: `sync-node-auth.test.ts`'s `upgradeTarget()` returned `true`
|
|
341
|
+
without ever calling `open`, so the bug was invisible to every test here — it now opens the socket
|
|
342
|
+
inside `upgrade()`, the way Bun does.
|
|
343
|
+
- **Every `socket.send` on the node reads its answer, and what a `false` costs is decided per
|
|
344
|
+
frame.** A subscribe reply is REPAIRABLE and was the silent one: `registry.subscribe` has already
|
|
345
|
+
seated the subscription and cleared its desync mark, so a dropped snapshot left the server
|
|
346
|
+
believing a client holding no rows was in sync, and every later change reached it as a patch
|
|
347
|
+
folded onto nothing — on a healthy socket, forever. It is marked desynced, exactly as
|
|
348
|
+
`live-fanout` marks a lost patch. A `rebase`/`ack` has nothing to mark (the node keeps no
|
|
349
|
+
per-mutation state) and a client only returns an `inflight` mutation to its queue when the
|
|
350
|
+
connection dies, so an undeliverable settlement **closes the socket**: the reconnect requeues and
|
|
351
|
+
replays it under the same idempotency key, and acking a rebase that never left is the
|
|
352
|
+
rebase-before-ack order defeated one frame later. A presence roster has no repair at all — the
|
|
353
|
+
membership is already on the shared set and the client's next heartbeat re-rosters — so it is
|
|
354
|
+
logged (`sync.presence_roster_dropped`). The drain's `reconnect` frame is the socket's slot in the
|
|
355
|
+
spread and nothing re-sends it, so `drain()` returns `DrainedSocket[]` with `notified` per socket
|
|
356
|
+
and logs `sync.drain_frames_dropped`.
|
|
314
357
|
- **A socket's actor comes from `createSyncNode({ authenticate })` and from nowhere else.** The node
|
|
315
358
|
imports no authenticator — the app supplies one, exactly as it supplies `onMutate` — and it runs
|
|
316
359
|
on the upgrade *before* `server.upgrade`, so a refused credential never costs a websocket.
|
|
@@ -418,20 +461,33 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
418
461
|
channel's lsn is the publishing hub's own per-node counter, so a client cannot tell a gap from a
|
|
419
462
|
message that came via another node. Declared in `socket.ts`, not core's `runtime-metrics.ts`:
|
|
420
463
|
that file is the series every process emits, this one exists only where channels do.
|
|
421
|
-
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
464
|
+
- **A qid is `@ultimat3/query`'s `queryHash(name, input)`, and this package derives none of its
|
|
465
|
+
own — `As of 2026-08`.** `qidOf` was the same two lines over a local copy of the canonical form
|
|
466
|
+
(`stableDigest(canonicalJson(input))`), and `canonicalJson`/`stableDigest` were this package's
|
|
467
|
+
third copy of what `@ultimat3/action` and `@ultimat3/query` also each held. They had already
|
|
468
|
+
diverged: `{ a: undefined, b: 1 }` gave `feed:eb8ed3ccb5023093` from `queryHash` and
|
|
469
|
+
`feed:c0bf82ad036cb0a5` from `qidOf`, because query's walk drops an `undefined`-valued key and
|
|
470
|
+
this one rendered `"a":null`. The two are COMPARED in one flow — `@ultimat3/query`'s `planResume`
|
|
471
|
+
decides refetch-vs-resume by comparing a cursor's `queryHash` against the query's, while
|
|
472
|
+
`liveQueryDefinition` keys the shared window by the qid — so keeping both correct was never the
|
|
473
|
+
option; the first time either moved, every resume decision and every window lookup were keyed
|
|
474
|
+
differently. `realtime -> query` is the one declared sideways edge and this package already
|
|
475
|
+
imports it. **`fnv1a` stays here**, and only it: its job is the cursor's result-set digest, where
|
|
476
|
+
a collision costs a missed re-sort and never one client served out of another's window.
|
|
477
|
+
`live-contract.test.ts` is the pin — it reads `registry.subscriberCount(queryHash(name, input))`
|
|
478
|
+
through a real subscribe, so a local derivation fails it. Cost of the move: none observable on the
|
|
479
|
+
server. Every qid a node computes comes from a DECODED frame, and `JSON.parse` produces no
|
|
480
|
+
`undefined`, no `Date`, no `Map` and no `Set` — the four values the two forms disagree about — so
|
|
481
|
+
no live subscription re-keyed and nothing re-snapshotted.
|
|
482
|
+
- **The canonical form is injective over the values it accepts, and `JSON.stringify` is not** —
|
|
483
|
+
the reason that survives the move, now `@ultimat3/core`'s to enforce. `JSON.stringify` answers
|
|
484
|
+
`"null"` for `NaN` and `±Infinity` and `"0"` for `-0`, so four distinct inputs hashed to one qid
|
|
485
|
+
— and a qid *hit* hands the joiner the first subscriber's compiled source, matcher and seated
|
|
486
|
+
window. Bare `NaN` / `Infinity` / `-Infinity` / `-0` tokens are emitted instead; they are not
|
|
487
|
+
valid JSON, which is correct, because that output is hashed and never parsed. Exposure is
|
|
488
|
+
narrower than it looks and the tests say so rather than overclaiming: `NaN` and `±Infinity` have
|
|
489
|
+
no JSON spelling and so cannot arrive on a `subscribe` frame — they reach the hash only from a
|
|
490
|
+
caller building `input` in JS. **`-0` is wire-reachable**: `JSON.parse('{"a":-0}')` answers `-0`.
|
|
435
491
|
- **Refusing new sockets and draining the ones you have are two shutdown phases.** `stopAccepting()`
|
|
436
492
|
is the `accept` phase: `ready = false`, `/readyz` 503, a late upgrade shed with `retry-after-ms`,
|
|
437
493
|
and every socket untouched — a draining node still owes its clients their patches, and `stop()` is
|
|
@@ -472,7 +528,7 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
472
528
|
every client on open and read by nobody — the node replied `resume: []` and decided resume per
|
|
473
529
|
subscription from the `subscribe` frame — so every reconnect shipped each cursor twice, up to 512
|
|
474
530
|
ids each, in the restart storm this package is measured on. Wiring it was the wrong half of the
|
|
475
|
-
choice: a cursor's `qid` is `` `${name}:${
|
|
531
|
+
choice: a cursor's `qid` is `` `${name}:${fingerprint(input)}` ``, so a node reading a resume list
|
|
476
532
|
recovers the query **name** — it is the plaintext prefix — but never the `input`, which is the half
|
|
477
533
|
every decision needs. Without it `definition.authorize({ actor, input })` cannot run and no entry
|
|
478
534
|
can be built; the qid names a window but not a decision, and the retained window holds pre-policy
|
|
@@ -492,9 +548,11 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
492
548
|
socket it opens against the *new* node. Two silent windows and the client closes with `4000` and
|
|
493
549
|
arms the reconnect. It is one
|
|
494
550
|
self-re-arming tick on the injected `Scheduler`, not an interval: a client is either beating on a
|
|
495
|
-
live socket or backing off toward a new one, never both. The 15s is
|
|
496
|
-
`realtime.heartbeatMs`
|
|
497
|
-
|
|
551
|
+
live socket or backing off toward a new one, never both. The 15s is the client's OWN number:
|
|
552
|
+
`realtime.heartbeatMs` was a `RealtimeConfig` key read by nothing and it is **deleted**
|
|
553
|
+
(2026-08-19). The server half of the beat stays derived — `PresenceRegistry.heartbeatMs` is
|
|
554
|
+
`max(1000, floor(ttlMs / 3))`, the same rule `idleSweepPeriodMs` follows, because a second knob
|
|
555
|
+
is a second number that can disagree with the one it is a fraction of.
|
|
498
556
|
- **Every question a hot path asks is indexed, never scanned.** `SubscriptionBook` keeps
|
|
499
557
|
`#bySocket` and a per-tenant count beside `#bySid`, and `SocketRegistry` keeps `#byTopic` beside
|
|
500
558
|
the socket table. Both replaced a walk of the whole node that ran once per socket or once per
|
|
@@ -587,6 +645,7 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
587
645
|
| `live-query.ts` / `live-definition.ts` / `changefeed.ts` / `changefeed-env.ts` / `replicator.ts` / `pg-advisory-lock.ts` / `fanout.ts` / `transport-env.ts` / `matcher-bridge.ts` | tier 2 |
|
|
588
646
|
| `pg-bytes.ts` / `pg-wire.ts` / `pg-auth.ts` / `pg-connection.ts` / `pg-socket.ts` | the Postgres v3 client: bytes, frames, SASL, session, socket |
|
|
589
647
|
| `pgoutput.ts` / `pg-entity-row.ts` / `pg-replication.ts` | WAL decode → `ChangeEvent`, and the lsn that orders it |
|
|
648
|
+
| `pg-preflight.ts` | the four questions asked before `START_REPLICATION` — `wal_level`, the publication, every entity's replica identity, the slot — plus `assertIdentifier`, the charset all four interpolate through |
|
|
590
649
|
| `nats-client.ts` | the bus port: publish/subscribe/request/requestMany/close/version/connected, and `parseNatsUrl` — the library takes `host:port` plus credentials and never reads a URL's userinfo |
|
|
591
650
|
| `nats-lib-client.ts` | the `nats` adapter — **the only file in the repo that imports `nats`** |
|
|
592
651
|
| `nats-jetstream.ts` / `nats-kv.ts` / `nats-transport.ts` | the JetStream KV bucket, presence over it, and the production `Transport` — all three written against the port |
|
|
@@ -618,8 +677,8 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
618
677
|
| `client-contract.ts` | the client's injected shapes — `ClientSocket`, `LiveClientOptions`, `LiveHandle` — declared apart from the class that consumes them |
|
|
619
678
|
| `policy-gate.ts` | the only authz seam |
|
|
620
679
|
| `subscriber-gate.ts` | the per-subscriber pass of a definition's row policy, and its two counters — `rowsDenied` and `gateFailures`. Evaluates no policy of its own |
|
|
621
|
-
| `live-contract.ts` | what a live query IS: `
|
|
622
|
-
| `json.ts` | the wire's value types
|
|
680
|
+
| `live-contract.ts` | what a live query IS: `LiveQueryDefinition`, `SnapshotResult`, `LiveSubscription`. Four modules need the shape and none of them needs the registry that runs it. The **id** is not here and not anywhere in this package — it is `@ultimat3/query`'s `queryHash` |
|
|
681
|
+
| `json.ts` | the wire's value types and `fnv1a` (drift), the one hash still owned here. The canonical form and the sharing-key hash are `@ultimat3/core`'s (`canonicalJson`, `fingerprint`) — `json.test.ts` pins that `fnv1a` is never mistakable for one |
|
|
623
682
|
| `live-definition.ts` | the only bridge from a declared `query({ live: true })` to a registrable definition — and `policy-gate.ts`'s only caller |
|
|
624
683
|
| `matcher-bridge.ts` | the only `@ultimat3/query` matcher seam |
|
|
625
684
|
|
package/README.md
CHANGED
|
@@ -265,15 +265,17 @@ new LiveClient({ signal, connect, buildId, heartbeatMs: 15_000 }); // 0 disables
|
|
|
265
265
|
|
|
266
266
|
| Property | Behaviour |
|
|
267
267
|
|---|---|
|
|
268
|
-
| Default | `DEFAULT_HEARTBEAT_MS`, 15s. The
|
|
268
|
+
| Default | `DEFAULT_HEARTBEAT_MS`, 15s. The client's own number and the only one: `realtime.heartbeatMs` in `app.config.ts` was deleted 2026-08-19 because nothing read it |
|
|
269
269
|
| One beat | a `hello` — which carries no cursors at all; `HelloFrame` has no resume list, so a beat and an opening frame are byte-identical — plus one subscribe frame per topic held |
|
|
270
270
|
| Why the topics | on the node, repeating the subscribe frame **is** the presence heartbeat; presence has no frame of its own in either direction |
|
|
271
271
|
| Not a deploy check | `update-available` answers a skew between the build id recorded at the upgrade and the node's own, and neither can change on an open socket — so every `hello` on one socket answers the same forever. A client hears about a deploy on the socket it opens against the **new** node |
|
|
272
272
|
| Silence | nothing received for **two** intervals ⇒ close `4000` (a private-use code, so it is distinguishable in a log) and arm the reconnect. Judged from the last frame of any kind, since the point is that bytes still cross |
|
|
273
273
|
| Not an interval | one armed tick, re-armed by itself, on the same injected `Scheduler` the reconnect uses — a client is either beating on a live socket or backing off toward a new one, never both |
|
|
274
274
|
|
|
275
|
-
`realtime.heartbeatMs` in `app.config.ts` is **
|
|
276
|
-
|
|
275
|
+
`realtime.heartbeatMs` in `app.config.ts` is **gone** `As of 2026-08-19` — it was read by nothing,
|
|
276
|
+
and an app that still sets it fails `x verify`'s typecheck step with TS2353 (`'heartbeatMs' does not
|
|
277
|
+
exist in type 'Input<RealtimeConfig>'`). Delete the line; this option is the only knob that changes
|
|
278
|
+
behaviour, and the node's presence beat is derived from its TTL rather than configured.
|
|
277
279
|
|
|
278
280
|
### A `send` that returned is not an acknowledgement
|
|
279
281
|
|
|
@@ -351,12 +353,18 @@ wire twice by a reconnect that raced an ack.
|
|
|
351
353
|
`mutate` is one lane per socket; `subscribe` is one lane per sid, or per topic name; `hello` and
|
|
352
354
|
the server-authored kinds are unlaned. A lane exists only while work is queued on it, because a
|
|
353
355
|
lane keyed by a client-chosen sid that outlived its work is an unbounded map one socket can grow.
|
|
354
|
-
- **`qid` is
|
|
355
|
-
where it was a 32-bit FNV-1a. It is a
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
356
|
+
- **`qid` is `@ultimat3/query`'s `queryHash(name, input)`** — `<name>:<first 16 hex of
|
|
357
|
+
SHA-256(canonicalJson(input))>`, 64 bits `As of 2026-08` where it was a 32-bit FNV-1a. It is a
|
|
358
|
+
*sharing* key: a hit is answered with the existing entry and the seated window, both holding the
|
|
359
|
+
first subscriber's input and rows, and input is client-chosen, so a collision is one client served
|
|
360
|
+
out of another's window. This package derives none of its own — `qidOf` was a second spelling of
|
|
361
|
+
`queryHash` while `@ultimat3/query`'s `planResume` compares a cursor's `queryHash` against the
|
|
362
|
+
query's, so the two had to be one function or every resume decision and every window lookup would
|
|
363
|
+
be keyed differently the first time either moved. A rolling deploy across the *hash* change costs
|
|
364
|
+
one bounded snapshot per subscription — a cursor minted under the old format names a ring entry
|
|
365
|
+
the new node never held, so the resume falls back correctly rather than silently; the `qidOf`
|
|
366
|
+
removal itself costs nothing, because every qid a node computes comes from a decoded frame and
|
|
367
|
+
`JSON.parse` produces none of the values the two spellings disagreed about.
|
|
360
368
|
- **A topic guard that *fails* keeps the topic.** On the re-auth pass, only a denial
|
|
361
369
|
(`X_TOPIC_FORBIDDEN`, or a policy denial) unsubscribes; anything else increments `hub.guardFailures`
|
|
362
370
|
and logs `channel.guard_failed`. `catch { unsubscribe }` reported a store that timed out as a
|
|
@@ -410,10 +418,16 @@ wire twice by a reconnect that raced an ack.
|
|
|
410
418
|
`undefined` and answer as if the row had said so. A patch whose row the shared window does not
|
|
411
419
|
hold is withheld — the window *is* the result set — and a subscriber holding that row gets the
|
|
412
420
|
one `delete` that says so. It counts as neither a denial nor a gate failure: nothing decided.
|
|
421
|
+
- **A `delete` is withheld too, and `holds` is the whole decision** (`As of 2026-08`). It carries no
|
|
422
|
+
row, so there is nothing to put in front of the rule — and it was forwarded unconditionally, so
|
|
423
|
+
every subscriber learned the id and the instant of every *other* tenant's row as it was deleted,
|
|
424
|
+
on a query whose `visible` rule had never let them see one. A subscriber that holds the row is
|
|
425
|
+
told it is gone; one that does not gets nothing, counted as `rowsDenied`.
|
|
413
426
|
- **`PgLogicalReplicationFeed` decodes `pgoutput` off a real slot** — its own Postgres v3 client
|
|
414
427
|
(SCRAM-SHA-256, in-band TLS, CopyBoth), no driver dependency. It preflights `wal_level`, the
|
|
415
|
-
publication
|
|
416
|
-
the
|
|
428
|
+
publication, every entity's replica identity and the slot — in that order, because the identity
|
|
429
|
+
check is worthless once the slot exists — creates the slot when there is none, and confirms the
|
|
430
|
+
slot as it goes so the WAL does not grow without bound. `InMemoryChangeFeed` + `InProcessTransport` remain the
|
|
417
431
|
defaults for `x dev` and every test.
|
|
418
432
|
- **`selectChangeFeed(env, { entities })` decides which feed a boot installs** — same law
|
|
419
433
|
`selectMailDriver` follows: an unset variable means the embedded default. It returns `{ feed,
|
|
@@ -456,8 +470,18 @@ wire twice by a reconnect that raced an ack.
|
|
|
456
470
|
*transactions* in commit order, so per-record WAL positions are not monotonic across them. The
|
|
457
471
|
pair sorts in delivery order and is byte-identical on replay, which is what turns at-least-once
|
|
458
472
|
redelivery into a drop instead of a duplicate.
|
|
459
|
-
- **A live query needs `REPLICA IDENTITY FULL
|
|
460
|
-
|
|
473
|
+
- **A live query needs `REPLICA IDENTITY FULL`, and the replicator now says so** (`As of
|
|
474
|
+
2026-08-19`). Deciding whether a row *left* a result set needs the old values; with the default
|
|
475
|
+
identity a delete replicates only the key columns, and `toRow` accepts that tuple because it only
|
|
476
|
+
requires a text `id`. `preflight` asks `pg_class.relreplident` for every entity in the list — the
|
|
477
|
+
fourth question it asks, and **before** `pg_create_logical_replication_slot`, since changing the
|
|
478
|
+
identity after a slot exists does not reach the rows that slot will decode. It is a **coded
|
|
479
|
+
warning**, `X_LIVE_REPLICA_IDENTITY`, whose `fix:` is the `ALTER TABLE <t> REPLICA IDENTITY FULL;`
|
|
480
|
+
per named table — not a throw, because every app on the default identity would otherwise stop
|
|
481
|
+
booting, which is worse than the partial rows. `ReplicationStreamStats.partialBefore` is the
|
|
482
|
+
running half: one per change delivered off a relation that is not FULL, so the decisions it
|
|
483
|
+
actually cost are countable rather than silent. A hard refusal at `x verify` time is the
|
|
484
|
+
follow-up.
|
|
461
485
|
- Tier 3's OPFS SQLite store is browser-only and throws until the browser entry ships; `MemoryLocalStore`
|
|
462
486
|
implements the full journal/rollback/replay semantics today. It holds membership and the journal;
|
|
463
487
|
the row values are the client's one `IdentityMap`, which is what a browser store has to inherit
|
|
@@ -472,8 +496,8 @@ wire twice by a reconnect that raced an ack.
|
|
|
472
496
|
`X_PROTOCOL_VERSION` · `X_CURSOR_STALE` ·
|
|
473
497
|
`X_REBASE_CONFLICT` · `X_TRANSPORT_UNAVAILABLE` · `X_TRANSPORT_PROTOCOL` ·
|
|
474
498
|
`X_REPLICATION_FAILED` · `X_REPLICATION_PROTOCOL` · `X_REPLICATOR_SLOT_HELD` ·
|
|
475
|
-
`X_LIVE_CLIENT_MISSING` · `X_LIVE_QUERY_UNKNOWN` · `
|
|
476
|
-
`X_SOCKET_AUTH_UNAVAILABLE` · `X_NOT_IMPLEMENTED`
|
|
499
|
+
`X_LIVE_CLIENT_MISSING` · `X_LIVE_QUERY_UNKNOWN` · `X_LIVE_REPLICA_IDENTITY` ·
|
|
500
|
+
`X_SOCKET_UNAUTHENTICATED` · `X_SOCKET_AUTH_UNAVAILABLE` · `X_NOT_IMPLEMENTED`
|
|
477
501
|
|
|
478
502
|
Topics deny by default: a topic with no matching guard is forbidden. An authz hole is not a config
|
|
479
503
|
option someone forgot to set.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/realtime",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0",
|
|
4
4
|
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
"test": "bun test"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@ultimat3/core": "
|
|
36
|
-
"@ultimat3/query": "
|
|
35
|
+
"@ultimat3/core": "4.0.0",
|
|
36
|
+
"@ultimat3/query": "4.0.0",
|
|
37
37
|
"nats": "2.29.3"
|
|
38
38
|
}
|
|
39
39
|
}
|
package/src/changefeed.ts
CHANGED
|
@@ -126,14 +126,26 @@ export class InMemoryChangeFeed implements ChangeFeed {
|
|
|
126
126
|
async #deliver(event: ChangeEvent): Promise<void> {
|
|
127
127
|
const handler = this.#handler;
|
|
128
128
|
if (!handler) return;
|
|
129
|
-
|
|
129
|
+
const result = this.#tail.then(async () => {
|
|
130
130
|
await handler(event);
|
|
131
131
|
this.#lastLsn = event.lsn;
|
|
132
132
|
});
|
|
133
|
-
|
|
133
|
+
// The lane chains on a SETTLED shadow, never on `result` — `window-lock.ts` solves the same
|
|
134
|
+
// problem the same way. Chained on the live tail, one rejected link poisoned every link behind
|
|
135
|
+
// it: later changes rejected with the FIRST error, the handler was never called again and
|
|
136
|
+
// `lastLsn()` froze. Reachable on any single-node deployment, because `createReplicator`'s
|
|
137
|
+
// `onChange` awaits `transport.publish(...)` and a closed `InProcessTransport` refuses — one
|
|
138
|
+
// transient publish failure ended change delivery for the life of the process. The rejection
|
|
139
|
+
// still reaches the caller that pushed THAT event, and only it; `stop()` awaits the shadow, so
|
|
140
|
+
// a teardown reports the teardown rather than re-raising a failure already handed over.
|
|
141
|
+
this.#tail = result.then(ignore, ignore);
|
|
142
|
+
await result;
|
|
134
143
|
}
|
|
135
144
|
}
|
|
136
145
|
|
|
146
|
+
/** Settles the shadow lane whichever way the delivery went. Nothing observes the value. */
|
|
147
|
+
const ignore = (): void => undefined;
|
|
148
|
+
|
|
137
149
|
export interface PgLogicalReplicationOptions {
|
|
138
150
|
/** Connection string for a role with REPLICATION: `postgres://user:pass@host:5432/db`. */
|
|
139
151
|
readonly url: string;
|
package/src/client-contract.ts
CHANGED
|
@@ -72,8 +72,9 @@ export interface LiveClientOptions<T extends TableMap = TableMap> {
|
|
|
72
72
|
readonly scheduler?: Scheduler;
|
|
73
73
|
/**
|
|
74
74
|
* How often a live socket re-announces itself, in ms. `0` disables it. Defaults to
|
|
75
|
-
* `DEFAULT_HEARTBEAT_MS
|
|
76
|
-
*
|
|
75
|
+
* `DEFAULT_HEARTBEAT_MS`, 15s. The one knob for the beat: `realtime.heartbeatMs` in
|
|
76
|
+
* `app.config.ts` was deleted 2026-08-19 because nothing read it, so this is not a restatement
|
|
77
|
+
* of a server value — browser code could never have reached one.
|
|
77
78
|
*/
|
|
78
79
|
readonly heartbeatMs?: number;
|
|
79
80
|
/** Where a dial failure inside the reconnect timer is reported. Defaults to `reportToConsole`. */
|
package/src/client-heartbeat.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
import type { Scheduler } from './thundering-herd';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
* The
|
|
9
|
-
*
|
|
10
|
-
*
|
|
8
|
+
* The client's own number, and `As of 2026-08-19` the only one. It used to be described as a
|
|
9
|
+
* restatement of `realtime.heartbeatMs` in `@ultimat3/core`'s config — that key was read by
|
|
10
|
+
* nothing and is deleted, so there is no second value to keep this equal to. The server side of
|
|
11
|
+
* the beat is DERIVED, never configured: `PresenceRegistry.heartbeatMs` is `ttlMs / 3`.
|
|
11
12
|
*/
|
|
12
13
|
export const DEFAULT_HEARTBEAT_MS = 15_000;
|
|
13
14
|
|
package/src/cursor.ts
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// and the budget exists to make the expensive answer (a snapshot) the *chosen* one, not the
|
|
4
4
|
// accidental one. See README "Reconnect is the hard part".
|
|
5
5
|
|
|
6
|
-
import { type Clock, systemClock } from '@ultimat3/core';
|
|
6
|
+
import { type Clock, canonicalJson, systemClock } from '@ultimat3/core';
|
|
7
7
|
import { CursorStaleError } from './errors';
|
|
8
|
-
import {
|
|
8
|
+
import { fnv1a, type Row, type RowPatch } from './json';
|
|
9
9
|
|
|
10
10
|
/** Ids are bounded so a cursor stays small enough to ship on every `subscribe` frame. */
|
|
11
11
|
export const CURSOR_ID_LIMIT = 512;
|
package/src/errors.ts
CHANGED
|
@@ -20,6 +20,7 @@ export const REALTIME_OWNED_ERROR_CODES = [
|
|
|
20
20
|
'X_LIVE_CLIENT_MISSING',
|
|
21
21
|
'X_LIVE_ROW_UNIDENTIFIED',
|
|
22
22
|
'X_LIVE_QUERY_UNKNOWN',
|
|
23
|
+
'X_LIVE_REPLICA_IDENTITY',
|
|
23
24
|
'X_QUERY_NOT_SUBSCRIBABLE',
|
|
24
25
|
'X_SOCKET_UNAUTHENTICATED',
|
|
25
26
|
'X_SOCKET_AUTH_UNAVAILABLE',
|
|
@@ -113,6 +114,7 @@ export const REALTIME_ERROR_TITLES: Readonly<Record<RealtimeOwnedErrorCode, stri
|
|
|
113
114
|
X_LIVE_CLIENT_MISSING: 'a realtime hook ran with no LiveClient registered',
|
|
114
115
|
X_LIVE_ROW_UNIDENTIFIED: 'a live query returned a row with no id',
|
|
115
116
|
X_LIVE_QUERY_UNKNOWN: 'no live query is registered under the name a subscribe frame asked for',
|
|
117
|
+
X_LIVE_REPLICA_IDENTITY: 'a replicated table sends a key-only row on delete',
|
|
116
118
|
X_QUERY_NOT_SUBSCRIBABLE: 'a hook was bound to a query that is not declared live',
|
|
117
119
|
X_SOCKET_UNAUTHENTICATED: 'the sync upgrade carried no credential this app accepts',
|
|
118
120
|
X_SOCKET_AUTH_UNAVAILABLE: 'the sync node could not decide who a connecting socket is',
|
|
@@ -327,6 +329,34 @@ export class ReplicatorSlotHeldError extends RealtimeError {
|
|
|
327
329
|
}
|
|
328
330
|
}
|
|
329
331
|
|
|
332
|
+
/**
|
|
333
|
+
* A table in the entity list replicates with a replica identity other than FULL, so its `delete`
|
|
334
|
+
* (and any key-changing `update`) carries the KEY COLUMNS ONLY. `toRow` accepts that tuple —
|
|
335
|
+
* it only requires a text `id` — so the live matcher decides "did this row leave the result set"
|
|
336
|
+
* from a one-column row, and a row policy written against `!row.private` reads `undefined`.
|
|
337
|
+
*
|
|
338
|
+
* **Raised at preflight and LOGGED, never thrown.** Every app running today on the default
|
|
339
|
+
* identity would stop booting, and the replicator refusing to start is a worse outcome than the
|
|
340
|
+
* partial rows it is warning about. The runtime half is `ReplicationStreamStats.partialBefore`,
|
|
341
|
+
* which counts the changes this actually affects. Refusing it at `x verify` time is the follow-up.
|
|
342
|
+
*
|
|
343
|
+
* The tables are named because the fix is per table, and they are the entity list's own names —
|
|
344
|
+
* every one has already passed `assertIdentifier`, so the `fix:` is SQL that can be pasted.
|
|
345
|
+
*/
|
|
346
|
+
export class ReplicaIdentityError extends RealtimeError {
|
|
347
|
+
constructor(args: { tables: readonly string[] }) {
|
|
348
|
+
super({
|
|
349
|
+
code: 'X_LIVE_REPLICA_IDENTITY',
|
|
350
|
+
cause:
|
|
351
|
+
`${args.tables.join(', ')} replicate with a replica identity other than FULL, so a ` +
|
|
352
|
+
'delete carries the key columns only and a live query decides visibility from a partial row',
|
|
353
|
+
fix:
|
|
354
|
+
`${args.tables.map((table) => `ALTER TABLE ${table} REPLICA IDENTITY FULL;`).join(' ')}` +
|
|
355
|
+
' -- rows already written to the WAL keep the identity they were written with',
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
330
360
|
/**
|
|
331
361
|
* A hook was called before the app entry registered its client. Never a transient fault: the
|
|
332
362
|
* registration is a single call in the entry, so the fix is the call itself rather than a retry.
|
package/src/index.ts
CHANGED
|
@@ -90,6 +90,7 @@ export {
|
|
|
90
90
|
RealtimeError,
|
|
91
91
|
type RealtimeErrorCode,
|
|
92
92
|
RebaseConflictError,
|
|
93
|
+
ReplicaIdentityError,
|
|
93
94
|
ReplicationFailedError,
|
|
94
95
|
ReplicationProtocolError,
|
|
95
96
|
ReplicatorSlotHeldError,
|
|
@@ -136,7 +137,6 @@ export {
|
|
|
136
137
|
} from './identity-map';
|
|
137
138
|
// ---- shared value domain ---------------------------------------------------------------------
|
|
138
139
|
export {
|
|
139
|
-
canonicalJson,
|
|
140
140
|
changedColumns,
|
|
141
141
|
fnv1a,
|
|
142
142
|
isJsonObject,
|
|
@@ -147,11 +147,10 @@ export {
|
|
|
147
147
|
type RowOp,
|
|
148
148
|
type RowPatch,
|
|
149
149
|
} from './json';
|
|
150
|
-
export {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
type SnapshotResult,
|
|
150
|
+
export type {
|
|
151
|
+
LiveQueryDefinition,
|
|
152
|
+
LiveSubscription,
|
|
153
|
+
SnapshotResult,
|
|
155
154
|
} from './live-contract';
|
|
156
155
|
export { type LiveDefinitionOptions, liveQueryDefinition } from './live-definition';
|
|
157
156
|
export {
|
|
@@ -377,6 +376,7 @@ export {
|
|
|
377
376
|
type AcceptBudgetOptions,
|
|
378
377
|
type BackoffPolicy,
|
|
379
378
|
backoffDelay,
|
|
379
|
+
type DrainedSocket,
|
|
380
380
|
type DrainPlanEntry,
|
|
381
381
|
type DrainPlanOptions,
|
|
382
382
|
defaultBackoff,
|
package/src/json.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
// The JSON value domain shared by the wire, the matcher, and the local store
|
|
2
|
-
//
|
|
1
|
+
// The JSON value domain shared by the wire, the matcher, and the local store, plus `fnv1a` — the
|
|
2
|
+
// one hash this package still owns.
|
|
3
|
+
//
|
|
4
|
+
// `canonicalJson` and `stableDigest` USED to live here and are `@ultimat3/core`'s now: they were a
|
|
5
|
+
// third copy of one injective canonical form and one sharing-key hash, beside `@ultimat3/action`'s
|
|
6
|
+
// and `@ultimat3/query`'s, and the copies had already diverged. `fnv1a` stays because its job is
|
|
7
|
+
// genuinely different — it is a cursor's result-set digest, where a collision costs a missed
|
|
8
|
+
// re-sort and never one client served out of another's window.
|
|
3
9
|
|
|
4
10
|
export type JsonValue =
|
|
5
11
|
| string
|
|
@@ -49,52 +55,6 @@ export function changedColumns(before: JsonObject | null, after: JsonObject): Js
|
|
|
49
55
|
return out;
|
|
50
56
|
}
|
|
51
57
|
|
|
52
|
-
/**
|
|
53
|
-
* Key-sorted JSON so a query id derived from input is stable across property order — and
|
|
54
|
-
* INJECTIVE, because that id decides who shares a window.
|
|
55
|
-
*
|
|
56
|
-
* `qidOf` is `stableDigest(canonicalJson(input))` and a qid HIT hands the joiner the existing
|
|
57
|
-
* entry: the first subscriber's compiled source, its matcher and its seated rows. Two inputs that
|
|
58
|
-
* canonicalise to one string are therefore two clients served out of one window. `JSON.stringify`
|
|
59
|
-
* is not injective over numbers — `NaN` and `±Infinity` are both `"null"`, which also collides
|
|
60
|
-
* with JSON `null` itself, and `-0` is `"0"` — so the number branch is spelled out here.
|
|
61
|
-
*
|
|
62
|
-
* `-0` is the one of those a client can put on the wire (`JSON.parse('{"a":-0}')` answers `-0`);
|
|
63
|
-
* the non-finite three have no JSON spelling and arrive only from a caller building `input` in JS,
|
|
64
|
-
* such as `useLive(feed, () => ({ limit: Number.parseInt(raw) }))` on an unparseable `raw`.
|
|
65
|
-
* The tokens are bare, never quoted: this output is only ever hashed, and the `string` branch
|
|
66
|
-
* always quotes, so an unquoted word cannot collide with the text that spells it.
|
|
67
|
-
*
|
|
68
|
-
* The twin of `@ultimat3/query`'s and `@ultimat3/action`'s rules in their own `stable.ts`. All
|
|
69
|
-
* three are tier 3, so no two of them can import each other; the shared home is `@ultimat3/core`
|
|
70
|
-
* if one is ever made.
|
|
71
|
-
*/
|
|
72
|
-
export function canonicalJson(value: JsonValue): string {
|
|
73
|
-
if (typeof value === 'number') return canonicalNumber(value);
|
|
74
|
-
if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
|
|
75
|
-
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
76
|
-
const keys = Object.keys(value).sort();
|
|
77
|
-
const parts = keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key] ?? null)}`);
|
|
78
|
-
return `{${parts.join(',')}}`;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** Ordinary numbers are `String(n)`, byte-identical to what this emitted before. */
|
|
82
|
-
function canonicalNumber(value: number): string {
|
|
83
|
-
if (Number.isNaN(value)) return 'NaN';
|
|
84
|
-
if (!Number.isFinite(value)) return value > 0 ? 'Infinity' : '-Infinity';
|
|
85
|
-
return Object.is(value, -0) ? '-0' : String(value);
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* SHA-256, first 16 hex characters. For the hashes that are also SHARING keys — a `qid` decides
|
|
90
|
-
* which subscribers are served from one window, and it is derived from input a client chooses, so
|
|
91
|
-
* the 32 bits `fnv1a` answers are a collision anyone can find offline in seconds. Same primitive
|
|
92
|
-
* and same width `@ultimat3/entity`'s `planScope` already chose for a cursor's scope.
|
|
93
|
-
*/
|
|
94
|
-
export function stableDigest(text: string): string {
|
|
95
|
-
return new Bun.CryptoHasher('sha256').update(text).digest('hex').slice(0, 16);
|
|
96
|
-
}
|
|
97
|
-
|
|
98
58
|
/** FNV-1a, 32-bit, hex. Not cryptographic — it identifies and detects drift, it does not protect. */
|
|
99
59
|
export function fnv1a(text: string): string {
|
|
100
60
|
let hash = 0x811c9dc5;
|
package/src/live-contract.ts
CHANGED
|
@@ -1,27 +1,20 @@
|
|
|
1
|
-
// What a live query IS: the
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// What a live query IS: the contract a definition satisfies and the subscription one socket holds.
|
|
2
|
+
// Split from `live-query.ts` because four modules need the shape and none of them needs the
|
|
3
|
+
// registry that runs it — and because one file runs one job.
|
|
4
|
+
//
|
|
5
|
+
// The id is NOT here, and no longer anywhere in this package: a `qid` is `@ultimat3/query`'s
|
|
6
|
+
// `queryHash(name, input)`, imported across the declared `realtime -> query` edge. `qidOf` was the
|
|
7
|
+
// same two lines over this package's own copy of the canonical form, and the two had already
|
|
8
|
+
// diverged on an `undefined`-valued key — while `planResume` compares a cursor's `queryHash` and
|
|
9
|
+
// `liveQueryDefinition` keys the shared window by the qid, so a divergence is every resume
|
|
10
|
+
// decision and every window lookup keyed differently.
|
|
4
11
|
|
|
5
12
|
import type { Actor } from '@ultimat3/core';
|
|
6
13
|
import type { LiveCursor } from './cursor';
|
|
7
|
-
import {
|
|
14
|
+
import type { JsonValue, Row } from './json';
|
|
8
15
|
import type { IncrementalMatcher } from './matcher-bridge';
|
|
9
16
|
import type { SyncSocket } from './socket';
|
|
10
17
|
|
|
11
|
-
/**
|
|
12
|
-
* `qid` = hash(query name, input). Fanout subjects and change windows are keyed by it.
|
|
13
|
-
*
|
|
14
|
-
* The hash is a **sharing** key, which is why it is `stableDigest` and not `fnv1a`: `#entryFor`
|
|
15
|
-
* answers a hit with the EXISTING entry and `liveQueryDefinition` answers with the seated
|
|
16
|
-
* `SharedWindow`, both carrying the first subscriber's input, compiled source and rows. Input is
|
|
17
|
-
* client-chosen, so a second input colliding with the first passes `authorize` against its own
|
|
18
|
-
* arguments and is then served out of somebody else's window — and 32 bits is a collision found
|
|
19
|
-
* offline in seconds.
|
|
20
|
-
*/
|
|
21
|
-
export function qidOf(name: string, input: JsonValue): string {
|
|
22
|
-
return `${name}:${stableDigest(canonicalJson(input))}`;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
18
|
export interface SnapshotResult<R extends Row = Row> {
|
|
26
19
|
readonly rows: readonly R[];
|
|
27
20
|
readonly lsn: string;
|
package/src/live-definition.ts
CHANGED
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
// every time. Collapsing the second onto the first is privilege escalation with a cache hit rate.
|
|
10
10
|
|
|
11
11
|
import type { Ctx } from '@ultimat3/core';
|
|
12
|
-
import { type AnyQuery, queryName } from '@ultimat3/query';
|
|
12
|
+
import { type AnyQuery, queryHash, queryName } from '@ultimat3/query';
|
|
13
13
|
import { LiveRowUnidentifiedError } from './errors';
|
|
14
14
|
import { isRow, type JsonValue, type Row } from './json';
|
|
15
|
-
import {
|
|
15
|
+
import type { LiveQueryDefinition, SnapshotResult } from './live-contract';
|
|
16
16
|
import { type IncrementalMatcher, matcherFor } from './matcher-bridge';
|
|
17
17
|
import { authorizeWithPolicy, visibleWithPolicy } from './policy-gate';
|
|
18
18
|
|
|
@@ -73,7 +73,7 @@ export function liveQueryDefinition(
|
|
|
73
73
|
const windows = new Map<string, SharedWindow>();
|
|
74
74
|
|
|
75
75
|
const resolve = async (input: JsonValue): Promise<SharedWindow> => {
|
|
76
|
-
const qid =
|
|
76
|
+
const qid = queryHash(name, input);
|
|
77
77
|
const seated = windows.get(qid);
|
|
78
78
|
if (seated !== undefined) return seated;
|
|
79
79
|
const live = await target.live(input, {
|
|
@@ -113,10 +113,10 @@ export function liveQueryDefinition(
|
|
|
113
113
|
const window = await resolve(input);
|
|
114
114
|
return { rows: await window.read(), lsn: options.lsn?.() ?? '' };
|
|
115
115
|
},
|
|
116
|
-
matcher: (input) => windows.get(
|
|
116
|
+
matcher: (input) => windows.get(queryHash(name, input))?.matcher ?? UNRESOLVED,
|
|
117
117
|
// Read off the same resolved window as the matcher, so the scope the client keys rows under and
|
|
118
118
|
// the entity the matcher patches them from can never be two different names.
|
|
119
|
-
rowEntity: (input) => windows.get(
|
|
119
|
+
rowEntity: (input) => windows.get(queryHash(name, input))?.rowEntity ?? null,
|
|
120
120
|
// The two per-subscriber gates, both through the package's one authz seam. Neither result is
|
|
121
121
|
// memoised anywhere: `authorize` runs on every subscribe, `visible` on every row of every
|
|
122
122
|
// delivery, and there is no key here an actor could share with another actor.
|
package/src/live-query.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// otherwise.
|
|
8
8
|
|
|
9
9
|
import { type Actor, type Clock, systemClock, uuid } from '@ultimat3/core';
|
|
10
|
+
import { queryHash } from '@ultimat3/query';
|
|
10
11
|
import type { ChangeEvent } from './changefeed';
|
|
11
12
|
import {
|
|
12
13
|
type LiveCursor,
|
|
@@ -17,12 +18,7 @@ import {
|
|
|
17
18
|
} from './cursor';
|
|
18
19
|
import { isPolicyDenial, LiveQueryUnknownError, SubscriptionLimitError } from './errors';
|
|
19
20
|
import type { JsonValue } from './json';
|
|
20
|
-
import {
|
|
21
|
-
type LiveQueryDefinition,
|
|
22
|
-
type LiveSubscription,
|
|
23
|
-
qidOf,
|
|
24
|
-
type SnapshotResult,
|
|
25
|
-
} from './live-contract';
|
|
21
|
+
import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
|
|
26
22
|
import { type FanoutDeps, fanoutChange, snapshotFrame } from './live-fanout';
|
|
27
23
|
import { createEntry, fillWindow, type QueryEntry } from './query-window';
|
|
28
24
|
import type { SyncSocket } from './socket';
|
|
@@ -176,7 +172,7 @@ export class LiveQueryRegistry {
|
|
|
176
172
|
// may not subscribe is work an unauthorized client gets to schedule.
|
|
177
173
|
await definition.prepare?.(args.input);
|
|
178
174
|
|
|
179
|
-
const qid =
|
|
175
|
+
const qid = queryHash(args.name, args.input);
|
|
180
176
|
const entry = this.#entryFor(qid, definition, args.input);
|
|
181
177
|
const now = this.#clock.now().getTime();
|
|
182
178
|
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// Single responsibility: the four questions asked of a database BEFORE `START_REPLICATION`, and
|
|
2
|
+
// the identifier charset every one of them interpolates through. Three refuse the boot with the
|
|
3
|
+
// exact statement that fixes them; the fourth warns, because refusing it would stop every app on
|
|
4
|
+
// the default replica identity from starting.
|
|
5
|
+
|
|
6
|
+
import { logger } from '@ultimat3/core';
|
|
7
|
+
import { ReplicaIdentityError, ReplicationFailedError } from './errors';
|
|
8
|
+
import type { PgConnection } from './pg-connection';
|
|
9
|
+
|
|
10
|
+
/** Identifiers reach a simple query unparameterised, so the charset is the injection boundary. */
|
|
11
|
+
const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The one gate between a caller-supplied name and a simple query. Exported because
|
|
15
|
+
* `PgReplicationStream` checks its slot, publication and entity names in its CONSTRUCTOR — a
|
|
16
|
+
* mistyped `REPLICATION_SLOT` is a boot-time fact, and finding it at the first WAL read means a
|
|
17
|
+
* replicator that reported itself started and then never delivered a change.
|
|
18
|
+
*/
|
|
19
|
+
export const assertIdentifier = (kind: string, value: string): string => {
|
|
20
|
+
if (IDENTIFIER.test(value)) return value;
|
|
21
|
+
throw new ReplicationFailedError({
|
|
22
|
+
stage: 'preflight',
|
|
23
|
+
detail: `${kind} "${value}" is not a lower-case postgres identifier`,
|
|
24
|
+
fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
|
|
25
|
+
});
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The four things that are always misconfigured. Three produce an unreadable server message if
|
|
30
|
+
* left to the server, so each gets its own `fix:` line; the fourth is `warnPartialIdentity` and
|
|
31
|
+
* only warns. `slot` and `publication` are interpolated into simple queries, so the `IDENTIFIER`
|
|
32
|
+
* charset is the injection boundary; re-asserted here rather than trusted, so the guarantee
|
|
33
|
+
* travels with the function instead of living only in `start()`.
|
|
34
|
+
*/
|
|
35
|
+
export async function preflight(
|
|
36
|
+
connection: PgConnection,
|
|
37
|
+
slot: string,
|
|
38
|
+
publication: string,
|
|
39
|
+
entities: ReadonlySet<string>,
|
|
40
|
+
): Promise<void> {
|
|
41
|
+
assertIdentifier('slot', slot);
|
|
42
|
+
assertIdentifier('publication', publication);
|
|
43
|
+
const [walLevel] = await connection.query('SHOW wal_level');
|
|
44
|
+
if (walLevel?.[0] !== 'logical') {
|
|
45
|
+
throw new ReplicationFailedError({
|
|
46
|
+
stage: 'preflight',
|
|
47
|
+
detail: `wal_level is "${walLevel?.[0] ?? 'unknown'}", so the server writes no logical WAL`,
|
|
48
|
+
fix: "ALTER SYSTEM SET wal_level = 'logical'; -- then restart postgres",
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
const publications = await connection.query(
|
|
52
|
+
`SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
|
|
53
|
+
);
|
|
54
|
+
if (publications.length === 0) {
|
|
55
|
+
throw new ReplicationFailedError({
|
|
56
|
+
stage: 'preflight',
|
|
57
|
+
detail: `no publication named "${publication}" exists`,
|
|
58
|
+
fix: `CREATE PUBLICATION ${publication} FOR ALL TABLES;`,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
await warnPartialIdentity(connection, entities);
|
|
62
|
+
const [existing] = await connection.query(
|
|
63
|
+
`SELECT plugin FROM pg_replication_slots WHERE slot_name = '${slot}'`,
|
|
64
|
+
);
|
|
65
|
+
if (existing === undefined) {
|
|
66
|
+
// Plain SQL rather than CREATE_REPLICATION_SLOT: the replication command exports a snapshot
|
|
67
|
+
// that pins xmin for the session, and its option syntax changed in postgres 15.
|
|
68
|
+
await connection.query(`SELECT pg_create_logical_replication_slot('${slot}', 'pgoutput')`);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
if (existing[0] !== 'pgoutput') {
|
|
72
|
+
throw new ReplicationFailedError({
|
|
73
|
+
stage: 'preflight',
|
|
74
|
+
detail: `slot "${slot}" decodes with "${existing[0] ?? 'unknown'}", not pgoutput`,
|
|
75
|
+
fix: `SELECT pg_drop_replication_slot('${slot}'); -- then start the replicator again`,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The fourth preflight question, and the one that does NOT refuse. A live query decides whether a
|
|
82
|
+
* row left its result set from `change.before`, and under any replica identity but FULL that tuple
|
|
83
|
+
* is the key columns alone — which `toRow` accepts, since it only requires a text `id`.
|
|
84
|
+
*
|
|
85
|
+
* It runs BEFORE `pg_create_logical_replication_slot`: a slot decodes with the identity the
|
|
86
|
+
* catalog held when the rows were written, so asking after the slot exists answers about a stream
|
|
87
|
+
* nobody is reading yet. It WARNS rather than throws because every app on the default identity
|
|
88
|
+
* would otherwise stop booting, and a replicator that will not start is worse than the partial
|
|
89
|
+
* rows it is complaining about — `ReplicationStreamStats.partialBefore` is the running half.
|
|
90
|
+
*
|
|
91
|
+
* Entity names are the ones the constructor already put through `assertIdentifier`, which is what
|
|
92
|
+
* makes both the interpolation and the `fix:` safe; a name postgres answers with that is not in
|
|
93
|
+
* that set is dropped rather than rendered.
|
|
94
|
+
*/
|
|
95
|
+
async function warnPartialIdentity(
|
|
96
|
+
connection: PgConnection,
|
|
97
|
+
entities: ReadonlySet<string>,
|
|
98
|
+
): Promise<void> {
|
|
99
|
+
if (entities.size === 0) return;
|
|
100
|
+
const names = [...entities].map((name) => `'${name}'`).join(', ');
|
|
101
|
+
const rows = await connection.query(
|
|
102
|
+
`SELECT relname FROM pg_class WHERE relkind = 'r' AND relreplident <> 'f' ` +
|
|
103
|
+
`AND relname IN (${names})`,
|
|
104
|
+
);
|
|
105
|
+
const tables = [
|
|
106
|
+
...new Set(
|
|
107
|
+
rows
|
|
108
|
+
.map((row) => row[0])
|
|
109
|
+
.filter((name): name is string => typeof name === 'string' && entities.has(name)),
|
|
110
|
+
),
|
|
111
|
+
].sort();
|
|
112
|
+
if (tables.length === 0) return;
|
|
113
|
+
const warning = new ReplicaIdentityError({ tables });
|
|
114
|
+
// FIELDS, never interpolation, and the message is the CODE alone — the same rule
|
|
115
|
+
// `@ultimat3/http`'s error-map stage follows, so a log index can be alerted on by code.
|
|
116
|
+
logger.warn(warning.code, { cause: warning.cause, fix: warning.fix, tables });
|
|
117
|
+
}
|
package/src/pg-replication.ts
CHANGED
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
// Single responsibility: turn a Postgres logical-replication slot into ordered `ChangeEvent`s —
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
2
|
+
// START_REPLICATION, decode pgoutput, and keep the slot confirmed. The preflight, the connection,
|
|
3
|
+
// the framing and the pgoutput decode live next door; what is decided here is *ordering*, because
|
|
4
|
+
// the lsn is the only authority the pipeline has.
|
|
5
5
|
|
|
6
6
|
import { type Clock, logger, systemClock } from '@ultimat3/core';
|
|
7
7
|
import type { ChangeEvent, ChangeOp, PgLogicalReplicationOptions } from './changefeed';
|
|
8
|
-
import {
|
|
8
|
+
import { ReplicationProtocolError } from './errors';
|
|
9
9
|
import { isRow, type JsonObject, type Row } from './json';
|
|
10
10
|
import { ByteReader, ByteWriter, epochMsToPgTimestamp, printLsn } from './pg-bytes';
|
|
11
11
|
import { PgConnection } from './pg-connection';
|
|
12
12
|
import { entityRow } from './pg-entity-row';
|
|
13
|
+
import { assertIdentifier, preflight } from './pg-preflight';
|
|
13
14
|
import { bunPgStream, parsePgUrl } from './pg-socket';
|
|
14
15
|
import { PgOutputDecoder, type PgOutputMessage, type PgRelation } from './pgoutput';
|
|
15
16
|
|
|
16
|
-
/** Identifiers reach a simple query unparameterised, so the charset is the injection boundary. */
|
|
17
|
-
const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
|
|
18
|
-
|
|
19
17
|
const DEFAULT_STATUS_INTERVAL_MS = 10_000;
|
|
20
18
|
|
|
21
19
|
/** `r` — the standby status update, the only frontend message a walsender listens for. */
|
|
@@ -27,6 +25,14 @@ export interface ReplicationStreamStats {
|
|
|
27
25
|
readonly skipped: number;
|
|
28
26
|
/** Rows replayed from before the resume position, dropped so `onChange` sees each one once. */
|
|
29
27
|
readonly replayed: number;
|
|
28
|
+
/**
|
|
29
|
+
* Changes delivered from a relation whose replica identity is not FULL — so the `before` row is
|
|
30
|
+
* the key columns alone (or absent), and the live matcher's "did this row leave the result set"
|
|
31
|
+
* is decided on a partial row. `X_LIVE_REPLICA_IDENTITY` warns about the CONFIGURATION once at
|
|
32
|
+
* preflight; this counts the decisions it actually cost, which is the half a running node can
|
|
33
|
+
* be alerted on. Inserts never count: there is no `before` to be partial.
|
|
34
|
+
*/
|
|
35
|
+
readonly partialBefore: number;
|
|
30
36
|
/**
|
|
31
37
|
* Why the pump stopped, or `null` while it is live. The read loop cannot throw into a caller —
|
|
32
38
|
* nothing awaits it — so this is the one place `/readyz` and a test can see that it died at all.
|
|
@@ -58,15 +64,6 @@ export const changeLsn = (commitLsn: bigint, sequence: number): string =>
|
|
|
58
64
|
/** The commit position inside a change lsn — where a resume asks the server to restart. */
|
|
59
65
|
export const commitPositionOf = (lsn: string): bigint => BigInt(`0x${lsn.slice(0, 16) || '0'}`);
|
|
60
66
|
|
|
61
|
-
const assertIdentifier = (kind: string, value: string): string => {
|
|
62
|
-
if (IDENTIFIER.test(value)) return value;
|
|
63
|
-
throw new ReplicationFailedError({
|
|
64
|
-
stage: 'preflight',
|
|
65
|
-
detail: `${kind} "${value}" is not a lower-case postgres identifier`,
|
|
66
|
-
fix: `rename the ${kind} to match [a-z_][a-z0-9_]* — it is interpolated into a replication command`,
|
|
67
|
-
});
|
|
68
|
-
};
|
|
69
|
-
|
|
70
67
|
export interface ReplicationStreamHandlers {
|
|
71
68
|
readonly from?: string | undefined;
|
|
72
69
|
onChange(event: ChangeEvent): void | Promise<void>;
|
|
@@ -94,6 +91,7 @@ export class PgReplicationStream {
|
|
|
94
91
|
#delivered = 0;
|
|
95
92
|
#skipped = 0;
|
|
96
93
|
#replayed = 0;
|
|
94
|
+
#partialBefore = 0;
|
|
97
95
|
#failure: string | null = null;
|
|
98
96
|
|
|
99
97
|
constructor(options: PgLogicalReplicationOptions) {
|
|
@@ -116,6 +114,7 @@ export class PgReplicationStream {
|
|
|
116
114
|
delivered: this.#delivered,
|
|
117
115
|
skipped: this.#skipped,
|
|
118
116
|
replayed: this.#replayed,
|
|
117
|
+
partialBefore: this.#partialBefore,
|
|
119
118
|
failure: this.#failure,
|
|
120
119
|
};
|
|
121
120
|
}
|
|
@@ -147,7 +146,7 @@ export class PgReplicationStream {
|
|
|
147
146
|
});
|
|
148
147
|
this.#connection = connection;
|
|
149
148
|
try {
|
|
150
|
-
await preflight(connection, slot, publication);
|
|
149
|
+
await preflight(connection, slot, publication, this.#entities);
|
|
151
150
|
const from = handlers.from;
|
|
152
151
|
this.#confirmed = from === undefined ? 0n : commitPositionOf(from);
|
|
153
152
|
await connection.startCopyBoth(
|
|
@@ -339,6 +338,10 @@ export class PgReplicationStream {
|
|
|
339
338
|
this.#replayed += 1;
|
|
340
339
|
return;
|
|
341
340
|
}
|
|
341
|
+
// Read off the Relation message rather than off the tuple: a DEFAULT-identity table whose
|
|
342
|
+
// non-key columns happen to be NULL sends the same bytes a FULL one does, so counting missing
|
|
343
|
+
// keys would undercount exactly the rows a policy is most likely to misjudge.
|
|
344
|
+
if (op !== 'insert' && relation.replicaIdentity !== 'f') this.#partialBefore += 1;
|
|
342
345
|
const before = toRow(relation, oldTuple);
|
|
343
346
|
const after = toRow(relation, newTuple);
|
|
344
347
|
const event: ChangeEvent = {
|
|
@@ -380,55 +383,6 @@ export class PgReplicationStream {
|
|
|
380
383
|
}
|
|
381
384
|
}
|
|
382
385
|
|
|
383
|
-
/**
|
|
384
|
-
* The three misconfigurations that produce an unreadable server message if left to the server.
|
|
385
|
-
* `slot` and `publication` are interpolated into simple queries, so the `IDENTIFIER` charset is the
|
|
386
|
-
* injection boundary; re-asserted here rather than trusted, so the guarantee travels with the
|
|
387
|
-
* function instead of living only in `start()`.
|
|
388
|
-
*/
|
|
389
|
-
async function preflight(
|
|
390
|
-
connection: PgConnection,
|
|
391
|
-
slot: string,
|
|
392
|
-
publication: string,
|
|
393
|
-
): Promise<void> {
|
|
394
|
-
assertIdentifier('slot', slot);
|
|
395
|
-
assertIdentifier('publication', publication);
|
|
396
|
-
const [walLevel] = await connection.query('SHOW wal_level');
|
|
397
|
-
if (walLevel?.[0] !== 'logical') {
|
|
398
|
-
throw new ReplicationFailedError({
|
|
399
|
-
stage: 'preflight',
|
|
400
|
-
detail: `wal_level is "${walLevel?.[0] ?? 'unknown'}", so the server writes no logical WAL`,
|
|
401
|
-
fix: "ALTER SYSTEM SET wal_level = 'logical'; -- then restart postgres",
|
|
402
|
-
});
|
|
403
|
-
}
|
|
404
|
-
const publications = await connection.query(
|
|
405
|
-
`SELECT 1 FROM pg_publication WHERE pubname = '${publication}'`,
|
|
406
|
-
);
|
|
407
|
-
if (publications.length === 0) {
|
|
408
|
-
throw new ReplicationFailedError({
|
|
409
|
-
stage: 'preflight',
|
|
410
|
-
detail: `no publication named "${publication}" exists`,
|
|
411
|
-
fix: `CREATE PUBLICATION ${publication} FOR ALL TABLES;`,
|
|
412
|
-
});
|
|
413
|
-
}
|
|
414
|
-
const [existing] = await connection.query(
|
|
415
|
-
`SELECT plugin FROM pg_replication_slots WHERE slot_name = '${slot}'`,
|
|
416
|
-
);
|
|
417
|
-
if (existing === undefined) {
|
|
418
|
-
// Plain SQL rather than CREATE_REPLICATION_SLOT: the replication command exports a snapshot
|
|
419
|
-
// that pins xmin for the session, and its option syntax changed in postgres 15.
|
|
420
|
-
await connection.query(`SELECT pg_create_logical_replication_slot('${slot}', 'pgoutput')`);
|
|
421
|
-
return;
|
|
422
|
-
}
|
|
423
|
-
if (existing[0] !== 'pgoutput') {
|
|
424
|
-
throw new ReplicationFailedError({
|
|
425
|
-
stage: 'preflight',
|
|
426
|
-
detail: `slot "${slot}" decodes with "${existing[0] ?? 'unknown'}", not pgoutput`,
|
|
427
|
-
fix: `SELECT pg_drop_replication_slot('${slot}'); -- then start the replicator again`,
|
|
428
|
-
});
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
|
|
432
386
|
/** A physical tuple becomes the row the matcher's predicates are written against, or nothing. */
|
|
433
387
|
function toRow(relation: PgRelation, physical: JsonObject | null): Row | null {
|
|
434
388
|
if (physical === null) return null;
|
package/src/rebase.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { RebaseConflictError } from './errors';
|
|
10
10
|
import type { Row } from './json';
|
|
11
11
|
import type { LocalStore, LocalTx, TableMap } from './local-store';
|
|
12
|
-
import { type ConflictStrategyName, type
|
|
12
|
+
import { type ConflictStrategyName, PROTOCOL_VERSION, type RebaseFrame } from './sync-protocol';
|
|
13
13
|
|
|
14
14
|
export interface MergeArgs {
|
|
15
15
|
/** Local row as the user last saw it, before any rollback. */
|
|
@@ -246,7 +246,12 @@ function numberAt(row: Row | null | undefined, field: string): number | null {
|
|
|
246
246
|
return typeof value === 'number' ? value : null;
|
|
247
247
|
}
|
|
248
248
|
|
|
249
|
-
|
|
249
|
+
/**
|
|
250
|
+
* `RebaseFrame`, not `Frame`: this builds exactly one member of the union and declaring the whole
|
|
251
|
+
* union threw that away, so every caller had to re-narrow a frame it had just constructed before it
|
|
252
|
+
* could read `strategy` or `row` back off it.
|
|
253
|
+
*/
|
|
254
|
+
export function rebaseFrame(ack: ServerAck, strategy: ConflictStrategy): RebaseFrame {
|
|
250
255
|
return {
|
|
251
256
|
type: 'rebase',
|
|
252
257
|
v: PROTOCOL_VERSION,
|
package/src/subscriber-gate.ts
CHANGED
|
@@ -129,7 +129,24 @@ export class SubscriberGate {
|
|
|
129
129
|
patch: RowPatch,
|
|
130
130
|
holds: boolean,
|
|
131
131
|
): Promise<RowPatch | null> {
|
|
132
|
-
|
|
132
|
+
// A delete carries no row, so there is nothing to put in front of the rule — `holds` IS the
|
|
133
|
+
// decision, the same one the two branches below take for a row a rule has just refused.
|
|
134
|
+
// Returned unconditionally it was a leak with no upper bound: the shared window is pre-policy,
|
|
135
|
+
// so every subscriber learned the id and the instant of every OTHER tenant's row as it was
|
|
136
|
+
// deleted, on a query whose `visible` rule had never let them see one of them.
|
|
137
|
+
//
|
|
138
|
+
// `holds` comes from `subscription.cursor.ids`, truncated at `CURSOR_ID_LIMIT` — so on a window
|
|
139
|
+
// wider than 512 rows a legitimate delete past position 512 is dropped and that row stays on
|
|
140
|
+
// screen until the subscriber re-snapshots. That is the trade the denied-update branch below
|
|
141
|
+
// already makes, and it is the right way round: a stale row is a bug, a row id leaked to
|
|
142
|
+
// another tenant is a breach.
|
|
143
|
+
if (patch.op === 'delete' || patch.row === null) {
|
|
144
|
+
if (holds) return patch;
|
|
145
|
+
// Counted, or a withheld delete is invisible in exactly the way `onRowDenied` exists to
|
|
146
|
+
// stop — and the rate of it is how an operator sees a window shared across tenants at all.
|
|
147
|
+
this.#denied(target.qid, who, patch.id);
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
133
150
|
const full = target.rows.find((row) => row.id === patch.id);
|
|
134
151
|
// No whole row means no decision to take. An update patch carries the changed columns only, so
|
|
135
152
|
// a rule reading `row.ownerId` on one reads `undefined` and answers as if the row had said so —
|
package/src/sync-frames.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// inbound surface the `sync` node exposes. Every dependency is injected, so the router is
|
|
3
3
|
// exercisable without a socket, a bus or a server.
|
|
4
4
|
|
|
5
|
+
import { logger } from '@ultimat3/core';
|
|
5
6
|
import type { ChannelHub } from './channel';
|
|
6
7
|
import { topic as makeTopic } from './channel';
|
|
7
8
|
import { FrameRateLimitError } from './errors';
|
|
@@ -9,7 +10,7 @@ import { FrameLanes, laneKeyOf } from './frame-lanes';
|
|
|
9
10
|
import type { JsonValue, Row } from './json';
|
|
10
11
|
import type { LiveQueryRegistry } from './live-query';
|
|
11
12
|
import { type PresenceRegistry, presenceFrame } from './presence';
|
|
12
|
-
import type
|
|
13
|
+
import { CLOSE, type SyncSocket } from './socket';
|
|
13
14
|
import { type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
14
15
|
|
|
15
16
|
/** Server-authoritative mutation execution. Injected: `sync` never owns business logic. */
|
|
@@ -104,7 +105,14 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
104
105
|
// Repeating the frame is therefore also the heartbeat — `join` re-`put`s the member.
|
|
105
106
|
if (presence) {
|
|
106
107
|
const roster = await presence.join(name, { id: socket.id, actorId: socket.actorId });
|
|
107
|
-
|
|
108
|
+
// The answer is read even though nothing here can repair it. A roster has no cursor and
|
|
109
|
+
// no re-send path of its own: the client renders an empty room until it repeats this
|
|
110
|
+
// very frame as its heartbeat, which re-joins and re-rosters. Membership on the shared
|
|
111
|
+
// set is already correct, so the drop costs one client one heartbeat of blank room —
|
|
112
|
+
// and the log is the only trace it leaves anywhere.
|
|
113
|
+
if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
|
|
114
|
+
logger.warn('sync.presence_roster_dropped', { topic: name, socketId: socket.id });
|
|
115
|
+
}
|
|
108
116
|
}
|
|
109
117
|
return;
|
|
110
118
|
}
|
|
@@ -121,12 +129,19 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
121
129
|
sid: frame.sid,
|
|
122
130
|
cursor: frame.target.cursor,
|
|
123
131
|
});
|
|
124
|
-
|
|
132
|
+
// `subscribe` has already seated the subscription and cleared its desync mark, so a reply
|
|
133
|
+
// the socket refuses leaves the server believing a client that holds no rows is in sync:
|
|
134
|
+
// the next change reaches it as a PATCH folded onto nothing, forever, on a socket that has
|
|
135
|
+
// since drained. Marked instead, which is the state it is actually in — the next delivery
|
|
136
|
+
// re-snapshots it out of the shared window, exactly as `live-fanout` does for a lost patch.
|
|
137
|
+
if (!socket.send(reply)) socket.markDesynced(frame.sid);
|
|
125
138
|
return;
|
|
126
139
|
}
|
|
127
140
|
case 'mutate': {
|
|
128
141
|
if (!options.onMutate) {
|
|
129
|
-
|
|
142
|
+
// A failure receipt is still a receipt: dropped, the client's mutation stays `inflight`
|
|
143
|
+
// and is neither rolled back nor retried, so this one reads the answer too.
|
|
144
|
+
const sent = socket.send({
|
|
130
145
|
type: 'ack',
|
|
131
146
|
v: PROTOCOL_VERSION,
|
|
132
147
|
ref: frame.key,
|
|
@@ -137,6 +152,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
137
152
|
fix: 'pass onMutate to createSyncNode({ onMutate })',
|
|
138
153
|
}),
|
|
139
154
|
});
|
|
155
|
+
if (!sent) undeliverable(socket, frame.key, 'ack');
|
|
140
156
|
return;
|
|
141
157
|
}
|
|
142
158
|
const result = await options.onMutate({
|
|
@@ -153,7 +169,7 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
153
169
|
// and no sequence to decide which later optimistic writes to replay over server truth.
|
|
154
170
|
// These are two frames on one socket, so the order is the only coordination there is.
|
|
155
171
|
if (result.entity !== undefined) {
|
|
156
|
-
socket.send({
|
|
172
|
+
const sent = socket.send({
|
|
157
173
|
type: 'rebase',
|
|
158
174
|
v: PROTOCOL_VERSION,
|
|
159
175
|
key: frame.key,
|
|
@@ -161,14 +177,22 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
161
177
|
strategy: 'server-wins',
|
|
162
178
|
row: result.row ?? null,
|
|
163
179
|
});
|
|
180
|
+
// The ack retires the client's rebase-log entry, so acking a rebase that never left is
|
|
181
|
+
// the divergence the ordering above exists to prevent — one frame later instead of one
|
|
182
|
+
// frame earlier. Nothing is acked; the mutation stays unsettled and is replayed.
|
|
183
|
+
if (!sent) return undeliverable(socket, frame.key, 'rebase');
|
|
184
|
+
}
|
|
185
|
+
if (
|
|
186
|
+
!socket.send({
|
|
187
|
+
type: 'ack',
|
|
188
|
+
v: PROTOCOL_VERSION,
|
|
189
|
+
ref: frame.key,
|
|
190
|
+
lsn: result.lsn ?? null,
|
|
191
|
+
error: null,
|
|
192
|
+
})
|
|
193
|
+
) {
|
|
194
|
+
undeliverable(socket, frame.key, 'ack');
|
|
164
195
|
}
|
|
165
|
-
socket.send({
|
|
166
|
-
type: 'ack',
|
|
167
|
-
v: PROTOCOL_VERSION,
|
|
168
|
-
ref: frame.key,
|
|
169
|
-
lsn: result.lsn ?? null,
|
|
170
|
-
error: null,
|
|
171
|
-
});
|
|
172
196
|
return;
|
|
173
197
|
}
|
|
174
198
|
// Server-authored frames are never received from a client.
|
|
@@ -183,3 +207,16 @@ export function createFrameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
183
207
|
}
|
|
184
208
|
}
|
|
185
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* A settlement the socket refused. There is nothing on the node to mark — the mutation is applied
|
|
213
|
+
* and the server keeps no per-mutation state — and a client only returns an `inflight` mutation to
|
|
214
|
+
* its queue when the connection dies (`requeueInflight`), so a receipt dropped on a socket that
|
|
215
|
+
* stays up is a write the client neither retires nor retries, with the server believing it settled.
|
|
216
|
+
* Closing IS the repair: the queue hands the mutation back and the reconnect replays it under the
|
|
217
|
+
* same idempotency key.
|
|
218
|
+
*/
|
|
219
|
+
function undeliverable(socket: SyncSocket, key: string, kind: 'rebase' | 'ack'): void {
|
|
220
|
+
logger.warn('sync.settlement_dropped', { socketId: socket.id, key, kind });
|
|
221
|
+
socket.close(CLOSE.overloaded, 'settlement undeliverable');
|
|
222
|
+
}
|
package/src/sync-node.ts
CHANGED
|
@@ -23,7 +23,13 @@ import { GrantBook, type SyncAuthenticator, sweepGrants } from './sync-auth';
|
|
|
23
23
|
import { ackRefOf, createFrameRouter, type MutationHandler } from './sync-frames';
|
|
24
24
|
import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
|
|
25
25
|
import { handleUpgrade, type UpgradeTarget, type WsData } from './sync-upgrade';
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
AcceptBudget,
|
|
28
|
+
type DrainedSocket,
|
|
29
|
+
drainPlan,
|
|
30
|
+
type Rng,
|
|
31
|
+
reconnectFrame,
|
|
32
|
+
} from './thundering-herd';
|
|
27
33
|
|
|
28
34
|
/** Declared with the upgrade that builds it — this file only ever reads one. */
|
|
29
35
|
export type { UpgradeTarget, WsData } from './sync-upgrade';
|
|
@@ -133,7 +139,7 @@ export interface SyncNode {
|
|
|
133
139
|
close(ws: SyncWs): void;
|
|
134
140
|
};
|
|
135
141
|
/** Sends every client a distinct reconnect delay, then closes. Returns the plan for tests/logs. */
|
|
136
|
-
drain(options?: { graceMs?: number }): Promise<readonly
|
|
142
|
+
drain(options?: { graceMs?: number }): Promise<readonly DrainedSocket[]>;
|
|
137
143
|
}
|
|
138
144
|
|
|
139
145
|
export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
@@ -247,8 +253,12 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
247
253
|
onRevoked: (socketId) => {
|
|
248
254
|
const socket = sockets.get(socketId);
|
|
249
255
|
if (!socket) return;
|
|
250
|
-
|
|
251
|
-
|
|
256
|
+
// Through `evict`, which closes BEFORE it releases. The other order was a close that never
|
|
257
|
+
// happened: `teardown` reaches `sockets.remove`, which closes the socket itself with
|
|
258
|
+
// `1001 connection closed`, and `SyncSocket.close` returns once `#closed` — so a revoked
|
|
259
|
+
// grant reached the client as a normal shutdown, which it retries against this node with
|
|
260
|
+
// the same dead credential, instead of the `1008` that tells it to re-dial with a new one.
|
|
261
|
+
evict(socket, CLOSE.policy, 'grant expired');
|
|
252
262
|
},
|
|
253
263
|
onRefreshFailed: (socketId, error) => {
|
|
254
264
|
// Not a denial: the grant is kept and retried next pass. Reported because a socket nobody
|
|
@@ -353,6 +363,9 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
353
363
|
newSocketId: () => uuid(),
|
|
354
364
|
authenticate: options.authenticate,
|
|
355
365
|
onGranted: (socketId, grant) => grants.set(socketId, grant),
|
|
366
|
+
// The other half of recording the grant before the upgrade: an upgrade that never took
|
|
367
|
+
// gets no `close` callback, so this is the only thing that can free its entry.
|
|
368
|
+
onUngranted: (socketId) => grants.delete(socketId),
|
|
356
369
|
},
|
|
357
370
|
request,
|
|
358
371
|
server,
|
|
@@ -440,15 +453,24 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
|
|
|
440
453
|
},
|
|
441
454
|
},
|
|
442
455
|
|
|
443
|
-
async drain(drainOptions = {}): Promise<readonly
|
|
456
|
+
async drain(drainOptions = {}): Promise<readonly DrainedSocket[]> {
|
|
444
457
|
ready = false;
|
|
445
458
|
const ids = [...sockets.all()].map((socket) => socket.id);
|
|
446
|
-
const
|
|
459
|
+
const spread = drainPlan(ids, {
|
|
447
460
|
spreadMs: options.drainSpreadMs ?? 30_000,
|
|
448
461
|
...(options.rng ? { rng: options.rng } : {}),
|
|
449
462
|
});
|
|
450
|
-
|
|
451
|
-
|
|
463
|
+
// The answer is read, not assumed: this frame IS the socket's slot, so a client that never
|
|
464
|
+
// received one reconnects on its own backoff — the herd the spread exists to break, minus
|
|
465
|
+
// that client. Nothing repairs it, so what the drop owes is a count.
|
|
466
|
+
const plan: DrainedSocket[] = spread.map((entry) => ({
|
|
467
|
+
...entry,
|
|
468
|
+
notified:
|
|
469
|
+
sockets.get(entry.socketId)?.send(reconnectFrame(entry.afterMs, 'drain')) === true,
|
|
470
|
+
}));
|
|
471
|
+
const notified = plan.reduce((total, entry) => total + (entry.notified ? 1 : 0), 0);
|
|
472
|
+
if (notified < plan.length) {
|
|
473
|
+
logger.warn('sync.drain_frames_dropped', { sockets: plan.length, notified });
|
|
452
474
|
}
|
|
453
475
|
const graceMs = drainOptions.graceMs ?? 5_000;
|
|
454
476
|
if (graceMs > 0) await new Promise((resolve) => setTimeout(resolve, graceMs));
|
package/src/sync-protocol.ts
CHANGED
|
@@ -72,7 +72,7 @@ export type SubscribeTarget =
|
|
|
72
72
|
* The opening frame, and the heartbeat's. It carries **no cursors**: resume is decided per
|
|
73
73
|
* subscription by `subscribe`, whose target already carries the cursor and whose `(name, input)`
|
|
74
74
|
* is what the node needs to authorize the read and reach the retained window at all. A cursor's
|
|
75
|
-
* `qid` is `
|
|
75
|
+
* `qid` is `queryHash(name, input)` — a digest, not an input — so a resume list here could never be
|
|
76
76
|
* more than a second, unauthorized restatement of that decision, and it cost every reconnect a
|
|
77
77
|
* duplicate copy of up to `CURSOR_ID_LIMIT` ids per subscription during the exact restart storm
|
|
78
78
|
* `thundering-herd.ts` exists to bound. Removing it needs no `PROTOCOL_VERSION` bump: `decode`
|
|
@@ -375,7 +375,7 @@ function list(obj: JsonObject, key: string, max: number, label = key): JsonValue
|
|
|
375
375
|
|
|
376
376
|
/**
|
|
377
377
|
* A client-supplied value, walked ITERATIVELY to its limits. Iteratively because the thing being
|
|
378
|
-
* refused is a stack overflow: `
|
|
378
|
+
* refused is a stack overflow: `queryHash` -> `canonicalJson` recurses over exactly this value, so a
|
|
379
379
|
* depth check that recursed would be the same crash one frame earlier.
|
|
380
380
|
*/
|
|
381
381
|
function bounded(value: JsonValue, label: string): JsonValue {
|
package/src/sync-upgrade.ts
CHANGED
|
@@ -37,8 +37,22 @@ export interface UpgradeDeps {
|
|
|
37
37
|
socketCount(): number;
|
|
38
38
|
newSocketId(): string;
|
|
39
39
|
readonly authenticate?: SyncAuthenticator | undefined;
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Recorded BEFORE `server.upgrade`, because Bun runs `websocket.open` synchronously inside it
|
|
42
|
+
* (measured on bun 1.3.14) and `open` is where the node reads this grant to build the socket's
|
|
43
|
+
* actor. Recorded after, every authenticated socket carried `actor: null` — the topic guard,
|
|
44
|
+
* `authorize`, `visible` and the per-tenant cap all deciding about nobody — and it never
|
|
45
|
+
* repaired, because the re-auth sweep only visits grants with an `expiresAt`.
|
|
46
|
+
*/
|
|
41
47
|
onGranted(socketId: string, grant: SyncGrant): void;
|
|
48
|
+
/**
|
|
49
|
+
* The grant given back on the one path that will never open a socket. Recording first is only
|
|
50
|
+
* safe because this exists: nothing but a `close` callback deletes a grant, and there is no
|
|
51
|
+
* callback for an upgrade that never took. Required, not optional — a host that reserves and
|
|
52
|
+
* cannot release is a leak the type refuses rather than a rule a reviewer has to remember. The
|
|
53
|
+
* same "reserve, then release" shape `channel.ts` uses for a topic slot.
|
|
54
|
+
*/
|
|
55
|
+
onUngranted(socketId: string): void;
|
|
42
56
|
}
|
|
43
57
|
|
|
44
58
|
/**
|
|
@@ -97,10 +111,14 @@ export async function handleUpgrade(
|
|
|
97
111
|
socketId: deps.newSocketId(),
|
|
98
112
|
clientBuildId: url.searchParams.get('build') ?? deps.buildId,
|
|
99
113
|
};
|
|
114
|
+
// Before the upgrade, never after: `server.upgrade` runs `websocket.open` synchronously and does
|
|
115
|
+
// not return until it has, so a grant recorded on the next line is one the socket was already
|
|
116
|
+
// built without.
|
|
117
|
+
if (grant) deps.onGranted(data.socketId, grant);
|
|
100
118
|
if (!server.upgrade(request, { data })) {
|
|
119
|
+
deps.onUngranted(data.socketId);
|
|
101
120
|
return new Response('expected websocket', { status: 426 });
|
|
102
121
|
}
|
|
103
|
-
if (grant) deps.onGranted(data.socketId, grant);
|
|
104
122
|
return undefined;
|
|
105
123
|
}
|
|
106
124
|
|
package/src/thundering-herd.ts
CHANGED
|
@@ -72,6 +72,16 @@ export interface DrainPlanEntry {
|
|
|
72
72
|
readonly afterMs: number;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
/**
|
|
76
|
+
* A plan entry and what became of it — what `SyncNode.drain` returns. `notified: false` means
|
|
77
|
+
* backpressure dropped that socket's `reconnect` frame: the frame is what carries the slot, nothing
|
|
78
|
+
* re-sends it, so that client reconnects on its own backoff and the count is the only place a log
|
|
79
|
+
* can say how much of the spread actually shipped.
|
|
80
|
+
*/
|
|
81
|
+
export interface DrainedSocket extends DrainPlanEntry {
|
|
82
|
+
readonly notified: boolean;
|
|
83
|
+
}
|
|
84
|
+
|
|
75
85
|
export interface DrainPlanOptions {
|
|
76
86
|
/** Window across which reconnects are spread. Must exceed the node's own drain grace period. */
|
|
77
87
|
readonly spreadMs?: number;
|