@ultimat3/realtime 19.2.0 → 19.3.1

package/CLAUDE.md

@@ -307,6 +307,29 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
   is entitled to its patches for the whole grace window, and to get them through a hub that is
   still open.
 - Exactly one `replicator` per DB, enforced by a session-level advisory lock.
+- **A start and an acquisition are MEMOISED, because `running` and `#connection` are both written
+  after an await (`As of 2026-09-06`).** `start()` read `if (running) return true` and then awaited
+  `lock.tryAcquire()`, so two overlapping calls both passed the guard, both were told they held the
+  lock — a holder's `tryAcquire()` answers `true` — and both ran `feed.start()`: one replication
+  slot, two pumps, every change published twice under two `seq` generations of one producer id,
+  which every sync node's `SeqGapDetector` reads as a gap and repairs by re-snapshotting the fleet.
+  `PgAdvisoryLock.tryAcquire` had the identical hole one layer down and a worse ending: two
+  concurrent calls opened two SESSIONS, the second assignment to `#connection` orphaned the first,
+  and `release()` closes one — so the key stayed held until the process died and no standby could
+  ever take the slot. Both are now the shape `packages/core/src/lifecycle.ts`'s `drain()` uses: a
+  synchronous guard-and-register, the promise published before anything is awaited, and the memo
+  cleared however it settles (a `false` is "somebody else holds it right now", which the takeover
+  loop asks again a backoff later). `stop()` and `release()` await the in-flight one rather than
+  reading their own flag — that flag is false for the whole of a start, so an unguarded teardown
+  returns "nothing to do" and leaves behind exactly what it was called to release.
+  **A start that FAILS hands the lock back, and `running` is set after the feed is pumping**
+  (`As of 2026-09-06`). It was set before `await feed.start()`, so a feed that rejected — a slot
+  already `active`, a preflight refusal — left this node holding the advisory lock and claiming to
+  run with nothing pumping: the takeover loop's next `start()` was answered `true` by the
+  `if (running)` guard without re-entering `begin`, and every standby stayed a standby of a slot
+  whose holder was not replicating. The release is best-effort, because the feed's failure is the
+  one the operator acts on and a session-scoped lock a dead connection cannot release is released
+  by Postgres when that session ends.
 - **The replication pump has one way out, and it closes what it held.** Both exits — a decode error
   and `nextCopyData()` returning `undefined`, which is the walsender ending the copy — run `#die`:
   record `stats().failure`, stop the confirm timer, close the connection and null it. Each one left
@@ -797,7 +820,24 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
   *replay* bound (what a delta resume costs to fold) and adds the byte budgets as the memory one —
   `packages/cache/src/lru.ts:1-2` states why: 4,096 queries x 1,024 patches is 4.19M retained rows
   and no number of bytes at all. `forget(qid)` is called by `LiveQueryRegistry.unsubscribe` when the
-  last subscriber of a query id goes; it had no caller, so the ring outlived the entry.
+  last subscriber of a query id goes; it had no caller, so the ring outlived the entry. It is
+  also called by `#dropIfUnheld` on the subscribe path, `As of 2026-09-06`: an entry is created
+  BEFORE the snapshot read that fills it, so a cold subscribe the database refused left an entry
+  with no subscriber and no removal path — `unsubscribe` can only reach one through a subscription
+  that was never attached. `maxEntries` cold failures were therefore enough to answer
+  `X_SUBSCRIPTION_LIMIT` to every later subscriber for the life of the process, after the database
+  had recovered, because `qid` derives from client-chosen input and distinct inputs mint distinct
+  orphans. Dropped only when the entry is still the one in the table, holds no subscriber and has
+  no read in flight — a read published on it belongs to a concurrent subscriber that has not
+  attached yet, and dropping it there would hand that subscriber a window no change reaches.
+- **A channel patch id carries the NODE that minted it** (`As of 2026-09-06`). `ChannelHub`'s
+  `#sequence` counts within one process, and the patch id was that counter alone — so two `sync`
+  replicas publishing to one topic minted the same id for the same subscriber, and a channel has
+  no cursor and no re-snapshot, so nothing downstream can repair a collision. `nodeId` defaults to
+  a per-hub `uuid()` and is declarable (the pod name) when an operator should be able to say which
+  node published a frame. The frame's `lsn` is deliberately left as the per-process counter:
+  nothing reads a channel frame's lsn as an order across nodes, and `client-frames.ts` advances a
+  cursor only for a registered live query, never for a topic.
 - **An error never renders a value that carries a credential.** `parsePgUrl` names `DATABASE_URL`
   rather than echoing the URL it refused — an error reaches a log, `--json`, an agent transcript
   and a ticket, and the password is in the string. Same rule as `packages/mail/src/driver-smtp.ts`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/realtime",
-  "version": "19.2.0",
+  "version": "19.3.1",
   "description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
   "license": "MIT",
   "type": "module",
@@ -36,8 +36,8 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/core": "19.2.0",
-    "@ultimat3/query": "19.2.0",
+    "@ultimat3/core": "19.3.1",
+    "@ultimat3/query": "19.3.1",
     "nats": "2.29.3"
   }
 }

package/src/channel.ts

@@ -4,7 +4,7 @@
 // append-only stream, so tier 1 needs no frame of its own. That is why climbing the ladder is a
 // config change: the client's frame handler is the same code at every rung.
 
-import { type Actor, finiteOption, logger, renderThrowable } from '@ultimat3/core';
+import { type Actor, finiteOption, logger, renderThrowable, uuid } from '@ultimat3/core';
 import { formatLsn } from './changefeed';
 import {
   isPolicyDenial,
@@ -57,6 +57,17 @@ export interface ChannelHubOptions {
    * admits unbounded distinct names inside one tenant, and a per-socket cap bounds nothing.
    */
   readonly maxTopicsPerNode?: number;
+  /**
+   * This node's mark on the patch ids it mints. Defaults to a per-hub random id, which is enough
+   * to keep two nodes apart; declare it (the pod name, the `sync` instance id) when an operator
+   * reading one frame should be able to say which node published it.
+   *
+   * A blank string is read as OMITTED, never as a mark: `??` only answers for `undefined`, so
+   * `nodeId: ''` — which is what an unset `POD_NAME` interpolates to — stored the empty mark and
+   * two hubs then minted the SAME first patch id, `:0000000000000001`. That is precisely the
+   * collision this field exists to prevent, arriving through the field itself.
+   */
+  readonly nodeId?: string;
 }
 
 /** Distinct topics one node bridges before `X_SUBSCRIPTION_LIMIT`. */
@@ -95,6 +106,12 @@ export class ChannelHub {
   readonly #maxTopicsPerNode: number;
   #guardFailures = 0;
   #sequence = 0n;
+  /**
+   * `#sequence` counts within one PROCESS, so it cannot identify a message across nodes: two
+   * `sync` replicas publishing to one topic minted the same id for the same subscriber, and a
+   * channel has no cursor and no re-snapshot, so nothing downstream could repair the collision.
+   */
+  readonly #nodeId: string;
   /** Set by `close()`. Read by `#open`, which is the only thing that can reach a late subscription. */
   #closed = false;
 
@@ -111,6 +128,9 @@ export class ChannelHub {
       'maxTopicsPerNode',
       options.maxTopicsPerNode ?? DEFAULT_MAX_TOPICS_PER_NODE,
     );
+    // Trimmed before the emptiness test: `nodeId: ' '` marks a frame with a space, which reads in
+    // a log as no mark at all and collides with the next hub that does the same.
+    this.#nodeId = options.nodeId?.trim() || uuid();
   }
 
   /** Sockets this node will deliver `name` to. The metric the fanout reads. */
@@ -230,7 +250,10 @@ export class ChannelHub {
   /** Publishes to every node. Local delivery happens via the transport bridge, never directly. */
   async publish(name: Topic, message: JsonObject): Promise<void> {
     this.#sequence += 1n;
-    const frame = channelFrame(name, formatLsn(this.#sequence), message);
+    const lsn = formatLsn(this.#sequence);
+    // The lsn stays this node's own counter — nothing reads a channel frame's lsn as an order
+    // across nodes — but the patch ID is what a client keys by, so it carries the node too.
+    const frame = channelFrame(name, lsn, message, `${this.#nodeId}:${lsn}`);
     await this.#transport.publish(`${CHANNEL_SUBJECT_PREFIX}.${name}`, encode(frame));
   }
 
@@ -360,12 +383,22 @@ function unsubscribeWhenOpen(bridge: Bridge): void {
   );
 }
 
-export function channelFrame(name: Topic, lsn: string, message: JsonObject): Frame {
+/**
+ * `id` identifies the MESSAGE and defaults to the lsn, which is what every caller outside this
+ * file already passes as one. `ChannelHub.publish` gives it the publishing node's mark instead:
+ * an lsn is a per-process counter, and two nodes on one topic mint the same one.
+ */
+export function channelFrame(
+  name: Topic,
+  lsn: string,
+  message: JsonObject,
+  id: string = lsn,
+): Frame {
   return {
     type: 'patch',
     v: PROTOCOL_VERSION,
     sid: name,
     lsn,
-    patches: [{ op: 'insert', id: lsn, row: message, lsn }],
+    patches: [{ op: 'insert', id, row: message, lsn }],
   };
 }

package/src/live-query.ts

@@ -189,6 +189,29 @@ export class LiveQueryRegistry {
 
     const qid = queryHash(args.name, args.input);
     const entry = this.#entryFor(qid, definition, args.input);
+    try {
+      return await this.#serve(entry, sid, args);
+    } catch (error) {
+      // The entry was born above, before anything could fill it. Everything below can throw — the
+      // snapshot, the resume, the window's own read deadline — and `unsubscribe` is the only other
+      // removal path, reachable only through a subscription that in this case was never attached.
+      this.#dropIfUnheld(qid, entry);
+      throw error;
+    }
+  }
+
+  /** The half of a subscribe that runs against a live entry, so its failures can be undone. */
+  async #serve(
+    entry: QueryEntry,
+    sid: string,
+    args: {
+      socket: SyncSocket;
+      name: string;
+      input: JsonValue;
+      cursor?: LiveCursor | null;
+    },
+  ): Promise<{ subscription: LiveSubscription; frame: Frame }> {
+    const qid = entry.qid;
     const now = this.#clock.now().getTime();
 
     if (args.cursor) {
@@ -260,6 +283,25 @@ export class LiveQueryRegistry {
     this.#options.source.forget?.(subscription.qid);
   }
 
+  /**
+   * An entry nothing is holding, dropped with the retained patches that were kept for it — the
+   * same three steps `unsubscribe` takes when the last subscriber leaves, for the case where a
+   * subscriber never arrived. Five cold failures against `maxEntries: 5` used to refuse every
+   * later subscribe on the node with `X_SUBSCRIPTION_LIMIT`, forever, after the database recovered.
+   */
+  #dropIfUnheld(qid: string, entry: QueryEntry): void {
+    // Identity, never presence: an entry this subscribe did not create may already have been
+    // dropped and re-created under the same qid.
+    if (this.#entries.get(qid) !== entry) return;
+    if (entry.subscribers.size !== 0) return;
+    // A read published on the entry belongs to a subscriber that has not attached yet — it owns
+    // the entry's fate, and dropping it here would leave that subscriber holding a window no
+    // change is ever fanned out to.
+    if (entry.reading !== null) return;
+    this.#entries.delete(qid);
+    this.#options.source.forget?.(qid);
+  }
+
   unsubscribeSocket(socketId: string): void {
     for (const subscription of this.#book.ofSocket(socketId)) {
       this.unsubscribe(socketId, subscription.sid);

package/src/pg-advisory-lock.ts

@@ -34,6 +34,17 @@ export class PgAdvisoryLock implements AdvisoryLock {
   readonly key: string;
   readonly #options: PgAdvisoryLockOptions;
   #connection: PgConnection | null = null;
+  /**
+   * The acquisition in flight, if any. `#connection` cannot answer "am I already taking this" —
+   * it is written at the END of an acquisition, with a dial, a handshake and a query awaited in
+   * between, so two concurrent callers both read `null` and both opened a session. Both then took
+   * a grant on the same key (a second `pg_try_advisory_lock` on a DIFFERENT session succeeds only
+   * if the first has not landed yet, and on the same key from two sessions one of them answers
+   * `f` — either way the loser's session was left open), and the second assignment to
+   * `#connection` orphaned the first: `release()` closes one, and the other holds the lock until
+   * the process dies. A standby then never takes over a slot whose owner has already stopped.
+   */
+  #acquiring: Promise<boolean> | null = null;
 
   constructor(options: PgAdvisoryLockOptions) {
     if (!KEY_PATTERN.test(options.key)) {
@@ -49,9 +60,27 @@ export class PgAdvisoryLock implements AdvisoryLock {
     this.#options = options;
   }
 
-  async tryAcquire(): Promise<boolean> {
+  /**
+   * Deliberately NOT `async`: the guard and the registration have to be one synchronous step, or
+   * the memo has the same check-then-act hole as the field it replaces. Same shape as
+   * `packages/core/src/lifecycle.ts`'s `drain()` and `packages/jobs/src/worker.ts`'s `round()`.
+   */
+  tryAcquire(): Promise<boolean> {
     // Already ours — see the class comment on why a second `pg_try_advisory_lock` must not run.
-    if (this.#connection !== null) return true;
+    if (this.#connection !== null) return Promise.resolve(true);
+    const inFlight = this.#acquiring;
+    if (inFlight !== null) return inFlight;
+    // Cleared however it settles: a `false` is "somebody else holds it right now", which the
+    // replicator's takeover loop asks again a backoff later, and a rejection is the database
+    // refusing THIS attempt. A memo either of those stuck to would answer for the process's life.
+    const attempt = this.#acquire().finally(() => {
+      if (this.#acquiring === attempt) this.#acquiring = null;
+    });
+    this.#acquiring = attempt;
+    return attempt;
+  }
+
+  async #acquire(): Promise<boolean> {
     const target = parsePgUrl(this.#options.url);
     const stream = await (this.#options.stream ?? bunPgStream)(target);
     // Plain SQL, not `replication: 'database'` — this session runs one statement, never a feed.
@@ -85,6 +114,13 @@ export class PgAdvisoryLock implements AdvisoryLock {
   }
 
   async release(): Promise<void> {
+    // A release that arrives mid-acquisition waits it out. `#connection` is still `null` there, so
+    // the early return below would answer "nothing to release" and the session would open behind
+    // it, holding the key with no object left intending to close it — the same orphan the memo
+    // exists to prevent, reached from the other side. The attempt's own failure is not this
+    // caller's to report: `tryAcquire`'s caller already has it.
+    const acquiring = this.#acquiring;
+    if (acquiring !== null) await acquiring.catch(() => false);
     const connection = this.#connection;
     if (connection === null) return;
     this.#connection = null;

package/src/replicator.ts

@@ -116,6 +116,8 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
   // the old one.
   let producer = uuid();
   let seq = 0;
+  /** The start in flight, if any — see `start()` on why `running` cannot answer that question. */
+  let starting: Promise<boolean> | undefined;
 
   const onChange = async (raw: ChangeEvent): Promise<void> => {
     const change = normalize(raw);
@@ -141,24 +143,66 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
     });
   };
 
-  return {
-    async start(): Promise<boolean> {
-      if (running) return true;
-      if (!(await options.lock.tryAcquire())) {
-        logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
-        return false;
-      }
-      running = true;
-      producer = uuid();
-      seq = 0;
+  const begin = async (): Promise<boolean> => {
+    if (!(await options.lock.tryAcquire())) {
+      logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
+      return false;
+    }
+    producer = uuid();
+    seq = 0;
+    try {
       await options.feed.start(
         options.from === undefined ? { onChange } : { from: options.from, onChange },
       );
-      logger.info('replicator started', { source: options.feed.source, key: options.lock.key });
-      return true;
+    } catch (thrown) {
+      // A start that failed holds NOTHING. `running` stays false so the takeover loop's next
+      // `start()` runs `begin` again instead of being answered `true` by a memo over a feed that
+      // never pumped, and the lock goes back so a standby can take the slot rather than waiting on
+      // a holder that is not replicating.
+      //
+      // The release is best-effort by design: the feed's failure is the one the caller must see,
+      // and a session-scoped advisory lock a dead connection cannot release is released by
+      // Postgres when that session ends.
+      await options.lock.release().catch(() => undefined);
+      throw thrown;
+    }
+    // AFTER the feed is pumping, never before: `running` is what `start()` answers `true` from and
+    // what `stop()` reads to decide there is anything to tear down.
+    running = true;
+    logger.info('replicator started', { source: options.feed.source, key: options.lock.key });
+    return true;
+  };
+
+  return {
+    /**
+     * Deliberately NOT `async`: `running` is set after an `await` on the lock, so a guard that
+     * awaited anything before registering has the same hole it is meant to close. Two overlapping
+     * starts both passed `if (running)`, both were told they held the lock — a holder's
+     * `tryAcquire()` answers `true` — and both ran `feed.start()`: one replication slot with two
+     * pumps, publishing every change twice under two `seq` generations of one producer id, which
+     * every sync node's `SeqGapDetector` reads as a gap and repairs by re-snapshotting the fleet.
+     * Same shape as `packages/core/src/lifecycle.ts`'s `drain()`.
+     */
+    start(): Promise<boolean> {
+      if (running) return Promise.resolve(true);
+      const inFlight = starting;
+      if (inFlight !== undefined) return inFlight;
+      // Cleared however it settles: a `false` is "another node holds the lock", and the takeover
+      // loop asks again a `retryDelayMs` later. A memo that stuck would make this node a permanent
+      // standby of a slot whose holder has already died.
+      const attempt = begin().finally(() => {
+        if (starting === attempt) starting = undefined;
+      });
+      starting = attempt;
+      return attempt;
     },
 
     async stop(): Promise<void> {
+      // A stop racing a start waits it out: `running` is false for the whole of `begin`, so the
+      // early return below would leave the feed it is about to start pumping into a replicator
+      // nothing intends to stop, with the advisory lock still held.
+      const inFlight = starting;
+      if (inFlight !== undefined) await inFlight.catch(() => false);
       if (!running) return;
       running = false;
       await options.feed.stop();

package/src/sync-listen.ts

@@ -20,6 +20,13 @@ export interface ListenOptions {
    * defeated by the listener beside it.
    */
   readonly hostname?: string;
+  /**
+   * The grace `drain()` gives the sockets it holds on SIGTERM, in ms; the node's default when
+   * omitted. `0` is for a process with nobody to hand its clients to — `x dev`, one node, whose
+   * clients reconnect to it once it is back and to nothing in the meantime: the 5s it kept their
+   * patches flowing was 5s of a Ctrl-C that had nothing else to wait for.
+   */
+  readonly drainGraceMs?: number;
 }
 
 export interface SyncListener {
@@ -61,7 +68,7 @@ export function listenSyncNode(node: SyncNode, options: ListenOptions = {}): Syn
     { phase: 'accept' },
   );
   const unregister = onShutdown('realtime:sync', async () => {
-    await node.drain();
+    await node.drain(options.drainGraceMs === undefined ? {} : { graceMs: options.drainGraceMs });
     await node.stop();
     server.stop();
     stopListening();

package/src/sync-node.ts

@@ -148,6 +148,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
   const grants = new GrantBook();
   const gaps = new SeqGapDetector();
   let ready = false;
+  /** Resolved by `teardown` when the last socket leaves, for a drain that is waiting its grace. */
+  let lastSocketLeft: (() => void) | null = null;
   let changes: TransportSubscription | null = null;
   let sweeping: ReturnType<typeof setInterval> | null = null;
   let reauthing: ReturnType<typeof setInterval> | null = null;
@@ -188,6 +190,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
     for (const name of topics) options.hub.unsubscribe(socket, name);
     sockets.remove(socket.id);
     grants.delete(socket.id);
+    if (sockets.count === 0) lastSocketLeft?.();
     // A closed socket is a leave, said now rather than left to TTL: everyone else would otherwise
     // keep rendering a member who is provably gone for the rest of its window. The write is on the
     // bus and the close callback is synchronous, so it cannot be awaited here.
@@ -210,6 +213,18 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
    * because dropping the socket from the table is three of `teardown`'s five steps and the two it
    * misses are the ones another node can see.
    */
+  /** `graceMs`, or until `teardown` reports the table empty — whichever comes first. */
+  const graceOrEmpty = (graceMs: number): Promise<void> =>
+    new Promise<void>((resolve) => {
+      const timer = setTimeout(done, graceMs);
+      function done(): void {
+        clearTimeout(timer);
+        lastSocketLeft = null;
+        resolve();
+      }
+      lastSocketLeft = done;
+    });
+
   const evict = (socket: SyncSocket, code: number, reason: string): readonly Promise<unknown>[] => {
     socket.close(code, reason);
     return teardown(socket);
@@ -447,8 +462,12 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
       if (notified < plan.length) {
         logger.warn('sync.drain_frames_dropped', { sockets: plan.length, notified });
       }
+      // The grace is what a socket is OWED after its reconnect frame — its patches, until its own
+      // delay is up — so it is waited only while there is a socket to owe it to, and ends the
+      // moment the last one leaves. Unconditional, it was 5.0s of a 5.1s Ctrl-C on `x dev` with
+      // no browser open (measured 2026-09-06): a drain of zero sockets, sleeping for nobody.
       const graceMs = drainGraceMs(drainOptions.graceMs);
-      if (graceMs > 0) await new Promise((resolve) => setTimeout(resolve, graceMs));
+      if (graceMs > 0 && sockets.count > 0) await graceOrEmpty(graceMs);
       // Through `evict`, never `sockets.remove` + `grants.delete`: those are three of `teardown`'s
       // five steps, and the two they skip are the ones the rest of the fleet can see. A drained
       // socket that never left its presence set is a member every other node renders for a full