@ultimat3/realtime 1.2.0 → 2.0.0

package/src/query-window.ts

@@ -0,0 +1,151 @@
+// The shared, pre-policy result window one query id is served from: how it is built, how it is
+// read once for N subscribers, and how one that is known to be wrong is replaced. The authz
+// decision is never here — `live-query.ts` owns that, once per subscriber, over what this returns.
+
+import type { JsonValue, Row } from './json';
+import type { LiveQueryDefinition, LiveSubscription, SnapshotResult } from './live-contract';
+import type { IncrementalMatcher, SubscriptionShape } from './matcher-bridge';
+import { WindowLock } from './window-lock';
+
+export interface QueryEntry {
+  readonly qid: string;
+  readonly definition: LiveQueryDefinition;
+  readonly input: JsonValue;
+  /** Told to the client on every snapshot: the identity scope its rows belong under. */
+  readonly rowEntity: string | null;
+  readonly shape: SubscriptionShape;
+  readonly matcher: IncrementalMatcher;
+  readonly subscribers: Map<string, LiveSubscription>;
+  /**
+   * The shared, *pre-policy* result window. One per query id, bounded by the query's `limit`, and
+   * the reason the matcher can run once for N subscribers: the read is shared, the authz is not.
+   */
+  rows: readonly Row[];
+  lsn: string;
+  /**
+   * This window is known to have missed at least one change, so patching it would compound the
+   * error. Set when the change stream skipped a sequence and when a matcher reports it lost the
+   * window's tail; cleared by the read that replaces the rows. It is *not* a subscriber's desync —
+   * that is per socket, and this is the window every one of them shares.
+   */
+  stale: boolean;
+  /** Serial lane over `rows`/`lsn`. Every fanout and every window assignment takes its turn here. */
+  readonly lock: WindowLock;
+  /** The read in flight, shared by every subscriber that arrives during it. `null` between reads. */
+  reading: Promise<SnapshotResult> | null;
+}
+
+export function createEntry(
+  qid: string,
+  definition: LiveQueryDefinition,
+  input: JsonValue,
+  matcher: IncrementalMatcher,
+): QueryEntry {
+  return {
+    qid,
+    definition,
+    input,
+    // Resolved with the matcher, from the same build: `prepare` has already run, so a definition
+    // that compiles its shape per input can answer.
+    rowEntity: definition.rowEntity?.(input) ?? null,
+    shape: {
+      qid,
+      // The matcher knows the dependency set this *input* produced; `definition.entities` is the
+      // static declaration and can only be a superset of it. Preferring the matcher is what lets a
+      // definition built from a real query carry no static list at all.
+      entities: matcher.entities.length > 0 ? matcher.entities : definition.entities,
+      orgId: orgIdOf(input),
+      ...(definition.columns ? { columns: definition.columns } : {}),
+    },
+    matcher,
+    subscribers: new Map(),
+    rows: [],
+    lsn: '',
+    stale: false,
+    lock: new WindowLock(),
+    reading: null,
+  };
+}
+
+/**
+ * The window this subscriber is served from, read once per entry. A subscriber arriving while
+ * another's read is in flight joins that read rather than issuing its own — N cold subscribers on
+ * one query id being N reads is the shared window not existing.
+ *
+ * The result lands in the lane, and never backwards. A snapshot that resolved after a newer change
+ * had already been fanned out would rewind every later subscriber to rows the window has moved
+ * past, so a stale read is discarded and its caller is served from the newer window instead.
+ */
+export async function fillWindow(
+  entry: QueryEntry,
+): Promise<{ rows: readonly Row[]; lsn: string }> {
+  // Read before `startRead` clears it: a second caller arriving during the read joins it and is
+  // not the one that forced it, which is what keeps one forced read from becoming N.
+  const forced = entry.stale;
+  const result = await (forced || entry.reading === null ? startRead(entry) : entry.reading);
+  return await entry.lock.run(async () => {
+    if (forced) {
+      // A forced read replaces the window whatever its lsn says: it was issued *because* what is
+      // under it is wrong, and a definition with no lsn provider answers `''` — which the
+      // never-backwards rule below would read as older than what we hold and discard, leaving
+      // every subscriber served from the window the gap already invalidated.
+      entry.rows = result.rows;
+      if (result.lsn > entry.lsn) entry.lsn = result.lsn;
+    } else if (result.lsn >= entry.lsn) {
+      entry.rows = result.rows;
+      entry.lsn = result.lsn;
+    }
+    return { rows: entry.rows, lsn: entry.lsn };
+  });
+}
+
+/**
+ * The same replacement, for a caller that is already holding the lane. A fanout cannot call
+ * `fillWindow` — that takes the entry's own lane, and a lane is not reentrant — so the one path
+ * that repairs a stale window mid-fanout is spelled here rather than deadlocking on the other.
+ */
+export async function refillWindowInLane(entry: QueryEntry): Promise<void> {
+  const result = await startRead(entry);
+  entry.rows = result.rows;
+  if (result.lsn > entry.lsn) entry.lsn = result.lsn;
+}
+
+/** Publishes the in-flight read, and clears it as it settles — the share is per read, not a cache. */
+function startRead(entry: QueryEntry): Promise<SnapshotResult> {
+  // Cleared here rather than when the read lands: the read about to be issued is the one that
+  // answers the staleness, so a second caller must join it instead of forcing another.
+  entry.stale = false;
+  const reading = readSnapshot(entry);
+  entry.reading = reading;
+  const done = (): void => {
+    if (entry.reading === reading) entry.reading = null;
+  };
+  void reading.then(done, done);
+  return reading;
+}
+
+/**
+ * The definition's read, with the staleness put back when it does not answer.
+ *
+ * Clearing the mark on the way in and never restoring it was the gap repair happening once and
+ * never again: the snapshot that was going to replace an invalidated window rejects — the pool is
+ * exhausted by the same incident that caused the gap — and the entry is left unmarked over rows it
+ * is known to have missed a change on. Nothing re-reads, `#resnapshot` serves every desynced
+ * subscriber out of that divergent window and clears their marks, and the divergence `stale` exists
+ * to prevent is now permanent and silent. `async` so a definition that throws synchronously takes
+ * the same path as one that rejects.
+ */
+async function readSnapshot(entry: QueryEntry): Promise<SnapshotResult> {
+  try {
+    return await entry.definition.snapshot({ input: entry.input });
+  } catch (error) {
+    entry.stale = true;
+    throw error;
+  }
+}
+
+export function orgIdOf(input: JsonValue): string | null {
+  if (typeof input !== 'object' || input === null || Array.isArray(input)) return null;
+  const value = input['orgId'];
+  return typeof value === 'string' ? value : null;
+}

package/src/rebase.ts

@@ -108,26 +108,86 @@ export function reconcile<T extends TableMap = TableMap>(
   const table = store.table(ack.entity);
   const local = table.get(ack.id);
 
-  // Everything at or after the acked sequence is optimistic and must be undone newest-first.
-  const affected = log.pending().filter((candidate) => candidate.seq >= (entry?.seq ?? 0));
+  const { affected, rolledBack } = undoFrom(store, log, entry?.seq ?? 0);
+
+  const base = store.table(ack.entity).get(ack.id);
+  const winner = land(store, ack, strategy, { local, base }, options);
+  log.drop(ack.key);
+
+  return {
+    strategy: strategyName(strategy),
+    rolledBack,
+    reapplied: replayExcept(store, affected, ack.key),
+    winner,
+  };
+}
+
+/**
+ * Everything at or after `from` is optimistic, so it is undone newest-first — and `reconcile` and
+ * `rollbackMutation` are one rule with different middles, not two. Spelled twice, the next change
+ * to the replay order has to be made twice, and the half that is missed diverges silently.
+ */
+function undoFrom<T extends TableMap>(
+  store: LocalStore<T>,
+  log: RebaseLog<T>,
+  from: number,
+): { affected: readonly RebaseEntry<T>[]; rolledBack: string[] } {
+  const affected = log.pending().filter((candidate) => candidate.seq >= from);
   const rolledBack: string[] = [];
   for (const candidate of [...affected].reverse()) {
     store.rollback(candidate.key);
     rolledBack.push(candidate.key);
   }
+  return { affected, rolledBack };
+}
 
-  const base = store.table(ack.entity).get(ack.id);
-  const winner = land(store, ack, strategy, { local, base }, options);
-  log.drop(ack.key);
-
+/**
+ * The other half: replay in sequence order, skipping the one the server has now settled. `local` is
+ * pure, which is what makes replaying it deterministic and therefore safe to do at all.
+ */
+function replayExcept<T extends TableMap>(
+  store: LocalStore<T>,
+  affected: readonly RebaseEntry<T>[],
+  settled: string,
+): string[] {
   const reapplied: string[] = [];
   for (const candidate of affected) {
-    if (candidate.key === ack.key) continue;
+    if (candidate.key === settled) continue;
     store.apply(candidate.key, (tx) => candidate.apply(tx));
     reapplied.push(candidate.key);
   }
+  return reapplied;
+}
+
+export interface RollbackResult {
+  readonly rolledBack: readonly string[];
+  readonly reapplied: readonly string[];
+}
+
+/**
+ * The other half of `reconcile`: the server **refused** a mutation, so there is no server truth to
+ * land — only an optimistic write to take back. Same shape as a reconcile, and for the same reason:
+ * the writes made after it may depend on it, so everything from its sequence onward is undone
+ * newest-first and then replayed without it. Replay is deterministic because `local` is pure.
+ *
+ * Idempotent for a key the log does not hold: a denial can arrive twice, and tier 2 records nothing
+ * to undo in the first place.
+ */
+export function rollbackMutation<T extends TableMap = TableMap>(args: {
+  store: LocalStore<T>;
+  log: RebaseLog<T>;
+  key: string;
+}): RollbackResult {
+  const { store, log, key } = args;
+  const entry = log.get(key);
+  if (!entry) return { rolledBack: [], reapplied: [] };
 
-  return { strategy: strategyName(strategy), rolledBack, reapplied, winner };
+  const { affected, rolledBack } = undoFrom(store, log, entry.seq);
+  // The one thing that differs from `reconcile`'s middle: there is no server truth to land. Dropped
+  // and never retried — a denial is a decision about this intent, so replaying it on the next
+  // reconcile would put the write the server refused back on the screen.
+  log.drop(key);
+  return { rolledBack, reapplied: replayExcept(store, affected, key) };
 }
 
 function land<T extends TableMap>(

package/src/replicator.ts

@@ -11,7 +11,7 @@
 // its feed and reports `/readyz` false; it retries with jittered backoff and takes over the moment
 // the holder dies. Scaling the replicator is therefore always vertical, and that is by design.
 
-import { logger, withSpan } from '@ultimat3/core';
+import { logger, uuid, withSpan } from '@ultimat3/core';
 import type { ChangeEvent, ChangeFeed } from './changefeed';
 import type { Transport } from './fanout';
 import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
@@ -72,6 +72,26 @@ export interface ReplicatorStats {
   readonly outOfOrder: number;
 }
 
+/**
+ * What the bus actually carries: one change, plus who published it and where in that publisher's
+ * stream it sits.
+ *
+ * Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
+ * cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
+ * cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
+ * arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
+ *
+ * Both fields are optional on the wire, so a node reading a publisher that predates them simply
+ * detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
+ */
+export interface ChangeEnvelope {
+  readonly change: ChangeEvent;
+  /** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
+  readonly seq: number | null;
+  /** Identifies one replicator *run*. A new one restarts `seq`, and that is not a gap. */
+  readonly producer: string | null;
+}
+
 export interface Replicator {
   /** `false` = another replicator holds the lock; this process must stay `/readyz` false. */
   start(): Promise<boolean>;
@@ -91,6 +111,11 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
   let published = 0;
   let skipped = 0;
   let outOfOrder = 0;
+  // One id per run, not per process: a replicator that took the lock back after a crash publishes
+  // from its persisted lsn, and a consumer must read that as a new stream rather than as a gap in
+  // the old one.
+  let producer = uuid();
+  let seq = 0;
 
   const onChange = async (raw: ChangeEvent): Promise<void> => {
     const change = normalize(raw);
@@ -104,7 +129,13 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
       return;
     }
     await withSpan('realtime.replicate', async () => {
-      await options.transport.publish(subjectOf(change), JSON.stringify(change));
+      seq += 1;
+      // The envelope is the change plus two fields, flat, so a consumer that only knows about the
+      // change reads it unchanged — `parseChange` still answers on the same payload.
+      await options.transport.publish(
+        subjectOf(change),
+        JSON.stringify({ ...change, seq, producer }),
+      );
       lastLsn = change.lsn;
       published += 1;
     });
@@ -118,6 +149,8 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
         return false;
       }
       running = true;
+      producer = uuid();
+      seq = 0;
       await options.feed.start(
         options.from === undefined ? { onChange } : { from: options.from, onChange },
       );
@@ -163,23 +196,63 @@ export function normalize(change: ChangeEvent): ChangeEvent | null {
 
 /** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
 export function parseChange(payload: string): ChangeEvent | null {
+  return parseEnvelope(payload)?.change ?? null;
+}
+
+/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
+export function parseEnvelope(payload: string): ChangeEnvelope | null {
   try {
     const parsed: unknown = JSON.parse(payload);
     if (typeof parsed !== 'object' || parsed === null) return null;
-    const shape = parsed as Partial<ChangeEvent>;
+    const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
     if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
     if (shape.op !== 'insert' && shape.op !== 'update' && shape.op !== 'delete') return null;
     return {
-      entity: shape.entity,
-      op: shape.op,
-      before: shape.before ?? null,
-      after: shape.after ?? null,
-      lsn: shape.lsn,
-      txid: typeof shape.txid === 'string' ? shape.txid : '',
-      orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
-      at: typeof shape.at === 'number' ? shape.at : 0,
+      change: {
+        entity: shape.entity,
+        op: shape.op,
+        before: shape.before ?? null,
+        after: shape.after ?? null,
+        lsn: shape.lsn,
+        txid: typeof shape.txid === 'string' ? shape.txid : '',
+        orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
+        at: typeof shape.at === 'number' ? shape.at : 0,
+      },
+      seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
+      producer: typeof shape.producer === 'string' ? shape.producer : null,
     };
   } catch {
     return null;
   }
 }
+
+/**
+ * The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
+ * say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
+ * there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
+ * simply never happened as far as every subscriber on this node could tell.
+ *
+ * Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
+ * reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
+ * the first message of a stream: a node that joined late has missed everything by definition, and
+ * every subscription it holds started after it did.
+ */
+export class SeqGapDetector {
+  readonly #next = new Map<string, number>();
+
+  /** `true` when at least one message between the last one and this one was never delivered. */
+  observe(envelope: ChangeEnvelope): boolean {
+    const { producer, seq } = envelope;
+    if (producer === null || seq === null) return false;
+    const expected = this.#next.get(producer);
+    // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
+    // into a gap of its own.
+    this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
+    return expected !== undefined && seq > expected;
+  }
+
+  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
+  forget(): void {
+    this.#next.clear();
+  }
+}

package/src/socket.ts

@@ -1,11 +1,22 @@
 // One WS connection. Bun's WS profile is what makes a million sockets affordable, so this object
-// is deliberately lean: ~8 fields plus two small sets. Budget is ~1KB of JS heap per connection on
-// top of Bun's own per-socket cost — anything richer (row caches, per-socket buffers) belongs in
-// the transport or the change buffer, never here. `sync` is stateless: nothing on this object
-// survives a restart, and nothing needs to.
+// is deliberately lean: ~8 fields, two small sets and one token bucket (the frame budget, which is
+// five numbers and the only thing standing between one authenticated socket and this node's whole
+// subscribe path). Budget is ~1KB of JS heap per connection on top of Bun's own per-socket cost —
+// anything richer (row caches, per-socket buffers) belongs in the transport or the change buffer,
+// never here. `sync` is stateless: nothing on this object survives a restart, and nothing needs to.
 
-import { type Actor, type Clock, recordConnection, systemClock, uuid } from '@ultimat3/core';
+import {
+  type Actor,
+  type Clock,
+  type Counter,
+  counter,
+  logger,
+  recordConnection,
+  systemClock,
+  uuid,
+} from '@ultimat3/core';
 import { encode, type Frame } from './sync-protocol';
+import { AcceptBudget } from './thundering-herd';
 
 export const CLOSE = {
   normal: 1000,
@@ -17,7 +28,16 @@ export const CLOSE = {
   drain: 4002,
 } as const;
 
-/** The slice of Bun's `ServerWebSocket` this package uses. Structural, so tests need no server. */
+/**
+ * The slice of Bun's `ServerWebSocket` this package uses. Structural, so tests need no server.
+ *
+ * `subscribe`/`unsubscribe` are Bun's native pub/sub and this package does NOT use them: nothing
+ * here publishes to a native topic, and nothing will — a native publish cannot be refused per
+ * socket, cannot report the frame it dropped and cannot mark a subscriber desynced, which is
+ * exactly what `SocketRegistry.deliver` and `SyncSocket.send` exist to do. They are still declared
+ * because `WsLike` is the slice of Bun's own object, and an app already implements it; deleting
+ * them is a separate, breaking edit to every implementer.
+ */
 export interface WsLike {
   send(data: string): number;
   close(code?: number, reason?: string): void;
@@ -37,8 +57,47 @@ export interface SyncSocketOptions {
   /** Frames are dropped rather than queued past this. See `desynced`. */
   readonly maxBufferedBytes?: number;
   readonly maxDroppedFrames?: number;
+  /** Sustained inbound frames this socket may have routed per second. See `frameBudget`. */
+  readonly maxFramesPerSecond?: number;
+  /** Burst allowance, so a client subscribing its whole cap at connect is never refused. */
+  readonly frameBurst?: number;
 }
 
+/**
+ * What one socket may ask this node to do per second, and how much of it may arrive at once.
+ *
+ * The burst clears `DEFAULT_MAX_PER_SOCKET` (128) plus a `hello`, because that is exactly what a
+ * legitimate client sends on connect; the sustained rate is well under the ~155 frames/s measured
+ * to consume a node through the subscribe path's amplifiers.
+ */
+export const DEFAULT_MAX_FRAMES_PER_SECOND = 64;
+export const DEFAULT_FRAME_BURST = 256;
+/**
+ * Queued-but-unwritten bytes on one server socket before `send` declines and marks the subscriber
+ * desynced. `sync-node.ts` hands the same number to Bun as `backpressureLimit` rather than spelling
+ * it again: they are one socket's one buffer, and a check the runtime's own limit fires before is a
+ * check that never runs. The client half (`client-mutations.ts`) is deliberately its own constant —
+ * it is browser code, and importing this file would pull the node's socket registry into the tab.
+ */
+export const DEFAULT_MAX_BUFFERED_BYTES = 1024 * 1024;
+
+/**
+ * Channel frames this process dropped under backpressure. A DATA-LOSS counter, not a saturation
+ * one: the live-query path repairs a dropped patch (the subscriber is marked desynced and the next
+ * change re-snapshots it), and a channel has no cursor, no mark and no re-snapshot — so this is the
+ * only trace a lost channel message leaves anywhere.
+ *
+ * Declared here rather than in `@ultimat3/core`'s `runtime-metrics.ts` because that file is the
+ * series EVERY Ultimate process emits and the deploy chart scales on; this one exists only where
+ * channels do. **No attributes**: a topic is client-chosen (`topic()` admits any
+ * `[A-Za-z0-9_-]+` segment), so a per-topic label is an unbounded series count one socket can mint
+ * — the topic goes in the log line, where cardinality is somebody else's index.
+ */
+const channelFramesDropped: Counter = counter('channel_frames_dropped_total', {
+  unit: '{frame}',
+  description: 'Channel frames dropped by socket backpressure — unrecoverable, nothing replays one',
+});
+
 export function actorIdOf(actor: Actor | null): string | null {
   return actor === null ? null : actor.id;
 }
@@ -58,6 +117,13 @@ export class SyncSocket {
    * flush re-snapshots instead of silently diverging.
    */
   readonly desynced = new Set<string>();
+  /**
+   * Inbound frames this socket may still have routed. The accept budget spends one token per
+   * UPGRADE, so nothing bounded what happened after: one authenticated socket reached a DB read,
+   * a shared-store presence write and a fleet-wide publish once per frame, unbounded and
+   * unawaited. Same token bucket, one per socket — the mechanism already existed, one rung down.
+   */
+  readonly frameBudget: AcceptBudget;
 
   actor: Actor | null;
   lastSeenAt: number;
@@ -77,8 +143,13 @@ export class SyncSocket {
     this.clientBuildId = options.clientBuildId;
     this.serverBuildId = options.serverBuildId;
     this.actor = options.actor ?? null;
-    this.#maxBufferedBytes = options.maxBufferedBytes ?? 1024 * 1024;
+    this.#maxBufferedBytes = options.maxBufferedBytes ?? DEFAULT_MAX_BUFFERED_BYTES;
     this.#maxDroppedFrames = options.maxDroppedFrames ?? 32;
+    this.frameBudget = new AcceptBudget({
+      perSecond: options.maxFramesPerSecond ?? DEFAULT_MAX_FRAMES_PER_SECOND,
+      burst: options.frameBurst ?? DEFAULT_FRAME_BURST,
+      clock: this.#clock,
+    });
     this.openedAt = this.#clock.now().getTime();
     this.lastSeenAt = this.openedAt;
   }
@@ -120,14 +191,18 @@ export class SyncSocket {
     this.desynced.delete(sid);
   }
 
+  /**
+   * This socket's own membership, and nothing else. It used to also call Bun's `ws.subscribe`,
+   * which built a second per-topic index nothing ever published to — the fanout is
+   * `SocketRegistry.deliver`, one filtered `send` per socket, because that is the only path that
+   * can count a dropped frame or close a socket that is drowning in them.
+   */
   subscribeTopic(topic: string): void {
     this.topics.add(topic);
-    this.#ws.subscribe(topic);
   }
 
   unsubscribeTopic(topic: string): void {
     this.topics.delete(topic);
-    this.#ws.unsubscribe(topic);
   }
 
   touch(): void {
@@ -154,8 +229,18 @@ export interface SocketRegistryOptions {
 /** Per-node socket table. Intentionally the only in-memory map on a `sync` node. */
 export class SocketRegistry {
   readonly #sockets = new Map<string, SyncSocket>();
+  /**
+   * Who is on each channel topic. `deliver` walked every socket on the node asking each whether
+   * it held the topic, so one message with one legitimate subscriber cost as many iterations as
+   * this node has connections — 50,000 at the scale this framework benchmarks. It lives here
+   * rather than on the hub because this is the only object that sees a socket die: a close, a
+   * drain and the idle sweep all pass through `remove`, and an index nobody cleans on those paths
+   * retains a dead socket per topic forever.
+   */
+  readonly #byTopic = new Map<string, Set<SyncSocket>>();
   readonly #clock: Clock;
   readonly #idleTimeoutMs: number;
+  #droppedChannelFrames = 0;
 
   constructor(options: SocketRegistryOptions = {}) {
     this.#clock = options.clock ?? systemClock;
@@ -178,8 +263,39 @@ export class SocketRegistry {
   }
 
   remove(id: string): void {
+    const socket = this.#sockets.get(id);
     // `Map.delete` answers "was it actually there", so a double close cannot decrement twice.
-    if (this.#sockets.delete(id)) recordConnection(-1);
+    if (!this.#sockets.delete(id)) return;
+    recordConnection(-1);
+    if (!socket) return;
+    // Leaving this table IS the close, whoever noticed first. Bun's `close` callback reports a
+    // connection that has already gone, so nothing called `close()` on this object and
+    // `socket.closed` stayed false — leaving a subscribe still awaiting its snapshot read with no
+    // way to tell that the socket it is about to attach to was torn down while it read.
+    socket.close(CLOSE.goingAway, 'connection closed');
+    for (const name of socket.topics) this.#dropFrom(name, socket);
+  }
+
+  /**
+   * Join a topic: the socket's own set, Bun's native pub/sub and this node's delivery index, in
+   * one call. `ChannelHub` is its only caller, and calls nothing else — two call sites for one
+   * membership is how an index goes wrong.
+   */
+  joinTopic(socket: SyncSocket, topic: string): void {
+    socket.subscribeTopic(topic);
+    const members = this.#byTopic.get(topic);
+    if (members) members.add(socket);
+    else this.#byTopic.set(topic, new Set([socket]));
+  }
+
+  leaveTopic(socket: SyncSocket, topic: string): void {
+    socket.unsubscribeTopic(topic);
+    this.#dropFrom(topic, socket);
+  }
+
+  /** Sockets this node would deliver `topic` to. */
+  subscriberCount(topic: string): number {
+    return this.#byTopic.get(topic)?.size ?? 0;
   }
 
   get(id: string): SyncSocket | undefined {
@@ -209,13 +325,53 @@ export class SocketRegistry {
     return closed;
   }
 
-  /** Local delivery for a channel topic. Bun's native pub/sub handles the common case; this is
-   *  the fallback used when a frame must be filtered per socket (policy, desync bookkeeping). */
+  /**
+   * Local delivery for a channel topic — every channel message on this node comes through here,
+   * so it reads the per-topic index rather than the socket table. A socket that closed without
+   * an unsubscribe is dropped as it is met, so the index cannot outlive the connection even on a
+   * path that forgot to `remove` it.
+   */
   deliver(topic: string, frame: Frame): number {
+    const members = this.#byTopic.get(topic);
+    if (!members) return 0;
     let sent = 0;
-    for (const socket of this.#sockets.values()) {
-      if (socket.topics.has(topic) && socket.send(frame)) sent += 1;
+    let dropped = 0;
+    for (const socket of members) {
+      if (socket.closed) {
+        members.delete(socket);
+        continue;
+      }
+      if (socket.send(frame)) sent += 1;
+      else dropped += 1;
+    }
+    if (members.size === 0) this.#byTopic.delete(topic);
+    if (dropped > 0) {
+      this.#droppedChannelFrames += dropped;
+      // Two readers, one event, one spelling: the series an operator alerts on and the line that
+      // says which topic it was. `deliver` ignored `send`'s answer and so did the hub above it, so
+      // until both existed a lost channel message left no trace at all.
+      channelFramesDropped.add(dropped);
+      logger.warn('channel.frames_dropped', { topic, dropped, total: this.#droppedChannelFrames });
     }
     return sent;
   }
+
+  /**
+   * Channel frames backpressure refused since boot, node-wide and cumulative — the in-process read
+   * of `channel_frames_dropped_total`, for a test or a benchmark that cannot scrape.
+   *
+   * Node-wide on purpose: a socket past `maxDroppedFrames` is closed and removed, so a per-socket
+   * count leaves with the socket exactly when loss is worst. Distinct from `SyncSocket.droppedFrames`,
+   * which counts every kind of frame one connection lost, channel and live-query patch alike.
+   */
+  get droppedChannelFrames(): number {
+    return this.#droppedChannelFrames;
+  }
+
+  #dropFrom(topic: string, socket: SyncSocket): void {
+    const members = this.#byTopic.get(topic);
+    if (!members) return;
+    members.delete(socket);
+    if (members.size === 0) this.#byTopic.delete(topic);
+  }
 }