@ultimat3/realtime 1.2.0 → 3.0.0

package/src/change-buffer.ts

@@ -1,50 +1,109 @@
-// The bounded per-query change window that makes reconnect a delta instead of a refetch.
-// Lives on the `replicator` (one per DB), so a reconnecting client costs zero DB work while its
-// gap is inside the window. Outside it, `resumeFrom` takes one snapshot — never WAL traversal.
+// The bounded per-query change window that makes reconnect a delta instead of a refetch. Inside
+// the window a reconnecting client costs zero DB work; outside it, `resumeFrom` takes one bounded
+// snapshot — never WAL traversal.
+//
+// **It is per `sync` node, and a `qid` window can only be.** The header used to say it lives on the
+// replicator; it does not, and it could not — a patch is query-scoped, so producing one needs that
+// query's compiled shape, its matcher and its current window, none of which the replicator has (it
+// is entity-scoped by construction). The consequence is real and is not fixed here: a client that
+// reconnects onto a node that never served its `qid` finds no ring, `shouldResnapshot` answers
+// `out-of-window`, and it takes the snapshot path. What that costs is one *shared* read per
+// (query, node) — `fillWindow` joins every subscriber arriving during a read into it — and not one
+// read per client. Making the delta path work across nodes means an **entity**-keyed window every
+// node fills from the change stream it already subscribes to, which is a `ResumeSource` shape
+// change, not a placement change.
 
 import type { ResumeSource } from './cursor';
 import type { RowPatch } from './json';
 
 export interface ChangeBufferOptions {
-  /** Retained patches per query hash. */
+  /** Retained patches per query hash — a REPLAY bound: what a delta resume may cost to fold. */
   readonly capacity?: number;
   /** Retained query hashes; the least-recently-written is dropped first. */
   readonly maxQueries?: number;
+  /** Retained bytes per query hash. The memory bound, and the one that actually holds. */
+  readonly maxBytesPerQuery?: number;
+  /** Retained bytes across every query on this node. */
+  readonly maxBytes?: number;
 }
 
+/**
+ * The node's retained-patch memory ceiling. `packages/cache/src/lru.ts:1-2` states the rule this
+ * exists to obey: bounded by BYTES, never by entry count — 4,096 queries x 1,024 patches is 4.19M
+ * retained `RowPatch` objects, each holding a whole row, and nothing in that product is memory.
+ */
+export const DEFAULT_MAX_BUFFER_BYTES = 64 * 1024 * 1024;
+export const DEFAULT_MAX_BUFFER_BYTES_PER_QUERY = 1024 * 1024;
+
 interface Ring {
   patches: RowPatch[];
+  bytes: number;
   /** Highest lsn already dropped. A cursor at or after this is still resumable. */
   evictedThrough: string | null;
 }
 
+const encoder = new TextEncoder();
+
+/** What one retained patch costs. Its serialised size: the row is the whole of it. */
+function patchBytes(patch: RowPatch): number {
+  return encoder.encode(JSON.stringify(patch)).length;
+}
+
 export class RingChangeBuffer implements ResumeSource {
   readonly #rings = new Map<string, Ring>();
   readonly #capacity: number;
   readonly #maxQueries: number;
+  readonly #maxBytesPerQuery: number;
+  readonly #maxBytes: number;
+  #bytes = 0;
 
   constructor(options: ChangeBufferOptions = {}) {
     this.#capacity = options.capacity ?? 1024;
     this.#maxQueries = options.maxQueries ?? 4096;
+    this.#maxBytesPerQuery = options.maxBytesPerQuery ?? DEFAULT_MAX_BUFFER_BYTES_PER_QUERY;
+    this.#maxBytes = options.maxBytes ?? DEFAULT_MAX_BUFFER_BYTES;
+  }
+
+  /** Retained bytes across every query on this node. The number the ceiling is about. */
+  get bytes(): number {
+    return this.#bytes;
   }
 
   append(qid: string, patch: RowPatch): void {
     const existing = this.#rings.get(qid);
-    const ring: Ring = existing ?? { patches: [], evictedThrough: null };
+    const ring: Ring = existing ?? { patches: [], bytes: 0, evictedThrough: null };
     ring.patches.push(patch);
-    while (ring.patches.length > this.#capacity) {
-      const dropped = ring.patches.shift();
-      if (dropped) ring.evictedThrough = dropped.lsn;
+    const cost = patchBytes(patch);
+    ring.bytes += cost;
+    this.#bytes += cost;
+    // Two ceilings, because they bound two different things: the count bounds what a resume has
+    // to fold, the bytes bound what this process holds. Whichever bites first, bites.
+    while (ring.patches.length > this.#capacity || ring.bytes > this.#maxBytesPerQuery) {
+      if (!this.#shift(ring)) break;
     }
     // Re-insert to move this qid to the tail of the LRU order.
     if (existing) this.#rings.delete(qid);
     this.#rings.set(qid, ring);
-    if (this.#rings.size > this.#maxQueries) {
+    while (this.#rings.size > this.#maxQueries || this.#bytes > this.#maxBytes) {
       const oldest = this.#rings.keys().next();
-      if (!oldest.done) this.#rings.delete(oldest.value);
+      // The only ring left is the one just written: evicting it would make a node under memory
+      // pressure retain nothing at all, and every reconnect a snapshot.
+      if (oldest.done || this.#rings.size === 1) break;
+      this.forget(oldest.value);
     }
   }
 
+  /** Drop the oldest patch of a ring, keeping both byte counters honest. Answers what it did. */
+  #shift(ring: Ring): boolean {
+    const dropped = ring.patches.shift();
+    if (!dropped) return false;
+    const cost = patchBytes(dropped);
+    ring.bytes -= cost;
+    this.#bytes -= cost;
+    ring.evictedThrough = dropped.lsn;
+    return true;
+  }
+
   since(qid: string, lsn: string): RowPatch[] | null {
     const ring = this.#rings.get(qid);
     if (!ring) return null;
@@ -58,8 +117,15 @@ export class RingChangeBuffer implements ResumeSource {
     return last ? last.lsn : null;
   }
 
-  /** Called when the last subscriber of a query goes away, so an idle query stops costing memory. */
+  /**
+   * Called when the last subscriber of a query goes away, so an idle query stops costing memory.
+   * It had no caller until `LiveQueryRegistry.unsubscribe` gained one: the entry was dropped and
+   * the ring behind it kept every patch it held until the LRU happened to reach it.
+   */
   forget(qid: string): void {
+    const ring = this.#rings.get(qid);
+    if (!ring) return;
+    this.#bytes -= ring.bytes;
     this.#rings.delete(qid);
   }
 

package/src/channel.ts

@@ -4,9 +4,14 @@
 // append-only stream, so tier 1 needs no frame of its own. That is why climbing the ladder is a
 // config change: the client's frame handler is the same code at every rung.
 
-import type { Actor } from '@ultimat3/core';
+import { type Actor, logger, renderThrowable } from '@ultimat3/core';
 import { formatLsn } from './changefeed';
-import { SubscriptionLimitError, TopicForbiddenError } from './errors';
+import {
+  isPolicyDenial,
+  SubscriptionLimitError,
+  TopicForbiddenError,
+  TransportUnavailableError,
+} from './errors';
 import { subjectMatches, type Transport, type TransportSubscription } from './fanout';
 import type { JsonObject } from './json';
 import type { SocketRegistry, SyncSocket } from './socket';
@@ -46,6 +51,30 @@ export interface ChannelHubOptions {
   readonly transport: Transport;
   readonly sockets: SocketRegistry;
   readonly maxTopicsPerSocket?: number;
+  /**
+   * Distinct topics this node will bridge at once. Each one is a live transport subscription, and
+   * `topic()` admits any `[A-Za-z0-9_-]+` segment — so even a guard as tight as `org.<myorg>.>`
+   * admits unbounded distinct names inside one tenant, and a per-socket cap bounds nothing.
+   */
+  readonly maxTopicsPerNode?: number;
+}
+
+/** Distinct topics one node bridges before `X_SUBSCRIPTION_LIMIT`. */
+export const DEFAULT_MAX_TOPICS_PER_NODE = 10_000;
+
+/**
+ * One topic's fanout into this node. `sub` is the transport subscription as a PROMISE, published
+ * into the table before it is awaited: looked up before the await and written after it, two sockets
+ * reaching one topic at once opened two transport subscriptions — the second replacing the first in
+ * the table, and the first then unreachable by `#release`, by a socket dying, by `close()` or by
+ * anything else, delivering every message on that topic a second time for the life of the process.
+ *
+ * `null` means the slot is taken and nothing is open yet: the node cap is decided before the guard
+ * runs, so the reservation has to exist before there is anything to reserve it with.
+ */
+interface Bridge {
+  sub: Promise<TransportSubscription> | null;
+  refs: number;
 }
 
 /**
@@ -56,14 +85,43 @@ export class ChannelHub {
   readonly #transport: Transport;
   readonly #sockets: SocketRegistry;
   readonly #guards: Array<{ pattern: string; guard: TopicGuard }> = [];
-  readonly #bridges = new Map<string, { sub: TransportSubscription; refs: number }>();
+  readonly #bridges = new Map<string, Bridge>();
+  /**
+   * Topics this socket has asked for and not yet joined. Weakly keyed, so it needs no teardown
+   * path of its own: a socket that dies mid-subscribe takes its claims with it.
+   */
+  readonly #claimed = new WeakMap<SyncSocket, number>();
   readonly #maxTopicsPerSocket: number;
+  readonly #maxTopicsPerNode: number;
+  #guardFailures = 0;
   #sequence = 0n;
+  /** Set by `close()`. Read by `#open`, which is the only thing that can reach a late subscription. */
+  #closed = false;
 
   constructor(options: ChannelHubOptions) {
     this.#transport = options.transport;
     this.#sockets = options.sockets;
     this.#maxTopicsPerSocket = options.maxTopicsPerSocket ?? 64;
+    this.#maxTopicsPerNode = options.maxTopicsPerNode ?? DEFAULT_MAX_TOPICS_PER_NODE;
+  }
+
+  /** Sockets this node will deliver `name` to. The metric the fanout reads. */
+  subscriberCount(name: Topic): number {
+    return this.#sockets.subscriberCount(name);
+  }
+
+  /** Distinct topics bridged from this node — one live transport subscription each. */
+  get topicCount(): number {
+    return this.#bridges.size;
+  }
+
+  /**
+   * `channel.guard_failed` for this node: guards that raised instead of deciding, during a re-auth.
+   * Never a denial — the same split `LiveQueryRegistry.reauthorize` makes one layer up, and an
+   * alert fires on one of them.
+   */
+  get guardFailures(): number {
+    return this.#guardFailures;
   }
 
   /** `pattern` uses NATS wildcards: `org.*.cursors`, `org.>`. First registered match wins. */
@@ -72,36 +130,90 @@ export class ChannelHub {
     return this;
   }
 
+  /**
+   * Both caps and the node's bridge slot are taken SYNCHRONOUSLY, before the guard is awaited: read
+   * at the top and acted on after two awaits, one WebSocket write carrying N subscribe frames
+   * passed each of them N times, and `maxTopicsPerSocket`/`maxTopicsPerNode` bounded nothing.
+   */
   async subscribe(socket: SyncSocket, name: Topic): Promise<void> {
     if (socket.topics.has(name)) return;
-    if (socket.topics.size >= this.#maxTopicsPerSocket) {
+    const claimed = this.#claimed.get(socket) ?? 0;
+    if (socket.topics.size + claimed >= this.#maxTopicsPerSocket) {
       throw new SubscriptionLimitError({
         scope: 'socket',
         id: socket.id,
         limit: this.#maxTopicsPerSocket,
+        // Named, never defaulted: the default for this scope is `maxPerSocket`, which is
+        // `LiveQueryRegistry`'s cap on live subscriptions — a different ceiling in a different
+        // constructor, so an operator following this fix line would have moved the wrong number.
+        knob: 'maxTopicsPerSocket',
       });
     }
-    await this.#authorize(socket.actor, name);
-    await this.#bridge(name);
-    socket.subscribeTopic(name);
+    // Refused before the guard runs and before a transport subscription is opened: a node that is
+    // out of topics has nothing to decide, and the answer must not depend on who asked.
+    const bridge = this.#reserve(name);
+    this.#claimed.set(socket, claimed + 1);
+    try {
+      await this.#authorize(socket.actor, name);
+      await this.#open(name, bridge);
+    } catch (error) {
+      // The slot this subscribe took, given back on the one path that will never fill it — and
+      // given back to the bridge this subscribe actually reserved. `close()` clears the table, so
+      // a later subscribe may have put a DIFFERENT bridge under this name in the meantime, and
+      // decrementing that one's refs releases a topic somebody else is holding.
+      this.#release(name, bridge);
+      throw error;
+    } finally {
+      const held = this.#claimed.get(socket) ?? 1;
+      if (held <= 1) this.#claimed.delete(socket);
+      else this.#claimed.set(socket, held - 1);
+    }
+    // A concurrent subscribe for this same socket and topic got there first: it holds the one
+    // membership this socket's close will give back, so the reference taken above has to go now or
+    // it is a bridge nothing will ever release.
+    if (socket.topics.has(name)) {
+      this.#release(name, bridge);
+      return;
+    }
+    // Through the registry, never `socket.subscribeTopic` directly: membership and the index the
+    // fanout reads are one fact, and two call sites for one fact is the drift that makes an index
+    // wrong. The registry owns it because it is the only thing that sees a socket die.
+    this.#sockets.joinTopic(socket, name);
   }
 
   unsubscribe(socket: SyncSocket, name: Topic): void {
     if (!socket.topics.has(name)) return;
-    socket.unsubscribeTopic(name);
+    this.#sockets.leaveTopic(socket, name);
     this.#release(name);
   }
 
-  /** Called when a socket's session changes (login, logout, role change, token refresh). */
+  /**
+   * Called when a socket's session changes (login, logout, role change, token refresh).
+   *
+   * A denial drops the topic; anything else keeps it. A guard is app code and may reach a database,
+   * so `catch { unsubscribe }` reported a store that timed out as a revoked grant — during one
+   * outage, every topic on every re-authenticated socket on the node, silently, with the client
+   * never told to resubscribe. The same split `LiveQueryRegistry.reauthorize` already makes, and
+   * for the same reason: a failure is not a decision.
+   */
   async onActorChange(socket: SyncSocket, actor: Actor | null): Promise<readonly Topic[]> {
     socket.actor = actor;
     const dropped: Topic[] = [];
     for (const name of [...socket.topics] as Topic[]) {
       try {
         await this.#authorize(actor, name);
-      } catch {
-        this.unsubscribe(socket, name);
-        dropped.push(name);
+      } catch (error) {
+        if (isPolicyDenial(error) || error instanceof TopicForbiddenError) {
+          this.unsubscribe(socket, name);
+          dropped.push(name);
+          continue;
+        }
+        this.#guardFailures += 1;
+        logger.warn('channel.guard_failed', {
+          topic: name,
+          socketId: socket.id,
+          error: renderThrowable(error),
+        });
       }
     }
     return dropped;
@@ -120,7 +232,14 @@ export class ChannelHub {
   }
 
   async close(): Promise<void> {
-    for (const bridge of this.#bridges.values()) bridge.sub.unsubscribe();
+    // Set BEFORE the table is walked, because the table is not the whole story: a reservation an
+    // in-flight `subscribe` has not opened yet is `sub === null`, so `unsubscribeWhenOpen` does
+    // nothing to it and `clear()` drops the entry. That open then lands on a `Bridge` nothing can
+    // name — `#release` looks the topic up, misses and returns — and its handler keeps calling
+    // `deliver` for the life of the process. The same orphan the `Bridge` comment describes, one
+    // state earlier, so the open that creates the subscription has to be the thing that closes it.
+    this.#closed = true;
+    for (const bridge of this.#bridges.values()) unsubscribeWhenOpen(bridge);
     this.#bridges.clear();
   }
 
@@ -147,29 +266,92 @@ export class ChannelHub {
     }
   }
 
-  /** One transport subscription per topic per node, refcounted across sockets. */
-  async #bridge(name: Topic): Promise<void> {
+  /**
+   * The node's slot for this topic, taken synchronously. One bridge per topic per node, refcounted
+   * across sockets — and the refcount includes the subscribes still deciding, so the count the node
+   * cap reads is the count that will exist.
+   */
+  #reserve(name: Topic): Bridge {
     const existing = this.#bridges.get(name);
     if (existing) {
       existing.refs += 1;
-      return;
+      return existing;
     }
-    const sub = await this.#transport.subscribe(`${CHANNEL_SUBJECT_PREFIX}.${name}`, (payload) => {
+    if (this.#bridges.size >= this.#maxTopicsPerNode) {
+      throw new SubscriptionLimitError({
+        scope: 'node',
+        id: 'topics',
+        limit: this.#maxTopicsPerNode,
+        knob: 'maxTopicsPerNode',
+      });
+    }
+    const created: Bridge = { sub: null, refs: 1 };
+    this.#bridges.set(name, created);
+    return created;
+  }
+
+  /** Opens the reserved bridge once, and shares the in-flight open with everyone else waiting. */
+  async #open(name: Topic, bridge: Bridge): Promise<void> {
+    // Published into the bridge before it is awaited: that is what makes a second subscriber join
+    // this open instead of starting a second one the table can never reach again.
+    bridge.sub ??= this.#transport.subscribe(`${CHANNEL_SUBJECT_PREFIX}.${name}`, (payload) => {
       this.#sockets.deliver(name, decode(payload));
     });
-    this.#bridges.set(name, { sub, refs: 1 });
+    await bridge.sub;
+    // The hub shut down while the transport was answering. `close()` either never saw this bridge
+    // or saw it with nothing to close, so this is the last reference to the subscription: it closes
+    // here or never. The entry goes with it, so a second post-close subscribe opens and closes its
+    // own rather than double-unsubscribing this one's handle.
+    if (this.#closed) {
+      unsubscribeWhenOpen(bridge);
+      if (this.#bridges.get(name) === bridge) this.#bridges.delete(name);
+      // RAISED, not returned. Returning let `subscribe` fall through to `joinTopic`, so the socket
+      // became a member of a topic nothing on this node is bridged to: silent for the life of the
+      // connection, with no error on either side and nothing telling the client to redial. The
+      // same refusal the transport itself answers when it is gone, because from the client's side
+      // that is what happened — this node's bus for that topic is closed.
+      throw new TransportUnavailableError({
+        transport: 'channel',
+        reason: `the hub closed while "${name}" was opening`,
+        // The reader is a browser websocket client, which cannot run a CLI — so pure command
+        // advice would be worse than prose here. The shape that satisfies axiom 4 anyway is
+        // `http/src/error-map.ts`'s: a command that SHIPS (`x errors explain`, unlike the planned
+        // `x logs tail`) for whoever is holding a terminal, then the instruction as a comment.
+        fix: 'x errors explain X_TRANSPORT_UNAVAILABLE --json   # then reconnect and resubscribe: this node is draining',
+      });
+    }
   }
 
-  #release(name: Topic): void {
+  /**
+   * `expected` is the bridge the caller reserved. Without it a release looks the topic up by name,
+   * and after a `close()` cleared the table that name may hold a bridge a LATER subscribe opened.
+   */
+  #release(name: Topic, expected?: Bridge): void {
     const bridge = this.#bridges.get(name);
     if (!bridge) return;
+    if (expected !== undefined && bridge !== expected) return;
     bridge.refs -= 1;
     if (bridge.refs > 0) return;
-    bridge.sub.unsubscribe();
+    unsubscribeWhenOpen(bridge);
     this.#bridges.delete(name);
   }
 }
 
+/**
+ * A bridge released while its subscription is still opening still has to be closed — the transport
+ * hands the handle back after the caller has gone, and dropping the promise would leave a live
+ * subscription this node can no longer name. An open that failed has nothing to unsubscribe and its
+ * rejection was already answered to the subscriber that caused it.
+ */
+function unsubscribeWhenOpen(bridge: Bridge): void {
+  void bridge.sub?.then(
+    (sub) => {
+      sub.unsubscribe();
+    },
+    () => undefined,
+  );
+}
+
 export function channelFrame(name: Topic, lsn: string, message: JsonObject): Frame {
   return {
     type: 'patch',

package/src/client-contract.ts

@@ -0,0 +1,81 @@
+// What a client IS, as types: the injected seams, the options, and the handles a subscription
+// gives back. Declared apart from the client that implements them for the same reason
+// `live-contract.ts` is — the hooks, the typed projection, the type pins and the mutation path all
+// need these shapes, and none of them needs the connection lifecycle that runs underneath.
+
+import type { Clock } from '@ultimat3/core';
+import type { LiveCursor } from './cursor';
+import type { JsonValue, Row } from './json';
+import type { LiveState } from './live-rows';
+import type { LocalStore, LocalTx, TableMap } from './local-store';
+import type { OfflineQueue } from './offline-queue';
+import type { ConflictStrategy, RebaseLog } from './rebase';
+import type { BackoffPolicy, Rng, Scheduler } from './thundering-herd';
+
+/** Injected reactive primitive. `createSignal` from Solid satisfies this exactly. */
+export type SignalFactory = <T>(initial: T) => [get: () => T, set: (next: T) => void];
+
+/** Injected socket, so tests drive the protocol without a network. */
+export interface ClientSocket {
+  send(data: string): void;
+  close(code?: number, reason?: string): void;
+  onOpen(handler: () => void): void;
+  onMessage(handler: (data: string) => void): void;
+  onClose(handler: (code: number) => void): void;
+  /**
+   * Bytes queued but not yet on the wire — `WebSocket.bufferedAmount`. Optional because a socket
+   * that cannot answer is treated as never backed up; supplying it is what lets the mutation drain
+   * stop instead of pushing a queue the tab is not draining into one it cannot see.
+   */
+  readonly bufferedAmount?: number;
+}
+
+export interface LiveHandle<R extends Row = Row> extends Disposable {
+  /** The reactive accessor. In an app this is the Solid signal `useLive` returns. */
+  readonly rows: () => readonly R[];
+  readonly state: () => LiveState;
+  readonly cursor: () => LiveCursor | null;
+  unsubscribe(): void;
+  /** The same call as `unsubscribe`, so `using sub = client.useLive(...)` just works. */
+  [Symbol.dispose](): void;
+}
+
+/** What `subscribe()` returns for a tier-1 topic: callable to unsubscribe, and `using`-able too. */
+export type Unsubscribe = (() => void) & Disposable;
+
+export interface LiveQueryRef {
+  readonly name: string;
+}
+
+export interface MutatorRef<T extends TableMap = TableMap> {
+  readonly name: string;
+  /** Optimistic twin. Pure — no I/O, no Date.now(), no Math.random(). */
+  local?: (tx: LocalTx<T>, input: JsonValue) => void;
+  readonly entity?: string;
+  readonly conflict?: ConflictStrategy;
+}
+
+export interface LiveClientOptions<T extends TableMap = TableMap> {
+  readonly signal: SignalFactory;
+  /** Called for every connect attempt; returning a fresh socket keeps reconnect logic here. */
+  readonly connect: () => ClientSocket;
+  readonly buildId: string;
+  readonly actorId?: string | null;
+  /** Tier 3 only. Without these, mutations are server-only and nothing is queued offline. */
+  readonly store?: LocalStore<T>;
+  readonly queue?: OfflineQueue;
+  readonly log?: RebaseLog<T>;
+  readonly backoff?: BackoffPolicy;
+  readonly rng?: Rng;
+  readonly clock?: Clock;
+  /** How a pending reconnect is armed. Defaults to `setTimeout`; tests fire theirs by hand. */
+  readonly scheduler?: Scheduler;
+  /**
+   * How often a live socket re-announces itself, in ms. `0` disables it. Defaults to
+   * `DEFAULT_HEARTBEAT_MS` — the same 15s as the server's `realtime.heartbeatMs`, which browser
+   * code cannot read, so it is an option here rather than a value shipped down the wire.
+   */
+  readonly heartbeatMs?: number;
+  /** Where a dial failure inside the reconnect timer is reported. Defaults to `reportToConsole`. */
+  readonly onError?: (error: unknown) => void;
+}