@ultimat3/realtime 1.2.0 → 3.0.0

package/src/socket.ts

@@ -1,11 +1,22 @@
 // One WS connection. Bun's WS profile is what makes a million sockets affordable, so this object
-// is deliberately lean: ~8 fields plus two small sets. Budget is ~1KB of JS heap per connection on
-// top of Bun's own per-socket cost — anything richer (row caches, per-socket buffers) belongs in
-// the transport or the change buffer, never here. `sync` is stateless: nothing on this object
-// survives a restart, and nothing needs to.
+// is deliberately lean: ~8 fields, two small sets and one token bucket (the frame budget, which is
+// five numbers and the only thing standing between one authenticated socket and this node's whole
+// subscribe path). Budget is ~1KB of JS heap per connection on top of Bun's own per-socket cost —
+// anything richer (row caches, per-socket buffers) belongs in the transport or the change buffer,
+// never here. `sync` is stateless: nothing on this object survives a restart, and nothing needs to.
 
-import { type Actor, type Clock, recordConnection, systemClock, uuid } from '@ultimat3/core';
+import {
+  type Actor,
+  type Clock,
+  type Counter,
+  counter,
+  logger,
+  recordConnection,
+  systemClock,
+  uuid,
+} from '@ultimat3/core';
 import { encode, type Frame } from './sync-protocol';
+import { AcceptBudget } from './thundering-herd';
 
 export const CLOSE = {
   normal: 1000,
@@ -17,7 +28,16 @@ export const CLOSE = {
   drain: 4002,
 } as const;
 
-/** The slice of Bun's `ServerWebSocket` this package uses. Structural, so tests need no server. */
+/**
+ * The slice of Bun's `ServerWebSocket` this package uses. Structural, so tests need no server.
+ *
+ * `subscribe`/`unsubscribe` are Bun's native pub/sub and this package does NOT use them: nothing
+ * here publishes to a native topic, and nothing will — a native publish cannot be refused per
+ * socket, cannot report the frame it dropped and cannot mark a subscriber desynced, which is
+ * exactly what `SocketRegistry.deliver` and `SyncSocket.send` exist to do. They are still declared
+ * because `WsLike` is the slice of Bun's own object, and an app already implements it; deleting
+ * them is a separate, breaking edit to every implementer.
+ */
 export interface WsLike {
   send(data: string): number;
   close(code?: number, reason?: string): void;
@@ -37,8 +57,47 @@ export interface SyncSocketOptions {
   /** Frames are dropped rather than queued past this. See `desynced`. */
   readonly maxBufferedBytes?: number;
   readonly maxDroppedFrames?: number;
+  /** Sustained inbound frames this socket may have routed per second. See `frameBudget`. */
+  readonly maxFramesPerSecond?: number;
+  /** Burst allowance, so a client subscribing its whole cap at connect is never refused. */
+  readonly frameBurst?: number;
 }
 
+/**
+ * What one socket may ask this node to do per second, and how much of it may arrive at once.
+ *
+ * The burst clears `DEFAULT_MAX_PER_SOCKET` (128) plus a `hello`, because that is exactly what a
+ * legitimate client sends on connect; the sustained rate is well under the ~155 frames/s measured
+ * to consume a node through the subscribe path's amplifiers.
+ */
+export const DEFAULT_MAX_FRAMES_PER_SECOND = 64;
+export const DEFAULT_FRAME_BURST = 256;
+/**
+ * Queued-but-unwritten bytes on one server socket before `send` declines and marks the subscriber
+ * desynced. `sync-node.ts` hands the same number to Bun as `backpressureLimit` rather than spelling
+ * it again: they are one socket's one buffer, and a check the runtime's own limit fires before is a
+ * check that never runs. The client half (`client-mutations.ts`) is deliberately its own constant —
+ * it is browser code, and importing this file would pull the node's socket registry into the tab.
+ */
+export const DEFAULT_MAX_BUFFERED_BYTES = 1024 * 1024;
+
+/**
+ * Channel frames this process dropped under backpressure. A DATA-LOSS counter, not a saturation
+ * one: the live-query path repairs a dropped patch (the subscriber is marked desynced and the next
+ * change re-snapshots it), and a channel has no cursor, no mark and no re-snapshot — so this is the
+ * only trace a lost channel message leaves anywhere.
+ *
+ * Declared here rather than in `@ultimat3/core`'s `runtime-metrics.ts` because that file is the
+ * series EVERY Ultimate process emits and the deploy chart scales on; this one exists only where
+ * channels do. **No attributes**: a topic is client-chosen (`topic()` admits any
+ * `[A-Za-z0-9_-]+` segment), so a per-topic label is an unbounded series count one socket can mint
+ * — the topic goes in the log line, where cardinality is somebody else's index.
+ */
+const channelFramesDropped: Counter = counter('channel_frames_dropped_total', {
+  unit: '{frame}',
+  description: 'Channel frames dropped by socket backpressure — unrecoverable, nothing replays one',
+});
+
 export function actorIdOf(actor: Actor | null): string | null {
   return actor === null ? null : actor.id;
 }
@@ -58,9 +117,22 @@ export class SyncSocket {
    * flush re-snapshots instead of silently diverging.
    */
   readonly desynced = new Set<string>();
+  /**
+   * Inbound frames this socket may still have routed. The accept budget spends one token per
+   * UPGRADE, so nothing bounded what happened after: one authenticated socket reached a DB read,
+   * a shared-store presence write and a fleet-wide publish once per frame, unbounded and
+   * unawaited. Same token bucket, one per socket — the mechanism already existed, one rung down.
+   */
+  readonly frameBudget: AcceptBudget;
 
   actor: Actor | null;
-  lastSeenAt: number;
+  /**
+   * MONOTONIC milliseconds, not an instant — the idle sweep measures a duration, and a duration
+   * read off the wall clock is decided by whatever NTP last wrote. A step forward evicts sockets
+   * that are talking; a step backward makes `idleFor` negative and spares sockets that are dead.
+   * Named for the units so nobody hands it to `new Date()`; `openedAt` is the wall-clock one.
+   */
+  lastSeenMonotonicMs: number;
   droppedFrames = 0;
   sentFrames = 0;
 
@@ -77,10 +149,17 @@ export class SyncSocket {
     this.clientBuildId = options.clientBuildId;
     this.serverBuildId = options.serverBuildId;
     this.actor = options.actor ?? null;
-    this.#maxBufferedBytes = options.maxBufferedBytes ?? 1024 * 1024;
+    this.#maxBufferedBytes = options.maxBufferedBytes ?? DEFAULT_MAX_BUFFERED_BYTES;
     this.#maxDroppedFrames = options.maxDroppedFrames ?? 32;
+    this.frameBudget = new AcceptBudget({
+      perSecond: options.maxFramesPerSecond ?? DEFAULT_MAX_FRAMES_PER_SECOND,
+      burst: options.frameBurst ?? DEFAULT_FRAME_BURST,
+      clock: this.#clock,
+    });
+    // Two clocks on purpose: `openedAt` is an instant a human reads, `lastSeenMonotonicMs` is the
+    // start of a duration only this process compares.
     this.openedAt = this.#clock.now().getTime();
-    this.lastSeenAt = this.openedAt;
+    this.lastSeenMonotonicMs = this.#clock.monotonic();
   }
 
   get actorId(): string | null {
@@ -120,22 +199,27 @@ export class SyncSocket {
     this.desynced.delete(sid);
   }
 
+  /**
+   * This socket's own membership, and nothing else. It used to also call Bun's `ws.subscribe`,
+   * which built a second per-topic index nothing ever published to — the fanout is
+   * `SocketRegistry.deliver`, one filtered `send` per socket, because that is the only path that
+   * can count a dropped frame or close a socket that is drowning in them.
+   */
   subscribeTopic(topic: string): void {
     this.topics.add(topic);
-    this.#ws.subscribe(topic);
   }
 
   unsubscribeTopic(topic: string): void {
     this.topics.delete(topic);
-    this.#ws.unsubscribe(topic);
   }
 
   touch(): void {
-    this.lastSeenAt = this.#clock.now().getTime();
+    this.lastSeenMonotonicMs = this.#clock.monotonic();
   }
 
-  idleFor(now: number): number {
-    return now - this.lastSeenAt;
+  /** `nowMonotonicMs` comes from the SAME clock — `Clock.monotonic()`, never `now().getTime()`. */
+  idleFor(nowMonotonicMs: number): number {
+    return nowMonotonicMs - this.lastSeenMonotonicMs;
   }
 
   close(code: number = CLOSE.normal, reason = ''): void {
@@ -145,21 +229,50 @@ export class SyncSocket {
   }
 }
 
+/**
+ * How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
+ * inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
+ * ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
+ * its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
+ * every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
+ */
+export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
+
+/**
+ * How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
+ * of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
+ * four times per window rather than once a second. Derived rather than configured — a second knob
+ * is a second number that can disagree with the one it is a fraction of.
+ */
+export function idleSweepPeriodMs(idleTimeoutMs: number): number {
+  return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
+}
+
 export interface SocketRegistryOptions {
   readonly clock?: Clock;
-  /** Bun also enforces its own `idleTimeout`; this sweep catches half-open connections. */
+  /** Bun's own `idleTimeout` is renewed by its ping/pong; this budget counts routed FRAMES. */
   readonly idleTimeoutMs?: number;
 }
 
 /** Per-node socket table. Intentionally the only in-memory map on a `sync` node. */
 export class SocketRegistry {
   readonly #sockets = new Map<string, SyncSocket>();
+  /**
+   * Who is on each channel topic. `deliver` walked every socket on the node asking each whether
+   * it held the topic, so one message with one legitimate subscriber cost as many iterations as
+   * this node has connections — 50,000 at the scale this framework benchmarks. It lives here
+   * rather than on the hub because this is the only object that sees a socket die: a close, a
+   * drain and the idle sweep all pass through `remove`, and an index nobody cleans on those paths
+   * retains a dead socket per topic forever.
+   */
+  readonly #byTopic = new Map<string, Set<SyncSocket>>();
   readonly #clock: Clock;
   readonly #idleTimeoutMs: number;
+  #droppedChannelFrames = 0;
 
   constructor(options: SocketRegistryOptions = {}) {
     this.#clock = options.clock ?? systemClock;
-    this.#idleTimeoutMs = options.idleTimeoutMs ?? 120_000;
+    this.#idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
   }
 
   /**
@@ -178,8 +291,39 @@ export class SocketRegistry {
   }
 
   remove(id: string): void {
+    const socket = this.#sockets.get(id);
     // `Map.delete` answers "was it actually there", so a double close cannot decrement twice.
-    if (this.#sockets.delete(id)) recordConnection(-1);
+    if (!this.#sockets.delete(id)) return;
+    recordConnection(-1);
+    if (!socket) return;
+    // Leaving this table IS the close, whoever noticed first. Bun's `close` callback reports a
+    // connection that has already gone, so nothing called `close()` on this object and
+    // `socket.closed` stayed false — leaving a subscribe still awaiting its snapshot read with no
+    // way to tell that the socket it is about to attach to was torn down while it read.
+    socket.close(CLOSE.goingAway, 'connection closed');
+    for (const name of socket.topics) this.#dropFrom(name, socket);
+  }
+
+  /**
+   * Join a topic: the socket's own set, Bun's native pub/sub and this node's delivery index, in
+   * one call. `ChannelHub` is its only caller, and calls nothing else — two call sites for one
+   * membership is how an index goes wrong.
+   */
+  joinTopic(socket: SyncSocket, topic: string): void {
+    socket.subscribeTopic(topic);
+    const members = this.#byTopic.get(topic);
+    if (members) members.add(socket);
+    else this.#byTopic.set(topic, new Set([socket]));
+  }
+
+  leaveTopic(socket: SyncSocket, topic: string): void {
+    socket.unsubscribeTopic(topic);
+    this.#dropFrom(topic, socket);
+  }
+
+  /** Sockets this node would deliver `topic` to. */
+  subscriberCount(topic: string): number {
+    return this.#byTopic.get(topic)?.size ?? 0;
   }
 
   get(id: string): SyncSocket | undefined {
@@ -194,28 +338,75 @@ export class SocketRegistry {
     return this.#sockets.size;
   }
 
-  /** Closes and returns everything past the idle budget. Called on an interval by `sync-node`. */
-  sweepIdle(): SyncSocket[] {
-    const now = this.#clock.now().getTime();
-    const closed: SyncSocket[] = [];
-    for (const socket of this.#sockets.values()) {
-      if (socket.idleFor(now) > this.#idleTimeoutMs) {
-        socket.close(CLOSE.idle, 'idle timeout');
-        // Through `remove`, not the map: the sweep is exactly the abnormal close a gauge leaks on.
-        this.remove(socket.id);
-        closed.push(socket);
-      }
-    }
-    return closed;
+  /** The budget `idle()` answers against, so a caller can size its own sweep from one number. */
+  get idleTimeoutMs(): number {
+    return this.#idleTimeoutMs;
   }
 
-  /** Local delivery for a channel topic. Bun's native pub/sub handles the common case; this is
-   *  the fallback used when a frame must be filtered per socket (policy, desync bookkeeping). */
+  /**
+   * Everything past the idle budget. A QUERY, and deliberately not an eviction: this table is three
+   * of the five things a socket holds, and the other two — its live subscriptions and its presence
+   * membership on the SHARED set — are only reachable from `sync-node`'s `teardown`. A sweep that
+   * closed and `remove`d here left a member every other node renders until its TTL and a
+   * `QueryEntry` whose `subscribers` map never empties. `sync-node` is the one caller and it
+   * releases each one the way the close callback does.
+   */
+  idle(): SyncSocket[] {
+    // Monotonic, because this is the one comparison an operator's clock could otherwise decide:
+    // `sync-node` hands this registry and every socket it builds the same `Clock`.
+    const now = this.#clock.monotonic();
+    return [...this.#sockets.values()].filter(
+      (socket) => socket.idleFor(now) > this.#idleTimeoutMs,
+    );
+  }
+
+  /**
+   * Local delivery for a channel topic — every channel message on this node comes through here,
+   * so it reads the per-topic index rather than the socket table. A socket that closed without
+   * an unsubscribe is dropped as it is met, so the index cannot outlive the connection even on a
+   * path that forgot to `remove` it.
+   */
   deliver(topic: string, frame: Frame): number {
+    const members = this.#byTopic.get(topic);
+    if (!members) return 0;
     let sent = 0;
-    for (const socket of this.#sockets.values()) {
-      if (socket.topics.has(topic) && socket.send(frame)) sent += 1;
+    let dropped = 0;
+    for (const socket of members) {
+      if (socket.closed) {
+        members.delete(socket);
+        continue;
+      }
+      if (socket.send(frame)) sent += 1;
+      else dropped += 1;
+    }
+    if (members.size === 0) this.#byTopic.delete(topic);
+    if (dropped > 0) {
+      this.#droppedChannelFrames += dropped;
+      // Two readers, one event, one spelling: the series an operator alerts on and the line that
+      // says which topic it was. `deliver` ignored `send`'s answer and so did the hub above it, so
+      // until both existed a lost channel message left no trace at all.
+      channelFramesDropped.add(dropped);
+      logger.warn('channel.frames_dropped', { topic, dropped, total: this.#droppedChannelFrames });
     }
     return sent;
   }
+
+  /**
+   * Channel frames backpressure refused since boot, node-wide and cumulative — the in-process read
+   * of `channel_frames_dropped_total`, for a test or a benchmark that cannot scrape.
+   *
+   * Node-wide on purpose: a socket past `maxDroppedFrames` is closed and removed, so a per-socket
+   * count leaves with the socket exactly when loss is worst. Distinct from `SyncSocket.droppedFrames`,
+   * which counts every kind of frame one connection lost, channel and live-query patch alike.
+   */
+  get droppedChannelFrames(): number {
+    return this.#droppedChannelFrames;
+  }
+
+  #dropFrom(topic: string, socket: SyncSocket): void {
+    const members = this.#byTopic.get(topic);
+    if (!members) return;
+    members.delete(socket);
+    if (members.size === 0) this.#byTopic.delete(topic);
+  }
 }

package/src/subscriber-gate.ts

@@ -0,0 +1,209 @@
+// The per-subscriber pass of a definition's row policy over the shared window, and the two numbers
+// it produces: rows denied, and gates that could not decide. It evaluates no policy of its own —
+// `policy-gate.ts` is this package's only authz seam — it calls `LiveQueryDefinition.visible` and
+// classifies what comes back, so a denial and a failure never arrive as the same event.
+
+import type { Actor } from '@ultimat3/core';
+import { isPolicyDenial } from './errors';
+import type { JsonValue, Row, RowPatch } from './json';
+import type { LiveQueryDefinition } from './live-contract';
+
+/**
+ * Who a decision is being made for. Every policy call in the live pipeline takes one, which is the
+ * shape of the rule: there is no path through the gate that reads a query id and no actor.
+ */
+export interface Subscriber {
+  readonly sid: string;
+  readonly actor: Actor | null;
+}
+
+/** One row withheld from one subscriber. Carries no row payload: the ids are the whole point. */
+export interface RowDenied {
+  readonly qid: string;
+  readonly sid: string;
+  readonly actorId: string | null;
+  readonly rowId: string;
+}
+
+/** Where a gate was standing when it failed. `authorize` is subscribe-time, the rest are rows. */
+export type GateStage = 'authorize' | 'snapshot' | 'patch';
+
+/**
+ * One gate that raised something other than a denial. `rowId` is absent for `authorize`, which
+ * decides about a subscription rather than a row, and `error` is passed through unwrapped so the
+ * node logs the driver's own message instead of a summary of it.
+ */
+export interface GateFailed {
+  readonly qid: string;
+  readonly sid: string;
+  readonly actorId: string | null;
+  readonly stage: GateStage;
+  readonly rowId?: string;
+  readonly error: unknown;
+}
+
+/**
+ * What the gate needs from a query entry and nothing more: the shared pre-policy window, the input
+ * the rules read, and the definition that owns `visible`. `QueryEntry` satisfies it structurally,
+ * so the registry passes its entry straight through and this file never learns what else is on it.
+ */
+export interface GateTarget {
+  readonly qid: string;
+  readonly input: JsonValue;
+  readonly definition: LiveQueryDefinition;
+  readonly rows: readonly Row[];
+}
+
+export interface SubscriberGateOptions {
+  /**
+   * `live.rows_denied`. A row an actor's policy refuses is dropped, never sent and never turned
+   * into an error — telling a client "there is a row you may not see" is itself the leak. Dropped
+   * silently it is also invisible, so the drop is a metric instead.
+   */
+  readonly onRowDenied?: (event: RowDenied) => void;
+  /**
+   * `live.gate_failed`. The gate raised something that is not a decision, so this subscriber's
+   * result set is unknown rather than empty. Separate from `onRowDenied` on purpose: an alert
+   * fires on this one, and a dashboard that summed them would show a permission change.
+   */
+  readonly onGateFailed?: (event: GateFailed) => void;
+}
+
+/** Both counters and the one call that classifies a throw. Owned per registry, never per query. */
+export class SubscriberGate {
+  readonly #options: SubscriberGateOptions;
+  #rowsDenied = 0;
+  #gateFailures = 0;
+
+  constructor(options: SubscriberGateOptions) {
+    this.#options = options;
+  }
+
+  /** Rows a subscriber's policy refused since boot. */
+  get rowsDenied(): number {
+    return this.#rowsDenied;
+  }
+
+  /** Gates that raised instead of deciding since boot. */
+  get gateFailures(): number {
+    return this.#gateFailures;
+  }
+
+  /**
+   * One snapshot, filtered for one subscriber. A failure raises rather than returning the rows it
+   * managed to admit: a short result set is indistinguishable from a correct one, and handing it
+   * over is the read silently losing rows.
+   */
+  async filterRows(target: GateTarget, who: Subscriber, rows: readonly Row[]): Promise<Row[]> {
+    const out: Row[] = [];
+    for (const row of rows) {
+      if (await this.#visible(target, who, row, 'snapshot')) out.push(row);
+      else this.#denied(target.qid, who, row.id);
+    }
+    return out;
+  }
+
+  /**
+   * Row-level authz over a patch list. A row that becomes invisible is converted to a `delete`
+   * when the subscriber holds it — otherwise a revoked grant would leave a stale row on screen
+   * forever.
+   */
+  async filterPatches(
+    target: GateTarget,
+    who: Subscriber,
+    patches: readonly RowPatch[],
+    held: ReadonlySet<string>,
+  ): Promise<RowPatch[]> {
+    const out: RowPatch[] = [];
+    for (const patch of patches) {
+      const allowed = await this.patch(target, who, patch, held.has(patch.id));
+      if (allowed !== null) out.push(allowed);
+    }
+    return out;
+  }
+
+  /** One patch, one decision. `holds` is whether this subscriber already has the row on screen. */
+  async patch(
+    target: GateTarget,
+    who: Subscriber,
+    patch: RowPatch,
+    holds: boolean,
+  ): Promise<RowPatch | null> {
+    if (patch.op === 'delete' || patch.row === null) return patch;
+    const full = target.rows.find((row) => row.id === patch.id);
+    // No whole row means no decision to take. An update patch carries the changed columns only, so
+    // a rule reading `row.ownerId` on one reads `undefined` and answers as if the row had said so —
+    // fail-closed for `=== actor.id`, and a leak for every `!row.private`. It is not a gate that
+    // failed either: the shared window *is* the result set, so a row it does not hold is a row this
+    // subscriber is not entitled to keep, and one that holds it is told so.
+    if (full === undefined) return holds ? withdrawn(patch) : null;
+    const row: Row = { ...full, ...patch.row, id: patch.id };
+    if (await this.#visible(target, who, row, 'patch')) return patch;
+    this.#denied(target.qid, who, patch.id);
+    return holds ? withdrawn(patch) : null;
+  }
+
+  /**
+   * `authorize` failed for a subscription that is being re-decided. Counted and reported here so
+   * every gate failure in the pipeline goes through one counter, whatever the caller then does
+   * with the subscription.
+   */
+  failedAuthorize(qid: string, who: Subscriber, error: unknown): void {
+    this.#failed(qid, who, 'authorize', undefined, error);
+  }
+
+  /** The definition's own predicate. A denial answers `false`; anything else is counted and raised. */
+  async #visible(
+    target: GateTarget,
+    who: Subscriber,
+    row: Row,
+    stage: GateStage,
+  ): Promise<boolean> {
+    try {
+      return await target.definition.visible({ actor: who.actor, row, input: target.input });
+    } catch (error) {
+      if (isPolicyDenial(error)) return false;
+      this.#failed(target.qid, who, stage, row.id, error);
+      throw error;
+    }
+  }
+
+  /** `live.rows_denied`. Counted here and nowhere else, so every drop is one increment. */
+  #denied(qid: string, who: Subscriber, rowId: string): void {
+    this.#rowsDenied += 1;
+    this.#options.onRowDenied?.({ qid, sid: who.sid, actorId: actorIdOf(who), rowId });
+  }
+
+  /** `live.gate_failed`. Same rule: one place counts, so the number and the events agree. */
+  #failed(
+    qid: string,
+    who: Subscriber,
+    stage: GateStage,
+    rowId: string | undefined,
+    error: unknown,
+  ): void {
+    this.#gateFailures += 1;
+    this.#options.onGateFailed?.({
+      qid,
+      sid: who.sid,
+      actorId: actorIdOf(who),
+      stage,
+      ...(rowId === undefined ? {} : { rowId }),
+      error,
+    });
+  }
+}
+
+const actorIdOf = (who: Subscriber): string | null => (who.actor === null ? null : who.actor.id);
+
+/**
+ * The one frame a subscriber gets for a row it may no longer keep, whether a rule refused it or the
+ * window stopped holding it. Written once so the two paths cannot answer differently: a client left
+ * holding the row instead renders a revoked grant until something else reconnects it.
+ */
+const withdrawn = (patch: RowPatch): RowPatch => ({
+  op: 'delete',
+  id: patch.id,
+  row: null,
+  lsn: patch.lsn,
+});