@ultimat3/realtime 23.0.0 → 25.0.0

package/src/sync-node-contract.ts

@@ -1,4 +1,4 @@
-// What a `sync` node IS, as types: the options `createSyncNode` takes, the node it returns and the
+// What a `sync` node IS, as types: the options `syncNode` takes, the node it returns and the
 // socket Bun hands its handlers. Apart from `sync-node.ts` so the lifecycle there stays under the
 // file ceiling, and so a host can name the shapes without reading the lifecycle.
 
@@ -33,7 +33,7 @@ export interface SyncNodeOptions {
   /**
    * When a socket starts dropping frames, and how many drops close it. On `SyncSocket` too, but
    * this node builds every socket it holds — so unforwarded they were reachable only by abandoning
-   * `createSyncNode`, and a dropped channel frame is the one loss nothing replays.
+   * `syncNode`, and a dropped channel frame is the one loss nothing replays.
    */
   readonly maxBufferedBytes?: number;
   readonly maxDroppedFrames?: number;
@@ -54,11 +54,35 @@ export interface SyncNodeOptions {
    */
   readonly authenticate?: SyncAuthenticator;
   /**
-   * Exact origins a page may open a socket from, besides this node's own host name — the page's
-   * origin when it is served on another host (`SYNC_URL` on a separate domain). Anything else is
+   * Exact origins a page may open a socket from — the app's public origin (`APP_URL`). Declared,
+   * they are the WHOLE list; with none, the node admits the origin it was reached on (its `Host`
+   * and port, and that host's https spelling when reached over plain http). Anything else is
    * refused `X_SOCKET_ORIGIN_REFUSED` before `authenticate` runs.
    */
   readonly allowedOrigins?: readonly string[];
+  /**
+   * Also admit the origin the node was reached on BESIDE a declared list. For a loopback dev
+   * server reached through a forwarded host; never a container, where `Host` is the caller's.
+   */
+  readonly admitReachedOrigin?: boolean;
+  /**
+   * The caller's address from the upgrade request — the boot injects `@ultimat3/http`'s
+   * `clientAddress()` under the deployment's `TRUSTED_PROXY_HOPS`. Absent, the socket's own
+   * address. It keys an anonymous subscriber's live rate limit (`UpgradeDeps.clientAddressOf`).
+   */
+  readonly clientAddressOf?: (request: Request, socketAddress: string | null) => string | null;
+  /**
+   * Peers this node's own `/healthz` and `/readyz` tell the whole report to: address classes
+   * (`loopback`, `private`, …) or exact IP literals — the app's `http.healthDetailPeers`. Defaults
+   * to core's `DEFAULT_HEALTH_DETAIL_PEERS` (the box itself); anyone else, and any request that
+   * says it was forwarded, gets `{ state, ready, role }`.
+   */
+  readonly healthDetailPeers?: readonly string[];
+  /**
+   * How long one grant's `refresh()` may hold the re-auth pass before it is reported and skipped
+   * (`DEFAULT_GRANT_REFRESH_DEADLINE_MS`). The grant stays expired and is asked again next pass.
+   */
+  readonly grantRefreshDeadlineMs?: number;
   /** How often an expired grant is re-decided. The clock a socket's authority runs on. */
   readonly reauthenticateIntervalMs?: number;
   readonly clock?: Clock;
@@ -66,6 +90,12 @@ export interface SyncNodeOptions {
   /** WS endpoint. One path, no negotiation — the protocol version lives in the frames. */
   readonly path?: string;
   readonly drainSpreadMs?: number;
+  /**
+   * Sockets one principal — an actor, or an anonymous caller's network (IPv4, IPv6 /64) — may hold
+   * on this node. Default `DEFAULT_MAX_SOCKETS_PER_ACTOR` (16); the boot passes the app's
+   * `realtime.maxSocketsPerActor`. The next upgrade is 429 `X_SOCKET_LIMIT`.
+   */
+  readonly maxSocketsPerActor?: number;
 }
 
 export interface SyncNode {
@@ -73,6 +103,8 @@ export interface SyncNode {
   readonly ready: boolean;
   /** The one path it answers an upgrade on: a HOST has to route it, and must not restate it. */
   readonly path: string;
+  /** Sockets one principal (`actor:<id>` / `address:<network>`) holds here — the cap's count. */
+  principalSockets(principal: string): number;
   start(): Promise<void>;
   /**
    * Refuse new connections, keep every one this node holds. The SIGTERM `accept` phase calls it —

package/src/sync-node.ts

@@ -4,18 +4,36 @@
 // client may reconnect to any node and resume from its cursor, which is why drain is allowed to
 // redistribute connections at all.
 
-import { logger, markReady, reportError, systemClock, uuid } from '@ultimat3/core';
+import {
+  DEFAULT_HEALTH_DETAIL_PEERS,
+  logger,
+  markReady,
+  reportError,
+  systemClock,
+  uuidV7,
+} from '@ultimat3/core';
 import type { Topic } from './channel';
+import { ChannelSids } from './channel-sids';
 import { detach } from './detach';
 import { evictInChunks } from './drain-evictions';
 import { isClientFault } from './errors';
 import type { TransportSubscription } from './fanout';
-import { CHANGE_SUBJECT_ALL, parseEnvelope, SeqGapDetector } from './replicator';
-import { CLOSE, DEFAULT_MAX_BUFFERED_BYTES, SocketRegistry, SyncSocket } from './socket';
+import { PrincipalSockets } from './principal-sockets';
+import { CHANGE_SUBJECT_ALL } from './replicator';
+import { SeqGapDetector } from './replicator-envelope';
+import { CLOSE, SocketRegistry, SyncSocket } from './socket';
+import { DEFAULT_MAX_BUFFERED_BYTES } from './socket-defaults';
 import { idleSweepPeriodMs } from './socket-idle';
+import { actorChangeHandler } from './sync-actor-change';
 import { GrantBook, sweepGrants } from './sync-auth';
-import { ackRefOf, createFrameRouter } from './sync-frames';
-import { drainGraceMs, socketCeilings, syncNodeBounds } from './sync-node-bounds';
+import { changeHandler, ProducerKinds, reconnectHandler } from './sync-bus-handlers';
+import { ackRefOf, frameRouter } from './sync-frames';
+import {
+  clientHeartbeatMs,
+  drainGraceMs,
+  socketCeilings,
+  syncNodeBounds,
+} from './sync-node-bounds';
 import type { SyncNode, SyncNodeOptions, SyncWs } from './sync-node-contract';
 import { decode, type Frame, PROTOCOL_VERSION, toWireError } from './sync-protocol';
 import { handleUpgrade, type UpgradeTarget } from './sync-upgrade';
@@ -34,7 +52,7 @@ export type { SyncNode, SyncNodeOptions, SyncWs } from './sync-node-contract';
 /** Declared with the upgrade that builds it — this file only ever reads one. */
 export type { UpgradeTarget, WsData } from './sync-upgrade';
 
-export function createSyncNode(options: SyncNodeOptions): SyncNode {
+export function syncNode(options: SyncNodeOptions): SyncNode {
   const sockets =
     options.sockets ??
     new SocketRegistry({
@@ -51,8 +69,22 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
   const path = options.path ?? '/_x/sync';
   const presence = options.presence;
   const grants = new GrantBook();
+  const principals = new PrincipalSockets(options.maxSocketsPerActor);
   const gaps = new SeqGapDetector();
+  // Kept across restarts of the bus subscription: a table's producer kind is a fact of the fleet.
+  const producers = new ProducerKinds();
+  const channelSids = new ChannelSids();
+  let reconnects: (() => void) | null = null;
+  /** The re-auth pass in flight, shared by every tick that lands while it runs. */
+  let reauthPass: Promise<void> | null = null;
   let ready = false;
+  /**
+   * The start in flight or done, and whether a stop/drain has run since. `start()` awaits the bus,
+   * and a stop inside that await found nothing to release: the subscription landed after it and
+   * the node came up `ready` and delivering after it was stopped. The `replicator.ts` pattern.
+   */
+  let started: Promise<void> | null = null;
+  let generation = 0;
   /** Resolved by `teardown` when the last socket leaves, for a drain that is waiting its grace. */
   let lastSocketLeft: (() => void) | null = null;
   let changes: TransportSubscription | null = null;
@@ -71,6 +103,8 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
   const release = (): void => {
     changes?.unsubscribe();
     changes = null;
+    reconnects?.();
+    reconnects = null;
     if (sweeping !== null) clearInterval(sweeping);
     sweeping = null;
     if (reauthing !== null) clearInterval(reauthing);
@@ -91,17 +125,23 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
    */
   const teardown = (socket: SyncSocket): readonly Promise<unknown>[] => {
     options.registry.unsubscribeSocket(socket.id);
-    const topics = [...socket.topics] as Topic[];
+    // Suspended seats included, and the presence rooms read BEFORE the seats are given back: the
+    // hub forgets a topic's declaration with its last member.
+    const topics = options.hub.topicsOf(socket);
+    const rooms = topics.filter(hasRoster);
     for (const name of topics) options.hub.unsubscribe(socket, name);
     sockets.remove(socket.id);
     grants.delete(socket.id);
+    principals.release(socket.id);
     if (sockets.count === 0) lastSocketLeft?.();
     // A closed socket is a leave, said now rather than left to TTL: everyone else would otherwise
     // keep rendering a member who is provably gone for the rest of its window. The write is on the
     // bus and the close callback is synchronous, so it cannot be awaited here.
     const leaves: Promise<unknown>[] = [];
     if (presence) {
-      for (const name of topics) {
+      // Only a channel declared `events: true` has a roster: a leave on any other was a KV read,
+      // a KV write and a presence event on a channel that never carried one.
+      for (const name of rooms) {
         const leave = presence.leave(name, socket.id);
         // Detached as well as returned: `detach` attaches the reporting catch, so a caller that
         // awaits this later is awaiting a promise whose rejection is already handled.
@@ -130,6 +170,11 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
       lastSocketLeft = done;
     });
 
+  /** Whether this topic's channel carries presence — asked while the hub still knows the topic. */
+  function hasRoster(name: Topic): boolean {
+    return options.hub.channelOf(name)?.channel.events === true;
+  }
+
   const evict = (socket: SyncSocket, code: number, reason: string): readonly Promise<unknown>[] => {
     socket.close(code, reason);
     return teardown(socket);
@@ -141,19 +186,30 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
    * was authorized for as long as it stayed open — and an active client's socket never idles out,
    * because every inbound frame touches it.
    */
-  const reauthenticate = async (): Promise<void> => {
+  const reauthenticate = (): Promise<void> => {
+    // One pass at a time. The interval does not wait for the pass it started: after a deploy every
+    // grant expires in one window, a pass over them can outlast the interval, and each overlapping
+    // tick then refreshed the same grants again and re-snapshotted the same subscriptions.
+    reauthPass ??= reauthPassOnce().finally(() => {
+      reauthPass = null;
+    });
+    return reauthPass;
+  };
+
+  const onActor = actorChangeHandler({
+    sockets,
+    hub: options.hub,
+    registry: options.registry,
+    channelSids,
+    presence,
+  });
+
+  const reauthPassOnce = async (): Promise<void> => {
     await sweepGrants({
       grants,
       clock,
-      onActor: async (socketId, actor) => {
-        const socket = sockets.get(socketId);
-        if (!socket) return;
-        // The hub sets `socket.actor` and drops the topics this actor may no longer read; the
-        // registry re-decides every live subscription and desyncs the survivors, so the next
-        // delivery re-snapshots them under the new authority rather than the old window.
-        await options.hub.onActorChange(socket, actor);
-        await options.registry.reauthorize(socket);
-      },
+      refreshDeadlineMs: options.grantRefreshDeadlineMs,
+      onActor,
       onRevoked: (socketId) => {
         const socket = sockets.get(socketId);
         if (!socket) return;
@@ -175,75 +231,92 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
     });
   };
 
-  const routeFrame = createFrameRouter({
+  const routeFrame = frameRouter({
     hub: options.hub,
     registry: options.registry,
     buildId: options.buildId,
     presence,
+    channelSids,
+    heartbeatMs: clientHeartbeatMs(presence?.heartbeatMs, sockets.idleTimeoutMs),
   });
 
+  /** Everything `start()` does, fenced on the generation it began under — see `started`. */
+  const begin = async (): Promise<void> => {
+    const run = generation;
+    const bus = { registry: options.registry, hub: options.hub, gaps, producers };
+    const subscription = await options.transport.subscribe(CHANGE_SUBJECT_ALL, changeHandler(bus));
+    // A stop or drain ran inside that await: what it released did not include this yet.
+    if (run !== generation) {
+      subscription.unsubscribe();
+      return;
+    }
+    changes = subscription;
+    reconnects = options.transport.onReconnect(reconnectHandler(bus));
+    // One pass per heartbeat window: a member is swept only once it has actually missed its
+    // window, and the interval never holds the process open — shutdown is the drain's job.
+    if (presence) {
+      sweeping = setInterval(
+        () => detach(presence.sweepAll(), 'presence.sweep'),
+        presence.heartbeatMs,
+      );
+      sweeping.unref();
+    }
+    // The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
+    // whose frame loop is wedged answers pings and keeps its grant, its subscriptions and its
+    // topic membership. `sweepIdle` was written for this and never called, so `touch()` and the
+    // 120s budget under it decided nothing.
+    idling = setInterval(() => {
+      for (const socket of sockets.idle()) evict(socket, CLOSE.idle, 'idle timeout');
+    }, idleSweepPeriodMs(sockets.idleTimeoutMs));
+    idling.unref();
+    if (options.authenticate) {
+      reauthing = setInterval(
+        () => detach(reauthenticate(), 'sync.reauthenticate'),
+        bounds.reauthenticateIntervalMs,
+      );
+      reauthing.unref();
+    } else {
+      // Enforced where it can be: nothing here can invent a credential, so the one honest signal
+      // is that every policy on this node is about to be asked about `null`.
+      logger.warn('sync node has no authenticator: every socket is anonymous', {
+        buildId: options.buildId,
+        fix: 'pass authenticate to syncNode({ authenticate })',
+      });
+    }
+    ready = true;
+    markReady();
+    logger.info('sync node ready', { buildId: options.buildId, path });
+  };
+
+  /** Ends the current generation and waits out a start in flight, so nothing it took outlives us. */
+  const halt = async (): Promise<void> => {
+    generation += 1;
+    ready = false;
+    const inFlight = started;
+    started = null;
+    if (inFlight !== null) await inFlight.catch(() => undefined);
+  };
+
   return {
     sockets,
     path,
+    principalSockets: (principal) => principals.count(principal),
 
     get ready(): boolean {
       return ready;
     },
 
-    async start(): Promise<void> {
-      changes = await options.transport.subscribe(CHANGE_SUBJECT_ALL, (payload) => {
-        const envelope = parseEnvelope(payload);
-        if (!envelope) return;
-        // Fanout is at-most-once over core NATS, so a reconnect is changes this node never saw.
-        // Nothing downstream could notice: no window's lsn moved, so no cursor moved, so nothing
-        // ever asked for a re-snapshot. A gap invalidates every window here instead, and the
-        // subscribers are re-served on the next change to each query.
-        if (gaps.observe(envelope)) {
-          const marked = options.registry.invalidate();
-          logger.warn('live.change_gap', { entity: envelope.change.entity, desynced: marked });
-        }
-        // Not awaited: the bus handler must return before the next change, and ordering is the
-        // registry's — one serial lane per query id. What this call site owes is the failure. An
-        // unhandled rejection here is a fanout that reached nobody, reported as a dead process.
-        detach(options.registry.deliver(envelope.change), 'live.deliver', envelope.change.entity);
-        // The same stream feeds the declared channels: a write names no channel (axiom 2), and the
-        // hub turns this change into `records` frames on every channel it touches.
-        options.hub.deliverChange(envelope.change);
+    start(): Promise<void> {
+      // Memoised: two overlapping starts were two subscriptions, every change fanned out twice.
+      if (started !== null) return started;
+      // Cleared only if it is still THIS attempt's memo: an older start failing after a stop and
+      // a newer start must not wipe the newer one, or a third start subscribes a second time.
+      const attempt: Promise<void> = begin().catch((error: unknown) => {
+        if (started === attempt) started = null;
+        throw error;
       });
-      // One pass per heartbeat window: a member is swept only once it has actually missed its
-      // window, and the interval never holds the process open — shutdown is the drain's job.
-      if (presence) {
-        sweeping = setInterval(
-          () => detach(presence.sweepAll(), 'presence.sweep'),
-          presence.heartbeatMs,
-        );
-        sweeping.unref();
-      }
-      // The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
-      // whose frame loop is wedged answers pings and keeps its grant, its subscriptions and its
-      // topic membership. `sweepIdle` was written for this and never called, so `touch()` and the
-      // 120s budget under it decided nothing.
-      idling = setInterval(() => {
-        for (const socket of sockets.idle()) evict(socket, CLOSE.idle, 'idle timeout');
-      }, idleSweepPeriodMs(sockets.idleTimeoutMs));
-      idling.unref();
-      if (options.authenticate) {
-        reauthing = setInterval(
-          () => detach(reauthenticate(), 'sync.reauthenticate'),
-          bounds.reauthenticateIntervalMs,
-        );
-        reauthing.unref();
-      } else {
-        // Enforced where it can be: nothing here can invent a credential, so the one honest signal
-        // is that every policy on this node is about to be asked about `null`.
-        logger.warn('sync node has no authenticator: every socket is anonymous', {
-          buildId: options.buildId,
-          fix: 'pass authenticate to createSyncNode({ authenticate })',
-        });
-      }
-      ready = true;
-      markReady();
-      logger.info('sync node ready', { buildId: options.buildId, path });
+      started = attempt;
+      return attempt;
     },
 
     stopAccepting(): void {
@@ -251,8 +324,13 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
     },
 
     async stop(): Promise<void> {
-      ready = false;
+      // `halt()` ends the generation synchronously, before its first await; what the current
+      // start installed is released right then. An older start still in flight is fenced by its
+      // generation and drops what it subscribes late itself, and a start() issued after this line
+      // owns a fresh bus that the wait below must not touch.
+      const waiting = halt();
       release();
+      await waiting;
     },
 
     async fetch(request: Request, server: UpgradeTarget): Promise<Response | undefined> {
@@ -267,13 +345,20 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
           // is parked inside `authenticate`, which is the whole reason they are functions.
           ready: () => ready,
           socketCount: () => sockets.count,
-          newSocketId: () => uuid(),
+          newSocketId: () => uuidV7(),
           authenticate: options.authenticate,
           allowedOrigins: options.allowedOrigins,
+          admitReachedOrigin: options.admitReachedOrigin,
+          clientAddressOf: options.clientAddressOf,
+          healthDetailPeers: options.healthDetailPeers ?? DEFAULT_HEALTH_DETAIL_PEERS,
           onGranted: (socketId, grant) => grants.set(socketId, grant),
           // The other half of recording the grant before the upgrade: an upgrade that never took
           // gets no `close` callback, so this is the only thing that can free its entry.
-          onUngranted: (socketId) => grants.delete(socketId),
+          onUngranted: (socketId) => {
+            grants.delete(socketId);
+            principals.release(socketId);
+          },
+          principals,
         },
         request,
         server,
@@ -303,6 +388,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
           clientBuildId: ws.data.clientBuildId,
           serverBuildId: options.buildId,
           actor: grants.get(ws.data.socketId)?.actor ?? null,
+          clientAddress: ws.data.clientAddress ?? null,
           clock,
           ...ceilings,
         });
@@ -353,6 +439,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
           // The socket is already gone, but a grant recorded for an upgrade whose `open` never ran
           // is not — and nothing else would ever reach it.
           grants.delete(ws.data.socketId);
+          principals.release(ws.data.socketId);
           return;
         }
         teardown(socket);
@@ -360,7 +447,7 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
     },
 
     async drain(drainOptions = {}): Promise<readonly DrainedSocket[]> {
-      ready = false;
+      await halt();
       const ids = [...sockets.all()].map((socket) => socket.id);
       const spread = drainPlan(ids, {
         spreadMs: bounds.drainSpreadMs,
@@ -403,9 +490,3 @@ export function createSyncNode(options: SyncNodeOptions): SyncNode {
     },
   };
 }
-
-/**
- * An upgrade refused before a socket exists, rendered as the error contract rather than as a word.
- * There is no frame to carry it — the client never got a connection — so the body is the only
- * channel, and `--json` on every error means this one too.
- */

package/src/sync-origin.ts

@@ -1,7 +1,7 @@
 // Single responsibility: whether a websocket upgrade may carry this app's ambient credential. No
-// CORS applies to a websocket and the session cookie rides it, so without this a page on a sibling
-// host (same-site, so `SameSite=Lax` does not stop it) opened a socket as its visitor. The rule is
-// core's `proveSameOrigin`, the one `@ultimat3/http`'s CSRF check asks too.
+// CORS applies to a websocket and the session cookie rides it, so without this a page that is not
+// this app opened a socket as its visitor. The rule is core's `proveSameOrigin`, the one
+// `@ultimat3/http`'s CSRF check asks too.
 
 import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
 
@@ -9,25 +9,40 @@ import { type OriginVerdict, proveSameOrigin } from '@ultimat3/core';
  * Admitted, in order:
  * - **no `Origin`** — RFC 6455 has a browser send one on every handshake, so a dial without it is
  *   not a browser and no visitor's cookie can be riding it;
- * - **the node's own host name**, whatever the port or scheme — cookies are not port-isolated, so
- *   the page on `:3000` dialling the Compose rung's `:3001` is the same credential boundary, and a
- *   TLS-terminating ingress means the node cannot know the page's scheme;
- * - otherwise core's rule, with `allowedOrigins` as the exact list (`APP_URL`'s origin, from the
- *   CLI, when the page is served on another host than the node).
+ * - **`allowedOrigins`, exactly, when any is declared** (`APP_URL`'s origin, from the CLI). A
+ *   declaration IS the app's origin, so it is the whole list: the origin the node sees is derived
+ *   from the `Host` header, which names whatever host the request was aimed at — a sibling
+ *   subdomain pointed at this node is "its own origin" by that reading, and a `Domain=`-wide
+ *   session cookie rides its socket;
+ * - **the node's own ORIGIN, only when nothing is declared** — host and port exactly as the
+ *   request reached it (`x dev`, a combined-role container). A host name alone is not this app:
+ *   cookies are not isolated by port. `admitReached` keeps it beside a declared list (`x dev`).
  */
 export function upgradeOrigin(
   request: Request,
   url: URL,
   allowedOrigins: readonly string[],
+  admitReached = false,
 ): OriginVerdict {
   const origin = request.headers.get('origin');
   if (origin === null) return { ok: true };
-  if (URL.parse(origin)?.hostname === url.hostname) return { ok: true };
+  // Normalised once (`https://x/` and `https://x` are one origin), then matched as whole strings.
+  const admitted = allowedOrigins.map((allowed) => URL.parse(allowed)?.origin);
   return proveSameOrigin({
-    selfOrigins: allowedOrigins,
+    selfOrigins: admitted.length > 0 && !admitReached ? [] : selfOrigins(url),
     origin,
     secFetchSite: request.headers.get('sec-fetch-site'),
-    listed: () => false,
-    listName: 'APP_URL or createSyncNode({ allowedOrigins })',
+    listed: (asked) => admitted.includes(asked),
+    listName: 'APP_URL or syncNode({ allowedOrigins })',
   });
 }
+
+/**
+ * The origins an UNDECLARED node is reached on. TLS usually ends at a proxy, so a node reached over
+ * plain http cannot know its pages' scheme: the https spelling of the same host and port is this
+ * app too. That also admits the plain-http page, which is why a deployment behind TLS declares
+ * `APP_URL` — the chart sets it from the ingress host.
+ */
+function selfOrigins(url: URL): readonly string[] {
+  return url.protocol === 'http:' ? [url.origin, `https://${url.host}`] : [url.origin];
+}