@ultimat3/realtime 21.0.0 → 22.1.0

package/src/replication-errors.ts

@@ -1,5 +1,5 @@
-// The four refusals the Postgres replication half raises: the wire, the connection, the slot, and
-// the replica identity it warns about.
+// The five refusals the Postgres replication half raises: the wire, the connection, its TLS, the
+// slot, and the replica identity it warns about.
 //
 // Split out of `errors.ts` on the one seam this package already draws — these are the only codes
 // no browser can reach, thrown by `pg-*.ts` and the replicator and by nothing on the client half.
@@ -40,6 +40,22 @@ export class ReplicationFailedError extends RealtimeError {
   }
 }
 
+/**
+ * The replication connection failed TLS: the certificate failed the verification the `sslmode`
+ * asked for, `sslrootcert` named nothing readable, or the handshake itself failed. Its own code
+ * because the fix is a TLS setting, never the network — a certificate refusal used to surface as
+ * `X_REPLICATION_FAILED` "the socket refused a 139-byte write", one step after a silent close.
+ */
+export class ReplicationTlsError extends RealtimeError {
+  constructor(args: { detail: string; fix: string }) {
+    super({
+      code: 'X_REPLICATION_TLS',
+      cause: `postgres replication tls failed: ${args.detail}`,
+      fix: args.fix,
+    });
+  }
+}
+
 /**
  * A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
  * nothing is wrong with this process: the database already has its one replicator, and a second
@@ -59,29 +75,26 @@ export class ReplicatorSlotHeldError extends RealtimeError {
 }
 
 /**
- * A table in the entity list replicates with a replica identity other than FULL, so its `delete`
- * (and any key-changing `update`) carries the KEY COLUMNS ONLY. `toRow` accepts that tuple —
- * it only requires a text `id` — so the live matcher decides "did this row leave the result set"
- * from a one-column row, and a row policy written against `!row.private` reads `undefined`.
- *
- * **Raised at preflight and LOGGED, never thrown.** Every app running today on the default
- * identity would stop booting, and the replicator refusing to start is a worse outcome than the
- * partial rows it is warning about. The runtime half is `ReplicationStreamStats.partialBefore`,
- * which counts the changes this actually affects. Refusing it at `x verify` time is the follow-up.
+ * A table in the entity list has NO replica identity — no primary key under DEFAULT, or
+ * `REPLICA IDENTITY NOTHING`. Once it is in the publication Postgres refuses its UPDATE and DELETE
+ * (`cannot update table … because it does not have a replica identity and publishes updates`),
+ * and a change the replicator did see could not be keyed. A keyed table under DEFAULT is correct
+ * and is never named: the shared window holds the whole row a live query decides on.
  *
- * The tables are named because the fix is per table, and they are the entity list's own names —
- * every one has already passed `assertIdentifier`, so the `fix:` is SQL that can be pasted.
+ * **Raised at preflight and LOGGED, never thrown**, as it always was: a replicator that will not
+ * start is worse than the tables it names. The tables are the entity list's own names — every one
+ * has already passed `assertIdentifier`, so the `fix:` is SQL that can be pasted.
  */
 export class ReplicaIdentityError extends RealtimeError {
   constructor(args: { tables: readonly string[] }) {
     super({
       code: 'X_LIVE_REPLICA_IDENTITY',
       cause:
-        `${args.tables.join(', ')} replicate with a replica identity other than FULL, so a ` +
-        'delete carries the key columns only and a live query decides visibility from a partial row',
+        `${args.tables.join(', ')} have no replica identity (no primary key, or REPLICA IDENTITY ` +
+        'NOTHING), so once published an UPDATE or DELETE on them fails and a change cannot be keyed',
       fix:
         `${args.tables.map((table) => `ALTER TABLE ${table} REPLICA IDENTITY FULL;`).join(' ')}` +
-        ' -- rows already written to the WAL keep the identity they were written with',
+        ' -- or give each a primary key; rows already in the WAL keep the identity they were written with',
     });
   }
 }

package/src/replicator.ts

@@ -14,7 +14,7 @@
 import { isWriteDigest, logger, uuid, withSpan } from '@ultimat3/core';
 import type { ChangeEvent, ChangeFeed } from './changefeed';
 import type { Transport } from './fanout';
-import { type BackoffPolicy, backoffDelay, defaultBackoff, type Rng } from './thundering-herd';
+import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
 
 export const CHANGE_SUBJECT_PREFIX = 'x.change';
 
@@ -225,13 +225,16 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
     },
 
     retryDelayMs(attempt: number): number {
-      return backoffDelay(attempt, backoff, options.rng ?? Math.random);
+      // This method's contract is 0-based (`retryDelayMs(0)` is the base); core counts from 1.
+      return policyDelay(backoff, attempt + 1, options.rng ?? Math.random);
     },
   };
 }
 
 /** Drops events the pipeline cannot use and hoists the tenant id out of the row. */
 export function normalize(change: ChangeEvent): ChangeEvent | null {
+  // The one change that carries no row by design; nothing to hoist a tenant out of.
+  if (change.op === 'truncate') return change;
   const row = change.after ?? change.before;
   if (!row) return null;
   if (change.op === 'insert' && change.after === null) return null;
@@ -253,7 +256,14 @@ export function parseEnvelope(payload: string): ChangeEnvelope | null {
     if (typeof parsed !== 'object' || parsed === null) return null;
     const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
     if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
-    if (shape.op !== 'insert' && shape.op !== 'update' && shape.op !== 'delete') return null;
+    if (
+      shape.op !== 'insert' &&
+      shape.op !== 'update' &&
+      shape.op !== 'delete' &&
+      shape.op !== 'truncate'
+    ) {
+      return null;
+    }
     return {
       change: {
         entity: shape.entity,

package/src/server.ts

@@ -41,6 +41,7 @@ export {
   DEFAULT_MAX_TOPICS_PER_NODE,
 } from './channel';
 export { type ChannelDescription, describeChannels } from './channel-describe';
+export { RealtimeTopologyError } from './errors';
 export {
   InProcessTransport,
   type InProcessTransportOptions,
@@ -62,6 +63,11 @@ export {
   LiveQueryRegistry,
   type LiveQueryRegistryOptions,
 } from './live-query';
+export {
+  type LiveReplicator,
+  type LiveReplicatorOptions,
+  startLiveReplicator,
+} from './live-replicator';
 export {
   applyToWindow,
   type BridgeResult,
@@ -109,7 +115,7 @@ export { openNatsClient } from './nats-lib-client';
 export { NatsTransport, type NatsTransportOptions } from './nats-transport';
 export { PgAdvisoryLock, type PgAdvisoryLockOptions } from './pg-advisory-lock';
 // ---- the postgres replication path ------------------------------------------------------------
-export { camel, entityRow } from './pg-entity-row';
+export { entityRow } from './pg-entity-row';
 export {
   changeLsn,
   commitPositionOf,
@@ -166,16 +172,15 @@ export {
   actorIdOf,
   CLOSE,
   DEFAULT_FRAME_BURST,
-  DEFAULT_IDLE_TIMEOUT_MS,
   DEFAULT_MAX_BUFFERED_BYTES,
   DEFAULT_MAX_FRAMES_PER_SECOND,
-  idleSweepPeriodMs,
   SocketRegistry,
   type SocketRegistryOptions,
   SyncSocket,
   type SyncSocketOptions,
   type WsLike,
 } from './socket';
+export { DEFAULT_IDLE_TIMEOUT_MS, idleSweepPeriodMs } from './socket-idle';
 export type {
   GateFailed,
   GateStage,

package/src/socket-drops.ts

@@ -0,0 +1,30 @@
+// How many dropped frames close a socket: more than `maxDroppedFrames` inside one window, never a
+// lifetime count. Split from `socket.ts`, which counts the drops; this decides what they mean.
+
+/**
+ * The window `maxDroppedFrames` is counted over. Ten seconds: long enough that a burst of
+ * backpressure drops closes a socket that cannot keep up, short enough that a connection open for
+ * hours is never closed for a lifetime's worth of isolated drops.
+ */
+export const DROP_WINDOW_MS = 10_000;
+
+/**
+ * The instants of the drops inside the current window. A burst of backpressure drops closes a
+ * socket that cannot keep up; the same number spread over a connection open for hours is a flaky
+ * link the cursor repairs, and closing for it was a healthy socket lost after 33 lifetime drops.
+ */
+export class DropWindow {
+  readonly #max: number;
+  readonly #recent: number[] = [];
+
+  constructor(max: number) {
+    this.#max = max;
+  }
+
+  /** Records one drop at `now` (monotonic ms) and answers whether the window is over its ceiling. */
+  overflowed(now: number): boolean {
+    this.#recent.push(now);
+    while ((this.#recent[0] ?? now) <= now - DROP_WINDOW_MS) this.#recent.shift();
+    return this.#recent.length > this.#max;
+  }
+}

package/src/socket-engine.ts

@@ -13,8 +13,8 @@ import { PortRouter } from './socket-routes';
 import { decode, encode, type Frame, PROTOCOL_VERSION } from './sync-protocol';
 import {
   type BackoffPolicy,
-  backoffDelay,
   browserBackoff,
+  policyDelay,
   type Rng,
   type Scheduler,
   timeoutScheduler,
@@ -250,7 +250,8 @@ export class SocketEngine {
     if (this.#reconnect !== null) return;
     const rng = this.#options.rng ?? Math.random;
     const delay =
-      afterMs ?? backoffDelay(this.#attempt, this.#options.backoff ?? browserBackoff, rng);
+      // `#attempt` counts reconnects already scheduled, from 0; the wait being armed is the next one.
+      afterMs ?? policyDelay(this.#options.backoff ?? browserBackoff, this.#attempt + 1, rng);
     this.#attempt += 1;
     this.#reconnect = this.#schedule(() => {
       this.#reconnect = null;
@@ -284,6 +285,17 @@ export class SocketEngine {
     socket?.close(1000, 'no tab left');
   }
 
+  /**
+   * A silent port, released — and TOLD first. Silent is usually a closed tab, but a hidden tab the
+   * browser throttled is silent too, and it was released without a word: its virtual socket stayed
+   * "open" over a port nobody read, and that tab's realtime was dead until a reload. Told, its
+   * client goes offline and redials, and the page re-hosts (`socket-host.ts`'s `rehosting`).
+   */
+  #reap(attached: AttachedPort): void {
+    if (attached.open) this.#post(attached, { t: 'close', code: 1006 });
+    this.#detach(attached);
+  }
+
   /** A `MessagePort` has no close event: a port silent for three beats is a closed tab. */
   #armReaper(): void {
     if (this.#reaper !== null) return;
@@ -291,7 +303,7 @@ export class SocketEngine {
       this.#reaper = null;
       const cutoff = this.#now() - REAP_AFTER_BEATS * this.#beatMs;
       for (const attached of [...this.#ports.values()]) {
-        if (attached.lastSeen < cutoff) this.#detach(attached);
+        if (attached.lastSeen < cutoff) this.#reap(attached);
       }
       if (this.#ports.size > 0) this.#armReaper();
     }, this.#beatMs);

package/src/socket-host.ts

@@ -7,12 +7,15 @@ import { browserSocket, dialUrl } from './browser-socket';
 import type { ClientSocket } from './client-contract';
 import type { SyncTarget } from './page-store';
 import { messagePort, type PortLike, SocketEngine } from './socket-engine';
+import { type Scheduler, timeoutScheduler } from './thundering-herd';
 
 export interface SocketHostOptions {
   /** The built worker script (`<meta name="ultimate-sync-worker">`); absent = in-page host. */
   readonly workerUrl?: string | undefined;
   /** The principal the page acts for: one worker per principal, never a shared socket across two. */
   readonly scope: string | null;
+  /** The build the page was rendered by: one worker per build, too. See `workerName`. */
+  readonly buildId?: string | undefined;
   /** Injected for tests; production reads the globals. */
   readonly sharedWorker?: SharedWorkerLike | undefined;
   readonly inPageEngine?: () => SocketEngine;
@@ -32,9 +35,13 @@ export interface SocketHost {
   bye(): void;
 }
 
-/** The worker's name carries the scope, so two principals in two tabs get two workers. */
-export function workerName(scope: string | null): string {
-  return `ultimate-sync:${encodeURIComponent(scope ?? '')}`;
+/**
+ * The worker's name carries the scope, so two principals in two tabs get two workers — and the
+ * BUILD, so two builds do too. A SharedWorker keeps the first tab's build id for its whole life, so
+ * a tab of the new build joined the old engine and was told "update available" about itself.
+ */
+export function workerName(scope: string | null, buildId: string | undefined): string {
+  return `ultimate-sync:${encodeURIComponent(scope ?? '')}:${encodeURIComponent(buildId ?? '')}`;
 }
 
 export function openHost(options: SocketHostOptions): SocketHost {
@@ -55,7 +62,9 @@ function workerPort(options: SocketHostOptions): PortLike | undefined {
     (typeof SharedWorker === 'function' ? (SharedWorker as SharedWorkerLike) : undefined);
   if (Worker === undefined || options.workerUrl === undefined) return undefined;
   try {
-    return messagePort(new Worker(options.workerUrl, { name: workerName(options.scope) }).port);
+    return messagePort(
+      new Worker(options.workerUrl, { name: workerName(options.scope, options.buildId) }).port,
+    );
   } catch {
     return undefined;
   }
@@ -124,3 +133,93 @@ class VirtualSocket implements ClientSocket {
     }
   }
 }
+
+/** A host that can be replaced under the page's one client. */
+export interface RehostingHost extends SocketHost {
+  /** Say bye to the current host and build a new one — the next `socket()` dials through it. */
+  rehost(): void;
+}
+
+export interface RehostingOptions {
+  /** How long a virtual socket may wait for `open` before its host is presumed dead. */
+  readonly openTimeoutMs: number;
+  readonly schedule?: Scheduler | undefined;
+}
+
+/**
+ * The page's host, replaceable. A port the engine reaped (a throttled hidden tab), or one the tab
+ * itself said `bye` on at `pagehide` before a bfcache restore, is a port nobody reads: the virtual
+ * socket over it waited for `open` forever and the tab's realtime was dead until a reload. So an
+ * `open` unanswered by `openTimeoutMs` closes that socket (1006, which the client redials on) and
+ * re-hosts on a fresh port; `page-socket.ts` also re-hosts on a bfcache `pageshow`.
+ */
+export function rehosting(make: () => SocketHost, options: RehostingOptions): RehostingHost {
+  const schedule = options.schedule ?? timeoutScheduler;
+  let current = make();
+  const self: RehostingHost = {
+    get kind(): 'worker' | 'in-page' {
+      return current.kind;
+    },
+    socket: (target) => {
+      const inner = current.socket(target);
+      return watchOpen(inner, schedule, options.openTimeoutMs, () => self.rehost());
+    },
+    bye: () => current.bye(),
+    rehost: () => {
+      current.bye();
+      current = make();
+    },
+  };
+  return self;
+}
+
+/** `inner`, with a deadline on its `open`: missed, the host is re-made and the socket closes. */
+function watchOpen(
+  inner: ClientSocket,
+  schedule: Scheduler,
+  ms: number,
+  rehost: () => void,
+): ClientSocket {
+  let opened: (() => void) | null = null;
+  let closed: ((code: number) => void) | null = null;
+  let settled = false;
+  const disarm = schedule(() => {
+    if (settled) return;
+    settled = true;
+    rehost();
+    closed?.(1006);
+  }, ms);
+  inner.onOpen(() => {
+    if (settled) return;
+    settled = true;
+    disarm();
+    opened?.();
+  });
+  inner.onClose((code) => {
+    if (!settled) {
+      settled = true;
+      disarm();
+    }
+    closed?.(code);
+  });
+  return {
+    get bufferedAmount(): number {
+      return inner.bufferedAmount ?? 0;
+    },
+    send: (data) => inner.send(data),
+    close: (code, reason) => {
+      if (!settled) {
+        settled = true;
+        disarm();
+      }
+      inner.close(code, reason);
+    },
+    onOpen: (handler) => {
+      opened = handler;
+    },
+    onMessage: (handler) => inner.onMessage(handler),
+    onClose: (handler) => {
+      closed = handler;
+    },
+  };
+}

package/src/socket-idle.ts

@@ -0,0 +1,21 @@
+// The application idle budget a sync node evicts a silent socket on, and how often it asks. Split
+// from `socket.ts` at its line ceiling.
+
+/**
+ * How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
+ * inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
+ * ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
+ * its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
+ * every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
+ */
+export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
+
+/**
+ * How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
+ * of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
+ * four times per window rather than once a second. Derived rather than configured — a second knob
+ * is a second number that can disagree with the one it is a fraction of.
+ */
+export function idleSweepPeriodMs(idleTimeoutMs: number): number {
+  return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
+}

package/src/socket.ts

@@ -19,6 +19,8 @@ import {
 import { GapRepairs } from './channel-gaps';
 import type { ChannelRecordsFrame } from './channel-wire';
 import { CLOSE } from './close-codes';
+import { DropWindow } from './socket-drops';
+import { DEFAULT_IDLE_TIMEOUT_MS } from './socket-idle';
 import { encode, type Frame } from './sync-protocol';
 import { AcceptBudget } from './thundering-herd';
 
@@ -156,6 +158,7 @@ export class SyncSocket {
   readonly #maxDroppedFrames: number;
   #clientBuildId: string;
   #closed = false;
+  readonly #drops: DropWindow;
 
   constructor(options: SyncSocketOptions) {
     this.#ws = options.ws;
@@ -168,6 +171,7 @@ export class SyncSocket {
     finiteOption('SyncSocket', 'maxBufferedBytes', this.#maxBufferedBytes);
     this.#maxDroppedFrames = options.maxDroppedFrames ?? 32;
     finiteOption('SyncSocket', 'maxDroppedFrames', this.#maxDroppedFrames);
+    this.#drops = new DropWindow(this.#maxDroppedFrames);
     this.frameBudget = new AcceptBudget({
       perSecond: finiteOption(
         'SyncSocket',
@@ -215,33 +219,34 @@ export class SyncSocket {
 
   /** `false` means the frame was dropped by backpressure — the caller must mark state stale. */
   send(frame: Frame): boolean {
+    return this.sendEncoded(encode(frame));
+  }
+
+  /** An `encode()`d frame — the fan-outs encode once for every socket. Adds no validation. */
+  sendEncoded(text: string): boolean {
     if (this.#closed) return false;
     if (this.#ws.getBufferedAmount() > this.#maxBufferedBytes) {
-      this.droppedFrames += 1;
-      if (this.droppedFrames > this.#maxDroppedFrames) {
-        this.close(CLOSE.overloaded, 'backpressure');
-      }
+      this.#dropped();
       return false;
     }
-    // `WsLike.send` is declared `: number` for this line and no other: Bun answers `0` for a
-    // message it DROPPED — the socket closed between the buffered-amount check above and this
-    // write — and `-1` under backpressure. Discarded, a dropped frame read as delivered, so
-    // `live-fanout` advanced the subscriber's cursor past a patch that never left and
-    // `sync-frames`' desync mark was never taken: permanently stale on a healthy socket, which is
-    // the exact outcome every other `socket.send` on this node reads its answer to prevent.
-    if (this.#ws.send(encode(frame)) <= 0) {
-      this.droppedFrames += 1;
-      // The same ceiling backpressure takes: a socket the runtime keeps refusing is one to close,
-      // and the two are one failure — the write went nowhere either way.
-      if (this.droppedFrames > this.#maxDroppedFrames) {
-        this.close(CLOSE.overloaded, 'backpressure');
-      }
+    // Bun answers `0` for a message it DROPPED (the socket closed after the check above) — the one
+    // drop. `-1` means QUEUED under backpressure, and it is delivered: counted as a drop it sent a
+    // spurious `replay-gap`, desynced a healthy subscriber, and closed the socket after 33.
+    if (this.#ws.send(text) === 0) {
+      this.#dropped();
       return false;
     }
     this.sentFrames += 1;
     return true;
   }
 
+  /** One frame that never left: counted for life, judged per `DROP_WINDOW_MS` (`socket-drops.ts`). */
+  #dropped(): void {
+    this.droppedFrames += 1;
+    if (this.#drops.overflowed(this.#clock.monotonic()))
+      this.close(CLOSE.overloaded, 'backpressure');
+  }
+
   /** Record a dropped or invalidated subscription so the next flush re-snapshots it. */
   markDesynced(sid: string): void {
     this.desynced.add(sid);
@@ -282,25 +287,6 @@ export class SyncSocket {
   }
 }
 
-/**
- * How long a socket may route no frame before `sync-node` evicts it. It is an APPLICATION
- * inactivity budget and not Bun's transport one: Bun's `idleTimeout` is renewed by its own
- * ping/pong, so a client whose TCP stack still answers pings while its frame loop is wedged holds
- * its grant, its subscriptions and its topic membership forever. A beating client sends a `hello`
- * every `DEFAULT_HEARTBEAT_MS` (15s), so this is eight missed beats.
- */
-export const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
-
-/**
- * How often to ask. A quarter of the budget, floored at a second: a socket is evicted within 25%
- * of its window of going quiet, and a node holding 50,000 of them pays one pass over the table
- * four times per window rather than once a second. Derived rather than configured — a second knob
- * is a second number that can disagree with the one it is a fraction of.
- */
-export function idleSweepPeriodMs(idleTimeoutMs: number): number {
-  return Math.max(1_000, Math.floor(idleTimeoutMs / 4));
-}
-
 export interface SocketRegistryOptions {
   readonly clock?: Clock;
   /** Bun's own `idleTimeout` is renewed by its ping/pong; this budget counts routed FRAMES. */
@@ -425,6 +411,9 @@ export class SocketRegistry {
   deliver(topic: string, frame: Frame): number {
     const members = this.#byTopic.get(topic);
     if (!members) return 0;
+    // Encoded ONCE for every member: per socket it was 16.9 ms against 0.6 ms for a 2 kB frame to
+    // 10,000 sockets, measured.
+    const text = encode(frame);
     let sent = 0;
     let dropped = 0;
     for (const socket of members) {
@@ -432,7 +421,7 @@ export class SocketRegistry {
         members.delete(socket);
         continue;
       }
-      if (socket.send(frame)) sent += 1;
+      if (socket.sendEncoded(text)) sent += 1;
       else dropped += 1;
     }
     if (members.size === 0) this.#byTopic.delete(topic);
@@ -447,6 +436,7 @@ export class SocketRegistry {
   deliverRecords(topic: string, epoch: string, frame: ChannelRecordsFrame): number {
     const members = this.#byTopic.get(topic);
     if (!members) return 0;
+    const text = encode(frame);
     let sent = 0;
     let dropped = 0;
     for (const socket of members) {
@@ -455,7 +445,7 @@ export class SocketRegistry {
         continue;
       }
       this.gapRepairs.repair(socket, topic);
-      if (socket.send(frame)) {
+      if (socket.sendEncoded(text)) {
         sent += 1;
         continue;
       }
@@ -467,6 +457,19 @@ export class SocketRegistry {
     return sent;
   }
 
+  /** Every member owes a re-read (a TRUNCATE): one `replay-gap` each now, or kept as a mark. */
+  announceGap(topic: string, epoch: string): number {
+    const members = this.#byTopic.get(topic);
+    if (!members) return 0;
+    let announced = 0;
+    for (const socket of members) {
+      if (socket.closed) continue;
+      socket.gaps.set(topic, epoch);
+      if (this.gapRepairs.repair(socket, topic)) announced += 1;
+    }
+    return announced;
+  }
+
   #countDropped(topic: string, dropped: number): void {
     this.#droppedChannelFrames += dropped;
     // Two readers, one event, one spelling: the series an operator alerts on and the line that