@ultimat3/realtime 30.9.0 → 31.1.0

package/src/server.ts

@@ -2,7 +2,11 @@
 // live-query registry it fans out through. Split from `index.ts` because `nats` require()s
 // `stream/web` and the WAL decoder is a Postgres client — one barrel carrying both made the browser
 // island realtime promises unbuildable. Every name here has exactly one home; the shared
-// vocabulary (the wire, the errors, `Row`, the backoff) stays on `@ultimat3/realtime`.
+// vocabulary (the wire, the errors, `LiveRow`, the backoff) stays on `@ultimat3/realtime`.
+
+// The server-only codes' titles (`sideEffects`): a process that loads this barrel titles them,
+// whichever class it constructs first.
+import './error-titles-server';
 
 export { type AdvisoryLock, type MemoryAdvisoryLock, memoryAdvisoryLock } from './advisory-lock';
 // ---- the retained change window one node fans out from ------------------------------------------
@@ -45,11 +49,8 @@ export {
 } from './channel';
 export { type ChannelDescription, describeChannels } from './channel-describe';
 // What a job or an action publishes an event with: the process's own bus, no hub needed.
-export {
-  publishChannelEvent,
-  resetChannelTransport,
-  setChannelTransport,
-} from './channel-publish';
+// Its installer is the boot's, on `./adapter` (`adapter.ts`), never here.
+export { publishChannelEvent } from './channel-publish';
 export {
   type AgentDial,
   type AgentSocket,
@@ -112,6 +113,7 @@ export type {
 export { type LiveDefinitionOptions, liveQueryDefinition } from './live-definition';
 export {
   DEFAULT_MAX_ENTRIES,
+  DEFAULT_MAX_LANE_DEPTH,
   LiveQueryRegistry,
   type LiveQueryRegistryOptions,
 } from './live-query';

package/src/socket-engine.ts

@@ -12,9 +12,9 @@ import type { AttachedPort, PortLike, PortMessage } from './socket-port';
 import { PortRouter } from './socket-routes';
 import { decode, encode, type Frame, PROTOCOL_VERSION } from './sync-protocol';
 import {
-  type BackoffPolicy,
   browserBackoff,
   policyDelay,
+  type ReconnectBackoff,
   type Rng,
   type Scheduler,
   timeoutScheduler,
@@ -28,7 +28,7 @@ export interface SocketEngineOptions {
   readonly dial: (target: SyncTarget) => ClientSocket;
   readonly scheduler?: Scheduler;
   readonly clock?: Clock;
-  readonly backoff?: BackoffPolicy;
+  readonly backoff?: ReconnectBackoff;
   readonly rng?: Rng;
   /** The engine's own beat on the real socket, and the unit a silent port is reaped in. */
   readonly heartbeatMs?: number;

package/src/subscriber-gate.ts

@@ -5,7 +5,7 @@
 
 import type { Actor } from '@ultimat3/core';
 import { isPolicyDenial } from './errors';
-import type { JsonValue, Row, RowPatch } from './json';
+import type { JsonValue, LiveRow, RowPatch } from './json';
 import type { LiveQueryDefinition } from './live-contract';
 import { narrowRow, projectionOf } from './matcher-bridge';
 
@@ -52,7 +52,7 @@ export interface GateTarget {
   readonly qid: string;
   readonly input: JsonValue;
   readonly definition: LiveQueryDefinition;
-  readonly rows: readonly Row[];
+  readonly rows: readonly LiveRow[];
 }
 
 /**
@@ -62,12 +62,12 @@ export interface GateTarget {
  * when a caller passes none.
  */
 export interface WindowIndex {
-  readonly byId: ReadonlyMap<string, Row>;
+  readonly byId: ReadonlyMap<string, LiveRow>;
   readonly position: ReadonlyMap<string, number>;
 }
 
-export function windowIndex(rows: readonly Row[]): WindowIndex {
-  const byId = new Map<string, Row>();
+export function windowIndex(rows: readonly LiveRow[]): WindowIndex {
+  const byId = new Map<string, LiveRow>();
   const position = new Map<string, number>();
   rows.forEach((row, at) => {
     byId.set(row.id, row);
@@ -144,8 +144,12 @@ export class SubscriberGate {
    * managed to admit: a short result set is indistinguishable from a correct one, and handing it
    * over is the read silently losing rows.
    */
-  async filterRows(target: GateTarget, who: Subscriber, rows: readonly Row[]): Promise<Row[]> {
-    const out: Row[] = [];
+  async filterRows(
+    target: GateTarget,
+    who: Subscriber,
+    rows: readonly LiveRow[],
+  ): Promise<LiveRow[]> {
+    const out: LiveRow[] = [];
     for (const row of rows) {
       if (await this.#visible(target, who, row, 'snapshot')) out.push(row);
       else this.#denied(target.qid, who, row.id);
@@ -219,7 +223,7 @@ export class SubscriberGate {
     // failed either: the shared window *is* the result set, so a row it does not hold is a row this
     // subscriber is not entitled to keep, and one that holds it is told so.
     if (full === undefined) return holds ? withdrawn(patch) : null;
-    const row: Row = { ...full, ...patch.row, id: patch.id };
+    const row: LiveRow = { ...full, ...patch.row, id: patch.id };
     if (await this.#visible(target, who, row, 'patch')) {
       // An update carries the changed columns only. A subscriber who was never sent this row —
       // denied at its snapshot, admitted by this change — would fold them onto nothing, and its
@@ -247,7 +251,7 @@ export class SubscriberGate {
   async #visible(
     target: GateTarget,
     who: Subscriber,
-    row: Row,
+    row: LiveRow,
     stage: GateStage,
   ): Promise<boolean> {
     try {
@@ -302,7 +306,7 @@ const actorIdOf = (who: Subscriber): string | null => (who.actor === null ? null
 function rebase(
   patch: RowPatch,
   holding: Holding,
-  rows: readonly Row[],
+  rows: readonly LiveRow[],
   index: WindowIndex,
 ): RowPatch {
   if (patch.index === undefined) return patch;

package/src/sync-bus-handlers.ts

@@ -8,6 +8,7 @@ import type { ChannelHub } from './channel';
 import { detach } from './detach';
 import type { LiveQueryRegistry } from './live-query';
 import { RealtimeError } from './realtime-error';
+import { parseResync } from './replication-resync';
 import {
   type ChangeEnvelope,
   type EnvelopeSource,
@@ -83,6 +84,16 @@ export class ProducerKinds {
 /** The handler a node subscribes to `CHANGE_SUBJECT_ALL` with. */
 export function changeHandler(bus: SyncBus): (payload: string) => void {
   return (payload) => {
+    // The replicator's own word that the stream has a hole no sequence shows (its slot was
+    // recreated): answered as a reconnect is, because it is the same fact — changes never seen.
+    const resync = parseResync(payload);
+    if (resync !== null) {
+      resyncAll(bus, 'live.replication_resync', {
+        reason: resync.reason,
+        producer: resync.producer,
+      });
+      return;
+    }
     const envelope = parseEnvelope(payload);
     if (!envelope) return;
     // Before the gap detector: a dropped producer's sequence is no stream this node follows.
@@ -127,9 +138,14 @@ export function changeHandler(bus: SyncBus): (payload: string) => void {
 export function reconnectHandler(bus: SyncBus): () => void {
   return () => {
     bus.outage?.answered();
-    bus.gaps.forget();
-    const marked = bus.registry.invalidate();
-    const channelGaps = bus.hub.invalidate();
-    logger.warn('live.bus_reconnected', { desynced: marked, channelGaps });
+    resyncAll(bus, 'live.bus_reconnected', {});
   };
 }
+
+/** Every window and every records topic re-read, the producer history forgotten, said once. */
+function resyncAll(bus: SyncBus, event: string, fields: Record<string, string>): void {
+  bus.gaps.forget();
+  const marked = bus.registry.invalidate();
+  const channelGaps = bus.hub.invalidate();
+  logger.warn(event, { ...fields, desynced: marked, channelGaps });
+}

package/src/sync-protocol.ts

@@ -20,7 +20,7 @@ import {
   isRow,
   type JsonObject,
   type JsonValue,
-  type Row,
+  type LiveRow,
   type RowPatch,
 } from './json';
 import { ProtocolVersionError } from './page-errors';
@@ -114,7 +114,7 @@ export interface SnapshotFrame {
   readonly type: 'snapshot';
   readonly v: number;
   readonly sid: string;
-  readonly rows: readonly Row[];
+  readonly rows: readonly LiveRow[];
   readonly cursor: LiveCursor;
   /**
    * The entity every row of this result set belongs to — the client's identity scope, so two
@@ -360,7 +360,7 @@ function writesOf(parsed: JsonObject, label: string): readonly string[] {
   });
 }
 
-function row(value: unknown): Row {
+function row(value: unknown): LiveRow {
   if (!isRow(value)) throw fail('row must be an object with a string "id"');
   return value;
 }

package/src/thundering-herd.ts

@@ -6,6 +6,7 @@
 //   2. policyDelay()  — the client's own jittered retry, for failures nobody scheduled
 //   3. AcceptBudget   — the receiving node's token bucket, so recovery sheds instead of collapsing
 
+import type { RetryPolicy } from '@ultimat3/core';
 import {
   backoffDelay,
   type Clock,
@@ -22,20 +23,20 @@ export type Rng = Random;
 /** Core's, re-exported under this package's name — never a second copy of the same three modes. */
 export type { JitterMode };
 
-export interface BackoffPolicy {
-  readonly baseMs: number;
-  readonly maxMs: number;
-  readonly factor: number;
-  readonly jitter: JitterMode;
-}
+/**
+ * A redial's wait: core's `RetryPolicy` without its two bounds. A socket, a NATS link and the
+ * replicator redial until they are closed, so `attempts` and `timeBudgetMs` would be fields read by
+ * nothing — the shape is core's, projected, never restated (`BackoffPolicy` until 31.0.0).
+ */
+export type ReconnectBackoff = Pick<RetryPolicy, 'base' | 'max' | 'factor' | 'curve' | 'jitter'>;
 
 /**
  * `full` jitter by default: it is the only mode that actually decorrelates a herd. `equal` keeps a
  * floor for latency-sensitive clients; `none` exists for tests and is never a production choice.
  */
-export const defaultBackoff: BackoffPolicy = {
-  baseMs: 500,
-  maxMs: 30_000,
+export const defaultBackoff: ReconnectBackoff = {
+  base: 500,
+  max: 30_000,
   factor: 2,
   jitter: 'full',
 };
@@ -51,16 +52,16 @@ export const BROWSER_RECONNECT_MAX_MS = 4_000;
  * window rather than at once, and the `AcceptBudget` sheds the excess before any query runs. A
  * PLANNED restart never reaches this: the drain's `reconnect` frame assigns each socket its slot.
  */
-export const browserBackoff: BackoffPolicy = {
-  baseMs: 500,
-  maxMs: BROWSER_RECONNECT_MAX_MS,
+export const browserBackoff: ReconnectBackoff = {
+  base: 500,
+  max: BROWSER_RECONNECT_MAX_MS,
   factor: 2,
   jitter: 'equal',
 };
 
 /**
- * A {@link BackoffPolicy} mapped onto `@ultimat3/core`'s `backoffDelay` — the one curve. Attempt is
- * 1-BASED, core's count: the wait after the first failure is `attempt: 1` and is `baseMs`.
+ * A {@link ReconnectBackoff} through `@ultimat3/core`'s `backoffDelay` — the one curve. Attempt is
+ * 1-BASED, core's count: the wait after the first failure is `attempt: 1` and is `base`.
  *
  * Internal to this package and never re-exported from the barrel. Until 22.0.0 `.` exported a
  * 0-based `backoffDelay` of its own that shifted by one before delegating, which made two counting
@@ -68,19 +69,11 @@ export const browserBackoff: BackoffPolicy = {
  * catch-up retry did) waited twice as long as it meant to, with no error anywhere.
  */
 export function policyDelay(
-  policy: BackoffPolicy,
+  policy: ReconnectBackoff,
   attempt: number,
   rng: Rng = Math.random,
 ): number {
-  return backoffDelay({
-    attempt,
-    base: policy.baseMs,
-    max: policy.maxMs,
-    factor: policy.factor,
-    curve: 'exponential',
-    jitter: policy.jitter,
-    random: rng,
-  });
+  return backoffDelay({ ...policy, attempt, random: rng });
 }
 
 /**

package/src/transport-env.ts

@@ -1,30 +1,27 @@
-// Single responsibility: `realtime.transport` + environment → fanout transport. The one place a boot
+// Single responsibility: environment → fanout transport. The one place a boot
 // decides whether this process fans changes out inside its own heap or over NATS, so `x dev`, a
 // `sync` container and any custom host resolve it identically. The KV bucket and the presence TTL are decided here too:
 // the bucket's whole-stream age limit and `PresenceRegistry`'s TTL are the same number seen from
 // two sides, and a caller that had to pass each one separately could quietly set them apart.
 
-import type { Clock, RealtimeConfig } from '@ultimat3/core';
-import { ConfigInvalidError, finiteOption } from '@ultimat3/core';
+import type { Clock } from '@ultimat3/core';
+import { finiteOption } from '@ultimat3/core';
 import type { Transport } from './fanout';
 import { InProcessTransport } from './fanout';
 import type { NatsConnect } from './nats-client';
 import { assertBucket } from './nats-jetstream';
 import { NatsTransport } from './nats-transport';
-import type { BackoffPolicy } from './thundering-herd';
+import type { ReconnectBackoff } from './thundering-herd';
 
 /**
  * The keys read here, and nothing else. Named once so docs and tests cannot drift from the code.
- * `NATS_URL` is the conventional bus variable: read under `transport: 'nats'` when `urlEnv` names
- * it, and under `'memory'` only to refuse it — see `selectTransport`.
+ * `NATS_URL` set is the bus; unset is the in-process one. Any other bus is `apps/<app>/runtime.ts`'s
+ * `transport`, which the boot takes INSTEAD of calling this.
  */
 export const TRANSPORT_ENV_KEYS = ['NATS_URL', 'NATS_KV_BUCKET'] as const;
 
-/** The two fields of `app.config.ts`'s `realtime` section that decide the bus. */
-export type RealtimeTopology = Pick<RealtimeConfig, 'transport' | 'urlEnv'>;
-
-/** Named in every refusal, so the reader edits the file the decision lives in. */
-const CONFIG_FILE = 'app.config.ts';
+/** Where a bus that is neither NATS nor in-process is handed over. */
+const RUNTIME_FILE = 'apps/<app>/runtime.ts';
 
 /**
  * One bucket per deployment, not per cluster: two apps sharing a nats-server would otherwise share
@@ -96,7 +93,7 @@ export interface SelectTransportOptions {
   /** Default `BUS_CONNECT_WAIT_MS`. Read under `'sockets'` and `'feed'` only. */
   readonly connectWithinMs?: number | undefined;
   /** The dial's retry curve; the transport's own default when unset. */
-  readonly backoff?: BackoffPolicy | undefined;
+  readonly backoff?: ReconnectBackoff | undefined;
   /** Where the transport's background failures go; its own logger line when unset. */
   readonly onError?: ((error: unknown, subject: string) => void) | undefined;
 }
@@ -105,22 +102,19 @@ const nonEmpty = (value: string | undefined): string | undefined =>
   value === undefined || value.trim().length === 0 ? undefined : value.trim();
 
 /**
- * The config decides the transport and the environment supplies its url — never the other way
- * round. Until 22.0.0 `NATS_URL` alone decided and `realtime.transport` / `realtime.urlEnv` were
- * read by nothing, so `transport: 'nats'` with the variable unset booted the in-process bus and
- * reached no other node, with no error on either side. Both mismatches are refused here:
+ * The environment decides the default bus: `NATS_URL` set (and not blank) is NATS, unset is the
+ * in-process bus. Until 31.0.0 `app.config.ts`'s `realtime.transport`/`urlEnv` decided and the env
+ * only had to agree — a closed union no app could add a bus to. A bus of the app's own is
+ * `transport` in `apps/<app>/runtime.ts`, and the boot never calls this for it.
  *
- * - `'nats'` with the variable `urlEnv` names unset or blank.
- * - `'memory'` with a bus url set (`NATS_URL`, or the variable `urlEnv` names). An operator who set
- *   one expected fanout across nodes; keeping every change in this heap instead is the same silent
- *   failure seen from the other side, so the two are refused rather than reconciled.
+ * The one silent failure left — a `sync` node on the in-process bus with its replicator in another
+ * process — is refused at boot by the caller, which knows the roles (`X_REALTIME_TOPOLOGY`).
  *
  * The bucket name is validated here rather than on first connect: a typo'd bucket is a boot that
  * reports a healthy bus and then fails every presence write.
  */
 export function selectTransport(
   env: TransportEnvironment,
-  topology: RealtimeTopology,
   options: SelectTransportOptions = {},
 ): TransportSelection {
   const presenceTtlMs = finiteOption(
@@ -129,16 +123,16 @@ export function selectTransport(
     options.presenceTtlMs ?? DEFAULT_PRESENCE_TTL_MS,
   );
   const use = options.use ?? 'sockets';
+  const url = nonEmpty(env['NATS_URL']);
 
-  if (topology.transport === 'memory') {
-    refuseStrayBusUrl(env, topology);
+  if (url === undefined) {
     const transport = new InProcessTransport(
       options.clock === undefined ? {} : { clock: options.clock },
     );
     return {
       transport,
       mode: 'embedded',
-      detail: `in-process fanout — set realtime.transport 'nats' in ${CONFIG_FILE} and NATS_URL to reach the other nodes`,
+      detail: `in-process fanout — set NATS_URL to reach the other nodes, or transport in ${RUNTIME_FILE} for another bus`,
       bucket: null,
       presenceTtlMs,
       use,
@@ -147,16 +141,6 @@ export function selectTransport(
     };
   }
 
-  const urlEnv = topology.urlEnv ?? 'NATS_URL';
-  const url = nonEmpty(env[urlEnv]);
-  if (url === undefined) {
-    throw new ConfigInvalidError({
-      cause: `realtime.transport is 'nats' and realtime.urlEnv names ${urlEnv}, which is unset in this process's environment, so no node would be reachable`,
-      fix: `set ${urlEnv} to the nats-server url for every realtime role (web, sync, replicator), or set realtime: { transport: 'memory' } in ${CONFIG_FILE} for a single node`,
-      meta: { key: 'realtime.urlEnv', variable: urlEnv },
-    });
-  }
-
   const bucket = nonEmpty(env['NATS_KV_BUCKET']) ?? DEFAULT_PRESENCE_BUCKET;
   assertBucket(bucket);
   const transport = new NatsTransport({
@@ -180,7 +164,7 @@ export function selectTransport(
   return {
     transport,
     mode: 'external',
-    detail: urlEnv,
+    detail: 'NATS_URL',
     bucket,
     presenceTtlMs,
     use,
@@ -192,21 +176,9 @@ export function selectTransport(
       await transport.connect({
         withinMs,
         // The variable, never the url behind it. `-js` only where the bucket is asked for.
-        fix: `x doctor — then check ${urlEnv} names a reachable nats-server${use === 'sockets' ? ' started with -js (JetStream)' : ''} — this process ${NEEDS[use]} and cannot start without it; the web, worker and scheduler roles boot without one and connect in the background`,
+        fix: `x doctor — then check NATS_URL names a reachable nats-server${use === 'sockets' ? ' started with -js (JetStream)' : ''} — this process ${NEEDS[use]} and cannot start without it; the web, worker and scheduler roles boot without one and connect in the background`,
       });
     },
     state: () => (transport.connected ? 'up' : 'connecting'),
   };
 }
-
-/** `'memory'` with a bus url in the environment: the conflict `selectTransport` refuses. */
-function refuseStrayBusUrl(env: TransportEnvironment, topology: RealtimeTopology): void {
-  const names = topology.urlEnv === undefined ? ['NATS_URL'] : ['NATS_URL', topology.urlEnv];
-  const set = names.find((name) => nonEmpty(env[name]) !== undefined);
-  if (set === undefined) return;
-  throw new ConfigInvalidError({
-    cause: `${set} is set but realtime.transport is 'memory', so this process would fan out in its own heap and reach no other node`,
-    fix: `set realtime: { transport: 'nats', urlEnv: '${set}' } in ${CONFIG_FILE} to use the bus, or unset ${set} for a single node`,
-    meta: { key: 'realtime.transport', variable: set },
-  });
-}

package/src/window-lock.ts

@@ -8,14 +8,26 @@
 /** FIFO, one task at a time. A task that rejects hands its rejection to its own caller and no one else. */
 export class WindowLock {
   #tail: Promise<unknown> = Promise.resolve();
+  #depth = 0;
+
+  /**
+   * Tasks queued or running. What a caller bounds the lane by (`live-deliver.ts`): every task is
+   * a closure holding its change, so a lane behind one slow policy pass is memory nobody capped.
+   */
+  get depth(): number {
+    return this.#depth;
+  }
 
   run<T>(work: () => Promise<T>): Promise<T> {
+    this.#depth += 1;
     const result = this.#tail.then(work);
     // The lane chains on a settled shadow, never on `result`: one delivery that threw must not
     // reject every delivery queued behind it, and an unwatched shadow must not look unhandled.
-    this.#tail = result.then(ignore, ignore);
+    this.#tail = result.then(this.#settled, this.#settled);
     return result;
   }
-}
 
-const ignore = (): void => undefined;
+  readonly #settled = (): void => {
+    this.#depth -= 1;
+  };
+}