@ultimat3/realtime 23.0.0 → 25.0.0

package/src/principal-sockets.ts

@@ -0,0 +1,75 @@
+// One principal's open sockets on this node, counted so the upgrade can refuse the next one. The
+// per-actor SUBSCRIPTION cap bounds what a principal holds; this bounds how many sockets it holds it
+// across — each one a grant, a frame budget, a heartbeat and a slot of `maxConnections`.
+
+import { ConfigInvalidError } from '@ultimat3/core';
+
+/**
+ * Sixteen sockets at the per-socket 128 subscriptions is 2,048 — past the per-actor 1,000, so a
+ * legitimate principal's tabs never hit this before they hit the subscription cap, while one
+ * principal can no longer take a node's `maxConnections` for itself.
+ */
+export const DEFAULT_MAX_SOCKETS_PER_ACTOR = 16;
+
+/**
+ * An ANONYMOUS principal is a network, not a person: an office behind one corporate NAT viewing a
+ * public live page is one `address:` key, and held to an actor's 16 it was 16 sockets for the whole
+ * floor. A network gets this many actors' worth — 128 at the default — while a signed-in actor keeps
+ * `maxSocketsPerActor`. The subscription cap is not multiplied.
+ */
+export const ANONYMOUS_SOCKET_MULTIPLIER = 8;
+
+const isNetwork = (principal: string): boolean => principal.startsWith('address:');
+
+/** A COUNT of at least 1, refused as the config key of the same name is. */
+function socketCeiling(value: number | undefined): number {
+  const max = value ?? DEFAULT_MAX_SOCKETS_PER_ACTOR;
+  if (Number.isSafeInteger(max) && max >= 1) return max;
+  throw new ConfigInvalidError({
+    cause: `maxSocketsPerActor (realtime.maxSocketsPerActor) must be a whole number of at least 1, not ${String(max)} — 0 refuses every upgrade, and a fraction or a NaN is a cap no count ever reaches`,
+    fix: `syncNode({ maxSocketsPerActor: ${String(DEFAULT_MAX_SOCKETS_PER_ACTOR)} })   # or realtime: { maxSocketsPerActor: ${String(DEFAULT_MAX_SOCKETS_PER_ACTOR)} } in app.config.ts`,
+    meta: { key: 'realtime.maxSocketsPerActor', value: max },
+  });
+}
+
+/**
+ * Taken at the upgrade — synchronously, right before `server.upgrade`, so a herd parked in
+ * `authenticate` cannot all pass one count — and given back on EVERY path a socket ends: its
+ * close, and an upgrade that threw or declined after the slot was taken. `release` is idempotent.
+ */
+export class PrincipalSockets {
+  readonly max: number;
+  readonly #bySocket = new Map<string, string>();
+  readonly #count = new Map<string, number>();
+
+  constructor(max: number | undefined) {
+    this.max = socketCeiling(max);
+  }
+
+  /** Admit `socketId` under `principal`, or `false` when that principal is at its cap. */
+  admit(socketId: string, principal: string): boolean {
+    const held = this.#count.get(principal) ?? 0;
+    if (held >= this.limitFor(principal)) return false;
+    this.#bySocket.set(socketId, principal);
+    this.#count.set(principal, held + 1);
+    return true;
+  }
+
+  /** The cap `principal` is held to: `max` for an actor, `max × 8` for an anonymous network. */
+  limitFor(principal: string): number {
+    return isNetwork(principal) ? this.max * ANONYMOUS_SOCKET_MULTIPLIER : this.max;
+  }
+
+  release(socketId: string): void {
+    const principal = this.#bySocket.get(socketId);
+    if (principal === undefined) return;
+    this.#bySocket.delete(socketId);
+    const next = (this.#count.get(principal) ?? 0) - 1;
+    if (next > 0) this.#count.set(principal, next);
+    else this.#count.delete(principal);
+  }
+
+  count(principal: string): number {
+    return this.#count.get(principal) ?? 0;
+  }
+}

package/src/query-window.ts

@@ -80,7 +80,7 @@ export interface PendingRead {
   readonly result: Promise<SnapshotResult>;
 }
 
-export function createEntry(
+export function queryEntry(
   qid: string,
   definition: LiveQueryDefinition,
   input: JsonValue,
@@ -131,9 +131,8 @@ export function createEntry(
  * had already been fanned out would rewind every later subscriber to rows the window has moved
  * past, so a stale read is discarded and its caller is served from the newer window instead.
  */
-export async function fillWindow(
-  entry: QueryEntry,
-): Promise<{ rows: readonly Row[]; lsn: string }> {
+export async function fillWindow(entry: QueryEntry): Promise<FilledWindow> {
+  let landed: FilledWindow['landed'] = null;
   for (let attempt = 0; ; attempt += 1) {
     // Read before `startRead` clears it: a second caller arriving during the read joins it and is
     // not the one that forced it, which is what keeps one forced read from becoming N.
@@ -155,15 +154,28 @@ export async function fillWindow(
       const first = entry.applied === 0;
       if (isNewestRead(entry, pending) && (forced || first || result.lsn >= entry.lsn)) {
         applyRead(entry, pending, result);
+        // A forced read outranks a first one: what it replaced was known to be wrong.
+        landed = forced || landed === 'forced' ? 'forced' : first ? 'first' : landed;
       }
       // A change reached this window while its first read was in flight and could not be folded,
       // so what just landed may predate it: read once more before serving anyone a partial window.
       return first && entry.stale && attempt < COLD_REREADS;
     });
-    if (!again) return { rows: entry.rows, lsn: entry.lsn };
+    if (!again) return { rows: entry.rows, lsn: entry.lsn, landed };
   }
 }
 
+/**
+ * The window a caller is served from, and whether THIS call put a read into it. `landed` is what
+ * decides the retained ring's floor (`change-buffer.ts`): `null` is a window that was already
+ * filled — its patch history is still true, and flooring it would evict every other subscriber's.
+ */
+export interface FilledWindow {
+  readonly rows: readonly Row[];
+  readonly lsn: string;
+  readonly landed: 'first' | 'forced' | null;
+}
+
 /**
  * How many times a cold window re-reads because writes kept landing during its read. Bounded: a
  * table written faster than it can be read would otherwise never serve a subscriber, and after the

package/src/record-persister.ts

@@ -88,10 +88,15 @@ export function recordPersister(options: RecordPersisterOptions): RecordPersiste
   });
 
   const restore = async (): Promise<number> => {
-    const scope = scopeKey(principal());
+    const owner = principal();
+    const scope = scopeKey(owner);
     if (scope === undefined) return 0;
+    const stored = await options.local.rows(scope);
+    // The read is async and a principal change can land inside it: its rescope has already cleared
+    // the store, so restoring now would hand the previous principal's rows to the next one.
+    if (principal() !== owner) return 0;
     let restored = 0;
-    for (const [type, rows] of await options.local.rows(scope)) {
+    for (const [type, rows] of stored) {
       if (!options.types.has(type)) continue;
       options.store.restore(type, rows);
       restored += Object.keys(rows).length;

package/src/record-publisher.ts

@@ -0,0 +1,178 @@
+// Committed rows an app hands over → change envelopes on the subject every sync node reads, for a
+// deployment that runs no replicator. Channels only: a publisher has no commit position, so its
+// changes feed the declared channels and never a live window (`sync-bus-handlers.ts`). One per
+// process, built at boot: it is one producer, and its seq is what a node sees a lost message in.
+
+import {
+  assert,
+  type Clock,
+  currentWriteOrigin,
+  isWriteDigest,
+  systemClock,
+  uuidV7,
+} from '@ultimat3/core';
+import { type RecordProjection, recordProjection } from '@ultimat3/entity/record';
+import { type ChangeEvent, formatLsn } from './changefeed';
+import type { ChannelEntity } from './channel-decl';
+import type { Transport } from './fanout';
+import type { Row } from './json';
+import { changeSubject } from './replicator';
+import { encodeEnvelope } from './replicator-envelope';
+
+export interface RecordPublisherOptions {
+  readonly transport: Transport;
+  /**
+   * Every entity this process publishes, claimed until `close()`. Claimed, `x dev`'s in-process
+   * bridge stops carrying that entity's repository writes to channels — the app's publish is then
+   * the one delivery, exactly as it is in production with no replicator.
+   */
+  readonly entities: readonly ChannelEntity[];
+  /** Tenant column, hoisted into the subject as the replicator hoists it. Default `orgId`. */
+  readonly tenantColumn?: string;
+  readonly clock?: Clock;
+}
+
+export interface PublishOptions {
+  /** `upsert` (default): the rows are adopted. `delete`: the rows are gone, by their key. */
+  readonly op?: 'upsert' | 'delete';
+  /**
+   * The keyed write these rows belong to — `writeDigest(idempotencyKey)` — so the page that made
+   * it settles its own echo. Default: the keyed request this call runs inside, else `null`.
+   */
+  readonly write?: string | null;
+}
+
+export interface RecordPublisher extends Disposable {
+  /** This publisher's producer id: the stream its `seq` counts in. */
+  readonly producer: string;
+  /**
+   * Put COMMITTED rows on the bus, one change per row, in call order. Call it after the
+   * transaction commits: a row published from inside one is on every page before a rollback.
+   * Rejects with the first refusal once every row was tried; a refused row's seq stays spent, so
+   * every node reads the hole and its channel members re-run their catch-up read.
+   */
+  publish(
+    entity: ChannelEntity,
+    rows: readonly Readonly<Record<string, unknown>>[],
+    options?: PublishOptions,
+  ): Promise<void>;
+  /** Releases the claim. Idempotent; a closed publisher refuses to publish. */
+  close(): void;
+}
+
+/** Tables a live publisher claims, counted: two publishers may claim one. */
+const claims = new Map<string, number>();
+
+/**
+ * Whether a `recordPublisher` in this process claims `table` — the relation, the vocabulary of
+ * `ChangeEvent.table` on every producer.
+ */
+export function isPublishedTable(table: string): boolean {
+  return claims.has(table);
+}
+
+export function recordPublisher(options: RecordPublisherOptions): RecordPublisher {
+  const clock = options.clock ?? systemClock;
+  const tenant = options.tenantColumn ?? 'orgId';
+  const projections = new Map<string, RecordProjection>();
+  for (const declared of options.entities) {
+    const projection = recordProjection(declared);
+    projections.set(projection.type, projection);
+  }
+  const names = [...projections.values()].map((projection) => projection.table);
+  for (const name of names) claims.set(name, (claims.get(name) ?? 0) + 1);
+  const producer = uuidV7();
+  let seq = 0;
+  let open = true;
+  // One chain, so seq order is bus order whatever the caller awaits: a node reads a seq that
+  // arrives out of order as a hole that was never there.
+  let tail: Promise<unknown> = Promise.resolve();
+
+  const close = (): void => {
+    if (!open) return;
+    open = false;
+    for (const name of names) {
+      const held = (claims.get(name) ?? 1) - 1;
+      if (held > 0) claims.set(name, held);
+      else claims.delete(name);
+    }
+  };
+
+  const changeOf = (
+    projection: RecordProjection,
+    row: Readonly<Record<string, unknown>>,
+    op: 'upsert' | 'delete',
+    write: string | null,
+  ): ChangeEvent => {
+    // The record's own properties and nothing else: the entity's record schema omits every sealed
+    // column, so neither a secret nor a property the entity never declared can ride.
+    const columns = projection.schema.properties ?? {};
+    const image = Object.fromEntries(
+      Object.entries(row).filter(([property]) => Object.hasOwn(columns, property)),
+    ) as Row;
+    // Refused here, before any row of the call is sequenced: a keyless row matches no record.
+    projection.key(image);
+    const orgId = image[tenant];
+    return {
+      table: projection.table,
+      op: op === 'delete' ? 'delete' : 'update',
+      before: op === 'delete' ? image : null,
+      after: op === 'delete' ? null : image,
+      // Stamped with the seq at send: it orders nothing outside this producer, which is why no
+      // live window is fed one.
+      lsn: '',
+      txid: producer,
+      orgId: typeof orgId === 'string' ? orgId : null,
+      at: clock.now().getTime(),
+      write,
+    };
+  };
+
+  const send = async (change: ChangeEvent): Promise<void> => {
+    seq += 1;
+    await options.transport.publish(
+      changeSubject(change),
+      encodeEnvelope({ ...change, lsn: formatLsn(seq) }, seq, producer, 'publisher'),
+    );
+  };
+
+  const publish: RecordPublisher['publish'] = async (entity, rows, publishOptions = {}) => {
+    assert(
+      open,
+      `recordPublisher ${producer} was closed and published "${entity.$name}" after it`,
+      'build one recordPublisher per process at boot and close it at shutdown',
+    );
+    const projection = projections.get(entity.$name);
+    assert(
+      projection !== undefined,
+      `recordPublisher was asked to publish "${entity.$name}", which it did not claim`,
+      `add ${entity.$name} to recordPublisher({ entities: [...] }) where the publisher is built`,
+    );
+    const write =
+      publishOptions.write === undefined ? (currentWriteOrigin() ?? null) : publishOptions.write;
+    assert(
+      write === null || isWriteDigest(write),
+      `publish({ write }) for "${entity.$name}" is not a write digest`,
+      "pass writeDigest(idempotencyKey) from '@ultimat3/core', or null for a write no request made",
+    );
+    const op = publishOptions.op ?? 'upsert';
+    const changes = rows.map((row) => changeOf(projection, row, op, write));
+    const sent = tail.then(async () => {
+      // Every row the bus takes goes, a refused one in the middle included: its spent seq is the
+      // hole the nodes repair, and the caller hears the first refusal once the rest are out.
+      let refused: { readonly error: unknown } | undefined;
+      for (const change of changes) {
+        try {
+          await send(change);
+        } catch (error) {
+          refused ??= { error };
+        }
+      }
+      if (refused !== undefined) throw refused.error;
+    });
+    tail = sent.catch(() => undefined);
+    await sent;
+  };
+
+  return { producer, publish, close, [Symbol.dispose]: close };
+}

package/src/record-store.ts

@@ -61,16 +61,21 @@ export class RecordStore implements RecordSink {
 
   /** Every record of one type, as seen — what a mutator's `tx.<type>.all()` walks. */
   all(type: string): readonly Row[] {
+    return this.keyed(type).map(([, row]) => row);
+  }
+
+  /** `all`, with each record's key — what a whole-type reader orders by. */
+  keyed(type: string): readonly (readonly [string, Row])[] {
     const prefix = `${type}:`;
-    const out: Row[] = [];
+    const out: (readonly [string, Row])[] = [];
     const seen = new Set<RecordKey>();
     for (const [rk, row] of this.#view) {
       if (!rk.startsWith(prefix)) continue;
       seen.add(rk);
-      if (row !== null) out.push(row);
+      if (row !== null) out.push([rk.slice(prefix.length), row]);
     }
     for (const [rk, row] of this.#synced.entries()) {
-      if (rk.startsWith(prefix) && !seen.has(rk)) out.push(row);
+      if (rk.startsWith(prefix) && !seen.has(rk)) out.push([rk.slice(prefix.length), row]);
     }
     return out;
   }
@@ -153,6 +158,18 @@ export class RecordStore implements RecordSink {
     if (this.#synced.release(rk)) this.#touchSynced(rk);
   }
 
+  /** Every record of `type` is shown by one reader: none is evicted while it is held. */
+  retainType(type: string): void {
+    this.#synced.retainType(type);
+  }
+
+  /** The last type hold leaving evicts the rows it alone kept (`SyncedLayer.releaseType`). */
+  releaseType(type: string): void {
+    this.batch(() => {
+      for (const rk of this.#synced.releaseType(type)) this.#touchSynced(rk);
+    });
+  }
+
   /**
    * An optimistic write, visible to every holder before this returns. `apply` is the mutator's
    * pure `local` half: it is REPLAYED over synced truth whenever synced truth moves, which is what
@@ -342,9 +359,13 @@ export class RecordStore implements RecordSink {
     this.#touched = result.touched;
   }
 
-  /** The one way an overlay leaves: its entry and the digest that named it go together. */
+  /**
+   * The one way an overlay leaves: its entry, the digest that named it and its server wait go
+   * together. A wait left behind ran for its whole bound after a later settle removed the overlay.
+   */
   #forget(key: string): boolean {
     this.#names.forget(key);
+    this.#wait.forget(key);
     return this.#overlays.delete(key);
   }
 

package/src/record-synced.ts

@@ -8,6 +8,10 @@ import type { RecordKey } from './record-key';
 export class SyncedLayer {
   readonly #rows = new Map<RecordKey, Row>();
   readonly #holds = new Map<RecordKey, number>();
+  /** Types a whole-type reader holds (`useRecords(type, selection)`): none of their rows is evicted. */
+  readonly #typeHolds = new Map<string, number>();
+  /** Per held type, the rows whose last key holder left while it was held: the hold's to evict. */
+  readonly #spared = new Map<string, Set<RecordKey>>();
   /** Keys restored from disk and not yet confirmed: the first server row REPLACES, never merges. */
   readonly #provisional = new Set<RecordKey>();
 
@@ -74,6 +78,8 @@ export class SyncedLayer {
   /** Answers whether a row was there to remove. */
   remove(rk: RecordKey): boolean {
     this.#provisional.delete(rk);
+    // Gone is no longer spared: a row the server sends again later is a new arrival, not this one.
+    this.#spared.get(rk.slice(0, rk.indexOf(':')))?.delete(rk);
     return this.#rows.delete(rk);
   }
 
@@ -81,7 +87,10 @@ export class SyncedLayer {
     this.#holds.set(rk, (this.#holds.get(rk) ?? 0) + 1);
   }
 
-  /** The last holder leaving evicts the row. Answers whether a row went. */
+  /**
+   * The last holder leaving evicts the row — unless a reader holds its whole type, which shows
+   * every row of it and would lose this one to another holder's unmount. Answers whether a row went.
+   */
   release(rk: RecordKey): boolean {
     const holds = this.#holds.get(rk);
     if (holds === undefined) return false;
@@ -90,11 +99,44 @@ export class SyncedLayer {
       return false;
     }
     this.#holds.delete(rk);
+    const type = rk.slice(0, rk.indexOf(':'));
+    if (this.#typeHolds.has(type)) {
+      const spared = this.#spared.get(type) ?? new Set<RecordKey>();
+      spared.add(rk);
+      this.#spared.set(type, spared);
+      return false;
+    }
     return this.remove(rk);
   }
 
+  retainType(type: string): void {
+    this.#typeHolds.set(type, (this.#typeHolds.get(type) ?? 0) + 1);
+  }
+
+  /**
+   * The last type hold leaving evicts what it spared — each row whose last key holder left while
+   * the type was held and that no key holder has taken since — exactly as that release would have.
+   * A row no key holder ever held is not the hold's to evict. Answers the rows that went.
+   */
+  releaseType(type: string): readonly RecordKey[] {
+    const holds = this.#typeHolds.get(type) ?? 0;
+    if (holds > 1) {
+      this.#typeHolds.set(type, holds - 1);
+      return [];
+    }
+    this.#typeHolds.delete(type);
+    const spared = this.#spared.get(type);
+    this.#spared.delete(type);
+    const evicted: RecordKey[] = [];
+    for (const rk of spared ?? []) {
+      if (!this.#holds.has(rk) && this.remove(rk)) evicted.push(rk);
+    }
+    return evicted;
+  }
+
   clear(): void {
     this.#rows.clear();
     this.#provisional.clear();
+    this.#spared.clear();
   }
 }

package/src/replication-errors.ts

@@ -17,10 +17,14 @@ export class ReplicationProtocolError extends RealtimeError {
   constructor(args: { stage: string; detail: string; fix?: string }) {
     super({
       code: 'X_REPLICATION_PROTOCOL',
-      cause: `postgres replication ${args.stage}: ${args.detail}`,
-      fix:
-        args.fix ??
-        'x doctor db — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical',
+      // The requirement rides the cause when the caller names no fix of its own: a `fix:` is one
+      // command, and `x doctor` has no `db` subcommand for the old line to have been one.
+      cause:
+        `postgres replication ${args.stage}: ${args.detail}` +
+        (args.fix === undefined
+          ? ' — the server must be postgres >= 14 with a pgoutput publication and wal_level=logical'
+          : ''),
+      fix: args.fix ?? 'x doctor --json',
     });
   }
 }
@@ -59,8 +63,8 @@ export class ReplicationTlsError extends RealtimeError {
 /**
  * A second replicator found the advisory lock held. Distinct from `X_REPLICATION_FAILED` because
  * nothing is wrong with this process: the database already has its one replicator, and a second
- * one that started anyway would publish every change twice. Terminal for a container whose whole
- * job is that role — the scheduler is the thing that has to change, not the connection.
+ * one that started anyway would publish every change twice. Thrown only under `x dev`: a container
+ * that loses the lock stays up unready and takes over when it frees (`cli/src/role-replicator.ts`).
  */
 export class ReplicatorSlotHeldError extends RealtimeError {
   constructor(args: { key: string; holder?: string | undefined }) {
@@ -69,7 +73,7 @@ export class ReplicatorSlotHeldError extends RealtimeError {
       cause:
         `advisory lock ${args.key} is held${args.holder === undefined ? '' : ` by ${args.holder}`}` +
         ' — one database has exactly one replicator',
-      fix: 'scale the replicator to 1 per database: kubectl scale deploy/replicator --replicas=1',
+      fix: 'x dev --role replicator   # rerun after the replicator already on this DATABASE_URL stops — one database has one replicator',
     });
   }
 }

package/src/replicator-envelope.ts

@@ -0,0 +1,160 @@
+// The bus envelope, both sides: the one encoder every producer writes with (`replicator.ts`,
+// `record-publisher.ts`), its decode back into a `ChangeEvent` (rows revived through their entity's
+// columns) with the producer and sequence a lost message is visible in, and the detector a `sync`
+// node runs over them.
+
+import { isWriteDigest } from '@ultimat3/core';
+import type { ChangeEvent } from './changefeed';
+import { busRow, reviveBusRow } from './replicator-row';
+
+/**
+ * What the bus actually carries: one change, plus who published it and where in that publisher's
+ * stream it sits.
+ *
+ * Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
+ * cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
+ * cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
+ * arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
+ *
+ * Both fields are optional on the wire, so a node reading a publisher that predates them simply
+ * detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
+ */
+export interface ChangeEnvelope {
+  readonly change: ChangeEvent;
+  /** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
+  readonly seq: number | null;
+  /** Identifies one replicator *run*. A new one restarts `seq` — and follows a stream that died. */
+  readonly producer: string | null;
+  /**
+   * Who sequenced it. Absent is the replicator (every publisher that predates the field). A
+   * `recordPublisher` says `'publisher'`: its changes carry no commit position, so they feed the
+   * declared channels and never a live window, and several publishers run side by side.
+   */
+  readonly source?: EnvelopeSource;
+}
+
+/** The two producers of the change bus: the one WAL replicator, and an app's record publishers. */
+export type EnvelopeSource = 'replicator' | 'publisher';
+
+/**
+ * Producer side: one change as the bus carries it. Flat — the change plus `seq`, `producer` and,
+ * for a record publisher, `source` — so a consumer that only knows the change reads it unchanged
+ * (`parseChange`). Rows cross as `busRow` writes them: bytes as base64, revived by the entity.
+ */
+export function encodeEnvelope(
+  change: ChangeEvent,
+  seq: number,
+  producer: string,
+  source?: EnvelopeSource,
+): string {
+  return JSON.stringify({
+    ...change,
+    before: busRow(change.before),
+    after: busRow(change.after),
+    seq,
+    producer,
+    ...(source === 'publisher' ? { source } : {}),
+  });
+}
+
+/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
+export function parseChange(payload: string): ChangeEvent | null {
+  return parseEnvelope(payload)?.change ?? null;
+}
+
+/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
+export function parseEnvelope(payload: string): ChangeEnvelope | null {
+  try {
+    const parsed: unknown = JSON.parse(payload);
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const shape = parsed as Partial<ChangeEvent> & {
+      seq?: unknown;
+      producer?: unknown;
+      source?: unknown;
+      entity?: unknown;
+    };
+    // `entity` is the field a 24.x replicator wrote the table under. Read, never written: across a
+    // rolling deploy to 25.0.0 the replicator and the sync nodes run different builds for minutes,
+    // and a node refusing every change of the old one is every live view frozen for that window.
+    const table = typeof shape.table === 'string' ? shape.table : shape.entity;
+    if (typeof table !== 'string' || typeof shape.lsn !== 'string') return null;
+    if (
+      shape.op !== 'insert' &&
+      shape.op !== 'update' &&
+      shape.op !== 'delete' &&
+      shape.op !== 'truncate'
+    ) {
+      return null;
+    }
+    return {
+      change: {
+        table,
+        op: shape.op,
+        before: reviveBusRow(table, shape.before ?? null),
+        after: reviveBusRow(table, shape.after ?? null),
+        lsn: shape.lsn,
+        txid: typeof shape.txid === 'string' ? shape.txid : '',
+        orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
+        at: typeof shape.at === 'number' ? shape.at : 0,
+        // Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
+        // label here would cost every member the whole frame rather than one page its echo.
+        write: isWriteDigest(shape.write) ? shape.write : null,
+        ...(isNames(shape.omitted) ? { omitted: shape.omitted } : {}),
+      },
+      seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
+      producer: typeof shape.producer === 'string' ? shape.producer : null,
+      ...(shape.source === 'publisher' ? { source: 'publisher' as const } : {}),
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** A non-empty list of property names — the only shape `ChangeEvent.omitted` crosses the bus in. */
+const isNames = (value: unknown): value is readonly string[] =>
+  Array.isArray(value) && value.length > 0 && value.every((name) => typeof name === 'string');
+
+/**
+ * The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
+ * say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
+ * there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
+ * simply never happened as far as every subscriber on this node could tell.
+ *
+ * Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
+ * reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
+ * the first message this node ever reads: a node that joined late has missed everything by
+ * definition, and every subscription it holds started after it did.
+ *
+ * A NEW producer after a known one IS a gap. The run before it died — that is why there is a new
+ * one — and nothing says its last messages reached this node: the tail of a dead stream has no
+ * later sequence number to be missed against. One re-read per replicator restart is the price.
+ *
+ * Only a REPLICATOR run succeeds another. Record publishers (`source: 'publisher'`) run side by
+ * side, one per app process, so a new one says nothing about the others; and a replicator run
+ * after a publisher is no successor either. A hole inside any one producer's sequence is a gap.
+ */
+export class SeqGapDetector {
+  readonly #next = new Map<string, number>();
+  /** Replicator runs read — the producers a NEW replicator run succeeds. */
+  readonly #runs = new Set<string>();
+
+  /** `true` when at least one message between the last one and this one was never delivered. */
+  observe(envelope: ChangeEnvelope): boolean {
+    const { producer, seq } = envelope;
+    if (producer === null || seq === null) return false;
+    const expected = this.#next.get(producer);
+    const replicator = envelope.source !== 'publisher';
+    const succeeds = replicator && expected === undefined && this.#runs.size > 0;
+    if (replicator) this.#runs.add(producer);
+    // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
+    // into a gap of its own.
+    this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
+    return succeeds || (expected !== undefined && seq > expected);
+  }
+
+  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
+  forget(): void {
+    this.#next.clear();
+    this.#runs.clear();
+  }
+}