@ultimat3/realtime 23.0.0 → 24.0.0

package/src/replicator.ts

@@ -8,51 +8,28 @@
 //
 // The lock is tied to the *session*, so a crashed replicator releases it automatically — no lease
 // renewal, no fencing token, no split brain. A process that fails to take the lock does not start
-// its feed and reports `/readyz` false; it retries with jittered backoff and takes over the moment
-// the holder dies. Scaling the replicator is therefore always vertical, and that is by design.
+// its feed and reports `/readyz` false. A stream that ENDS — or a lock that is LOST — is this file's
+// to answer: `running` goes false, everything is let go, and the process asks again on a backoff.
 
-import { isWriteDigest, logger, uuid, withSpan } from '@ultimat3/core';
+import { logger, renderThrowable, uuid, withSpan } from '@ultimat3/core';
+import type { AdvisoryLock } from './advisory-lock';
 import type { ChangeEvent, ChangeFeed } from './changefeed';
+import { ReplicationFailedError } from './errors';
 import type { Transport } from './fanout';
-import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
+import { busRow } from './replicator-row';
+import {
+  type BackoffPolicy,
+  defaultBackoff,
+  policyDelay,
+  type Rng,
+  type Scheduler,
+} from './thundering-herd';
 
 export const CHANGE_SUBJECT_PREFIX = 'x.change';
 
 /** Every change subject: what a `sync` node subscribes to. A changefeed subject, never a channel. */
 export const CHANGE_SUBJECT_ALL = `${CHANGE_SUBJECT_PREFIX}.>`;
 
-/** Session-scoped mutual exclusion. Postgres-backed in production, in-memory for `x dev`. */
-export interface AdvisoryLock {
-  readonly key: string;
-  /** `false` means another process holds it — never block, never steal. */
-  tryAcquire(): Promise<boolean>;
-  release(): Promise<void>;
-}
-
-/** Single-process default: correct for `x dev` and tests, useless across containers (by design). */
-export class InMemoryAdvisoryLock implements AdvisoryLock {
-  static readonly #held = new Set<string>();
-  readonly key: string;
-  #mine = false;
-
-  constructor(key: string) {
-    this.key = key;
-  }
-
-  async tryAcquire(): Promise<boolean> {
-    if (InMemoryAdvisoryLock.#held.has(this.key)) return false;
-    InMemoryAdvisoryLock.#held.add(this.key);
-    this.#mine = true;
-    return true;
-  }
-
-  async release(): Promise<void> {
-    if (!this.#mine) return;
-    InMemoryAdvisoryLock.#held.delete(this.key);
-    this.#mine = false;
-  }
-}
-
 /** `x.change.<entity>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
 export function changeSubject(change: ChangeEvent): string {
   return `${CHANGE_SUBJECT_PREFIX}.${change.entity}.${change.orgId ?? '_'}`;
@@ -67,38 +44,34 @@ export interface ReplicatorOptions {
   readonly subjectOf?: (change: ChangeEvent) => string;
   readonly backoff?: BackoffPolicy;
   readonly rng?: Rng;
+  /** The timer a restart waits on. Injected so a test proves the takeover loop without sleeping. */
+  readonly schedule?: Scheduler;
 }
 
 export interface ReplicatorStats {
   readonly published: number;
   readonly skipped: number;
   readonly outOfOrder: number;
-}
-
-/**
- * What the bus actually carries: one change, plus who published it and where in that publisher's
- * stream it sits.
- *
- * Fanout is core NATS — `publish`, no ack, at most once — so a consumer that reads only the change
- * cannot tell "nothing happened" from "eleven changes went past while I was reconnecting". An lsn
- * cannot answer it either: a WAL position is a byte offset, so a legitimate next change is an
- * arbitrary jump forwards. A per-publisher counter is the one number a gap is visible in.
- *
- * Both fields are optional on the wire, so a node reading a publisher that predates them simply
- * detects nothing — the same rule the `snapshot.entity` field follows, on a subject no client sees.
- */
-export interface ChangeEnvelope {
-  readonly change: ChangeEvent;
-  /** Monotonic within `producer`, from 1. `null` from a publisher that does not sequence. */
-  readonly seq: number | null;
-  /** Identifies one replicator *run*. A new one restarts `seq`, and that is not a gap. */
-  readonly producer: string | null;
+  /** Streams this replicator brought back after one ended on its own. */
+  readonly restarts: number;
+  /**
+   * Why this replicator is not running although nothing stopped it — the stream's own reason, the
+   * last restart's, or the change (entity, lsn, how many times) the bus keeps refusing. `null`
+   * while it runs, and after a `stop()`.
+   */
+  readonly failure: string | null;
 }
 
 export interface Replicator {
   /** `false` = another replicator holds the lock; this process must stay `/readyz` false. */
   start(): Promise<boolean>;
   stop(): Promise<void>;
+  /**
+   * `true` only while this process is REPLICATING: a stream is pumping under the lock, and no
+   * change is waiting on a publish the bus keeps refusing. `false` the moment the stream ends or
+   * the lock is lost, and through every restart that has yet to publish the change it owes — the
+   * one fact a readiness check has to read.
+   */
   readonly running: boolean;
   lastLsn(): string | null;
   stats(): ReplicatorStats;
@@ -106,10 +79,39 @@ export interface Replicator {
   retryDelayMs(attempt: number): number;
 }
 
+/** One `begin()`: the stream and the lock grant that belong together. */
+interface Run {
+  /** Why this run is over — the stream ended, the lock was lost, or it was stopped. */
+  over: string | null;
+}
+
+/** The change a run could not publish, kept until one goes out. */
+interface Refusal {
+  readonly entity: string;
+  readonly lsn: string;
+  readonly error: string;
+  times: number;
+}
+
+/**
+ * How long each polite goodbye of a `stop()` may take — the feed's, then the lock's — before the
+ * thing is abandoned instead. Both are conversations with a socket; a black-holed one never
+ * answers, and a SIGTERM that waited on it never finished.
+ */
+export const STOP_DEADLINE_MS = 5_000;
+
+const fencedOut = (reason: string): ReplicationFailedError =>
+  new ReplicationFailedError({
+    stage: 'stream',
+    detail: `a change arrived for a run that is over (${reason}), and was not published`,
+    fix: 'x doctor --json',
+  });
+
 export function createReplicator(options: ReplicatorOptions): Replicator {
   const subjectOf = options.subjectOf ?? changeSubject;
   const backoff = options.backoff ?? defaultBackoff;
-  let running = false;
+  /** A stream is pumping under a lock this process holds. Internal: `running` is stricter. */
+  let pumping = false;
   let lastLsn: string | null = null;
   let published = 0;
   let skipped = 0;
@@ -119,33 +121,126 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
   // the old one.
   let producer = uuid();
   let seq = 0;
-  /** The start in flight, if any — see `start()` on why `running` cannot answer that question. */
+  /** The start in flight, if any — see `start()` on why `pumping` cannot answer that question. */
   let starting: Promise<boolean> | undefined;
+  const schedule = options.schedule ?? unrefScheduler;
+  let restarts = 0;
+  let failure: string | null = null;
+  /** Set by `stop()`: the one thing that ends the takeover loop. */
+  let stopped = false;
+  /** The run whose stream and lock still speak for this replicator. */
+  let current: Run | undefined;
+  let cancelRetry: (() => void) | undefined;
+  /** Stops listening for the lock being lost. Held only while this replicator holds the lock. */
+  let unwatchLock: (() => void) | undefined;
+  const unwatch = (): void => {
+    unwatchLock?.();
+    unwatchLock = undefined;
+  };
+  /**
+   * Deaths since a change was last PUBLISHED. "The stream started" is not progress: a change the
+   * bus refuses every time kills each new stream on its first delivery, and counting from the
+   * start made that a redial — two dials, a preflight, a `START_REPLICATION` — at the base delay
+   * forever, with the WAL retained behind it.
+   */
+  let attempt = 0;
+  let refusal: Refusal | null = null;
 
-  const onChange = async (raw: ChangeEvent): Promise<void> => {
-    const change = normalize(raw);
-    if (!change) {
-      skipped += 1;
-      return;
-    }
-    if (lastLsn !== null && change.lsn <= lastLsn) {
-      // At-least-once delivery is the feed's contract, so a repeat is expected, not an error.
-      outOfOrder += 1;
-      return;
-    }
-    await withSpan('realtime.replicate', async () => {
-      seq += 1;
-      // The envelope is the change plus two fields, flat, so a consumer that only knows about the
-      // change reads it unchanged — `parseChange` still answers on the same payload.
-      await options.transport.publish(
-        subjectOf(change),
-        JSON.stringify({ ...change, seq, producer }),
-      );
-      lastLsn = change.lsn;
-      published += 1;
+  /** One goodbye against the deadline, on the `schedule` seam. Never rejects. */
+  const bounded = async (
+    goodbye: Promise<void>,
+  ): Promise<
+    { readonly how: 'done' | 'late' } | { readonly how: 'failed'; readonly error: unknown }
+  > => {
+    let cancel = (): void => undefined;
+    const deadline = new Promise<{ readonly how: 'late' }>((resolve) => {
+      cancel = schedule(() => resolve({ how: 'late' }), STOP_DEADLINE_MS);
     });
+    const said = goodbye.then(
+      () => ({ how: 'done' as const }),
+      (error: unknown) => ({ how: 'failed' as const, error }),
+    );
+    const outcome = await Promise.race([said, deadline]);
+    cancel();
+    return outcome;
+  };
+
+  /**
+   * The orderly unlock, bounded. Past the deadline — or refused — the session is dropped instead:
+   * closing it releases the lock just the same.
+   */
+  const release = async (): Promise<void> => {
+    if ((await bounded(options.lock.release())).how !== 'done') options.lock.abandon();
   };
 
+  /** The orderly stream stop, bounded the same way. A refusal is handed back for the caller. */
+  const quiet = async (): Promise<unknown> => {
+    const outcome = await bounded(options.feed.stop());
+    if (outcome.how !== 'done') options.feed.abandon();
+    return outcome.how === 'failed' ? outcome.error : undefined;
+  };
+
+  const changeHandler =
+    (run: Run) =>
+    async (raw: ChangeEvent): Promise<void> => {
+      // Fenced by run, and REFUSED rather than dropped: after a lock loss the socket still holds
+      // changes, and publishing them is one of two replicators speaking. A handler that merely
+      // returned would let the stream count the change delivered and confirm it.
+      if (current !== run || run.over !== null) throw fencedOut(run.over ?? 'a newer run started');
+      const change = normalize(raw);
+      if (!change) {
+        skipped += 1;
+        return;
+      }
+      if (lastLsn !== null && change.lsn <= lastLsn) {
+        // At-least-once delivery is the feed's contract, so a repeat is expected, not an error.
+        outOfOrder += 1;
+        return;
+      }
+      await withSpan('realtime.replicate', async () => {
+        seq += 1;
+        try {
+          // The envelope is the change plus two fields, flat, so a consumer that only knows about
+          // the change reads it unchanged — `parseChange` still answers on the same payload.
+          await options.transport.publish(
+            subjectOf(change),
+            JSON.stringify({
+              ...change,
+              before: busRow(change.before),
+              after: busRow(change.after),
+              seq,
+              producer,
+            }),
+          );
+        } catch (thrown) {
+          const error = renderThrowable(thrown);
+          const same = refusal !== null && refusal.lsn === change.lsn;
+          refusal = {
+            entity: change.entity,
+            lsn: change.lsn,
+            error,
+            times: same ? (refusal?.times ?? 0) + 1 : 1,
+          };
+          throw thrown;
+        }
+        lastLsn = change.lsn;
+        published += 1;
+        // Progress: the next death waits the base delay again, and nothing is owed any more.
+        attempt = 0;
+        if (refusal !== null) {
+          refusal = null;
+          if (pumping) failure = null;
+        }
+      });
+    };
+
+  /** What `stats().failure` says while a change keeps being refused: which one, and how often. */
+  const describeFailure = (reason: string): string =>
+    refusal === null
+      ? reason
+      : `${refusal.entity} at lsn ${refusal.lsn} could not be published ` +
+        `(${refusal.times} time(s) in a row): ${refusal.error}`;
+
   const begin = async (): Promise<boolean> => {
     if (!(await options.lock.tryAcquire())) {
       logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
@@ -153,67 +248,172 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
     }
     producer = uuid();
     seq = 0;
+    const run: Run = { over: null };
+    current = run;
+    const end = (reason: string): void => {
+      if (run.over !== null) return;
+      run.over = reason;
+      // Not yet `pumping`: `begin` is still between the feed's start and its own last line, and
+      // reads `run.over` there. Not `current`: a stop or a newer run owns the replicator now.
+      if (current === run && pumping) lost(reason);
+    };
+    // The lock is a session, and a session can die while the stream stays up — or while it is
+    // still being dialled. Postgres then gives the lock to whoever asks next, and the loss is said
+    // ONCE: recorded on the run, so a start that was mid-dial reads it below instead of going on
+    // to stream, ready and unlocked, for the rest of the process's life.
+    unwatch();
+    unwatchLock = options.lock.onLost((reason) => end(`the advisory lock was lost: ${reason}`));
+    // A restart resumes where THIS process got to. `options.from` is where the first run began;
+    // handing it to the second would have the feed re-deliver everything published since.
+    const from = lastLsn ?? options.from;
+    const onChange = changeHandler(run);
+    let fed = false;
     try {
       await options.feed.start(
-        options.from === undefined ? { onChange } : { from: options.from, onChange },
+        from === undefined ? { onChange, onEnd: end } : { from, onChange, onEnd: end },
       );
+      fed = true;
+      if (run.over !== null) {
+        throw new ReplicationFailedError({
+          stage: 'stream',
+          detail: `the run was over as it started: ${run.over}`,
+          fix: 'x doctor --json',
+        });
+      }
     } catch (thrown) {
-      // A start that failed holds NOTHING. `running` stays false so the takeover loop's next
+      // A start that failed holds NOTHING. `pumping` stays false so the takeover loop's next
       // `start()` runs `begin` again instead of being answered `true` by a memo over a feed that
       // never pumped, and the lock goes back so a standby can take the slot rather than waiting on
-      // a holder that is not replicating.
-      //
-      // The release is best-effort by design: the feed's failure is the one the caller must see,
-      // and a session-scoped advisory lock a dead connection cannot release is released by
-      // Postgres when that session ends.
-      await options.lock.release().catch(() => undefined);
+      // a holder that is not replicating. A feed that DID start — the lock went while it dialled —
+      // is stopped first: it must not outlive the grant it was started under.
+      run.over ??= renderThrowable(thrown);
+      unwatch();
+      if (fed) await quiet();
+      await release();
       throw thrown;
     }
-    // AFTER the feed is pumping, never before: `running` is what `start()` answers `true` from and
+    // AFTER the feed is pumping, never before: `pumping` is what `start()` answers `true` from and
     // what `stop()` reads to decide there is anything to tear down.
-    running = true;
+    pumping = true;
+    if (refusal === null) failure = null;
     logger.info('replicator started', { source: options.feed.source, key: options.lock.key });
     return true;
   };
 
+  /** Memoised — see `start()`. The takeover loop calls this, never `start()`: it must not un-stop. */
+  const launch = (): Promise<boolean> => {
+    if (pumping) return Promise.resolve(true);
+    const inFlight = starting;
+    if (inFlight !== undefined) return inFlight;
+    // Cleared however it settles: a `false` is "another node holds the lock", and the takeover
+    // loop asks again a `retryDelayMs` later. A memo that stuck would make this node a permanent
+    // standby of a slot whose holder has already died.
+    const begun = begin().finally(() => {
+      if (starting === begun) starting = undefined;
+    });
+    starting = begun;
+    return begun;
+  };
+
+  const retryDelayMs = (count: number): number =>
+    // This method's contract is 0-based (`retryDelayMs(0)` is the base); core counts from 1.
+    policyDelay(backoff, count + 1, options.rng ?? Math.random);
+
+  /**
+   * The takeover loop. One timer at a time, each on a longer delay than the last until a change
+   * is PUBLISHED — which is what resets the count (`changeHandler`), not a stream coming up.
+   * `false` from `launch` is another process holding the lock, which is this loop working: that
+   * process replicates, and this one keeps asking in case it dies too.
+   */
+  const retry = (): void => {
+    if (stopped) return;
+    const wait = retryDelayMs(attempt);
+    attempt += 1;
+    cancelRetry = schedule(() => {
+      cancelRetry = undefined;
+      // `pumping` already: the owner called `start()` itself while this timer was armed.
+      if (stopped || pumping) return;
+      // Voided: both outcomes are handled right here, and a timer has nobody to hand a promise to.
+      void launch().then(
+        (started) => {
+          if (started) restarts += 1;
+          else retry();
+        },
+        (thrown: unknown) => {
+          failure = describeFailure(renderThrowable(thrown));
+          logger.warn('replicator.restart_failed', { key: options.lock.key, error: failure });
+          retry();
+        },
+      );
+    }, wait);
+  };
+
+  /**
+   * A stream that ended on its own, or a lock that was lost. It used to be recorded where nothing
+   * read it: `running` stayed `true`, the lock stayed held, and the fleet's live windows froze
+   * until the pod was restarted. Everything the run held is let go here and the loop above asks
+   * for it back.
+   *
+   * Both are ABANDONED, never asked, and synchronously: a black-holed socket is what produces a
+   * lost stream, and a goodbye to one — the feed's or the lock's — is a recovery that never
+   * reaches its retry. Being synchronous is also what leaves no window for a `start()` to be
+   * answered over a run still being torn down.
+   */
+  const lost = (reason: string): void => {
+    pumping = false;
+    failure = describeFailure(reason);
+    unwatch();
+    logger.error('replicator.stream_ended', { key: options.lock.key, error: failure });
+    // The feed first: it must be silent before the grant it ran under is given up. Anything its
+    // socket still held is refused by the run's own fence.
+    options.feed.abandon();
+    options.lock.abandon();
+    retry();
+  };
+
   return {
     /**
-     * Deliberately NOT `async`: `running` is set after an `await` on the lock, so a guard that
+     * Deliberately NOT `async`: `pumping` is set after an `await` on the lock, so a guard that
      * awaited anything before registering has the same hole it is meant to close. Two overlapping
-     * starts both passed `if (running)`, both were told they held the lock — a holder's
+     * starts both passed the guard, both were told they held the lock — a holder's
      * `tryAcquire()` answers `true` — and both ran `feed.start()`: one replication slot with two
      * pumps, publishing every change twice under two `seq` generations of one producer id, which
      * every sync node's `SeqGapDetector` reads as a gap and repairs by re-snapshotting the fleet.
      * Same shape as `packages/core/src/lifecycle.ts`'s `drain()`.
      */
     start(): Promise<boolean> {
-      if (running) return Promise.resolve(true);
-      const inFlight = starting;
-      if (inFlight !== undefined) return inFlight;
-      // Cleared however it settles: a `false` is "another node holds the lock", and the takeover
-      // loop asks again a `retryDelayMs` later. A memo that stuck would make this node a permanent
-      // standby of a slot whose holder has already died.
-      const attempt = begin().finally(() => {
-        if (starting === attempt) starting = undefined;
-      });
-      starting = attempt;
-      return attempt;
+      stopped = false;
+      return launch();
     },
 
     async stop(): Promise<void> {
-      // A stop racing a start waits it out: `running` is false for the whole of `begin`, so the
+      // A stop racing a start waits it out: `pumping` is false for the whole of `begin`, so the
       // early return below would leave the feed it is about to start pumping into a replicator
       // nothing intends to stop, with the advisory lock still held.
+      stopped = true;
+      cancelRetry?.();
+      cancelRetry = undefined;
       const inFlight = starting;
       if (inFlight !== undefined) await inFlight.catch(() => false);
-      if (!running) return;
-      running = false;
-      await options.feed.stop();
-      await options.lock.release();
+      // An orderly stop is not a failure, whatever the last stream died of.
+      failure = null;
+      refusal = null;
+      if (!pumping) return;
+      pumping = false;
+      if (current !== undefined) current.over ??= 'the replicator was stopped';
+      current = undefined;
+      unwatch();
+      // Whatever the feed said: a lock kept because the stream would not close politely is a
+      // slot no standby can take. Its refusal is still the one the caller is owed.
+      const refused = await quiet();
+      await release();
+      if (refused !== undefined) throw refused;
     },
 
     get running(): boolean {
-      return running;
+      // Replicating, not merely connected: a stream that is up while the change it owes is still
+      // being refused has published nothing, and must not read as ready between two of its deaths.
+      return pumping && refusal === null;
     },
 
     lastLsn(): string | null {
@@ -221,16 +421,22 @@ export function createReplicator(options: ReplicatorOptions): Replicator {
     },
 
     stats(): ReplicatorStats {
-      return { published, skipped, outOfOrder };
+      return { published, skipped, outOfOrder, restarts, failure };
     },
 
-    retryDelayMs(attempt: number): number {
-      // This method's contract is 0-based (`retryDelayMs(0)` is the base); core counts from 1.
-      return policyDelay(backoff, attempt + 1, options.rng ?? Math.random);
-    },
+    retryDelayMs,
   };
 }
 
+/** The production timer: a pending restart must never be what keeps a draining process alive. */
+const unrefScheduler: Scheduler = (fn, ms) => {
+  const handle = setTimeout(fn, ms);
+  handle.unref?.();
+  return () => {
+    clearTimeout(handle);
+  };
+};
+
 /** Drops events the pipeline cannot use and hoists the tenant id out of the row. */
 export function normalize(change: ChangeEvent): ChangeEvent | null {
   // The one change that carries no row by design; nothing to hoist a tenant out of.
@@ -243,76 +449,3 @@ export function normalize(change: ChangeEvent): ChangeEvent | null {
   const orgId = row['orgId'];
   return typeof orgId === 'string' ? { ...change, orgId } : change;
 }
-
-/** Sync-node side of the bus: decode a published change back into a `ChangeEvent`. */
-export function parseChange(payload: string): ChangeEvent | null {
-  return parseEnvelope(payload)?.change ?? null;
-}
-
-/** The same decode, keeping the two fields a gap is visible in. `parseChange` is this, narrowed. */
-export function parseEnvelope(payload: string): ChangeEnvelope | null {
-  try {
-    const parsed: unknown = JSON.parse(payload);
-    if (typeof parsed !== 'object' || parsed === null) return null;
-    const shape = parsed as Partial<ChangeEvent> & { seq?: unknown; producer?: unknown };
-    if (typeof shape.entity !== 'string' || typeof shape.lsn !== 'string') return null;
-    if (
-      shape.op !== 'insert' &&
-      shape.op !== 'update' &&
-      shape.op !== 'delete' &&
-      shape.op !== 'truncate'
-    ) {
-      return null;
-    }
-    return {
-      change: {
-        entity: shape.entity,
-        op: shape.op,
-        before: shape.before ?? null,
-        after: shape.after ?? null,
-        lsn: shape.lsn,
-        txid: typeof shape.txid === 'string' ? shape.txid : '',
-        orgId: typeof shape.orgId === 'string' ? shape.orgId : null,
-        at: typeof shape.at === 'number' ? shape.at : 0,
-        // Only the minted shape crosses: a frame decoder refuses anything else, so a malformed
-        // label here would cost every member the whole frame rather than one page its echo.
-        ...(isWriteDigest(shape.write) ? { write: shape.write } : {}),
-      },
-      seq: typeof shape.seq === 'number' && Number.isFinite(shape.seq) ? shape.seq : null,
-      producer: typeof shape.producer === 'string' ? shape.producer : null,
-    };
-  } catch {
-    return null;
-  }
-}
-
-/**
- * The consume-side twin of the publisher's counter, and the only thing on a `sync` node that can
- * say "this node missed changes". Publish-side duplicate and out-of-order guards already existed;
- * there was no equivalent here, so a NATS blip during a rolling restart was eleven changes that
- * simply never happened as far as every subscriber on this node could tell.
- *
- * Per producer, because a replicator restart legitimately rewinds the counter. A repeat or a
- * reordering is *not* reported as a gap — the window's own lsn guard refuses those — and neither is
- * the first message of a stream: a node that joined late has missed everything by definition, and
- * every subscription it holds started after it did.
- */
-export class SeqGapDetector {
-  readonly #next = new Map<string, number>();
-
-  /** `true` when at least one message between the last one and this one was never delivered. */
-  observe(envelope: ChangeEnvelope): boolean {
-    const { producer, seq } = envelope;
-    if (producer === null || seq === null) return false;
-    const expected = this.#next.get(producer);
-    // Never backwards: a redelivery must not lower the bar and turn the next legitimate message
-    // into a gap of its own.
-    this.#next.set(producer, Math.max(expected ?? 0, seq + 1));
-    return expected !== undefined && seq > expected;
-  }
-
-  /** Producers this node has read. Bounded by replicator restarts, so it is swept on a drain. */
-  forget(): void {
-    this.#next.clear();
-  }
-}

package/src/server.ts

@@ -4,6 +4,7 @@
 // island realtime promises unbuildable. Every name here has exactly one home; the shared
 // vocabulary (the wire, the errors, `Row`, the backoff) stays on `@ultimat3/realtime`.
 
+export { type AdvisoryLock, InMemoryAdvisoryLock } from './advisory-lock';
 // ---- the retained change window one node fans out from ------------------------------------------
 export {
   type ChangeBufferOptions,
@@ -116,12 +117,15 @@ export { NatsTransport, type NatsTransportOptions } from './nats-transport';
 export { PgAdvisoryLock, type PgAdvisoryLockOptions } from './pg-advisory-lock';
 // ---- the postgres replication path ------------------------------------------------------------
 export { entityRow } from './pg-entity-row';
+export { paramsChannelTables } from './pg-identity-tables';
+export { CHANNEL_IDENTITY_EVENT } from './pg-preflight';
 export {
   changeLsn,
   commitPositionOf,
   type ReplicationStreamStats,
 } from './pg-replication';
 export { bunPgStream, type PgTarget, parsePgUrl, type SslMode } from './pg-socket';
+export { answersWeakAuth } from './pg-tls';
 // The value domain a WAL tuple lands in. Public because it is `PgOutputMessage`'s and
 // `entityRow`'s: a caller naming either type has to be able to name what is inside one.
 export { decodeValue, type PhysicalRow, type PhysicalValue } from './pg-values';
@@ -154,20 +158,20 @@ export {
   refillWindowInLane,
 } from './query-window';
 export {
-  type AdvisoryLock,
   CHANGE_SUBJECT_PREFIX,
-  type ChangeEnvelope,
   changeSubject,
   createReplicator,
-  InMemoryAdvisoryLock,
   normalize,
-  parseChange,
-  parseEnvelope,
   type Replicator,
   type ReplicatorOptions,
   type ReplicatorStats,
-  SeqGapDetector,
 } from './replicator';
+export {
+  type ChangeEnvelope,
+  parseChange,
+  parseEnvelope,
+  SeqGapDetector,
+} from './replicator-envelope';
 export {
   actorIdOf,
   CLOSE,