@ultimat3/realtime 30.8.0 → 31.0.0

package/src/record-publisher.ts

@@ -15,7 +15,7 @@ import { type RecordProjection, recordProjection } from '@ultimat3/entity/record
 import { type ChangeEvent, formatLsn } from './changefeed';
 import type { ChannelEntity } from './channel-decl';
 import type { Transport } from './fanout';
-import type { Row } from './json';
+import type { LiveRow } from './json';
 import { changeSubject } from './replicator';
 import { encodeEnvelope } from './replicator-envelope';
 
@@ -109,7 +109,7 @@ export function recordPublisher(options: RecordPublisherOptions): RecordPublishe
     const columns = projection.schema.properties ?? {};
     const image = Object.fromEntries(
       Object.entries(row).filter(([property]) => Object.hasOwn(columns, property)),
-    ) as Row;
+    ) as LiveRow;
     // Refused here, before any row of the call is sequenced: a keyless row matches no record.
     projection.key(image);
     const orgId = image[tenant];

package/src/record-store.ts

@@ -258,19 +258,21 @@ export class RecordStore implements RecordSink {
           if (carried === undefined || carried.has(rk)) this.#resolve(entry, rk);
       }
       const heard = this.#wait.takeHeard(key);
-      const missing =
+      // A Set: the split below asks it once per touched row, and an array made that O(t²).
+      const missing = new Set(
         carried === undefined
           ? []
-          : [...touched].filter((rk) => !carried.has(rk) && heard?.has(rk) !== true);
-      if (missing.length > 0) {
+          : [...touched].filter((rk) => !carried.has(rk) && heard?.has(rk) !== true),
+      );
+      if (missing.size > 0) {
         // What the server did answer holds this write already: the twin stays on the rest only.
-        const answered = [...touched].filter((rk) => !missing.includes(rk));
+        const answered = [...touched].filter((rk) => !missing.has(rk));
         if (answered.length > 0) {
           const confirmed = new Set([...(entry.confirmed ?? []), ...answered]);
           this.#overlays.set(key, { ...entry, confirmed });
           this.#replay();
         }
-        this.#wait.start(key, new Set(missing), () => this.drop(key));
+        this.#wait.start(key, missing, () => this.drop(key));
         return;
       }
       this.#forget(key);

package/src/replication-errors.ts

@@ -1,5 +1,5 @@
-// The five refusals the Postgres replication half raises: the wire, the connection, its TLS, the
-// slot, and the replica identity it warns about.
+// The refusals the Postgres replication half raises: the wire, the connection, its TLS, the lock,
+// the slot (invalidated, or recreated under a resume), and the replica identity it warns about.
 //
 // Split out of `errors.ts` on the one seam this package already draws — these are the only codes
 // no browser can reach, thrown by `pg-*.ts` and the replicator and by nothing on the client half.
@@ -102,3 +102,41 @@ export class ReplicaIdentityError extends RealtimeError {
     });
   }
 }
+
+/**
+ * Postgres INVALIDATED the slot (`wal_status = 'lost'`): it fell further behind than
+ * `max_slot_wal_keep_size` allows and the WAL it needed was removed. It can never stream again,
+ * and every restart failed on a generic `X_REPLICATION_FAILED`. The slot name has passed
+ * `assertIdentifier`, so the `fix:` is SQL that can be pasted.
+ */
+export class ReplicationSlotInvalidatedError extends RealtimeError {
+  constructor(args: { slot: string }) {
+    super({
+      code: 'X_REPLICATION_SLOT_INVALIDATED',
+      cause:
+        `postgres invalidated replication slot "${args.slot}" (wal_status = lost): it fell ` +
+        'further behind than max_slot_wal_keep_size allows, the WAL it needed is gone, and it ' +
+        'can never stream again',
+      fix: `SELECT pg_drop_replication_slot('${args.slot}');   # then restart the replicator: it recreates the slot and every live window resyncs`,
+    });
+  }
+}
+
+/**
+ * The replicator resumed from a position, and its slot was not there — dropped by hand, or a
+ * failover to a server that never had it. A new slot starts at the server's current WAL, so the
+ * changes committed in between were never streamed. **LOGGED, never thrown**: the stream goes on
+ * from the new slot, and every live window and channel is resynced from the database at once.
+ */
+export class ReplicationSlotRecreatedError extends RealtimeError {
+  constructor(args: { slot: string; from: string }) {
+    super({
+      code: 'X_REPLICATION_SLOT_RECREATED',
+      cause:
+        `replication slot "${args.slot}" was gone when the replicator resumed from ${args.from}, ` +
+        'so a new one was created and the changes committed in between were never streamed — ' +
+        'every live window and channel is resynced from the database',
+      fix: 'x doctor --json   # find what drops the slot: a failover without slot sync (postgres 17 sync_replication_slots), or a manual pg_drop_replication_slot',
+    });
+  }
+}

package/src/replication-resync.ts

@@ -0,0 +1,38 @@
+// Single responsibility: the bus's one control message — "the change stream has a hole no sequence
+// can show; every reader re-read now" — its subject, its bytes and its decode. The replicator says
+// it when its slot was recreated under a resume position; a sync node answers it as a reconnect.
+
+/**
+ * Under `x.change.>`, so every node already subscribed hears it — and on a token no table can
+ * spell (`[a-z_][a-z0-9_]*` refuses `-`), so it is never a table's subject. A node that predates it
+ * reads no `table` and no `lsn` and drops it, as it drops any payload it cannot parse.
+ */
+export const RESYNC_SUBJECT = 'x.change.-resync';
+
+export interface ResyncSignal {
+  /** The replicator run that said it. */
+  readonly producer: string;
+  readonly reason: string;
+}
+
+export function encodeResync(signal: ResyncSignal): string {
+  return JSON.stringify({ resync: true, producer: signal.producer, reason: signal.reason });
+}
+
+/** The signal, or `null` for every other payload — a change envelope included. */
+export function parseResync(payload: string): ResyncSignal | null {
+  // A change envelope never opens with this key: cheap to refuse before parsing every change twice.
+  if (!payload.startsWith('{"resync":true')) return null;
+  try {
+    const parsed: unknown = JSON.parse(payload);
+    if (typeof parsed !== 'object' || parsed === null) return null;
+    const shape = parsed as { resync?: unknown; producer?: unknown; reason?: unknown };
+    if (shape.resync !== true || typeof shape.producer !== 'string') return null;
+    return {
+      producer: shape.producer,
+      reason: typeof shape.reason === 'string' ? shape.reason : 'the change stream has a gap',
+    };
+  } catch {
+    return null;
+  }
+}

package/src/replicator-deadline.ts

@@ -0,0 +1,33 @@
+// Single responsibility: the one deadline every goodbye of a replicator `stop()` is held to — the
+// feed's, the lock's, and a start still in flight — on the injected `Scheduler`, so a test fires it.
+
+import type { Scheduler } from './thundering-herd';
+
+/**
+ * How long each polite goodbye of a `stop()` may take — the feed's, then the lock's — before the
+ * thing is abandoned instead. Both are conversations with a socket; a black-holed one never
+ * answers, and a SIGTERM that waited on it never finished.
+ */
+export const STOP_DEADLINE_MS = 5_000;
+
+export type DeadlineOutcome =
+  | { readonly how: 'done' | 'late' }
+  | { readonly how: 'failed'; readonly error: unknown };
+
+/** `goodbye` raced against `STOP_DEADLINE_MS`. Never rejects; the timer is cancelled either way. */
+export async function withinStopDeadline(
+  schedule: Scheduler,
+  goodbye: Promise<void>,
+): Promise<DeadlineOutcome> {
+  let cancel = (): void => undefined;
+  const deadline = new Promise<{ readonly how: 'late' }>((resolve) => {
+    cancel = schedule(() => resolve({ how: 'late' }), STOP_DEADLINE_MS);
+  });
+  const said = goodbye.then(
+    () => ({ how: 'done' as const }),
+    (error: unknown) => ({ how: 'failed' as const, error }),
+  );
+  const outcome = await Promise.race([said, deadline]);
+  cancel();
+  return outcome;
+}

package/src/replicator-reconnect.ts

@@ -0,0 +1,85 @@
+// Single responsibility: which replicator failures are the CONNECTION going away — a server shut
+// down or restarting, a reset socket, a lock session that died — and the short curve they redial
+// on. Measured on the demo: a database restart cost ~73 s of frozen live windows, because every
+// refused redial while the server was down grew the takeover backoff toward its 30 s cap.
+
+import { renderThrowable } from '@ultimat3/core';
+import { policyDelay, type ReconnectBackoff } from './thundering-herd';
+
+/** The longest the replicator waits between two redials while the connection is what failed. */
+export const REPLICATOR_RECONNECT_MAX_MS = 2_000;
+
+/**
+ * The redial curve after a lost connection: the FIRST redial is immediate (`replicator.ts`), the
+ * rest climb from 250 ms to two seconds. `equal` jitter keeps a floor, so a server still starting
+ * is not asked again at once; there is one replicator per database, so there is no herd to spread.
+ */
+export const reconnectBackoff: ReconnectBackoff = {
+  base: 250,
+  max: REPLICATOR_RECONNECT_MAX_MS,
+  factor: 2,
+  jitter: 'equal',
+};
+
+/** What a socket error carries in `code` (Bun's `connect` rejects with `{ code: 'ECONNREFUSED' }`). */
+const SOCKET_CODES: ReadonlySet<string> = new Set(['ECONNRESET', 'ECONNREFUSED', 'EPIPE']);
+
+/**
+ * In the rendered failure: an admin, crash or startup shutdown (SQLSTATE 57P01/57P02/57P03, as
+ * `describeFields` leads with it), a socket code in a message, and the phrases this package's own
+ * Postgres client says when the far end went away — the dial found no listener
+ * (`pg-socket.ts`'s `connectFailure`), a write the dead socket refused, an idle session closed, and
+ * any lost advisory-lock session (it is lost only when its connection ends).
+ */
+const CONNECTION_TEXT =
+  /(?<![\w/])(?:57P0[123]|ECONNRESET|ECONNREFUSED|EPIPE)\b|postgres replication (?:connect|write) failed|the server closed the connection|the advisory lock was lost/;
+
+/**
+ * `true` when `failure` — a stream's end reason or what a restart threw — is the connection ending
+ * underneath the replicator rather than something a redial cannot fix (a refused publish, a
+ * privilege, a decode the package does not speak). Those keep the long curve; this one redials.
+ */
+export function connectionLost(failure: unknown): boolean {
+  if (typeof failure === 'object' && failure !== null && 'code' in failure) {
+    const { code } = failure;
+    if (typeof code === 'string' && SOCKET_CODES.has(code)) return true;
+  }
+  const text = typeof failure === 'string' ? failure : renderThrowable(failure);
+  return CONNECTION_TEXT.test(text);
+}
+
+/**
+ * The takeover loop's two curves and their counts. `long` is the replicator's own `retryDelayMs`
+ * (a refused change, a standby, a stream that ended politely); a lost connection redials at once,
+ * then on `reconnectBackoff`. Both counts reset only on a PUBLISHED change — "a stream came up" is
+ * not progress, so a server that kills every new stream cannot hold the loop at zero.
+ */
+export class RedialCurve {
+  readonly #long: (attempt: number) => number;
+  readonly #rng: () => number;
+  #attempt = 0;
+  #reconnects = 0;
+
+  constructor(long: (attempt: number) => number, rng: () => number) {
+    this.#long = long;
+    this.#rng = rng;
+  }
+
+  /** The wait before the next redial, counted. */
+  next(lost: boolean): number {
+    if (!lost) {
+      const wait = this.#long(this.#attempt);
+      this.#attempt += 1;
+      return wait;
+    }
+    const count = this.#reconnects;
+    this.#reconnects += 1;
+    return count === 0 ? 0 : policyDelay(reconnectBackoff, count, this.#rng);
+  }
+
+  /** A change went out: the next death is a first one again, on either curve. */
+  progress(): void {
+    this.#attempt = 0;
+    this.#reconnects = 0;
+  }
+}

package/src/replicator-resync.ts

@@ -0,0 +1,30 @@
+// Single responsibility: the resync the replicator OWES the bus — set when its feed's slot was
+// recreated under a resume position, paid by one publish, kept owed across runs until one lands.
+// What the message is lives in `replication-resync.ts`; when it is said, in `replicator.ts`.
+
+export class ResyncDebt<Run> {
+  #owed: string | null = null;
+  /** The publish in flight, per run: the boot and a change asking at once publish ONE resync. */
+  #paying: { readonly run: Run; readonly paid: Promise<void> } | undefined;
+
+  /** Owed until a publish lands — across runs, since the next run finds the slot there. */
+  owe(reason: string): void {
+    this.#owed = reason;
+  }
+
+  /** Publishes what is owed, if anything. A refusal rejects and leaves it owed. */
+  pay(run: Run, publish: (reason: string) => Promise<void>): Promise<void> {
+    const reason = this.#owed;
+    if (reason === null) return Promise.resolve();
+    if (this.#paying?.run === run) return this.#paying.paid;
+    const paid = publish(reason)
+      .then(() => {
+        if (this.#owed === reason) this.#owed = null;
+      })
+      .finally(() => {
+        if (this.#paying?.paid === paid) this.#paying = undefined;
+      });
+    this.#paying = { run, paid };
+    return paid;
+  }
+}

package/src/replicator-row.ts

@@ -3,7 +3,7 @@
 // form JSON can carry and the sync node reads it back through the ENTITY's own columns.
 
 import { entityForTable } from '@ultimat3/entity';
-import type { Row } from './json';
+import type { LiveRow } from './json';
 
 const CHUNK = 0x8000;
 
@@ -28,7 +28,7 @@ function fromBase64(text: string): Uint8Array {
  * serialises to `{"0":1,"1":2}`, which nothing reads back, so it crosses as base64. Only a
  * top-level value can be one: `arrayOf()` refuses `bytes()` at declaration.
  */
-export function busRow(row: Row | null): Row | null {
+export function busRow(row: LiveRow | null): LiveRow | null {
   if (row === null) return null;
   let out: Record<string, unknown> | undefined;
   for (const [property, value] of Object.entries(row as Record<string, unknown>)) {
@@ -36,7 +36,7 @@ export function busRow(row: Row | null): Row | null {
     out ??= { ...row };
     out[property] = toBase64(value);
   }
-  return out === undefined ? row : (out as unknown as Row);
+  return out === undefined ? row : (out as unknown as LiveRow);
 }
 
 /**
@@ -52,7 +52,7 @@ export function busRow(row: Row | null): Row | null {
  * its column refuses are all handed back as they arrived — a change that crosses unrevived is the
  * old behaviour; a change dropped is a window that silently diverges.
  */
-export function reviveBusRow(table: string, row: Row | null): Row | null {
+export function reviveBusRow(table: string, row: LiveRow | null): LiveRow | null {
   if (row === null) return null;
   const entity = entityForTable(table);
   if (entity === undefined) return row;
@@ -71,5 +71,5 @@ export function reviveBusRow(table: string, row: Row | null): Row | null {
       // Kept as it arrived — see the function comment.
     }
   }
-  return out as unknown as Row;
+  return out as unknown as LiveRow;
 }

package/src/replicator-subject.ts

@@ -0,0 +1,28 @@
+// Single responsibility: where a replicated change goes on the bus and what makes it publishable —
+// the subject a change is published under, and the normalize step that drops the unusable and
+// hoists the tenant out of the row. Split from `replicator.ts`, which re-exports both.
+
+import type { ChangeEvent } from './changefeed';
+
+export const CHANGE_SUBJECT_PREFIX = 'x.change';
+
+/** Every change subject: what a `sync` node subscribes to. A changefeed subject, never a channel. */
+export const CHANGE_SUBJECT_ALL = `${CHANGE_SUBJECT_PREFIX}.>`;
+
+/** `x.change.<table>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
+export function changeSubject(change: ChangeEvent): string {
+  return `${CHANGE_SUBJECT_PREFIX}.${change.table}.${change.orgId ?? '_'}`;
+}
+
+/** Drops events the pipeline cannot use and hoists the tenant id out of the row. */
+export function normalize(change: ChangeEvent): ChangeEvent | null {
+  // The one change that carries no row by design; nothing to hoist a tenant out of.
+  if (change.op === 'truncate') return change;
+  const row = change.after ?? change.before;
+  if (!row) return null;
+  if (change.op === 'insert' && change.after === null) return null;
+  if (change.op === 'delete' && change.before === null) return null;
+  if (change.orgId !== null) return change;
+  const orgId = row['orgId'];
+  return typeof orgId === 'string' ? { ...change, orgId } : change;
+}

package/src/replicator.ts

@@ -16,24 +16,27 @@ import type { AdvisoryLock } from './advisory-lock';
 import type { ChangeEvent, ChangeFeed } from './changefeed';
 import { ReplicationFailedError } from './errors';
 import type { Transport } from './fanout';
+import { encodeResync, RESYNC_SUBJECT } from './replication-resync';
+import { withinStopDeadline } from './replicator-deadline';
 import { encodeEnvelope } from './replicator-envelope';
+import { connectionLost, RedialCurve } from './replicator-reconnect';
+import { ResyncDebt } from './replicator-resync';
+import { changeSubject, normalize } from './replicator-subject';
 import {
-  type BackoffPolicy,
   defaultBackoff,
   policyDelay,
+  type ReconnectBackoff,
   type Rng,
   type Scheduler,
 } from './thundering-herd';
 
-export const CHANGE_SUBJECT_PREFIX = 'x.change';
-
-/** Every change subject: what a `sync` node subscribes to. A changefeed subject, never a channel. */
-export const CHANGE_SUBJECT_ALL = `${CHANGE_SUBJECT_PREFIX}.>`;
-
-/** `x.change.<table>.<orgId>` — tenant in the subject, so fanout filters without parsing a row. */
-export function changeSubject(change: ChangeEvent): string {
-  return `${CHANGE_SUBJECT_PREFIX}.${change.table}.${change.orgId ?? '_'}`;
-}
+// The subjects and the normalize step live next door; their importers keep this path.
+export {
+  CHANGE_SUBJECT_ALL,
+  CHANGE_SUBJECT_PREFIX,
+  changeSubject,
+  normalize,
+} from './replicator-subject';
 
 export interface ReplicatorOptions {
   readonly feed: ChangeFeed;
@@ -42,7 +45,7 @@ export interface ReplicatorOptions {
   /** Resume position, normally the last lsn this replicator persisted. */
   readonly from?: string;
   readonly subjectOf?: (change: ChangeEvent) => string;
-  readonly backoff?: BackoffPolicy;
+  readonly backoff?: ReconnectBackoff;
   readonly rng?: Rng;
   /** The timer a restart waits on. Injected so a test proves the takeover loop without sleeping. */
   readonly schedule?: Scheduler;
@@ -93,12 +96,7 @@ interface Refusal {
   times: number;
 }
 
-/**
- * How long each polite goodbye of a `stop()` may take — the feed's, then the lock's — before the
- * thing is abandoned instead. Both are conversations with a socket; a black-holed one never
- * answers, and a SIGTERM that waited on it never finished.
- */
-export const STOP_DEADLINE_MS = 5_000;
+export { STOP_DEADLINE_MS } from './replicator-deadline';
 
 /**
  * How long a publish refused because the bus is AWAY is waited out, in the handler, before the run
@@ -125,6 +123,8 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
   /** A stream is pumping under a lock this process holds. Internal: `running` is stricter. */
   let pumping = false;
   let lastLsn: string | null = null;
+  /** Where a run with no `lastLsn` resumes: `options.from`, until a resync names a new timeline. */
+  let resumeFrom = options.from;
   let published = 0;
   let skipped = 0;
   let outOfOrder = 0;
@@ -152,32 +152,18 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     unwatchLock = undefined;
   };
   /**
-   * Deaths since a change was last PUBLISHED. "The stream started" is not progress: a change the
-   * bus refuses every time kills each new stream on its first delivery, and counting from the
-   * start made that a redial — two dials, a preflight, a `START_REPLICATION` — at the base delay
-   * forever, with the WAL retained behind it.
+   * Deaths since a change was last PUBLISHED, per curve. "The stream started" is not progress: a
+   * change the bus refuses every time kills each new stream on its first delivery, and counting
+   * from the start made that a redial at the base delay forever, with the WAL retained behind it.
+   * A lost CONNECTION (a database restart) redials at once, then on a seconds cap.
    */
-  let attempt = 0;
+  const curve = new RedialCurve((count) => retryDelayMs(count), options.rng ?? Math.random);
+  /** Counts the starts a `stop()` stopped waiting for — read by the `begin` it gave up on. */
+  let givenUp = 0;
   let refusal: Refusal | null = null;
+  const resync = new ResyncDebt<Run>();
 
-  /** One goodbye against the deadline, on the `schedule` seam. Never rejects. */
-  const bounded = async (
-    goodbye: Promise<void>,
-  ): Promise<
-    { readonly how: 'done' | 'late' } | { readonly how: 'failed'; readonly error: unknown }
-  > => {
-    let cancel = (): void => undefined;
-    const deadline = new Promise<{ readonly how: 'late' }>((resolve) => {
-      cancel = schedule(() => resolve({ how: 'late' }), STOP_DEADLINE_MS);
-    });
-    const said = goodbye.then(
-      () => ({ how: 'done' as const }),
-      (error: unknown) => ({ how: 'failed' as const, error }),
-    );
-    const outcome = await Promise.race([said, deadline]);
-    cancel();
-    return outcome;
-  };
+  const bounded = (goodbye: Promise<void>) => withinStopDeadline(schedule, goodbye);
 
   /**
    * The orderly unlock, bounded. Past the deadline — or refused — the session is dropped instead:
@@ -227,6 +213,11 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     }
   };
 
+  const sayResync = (run: Run): Promise<void> =>
+    resync.pay(run, (reason) =>
+      publishThroughBlip(run, RESYNC_SUBJECT, encodeResync({ producer, reason })),
+    );
+
   const changeHandler =
     (run: Run) =>
     async (raw: ChangeEvent): Promise<void> => {
@@ -234,6 +225,8 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
       // changes, and publishing them is one of two replicators speaking. A handler that merely
       // returned would let the stream count the change delivered and confirm it.
       if (current !== run || run.over !== null) throw fencedOut(run.over ?? 'a newer run started');
+      // Before any change of a stream with a hole, or the run ends as for a refused change.
+      await sayResync(run);
       const change = normalize(raw);
       if (!change) {
         skipped += 1;
@@ -262,7 +255,7 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
         lastLsn = change.lsn;
         published += 1;
         // Progress: the next death waits the base delay again, and nothing is owed any more.
-        attempt = 0;
+        curve.progress();
         if (refusal !== null) {
           refusal = null;
           if (pumping) failure = null;
@@ -278,10 +271,17 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
         `(${refusal.times} time(s) in a row): ${refusal.error}`;
 
   const begin = async (): Promise<boolean> => {
+    const givenUpAtEntry = givenUp;
     if (!(await options.lock.tryAcquire())) {
       logger.warn('replicator standby: advisory lock held elsewhere', { key: options.lock.key });
       return false;
     }
+    // A stop that gave up waiting for THIS start (`stop()`'s deadline) owns the replicator now; a
+    // stop inside the deadline still waits the start out and stops what it began.
+    if (stopped && givenUp !== givenUpAtEntry) {
+      await release();
+      return false;
+    }
     producer = uuidV7();
     seq = 0;
     const run: Run = { over: null };
@@ -301,13 +301,18 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     unwatchLock = options.lock.onLost((reason) => end(`the advisory lock was lost: ${reason}`));
     // A restart resumes where THIS process got to. `options.from` is where the first run began;
     // handing it to the second would have the feed re-deliver everything published since.
-    const from = lastLsn ?? options.from;
+    const from = lastLsn ?? resumeFrom;
     const onChange = changeHandler(run);
+    // A recreated slot is a NEW timeline: no old position may drop its changes or resume it.
+    const onResync = (reason: string): void => {
+      resync.owe(reason);
+      lastLsn = null;
+      resumeFrom = undefined;
+    };
     let fed = false;
     try {
-      await options.feed.start(
-        from === undefined ? { onChange, onEnd: end } : { from, onChange, onEnd: end },
-      );
+      const handlers = { onChange, onEnd: end, onResync };
+      await options.feed.start(from === undefined ? handlers : { from, ...handlers });
       fed = true;
       if (run.over !== null) {
         throw new ReplicationFailedError({
@@ -332,6 +337,13 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     // what `stop()` reads to decide there is anything to tear down.
     pumping = true;
     if (refusal === null) failure = null;
+    // Now, not on the next change: a quiet table would stay stale until its next write.
+    void sayResync(run).catch((thrown: unknown) => {
+      logger.warn('replicator.resync_unpublished', {
+        key: options.lock.key,
+        error: renderThrowable(thrown),
+      });
+    });
     logger.info('replicator started', { source: options.feed.source, key: options.lock.key });
     return true;
   };
@@ -356,15 +368,14 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     policyDelay(backoff, count + 1, options.rng ?? Math.random);
 
   /**
-   * The takeover loop. One timer at a time, each on a longer delay than the last until a change
-   * is PUBLISHED — which is what resets the count (`changeHandler`), not a stream coming up.
-   * `false` from `launch` is another process holding the lock, which is this loop working: that
-   * process replicates, and this one keeps asking in case it dies too.
+   * The takeover loop. One timer at a time, each longer than the last until a change is PUBLISHED
+   * (`RedialCurve`). `lost` — the connection went (`connectionLost`) — redials at once, then within
+   * seconds, unless a change is owed: that death is the bus's. `false` from `launch` is another
+   * process holding the lock: it replicates, and this one keeps asking in case it dies too.
    */
-  const retry = (): void => {
+  const retry = (lost: boolean): void => {
     if (stopped) return;
-    const wait = retryDelayMs(attempt);
-    attempt += 1;
+    const wait = curve.next(lost && refusal === null);
     cancelRetry = schedule(() => {
       cancelRetry = undefined;
       // `pumping` already: the owner called `start()` itself while this timer was armed.
@@ -373,12 +384,12 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
       void launch().then(
         (started) => {
           if (started) restarts += 1;
-          else retry();
+          else retry(false);
         },
         (thrown: unknown) => {
           failure = describeFailure(renderThrowable(thrown));
           logger.warn('replicator.restart_failed', { key: options.lock.key, error: failure });
-          retry();
+          retry(connectionLost(thrown));
         },
       );
     }, wait);
@@ -404,7 +415,7 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
     // socket still held is refused by the run's own fence.
     options.feed.abandon();
     options.lock.abandon();
-    retry();
+    retry(connectionLost(reason));
   };
 
   return {
@@ -429,8 +440,20 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
       stopped = true;
       cancelRetry?.();
       cancelRetry = undefined;
+      // Bounded like every other goodbye: a dial can sit in preflight (a slot its last walsender
+      // has not let go, a silent server) and a SIGTERM must not wait on it. Past the deadline its
+      // run is marked over — so the dial, landing later, stops what it started — and both
+      // sessions are dropped now.
       const inFlight = starting;
-      if (inFlight !== undefined) await inFlight.catch(() => false);
+      if (
+        inFlight !== undefined &&
+        (await bounded(inFlight.then(() => undefined))).how === 'late'
+      ) {
+        givenUp += 1;
+        if (current !== undefined) current.over ??= 'the replicator was stopped';
+        options.feed.abandon();
+        options.lock.abandon();
+      }
       // An orderly stop is not a failure, whatever the last stream died of.
       failure = null;
       refusal = null;
@@ -474,16 +497,3 @@ const unrefScheduler: Scheduler = (fn, ms) => {
     clearTimeout(handle);
   };
 };
-
-/** Drops events the pipeline cannot use and hoists the tenant id out of the row. */
-export function normalize(change: ChangeEvent): ChangeEvent | null {
-  // The one change that carries no row by design; nothing to hoist a tenant out of.
-  if (change.op === 'truncate') return change;
-  const row = change.after ?? change.before;
-  if (!row) return null;
-  if (change.op === 'insert' && change.after === null) return null;
-  if (change.op === 'delete' && change.before === null) return null;
-  if (change.orgId !== null) return change;
-  const orgId = row['orgId'];
-  return typeof orgId === 'string' ? { ...change, orgId } : change;
-}