@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/services/replicator/drain-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"drain-handler.js","names":["#lc","#changeStreamer","#state","#options","#committedContext","#restoreGranted","#draining","#isActive","#switchover","#next","#attempt","#holdForRestore"],"sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {readHead, type Sized, type Source} from '../../types/streams.ts';\nimport type {\n  ChangeStreamer,\n  Status,\n  SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport type {RunningState} from '../running-state.ts';\n\n// Bounds the time to wait for a switchover subscription to be established\n// and to receive its first message.\nconst DEFAULT_SWITCHOVER_TIMEOUT_MS = 10_000;\nconst MIN_SWITCHOVER_RETRY_DELAY_MS = 1_000;\nconst MAX_SWITCHOVER_RETRY_DELAY_MS = 30_000;\n\nexport type SwitchoverOptions = {\n  /**\n   * Called when a replication-manager with a newer replica version grants\n   * permission to restore its replica. This should initiate a graceful\n   * restart of the server (during which the current subscription continues\n   * to be served).\n   */\n  onRestoreGranted: () => void;\n\n  /**\n   * The replica file, from which the state of the last committed\n   * transaction is read for switchover subscriptions.\n   */\n  replicaFile: string;\n\n  /** For testing. */\n  timeoutMs?: number | undefined;\n};\n\nexport type Downstream = Source<Sized<SubscribeDownstream>>;\n\ntype AttemptOutcome = 'switched' | 'restoring' | 'retry';\n\n/**\n * Handles the switchover of a serving replica's subscription from a draining\n * replication-manager to another one, without interrupting the current\n * subscription.\n *\n * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)\n * opens a new `switchover` subscription, at the watermark of the replica's\n * last committed transaction, to whichever replication-manager the\n * connection is routed to. The first message on the new subscription\n * determines the outcome:\n * * `status`: The subscription was accepted (i.e. the replica is compatible).\n *   The current subscription is cancelled (aborting any transaction in\n *   progress), and the new one is made available via\n *   {@link getNextConnection}. Note that the new subscription may start with\n *   transactions that were committed from the current subscription in the\n *   meantime, which the consumer must skip.\n * * `status` with `restore: queued`: The replication-manager has a newer\n *   replica version, and is holding the subscription until it grants\n *   permission to restore. The held subscription is monitored for the grant.\n * * `status` with `restore: granted`: Permission to restore is granted, and\n *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted\n *   subscription is kept open until the process exits, so that the\n *   replication-manager can track the grant until the restore begins.\n * * Anything else (e.g. an error, or a `draining` replication-manager):\n *   The switchover is retried later.\n *\n * The current subscription continues to be consumed throughout. A\n * switchover is scoped to the subscription from which it switches: If that\n * subscription ends first (e.g. the replication-manager shuts down), the\n * switchover is abandoned, including any held (but not granted)\n * subscription, as the consumer reconnects on its own.\n */\nexport class DrainHandler {\n  readonly #lc: LogContext;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #state: RunningState;\n  readonly #options: SwitchoverOptions;\n  readonly #committedContext: () => SubscriberContext;\n\n  // The subscription currently being switched from.\n  #draining: Downstream | undefined;\n  #restoreGranted = false;\n  #next: Downstream | undefined;\n\n  /**\n   * @param committedContext Returns the subscriber context as of the\n   *        replica's last committed transaction.\n   */\n  constructor(\n    lc: LogContext,\n    changeStreamer: ChangeStreamer,\n    state: RunningState,\n    options: SwitchoverOptions,\n    committedContext: () => SubscriberContext,\n  ) {\n    this.#lc = lc.withContext('component', 'drain-handler');\n    this.#changeStreamer = changeStreamer;\n    this.#state = state;\n    this.#options = options;\n    this.#committedContext = committedContext;\n  }\n\n  /**\n   * Called when the change-streamer of the `current` subscription is\n   * draining.\n   */\n  onChangeStreamerDrain(current: Downstream) {\n    if (\n      this.#restoreGranted ||\n      this.#draining === current ||\n      !this.#isActive(current)\n    ) {\n      return;\n    }\n    this.#draining = current;\n    this.#lc.info?.(\n      `replication-manager is draining. switching over to another`,\n    );\n    void this.#switchover(current);\n  }\n\n  get hasNextConnection(): boolean {\n    return this.#next !== undefined;\n  }\n\n  /**\n   * Returns (and clears) the subscription that replaces the (now cancelled)\n   * current one, if a switchover succeeded.\n   */\n  getNextConnection(): Downstream | undefined {\n    const next = this.#next;\n    this.#next = undefined;\n    return next;\n  }\n\n  #isActive(current: Downstream) {\n    return !current.signal.aborted && this.#state.shouldRun();\n  }\n\n  async #switchover(current: Downstream) {\n    let failures = 0;\n    while (this.#isActive(current)) {\n      const outcome = await this.#attempt(current);\n      if (outcome !== 'retry') {\n        return;\n      }\n      const delay = Math.min(\n        MIN_SWITCHOVER_RETRY_DELAY_MS * 2 ** failures++,\n        MAX_SWITCHOVER_RETRY_DELAY_MS,\n      );\n      await this.#state.sleep(delay);\n    }\n  }\n\n  async #attempt(current: Downstream): Promise<AttemptOutcome> {\n    const lc = this.#lc;\n    const timeoutMs = this.#options.timeoutMs ?? DEFAULT_SWITCHOVER_TIMEOUT_MS;\n    let candidate: Downstream | undefined;\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    let timedOut = false;\n    try {\n      const ctx = this.#committedContext();\n      lc.info?.(`attempting switchover at ${ctx.watermark}`);\n      const timeout = new Promise<never>((_, reject) => {\n        timer = setTimeout(() => {\n          timedOut = true;\n          reject(new Error(`timed out after ${timeoutMs} ms`));\n        }, timeoutMs);\n      });\n      const subscribing = this.#changeStreamer.subscribe({\n        ...ctx,\n        switchover: true,\n      });\n      // Clean up a connection that is established after a timeout.\n      void subscribing.then(\n        sub => (timedOut ? sub.cancel() : undefined),\n        () => {},\n      );\n      candidate = await Promise.race([subscribing, timeout]);\n      const {head, rest} = readHead(candidate);\n      const first = await Promise.race([head.next(), timeout]);\n      const msg = first.done ? undefined : first.value.data;\n\n      if (!this.#isActive(current)) {\n        // e.g. The current subscription failed in the meantime.\n      } else if (msg?.[0] === 'status') {\n        const {restore, draining} = msg[1];\n        if (restore) {\n          const granted = await this.#holdForRestore(current, rest, restore);\n          if (granted) {\n            return 'restoring';\n          }\n          lc.info?.(`held switchover subscription closed. will retry`);\n          return 'retry';\n        }\n        if (!draining) {\n          lc.info?.(`switched over to a new subscription at ${ctx.watermark}`);\n          this.#next = rest;\n          current.cancel();\n          return 'switched';\n        }\n      }\n      lc.warn?.(`switchover rejected. continuing current subscription`, msg);\n    } catch (e) {\n      lc.warn?.(`switchover failed. continuing current subscription`, e);\n    } finally {\n      clearTimeout(timer);\n    }\n    candidate?.cancel();\n    return 'retry';\n  }\n\n  /**\n   * Keeps the held (or granted) subscription open, and initiates a graceful\n   * restart when permission to restore is granted. A held subscription that\n   * has yet to be granted is cancelled if the `current` subscription ends.\n   *\n   * @returns `true` once permission is granted, or `false` if the held\n   *          subscription closes before then.\n   */\n  #holdForRestore(\n    current: Downstream,\n    held: Downstream,\n    restore: NonNullable<Status['restore']>,\n  ): Promise<boolean> {\n    const lc = this.#lc;\n    const result = resolver<boolean>();\n    const unregister = this.#state.cancelOnStop(held);\n    const onCurrentEnded = () => {\n      if (!this.#restoreGranted) {\n        lc.info?.(`abandoning held switchover subscription`);\n        held.cancel();\n      }\n    };\n    current.signal.addEventListener('abort', onCurrentEnded, {once: true});\n\n    const onRestore = (r: NonNullable<Status['restore']>) => {\n      if (r.state === 'queued') {\n        lc.info?.(\n          `waiting for permission to restore a new replica (position ${r.position})`,\n        );\n      } else if (!this.#restoreGranted) {\n        lc.info?.(\n          `granted permission to restore a new replica. restarting gracefully`,\n        );\n        this.#restoreGranted = true;\n        this.#options.onRestoreGranted();\n        result.resolve(true);\n      }\n    };\n    onRestore(restore);\n    void (async () => {\n      try {\n        // Note: The loop must not be exited early, as that would close the\n        // connection.\n        for await (const {data: msg} of held) {\n          if (msg[0] === 'status' && msg[1].restore) {\n            onRestore(msg[1].restore);\n          }\n        }\n      } catch (e) {\n        lc.warn?.(`held switchover subscription failed`, e);\n      } finally {\n        unregister();\n        current.signal.removeEventListener('abort', onCurrentEnded);\n        result.resolve(false);\n      }\n    })();\n    return result.promise;\n  }\n}\n"],"mappings":";;;AAaA,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;AACtC,IAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDtC,IAAa,eAAb,MAA0B;CACxB;CACA;CACA;CACA;CACA;CAGA;CACA,kBAAkB;CAClB;;;;;CAMA,YACE,IACA,gBACA,OACA,SACA,kBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,kBAAkB;EACvB,KAAKC,SAAS;EACd,KAAKC,WAAW;EAChB,KAAKC,oBAAoB;CAC3B;;;;;CAMA,sBAAsB,SAAqB;EACzC,IACE,KAAKC,mBACL,KAAKC,cAAc,WACnB,CAAC,KAAKC,UAAU,OAAO,GAEvB;EAEF,KAAKD,YAAY;EACjB,KAAKN,IAAI,OACP,4DACF;EACA,KAAUQ,YAAY,OAAO;CAC/B;CAEA,IAAI,oBAA6B;EAC/B,OAAO,KAAKC,UAAU,KAAA;CACxB;;;;;CAMA,oBAA4C;EAC1C,MAAM,OAAO,KAAKA;EAClB,KAAKA,QAAQ,KAAA;EACb,OAAO;CACT;CAEA,UAAU,SAAqB;EAC7B,OAAO,CAAC,QAAQ,OAAO,WAAW,KAAKP,OAAO,UAAU;CAC1D;CAEA,MAAMM,YAAY,SAAqB;EACrC,IAAI,WAAW;EACf,OAAO,KAAKD,UAAU,OAAO,GAAG;GAE9B,IAAI,MADkB,KAAKG,SAAS,OAAO,MAC3B,SACd;GAEF,MAAM,QAAQ,KAAK,IACjB,gCAAgC,KAAK,YACrC,6BACF;GACA,MAAM,KAAKR,OAAO,MAAM,KAAK;EAC/B;CACF;CAEA,MAAMQ,SAAS,SAA8C;EAC3D,MAAM,KAAK,KAAKV;EAChB,MAAM,YAAY,KAAKG,SAAS,aAAa;EAC7C,IAAI;EACJ,IAAI;EACJ,IAAI,WAAW;EACf,IAAI;GACF,MAAM,MAAM,KAAKC,kBAAkB;GACnC,GAAG,OAAO,4BAA4B,IAAI,WAAW;GACrD,MAAM,UAAU,IAAI,SAAgB,GAAG,WAAW;IAChD,QAAQ,iBAAiB;KACvB,WAAW;KACX,uBAAO,IAAI,MAAM,mBAAmB,UAAU,IAAI,CAAC;IACrD,GAAG,SAAS;GACd,CAAC;GACD,MAAM,cAAc,KAAKH,gBAAgB,UAAU;IACjD,GAAG;IACH,YAAY;GACd,CAAC;GAED,YAAiB,MACf,QAAQ,WAAW,IAAI,OAAO,IAAI,KAAA,SAC5B,CAAC,CACT;GACA,YAAY,MAAM,QAAQ,KAAK,CAAC,aAAa,OAAO,CAAC;GACrD,MAAM,EAAC,MAAM,SAAQ,SAAS,SAAS;GACvC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;GACvD,MAAM,MAAM,MAAM,OAAO,KAAA,IAAY,MAAM,MAAM;GAEjD,IAAI,CAAC,KAAKM,UAAU,OAAO,GAAG,CAE9B,OAAO,IAAI,MAAM,OAAO,UAAU;IAChC,MAAM,EAAC,SAAS,aAAY,IAAI;IAChC,IAAI,SAAS;KAEX,IAAI,MADkB,KAAKI,gBAAgB,SAAS,MAAM,OAAO,GAE/D,OAAO;KAET,GAAG,OAAO,iDAAiD;KAC3D,OAAO;IACT;IACA,IAAI,CAAC,UAAU;KACb,GAAG,OAAO,0CAA0C,IAAI,WAAW;KACnE,KAAKF,QAAQ;KACb,QAAQ,OAAO;KACf,OAAO;IACT;GACF;GACA,GAAG,OAAO,wDAAwD,GAAG;EACvE,SAAS,GAAG;GACV,GAAG,OAAO,sDAAsD,CAAC;EACnE,UAAU;GACR,aAAa,KAAK;EACpB;EACA,WAAW,OAAO;EAClB,OAAO;CACT;;;;;;;;;CAUA,gBACE,SACA,MACA,SACkB;EAClB,MAAM,KAAK,KAAKT;EAChB,MAAM,SAAS,SAAkB;EACjC,MAAM,aAAa,KAAKE,OAAO,aAAa,IAAI;EAChD,MAAM,uBAAuB;GAC3B,IAAI,CAAC,KAAKG,iBAAiB;IACzB,GAAG,OAAO,yCAAyC;IACnD,KAAK,OAAO;GACd;EACF;EACA,QAAQ,OAAO,iBAAiB,SAAS,gBAAgB,EAAC,MAAM,KAAI,CAAC;EAErE,MAAM,aAAa,MAAsC;GACvD,IAAI,EAAE,UAAU,UACd,GAAG,OACD,6DAA6D,EAAE,SAAS,EAC1E;QACK,IAAI,CAAC,KAAKA,iBAAiB;IAChC,GAAG,OACD,oEACF;IACA,KAAKA,kBAAkB;IACvB,KAAKF,SAAS,iBAAiB;IAC/B,OAAO,QAAQ,IAAI;GACrB;EACF;EACA,UAAU,OAAO;EACjB,CAAM,YAAY;GAChB,IAAI;IAGF,WAAW,MAAM,EAAC,MAAM,SAAQ,MAC9B,IAAI,IAAI,OAAO,YAAY,IAAI,GAAG,SAChC,UAAU,IAAI,GAAG,OAAO;GAG9B,SAAS,GAAG;IACV,GAAG,OAAO,uCAAuC,CAAC;GACpD,UAAU;IACR,WAAW;IACX,QAAQ,OAAO,oBAAoB,SAAS,cAAc;IAC1D,OAAO,QAAQ,KAAK;GACtB;EACF,GAAG;EACH,OAAO,OAAO;CAChB;AACF"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts

@@ -1,8 +1,11 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { Source } from '../../types/streams.ts';
+import type { ReservationFollowup } from '../change-streamer/change-streamer-http.ts';
 import { type ChangeStreamer } from '../change-streamer/change-streamer.ts';
-import type { ReplicationStatusPublisher } from './replication-status.ts';
+import { type SwitchoverOptions } from './drain-handler.ts';
+import { type ReplicationStatusPublisher } from './replication-status.ts';
 import type { ReplicaState, ReplicatorMode } from './replicator.ts';
+import { type ReplicationLagReadinessOptions } from './reporter/recorder.ts';
 import type { WriteWorkerClient } from './write-worker-client.ts';
 /**
  * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
@@ -13,7 +16,8 @@ import type { WriteWorkerClient } from './write-worker-client.ts';
  */
 export declare class IncrementalSyncer {
     #private;
-    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null);
+    constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null, replicationLagReadinessOptions?: ReplicationLagReadinessOptions, initialConnection?: ReservationFollowup, switchover?: SwitchoverOptions);
+    ready(): Promise<void>;
     run(): Promise<void>;
     subscribe(): Source<ReplicaState>;
     stop(lc: LogContext, err?: unknown): void;

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAOhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IAmB5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAWnD;IAEK,GAAG,kBAiLR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAED,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAEL,KAAK,8BAA8B,EACpC,MAAM,wBAAwB,CAAC;AAIhC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAchE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IA2B5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAClD,8BAA8B,GAAE,8BAAmC,EACnE,iBAAiB,CAAC,EAAE,mBAAmB,EACvC,UAAU,CAAC,EAAE,iBAAiB,EAuB/B;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAEK,GAAG,kBAwSR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAqCD,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -1,12 +1,20 @@
 import { AbortError } from "../../../../shared/src/abort-error.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { mapPostgresToLiteIndex } from "../../db/pg-to-lite.js";
 import { getOrCreateCounter } from "../../observability/metrics.js";
+import { readReplicaBackfills } from "./schema/backfilling.js";
+import { getSubscriptionState } from "./schema/replication-state.js";
+import { StatementRunner } from "../../db/statements.js";
 import { RunningState } from "../running-state.js";
+import { IndexingProgress } from "./replication-status.js";
 import { errorTypeToReadableName } from "../change-streamer/change-streamer.js";
 import { Notifier } from "./notifier.js";
+import { DrainHandler } from "./drain-handler.js";
 import { ReplicationReportRecorder } from "./reporter/recorder.js";
 //#region ../zero-cache/src/services/replicator/incremental-sync.ts
 var MAX_WORKER_BATCH_MESSAGES = 256;
 var MAX_WORKER_BATCH_SIZE = 1024 * 1024;
+var MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;
 /**
 * The {@link IncrementalSyncer} manages a logical replication stream from upstream,
 * handling application lifecycle events (start, stop) and retrying the
@@ -24,9 +32,12 @@ var IncrementalSyncer = class {
 	#statusPublisher;
 	#notifier;
 	#reporter;
+	#initialConnection;
+	#drainHandler;
+	#olderReplicaVersionRejections = 0;
 	#state = new RunningState("IncrementalSyncer");
 	#replicationEvents = getOrCreateCounter("replication", "events", "Number of replication events processed");
-	constructor(lc, taskID, id, changeStreamer, worker, mode, statusPublisher) {
+	constructor(lc, taskID, id, changeStreamer, worker, mode, statusPublisher, replicationLagReadinessOptions = {}, initialConnection, switchover) {
 		this.#lc = lc;
 		this.#taskID = taskID;
 		this.#id = id;
@@ -35,7 +46,12 @@ var IncrementalSyncer = class {
 		this.#mode = mode;
 		this.#statusPublisher = statusPublisher;
 		this.#notifier = new Notifier();
-		this.#reporter = new ReplicationReportRecorder(lc);
+		this.#reporter = new ReplicationReportRecorder(lc, replicationLagReadinessOptions);
+		this.#initialConnection = initialConnection;
+		this.#drainHandler = mode === "serving" && switchover ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () => this.#committedContext(switchover.replicaFile)) : void 0;
+	}
+	ready() {
+		return this.#reporter.ready();
 	}
 	async run() {
 		const lc = this.#lc;
@@ -51,28 +67,28 @@ var IncrementalSyncer = class {
 			watermark: initialWatermark
 		});
 		while (this.#state.shouldRun()) {
-			const { replicaVersion, watermark } = await this.#worker.getSubscriptionState();
+			const { replicaVersion, watermark, backfills } = await this.#worker.getSubscriptionState();
 			let downstream;
 			let unregister = () => {};
 			let err;
 			try {
-				downstream = await this.#changeStreamer.subscribe({
-					protocolVersion: 6,
-					taskID: this.#taskID,
-					id: this.#id,
-					mode: this.#mode,
-					watermark,
-					replicaVersion,
-					initial: watermark === initialWatermark,
-					logsChangeStream: false
-				});
-				this.#state.resetBackoff();
+				const ctx = this.#subscriberContext(replicaVersion, watermark, backfills);
+				const switched = this.#drainHandler?.getNextConnection();
+				const initialConnection = this.#initialConnection;
+				this.#initialConnection = void 0;
+				downstream = switched ?? (initialConnection ? await initialConnection.subscribe(ctx) : await this.#changeStreamer.subscribe(ctx));
 				unregister = this.#state.cancelOnStop(downstream);
 				this.#statusPublisher?.publish(lc, "Replicating", `Replicating from ${watermark}`);
-				let backfillStatus;
+				let backfill;
 				let writeBatch = [];
 				let writeBatchSize = 0;
 				let inTransaction = false;
+				let indexing;
+				const publishBackfillStatus = (table) => this.#statusPublisher?.publish(lc, "Replicating", `Backfilling ${table} table`, 3e3, () => backfill ? { downloadStatus: [{
+					...backfill.status,
+					table: backfill.table,
+					columns: backfill.columns
+				}] } : {});
 				const flushWrites = async () => {
 					if (writeBatch.length === 0) return;
 					const batch = writeBatch;
@@ -80,13 +96,33 @@ var IncrementalSyncer = class {
 					writeBatchSize = 0;
 					const result = await this.#worker.processMessages(batch);
 					this.#handleResult(lc, result);
-					if (result?.completedBackfill) backfillStatus = void 0;
+					if (result?.completedBackfill) backfill = void 0;
+				};
+				let skipping = false;
+				const shouldSkip = (message) => {
+					const type = message[0];
+					if (type === "begin" && message[2].commitWatermark <= watermark) {
+						lc.debug?.(`skipping committed transaction ${message[2].commitWatermark}`);
+						skipping = true;
+					}
+					if (skipping && (type === "begin" || type === "data" || type === "commit" || type === "rollback")) {
+						skipping = type !== "commit" && type !== "rollback";
+						return true;
+					}
+					return false;
 				};
 				for await (const { data: message, size } of downstream) {
+					if (shouldSkip(message)) continue;
+					this.#state.resetBackoff();
 					this.#replicationEvents.add(1);
 					switch (message[0]) {
+						case "reserved":
+							lc.warn?.(`unexpected snapshot reservation message received`, { message: message[0] });
+							break;
 						case "status": {
-							const { lagReport } = message[1];
+							const { lagReport, draining } = message[1];
+							this.#olderReplicaVersionRejections = 0;
+							if (draining) this.#drainHandler?.onChangeStreamerDrain(downstream);
 							if (lagReport) {
 								const report = { nextSendTimeMs: lagReport.nextSendTimeMs };
 								if (lagReport.lastTimings) report.lastTimings = {
@@ -98,24 +134,43 @@ var IncrementalSyncer = class {
 							break;
 						}
 						case "error": {
-							const { type, message: msg } = message[1];
+							const { type, message: msg, replicaVersion: rmVersion } = message[1];
+							if (type === 1 && rmVersion !== void 0 && rmVersion < replicaVersion && this.#olderReplicaVersionRejections++ < MAX_OLDER_REPLICA_VERSION_REJECTIONS) {
+								lc.warn?.(`replication-manager is at an older replica version (${rmVersion}) than ${replicaVersion}. reconnecting`);
+								break;
+							}
 							this.stop(lc, new AbortError(`${errorTypeToReadableName(type)}: ${msg}`));
 							break;
 						}
 						default: {
 							const msg = message[1];
 							if (msg.tag === "backfill" && msg.status) {
-								const { status } = msg;
-								if (!backfillStatus) {
-									backfillStatus = status;
-									this.#statusPublisher?.publish(lc, "Replicating", `Backfilling ${msg.relation.name} table`, 3e3, () => backfillStatus ? { downloadStatus: [{
-										...backfillStatus,
-										table: msg.relation.name,
-										columns: [...msg.relation.rowKey.columns, ...msg.columns]
-									}] } : {});
-								}
-								backfillStatus = status;
+								const { status, relation } = msg;
+								const first = !backfill;
+								backfill = {
+									status,
+									table: relation.name,
+									columns: [...relation.rowKey.columns, ...msg.columns]
+								};
+								if (first) publishBackfillStatus(relation.name);
+							}
+							if (msg.tag === "create-index" && this.#statusPublisher) {
+								await flushWrites();
+								const index = mapPostgresToLiteIndex(msg.spec);
+								indexing ??= new IndexingProgress();
+								indexing.start(index);
+								this.#statusPublisher.publish(lc, "Replicating", `Creating index ${index.name} on ${index.tableName}`, 3e3, indexing.state);
+								indexing.restartTimer();
+								writeBatch.push(message);
+								writeBatchSize += size;
+								await flushWrites();
+								const elapsed = indexing.finish();
+								lc.info?.(`Created index ${index.name} (${elapsed.toFixed(3)} ms)`);
+								this.#statusPublisher.publish(lc, "Replicating", `Created index ${index.name} on ${index.tableName}`, 0, indexing.state);
+								if (backfill) publishBackfillStatus(backfill.table);
+								break;
 							}
+							indexing = void 0;
 							const type = message[0];
 							const invalidTransactionSequence = type === "begin" ? inTransaction : !inTransaction;
 							if (type === "begin") inTransaction = true;
@@ -136,6 +191,8 @@ var IncrementalSyncer = class {
 				unregister();
 				this.#statusPublisher?.stop();
 			}
+			if (err === void 0 && this.#drainHandler?.hasNextConnection) continue;
+			this.#drainHandler?.getNextConnection()?.cancel();
 			await this.#state.backoff(lc, err);
 		}
 		lc.info?.("IncrementalSyncer stopped");
@@ -157,6 +214,35 @@ var IncrementalSyncer = class {
 	subscribe() {
 		return this.#notifier.subscribe();
 	}
+	#subscriberContext(replicaVersion, watermark, backfills) {
+		return {
+			protocolVersion: 8,
+			taskID: this.#taskID,
+			id: this.#id,
+			mode: this.#mode,
+			watermark,
+			replicaVersion,
+			backfills,
+			...this.#drainHandler ? { drainable: true } : {}
+		};
+	}
+	/**
+	* Returns the subscriber context as of the last committed transaction,
+	* read from the replica file (i.e. not from the write worker, whose
+	* connection may be in the middle of a transaction).
+	*/
+	#committedContext(replicaFile) {
+		const db = new Database(this.#lc, replicaFile, { readonly: true });
+		try {
+			const { replicaVersion, watermark, backfills } = db.transaction(() => ({
+				...getSubscriptionState(new StatementRunner(db)),
+				backfills: readReplicaBackfills(db)
+			}));
+			return this.#subscriberContext(replicaVersion, watermark, backfills);
+		} finally {
+			db.close();
+		}
+	}
 	stop(lc, err) {
 		this.#state.stop(lc, err);
 		this.#worker.abort();

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SizedDownstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<SizedDownstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n          // The SQLite change log is written by the change-streamer itself, so\n          // no replicator logs the change stream any more. The parameter stays\n          // on the wire for change-streamers that still exclude a writer from\n          // SQLite catchup.\n          logsChangeStream: false,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfillStatus = undefined;\n          }\n        };\n\n        for await (const {data: message, size} of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;AAyBA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;KAKvB,kBAAkB;IACpB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IACJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKO,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;IAErB;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,KAAKD,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}
+{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#drainHandler","#state","#replicationEvents","#initialConnection","#committedContext","#subscriberContext","#handleResult","#olderReplicaVersionRejections"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type SubscriberContext,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {WrongReplicaVersion} from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {\n  DrainHandler,\n  type Downstream,\n  type SwitchoverOptions,\n} from './drain-handler.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n  IndexingProgress,\n  type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {\n  ReplicationReportRecorder,\n  type ReplicationLagReadinessOptions,\n} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport {readReplicaBackfills} from './schema/backfilling.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n// The number of times to reconnect after being rejected by a\n// replication-manager with an older replica version (e.g. one that has yet\n// to be replaced by a resync), before accepting the rejection and restarting\n// (e.g. if the upstream was legitimately replaced by one with a lower replica\n// version).\nconst MAX_OLDER_REPLICA_VERSION_REJECTIONS = 20;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n  // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n  // subsequent (re)connection goes through #changeStreamer.subscribe().\n  #initialConnection: ReservationFollowup | undefined;\n\n  // Only set for serving replicas that switch over between\n  // replication-managers.\n  readonly #drainHandler: DrainHandler | undefined;\n  #olderReplicaVersionRejections = 0;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n    replicationLagReadinessOptions: ReplicationLagReadinessOptions = {},\n    initialConnection?: ReservationFollowup,\n    switchover?: SwitchoverOptions,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(\n      lc,\n      replicationLagReadinessOptions,\n    );\n    this.#initialConnection = initialConnection;\n    // Only serving replicas switch over between replication-managers. In\n    // particular, the in-RM backup-replicator never does.\n    this.#drainHandler =\n      mode === 'serving' && switchover\n        ? new DrainHandler(lc, changeStreamer, this.#state, switchover, () =>\n            this.#committedContext(switchover.replicaFile),\n          )\n        : undefined;\n  }\n\n  ready(): Promise<void> {\n    return this.#reporter.ready();\n  }\n\n  async run() {\n    const lc = this.#lc;\n    let workerError: Error | undefined;\n    this.#worker.onError(err => {\n      workerError ??= err;\n      this.#state.stop(lc, err);\n    });\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark, backfills} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Downstream | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        const ctx = this.#subscriberContext(\n          replicaVersion,\n          watermark,\n          backfills,\n        );\n        // A successful switchover has already established the subscription\n        // on a new connection.\n        const switched = this.#drainHandler?.getNextConnection();\n        // The first connection reuses the still-open reservation connection\n        // (if any) from the backup restore, so the replication-manager that\n        // reserved the change log is the one that serves the subscription.\n        // Every reconnect after that opens a fresh connection.\n        const initialConnection = this.#initialConnection;\n        this.#initialConnection = undefined;\n        downstream =\n          switched ??\n          (initialConnection\n            ? await initialConnection.subscribe(ctx)\n            : await this.#changeStreamer.subscribe(ctx));\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfill:\n          | {status: DownloadStatus; table: string; columns: string[]}\n          | undefined;\n        let writeBatch: ChangeStreamData[] = [];\n        let writeBatchSize = 0;\n        let inTransaction = false;\n        // Indexes created in the current run of consecutive index creations.\n        let indexing: IndexingProgress | undefined;\n\n        const publishBackfillStatus = (table: string) =>\n          this.#statusPublisher?.publish(\n            lc,\n            'Replicating',\n            `Backfilling ${table} table`,\n            3000,\n            () =>\n              backfill\n                ? {\n                    downloadStatus: [\n                      {\n                        ...backfill.status,\n                        table: backfill.table,\n                        columns: backfill.columns,\n                      },\n                    ],\n                  }\n                : {},\n          );\n\n        const flushWrites = async () => {\n          if (writeBatch.length === 0) {\n            return;\n          }\n          const batch = writeBatch;\n          writeBatch = [];\n          writeBatchSize = 0;\n\n          const result = await this.#worker.processMessages(batch);\n          this.#handleResult(lc, result);\n          if (result?.completedBackfill) {\n            backfill = undefined;\n          }\n        };\n\n        // Whether the current transaction is being skipped.\n        let skipping = false;\n\n        // A subscription established by a switchover starts at the\n        // watermark at which it was requested, and may thus include\n        // transactions that were since committed from the previous\n        // subscription.\n        const shouldSkip = (message: SubscribeDownstream) => {\n          const type = message[0];\n          if (type === 'begin' && message[2].commitWatermark <= watermark) {\n            lc.debug?.(\n              `skipping committed transaction ${message[2].commitWatermark}`,\n            );\n            skipping = true;\n          }\n          if (\n            skipping &&\n            (type === 'begin' ||\n              type === 'data' ||\n              type === 'commit' ||\n              type === 'rollback')\n          ) {\n            skipping = type !== 'commit' && type !== 'rollback';\n            return true;\n          }\n          return false;\n        };\n\n        for await (const {data: message, size} of downstream) {\n          if (shouldSkip(message)) {\n            continue;\n          }\n          // Only reset backoff after receiving an actual message.\n          this.#state.resetBackoff();\n\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'reserved':\n              lc.warn?.(`unexpected snapshot reservation message received`, {\n                message: message[0],\n              });\n              break;\n            case 'status': {\n              const {lagReport, draining} = message[1];\n              this.#olderReplicaVersionRejections = 0;\n              if (draining) {\n                this.#drainHandler?.onChangeStreamerDrain(downstream);\n              }\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              const {\n                type,\n                message: msg,\n                replicaVersion: rmVersion,\n              } = message[1];\n              if (\n                type === WrongReplicaVersion &&\n                rmVersion !== undefined &&\n                rmVersion < replicaVersion &&\n                this.#olderReplicaVersionRejections++ <\n                  MAX_OLDER_REPLICA_VERSION_REJECTIONS\n              ) {\n                // The replication-manager is behind this replica (e.g. it\n                // has yet to be replaced by a resync). Rather than restoring\n                // its older replica, reconnect (with backoff), which may\n                // reach a different replication-manager.\n                lc.warn?.(\n                  `replication-manager is at an older replica version ` +\n                    `(${rmVersion}) than ${replicaVersion}. reconnecting`,\n                );\n                break;\n              }\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status, relation} = msg;\n                const first = !backfill;\n                backfill = {\n                  status, // Update the current status\n                  table: relation.name,\n                  columns: [...relation.rowKey.columns, ...msg.columns],\n                };\n                if (first) {\n                  // Start publishing the status every 3 seconds.\n                  publishBackfillStatus(relation.name);\n                }\n              }\n\n              if (msg.tag === 'create-index' && this.#statusPublisher) {\n                // Creating an index on an existing table can take a long\n                // time. Flush preceding changes so that the index creation\n                // is processed (and timed) on its own.\n                await flushWrites();\n                const index = mapPostgresToLiteIndex(msg.spec);\n                indexing ??= new IndexingProgress();\n                indexing.start(index);\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Creating index ${index.name} on ${index.tableName}`,\n                  3000,\n                  indexing.state,\n                );\n                // Exclude the time spent publishing the status.\n                indexing.restartTimer();\n                writeBatch.push(message as ChangeStreamData);\n                writeBatchSize += size;\n                await flushWrites();\n                const elapsed = indexing.finish();\n                lc.info?.(\n                  `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n                );\n                this.#statusPublisher.publish(\n                  lc,\n                  'Replicating',\n                  `Created index ${index.name} on ${index.tableName}`,\n                  0,\n                  indexing.state,\n                );\n                if (backfill) {\n                  // Resume reporting the progress of the ongoing backfill.\n                  publishBackfillStatus(backfill.table);\n                }\n                break;\n              }\n              indexing = undefined;\n\n              const type = message[0];\n              const invalidTransactionSequence =\n                type === 'begin' ? inTransaction : !inTransaction;\n              if (type === 'begin') {\n                inTransaction = true;\n              } else if (type === 'commit' || type === 'rollback') {\n                inTransaction = false;\n              }\n\n              writeBatch.push(message as ChangeStreamData);\n              writeBatchSize += size;\n              if (\n                // Waiting here ensures that consuming the commit, which ACKs\n                // it upstream, only happens after the SQLite commit finishes.\n                type === 'commit' ||\n                type === 'rollback' ||\n                // Promptly surface malformed transaction sequences instead\n                // of leaving a partial batch waiting for another message.\n                invalidTransactionSequence ||\n                writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n                writeBatchSize >= MAX_WORKER_BATCH_SIZE\n              ) {\n                await flushWrites();\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      if (err === undefined && this.#drainHandler?.hasNextConnection) {\n        continue; // Switched over to a new subscription; no backoff.\n      }\n      this.#drainHandler?.getNextConnection()?.cancel();\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n    if (workerError) {\n      throw workerError;\n    }\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n        upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  #subscriberContext(\n    replicaVersion: string,\n    watermark: string,\n    backfills: SubscriberContext['backfills'],\n  ): SubscriberContext {\n    return {\n      protocolVersion: PROTOCOL_VERSION,\n      taskID: this.#taskID,\n      id: this.#id,\n      mode: this.#mode,\n      watermark,\n      replicaVersion,\n      backfills,\n      ...(this.#drainHandler ? {drainable: true} : {}),\n    };\n  }\n\n  /**\n   * Returns the subscriber context as of the last committed transaction,\n   * read from the replica file (i.e. not from the write worker, whose\n   * connection may be in the middle of a transaction).\n   */\n  #committedContext(replicaFile: string): SubscriberContext {\n    const db = new Database(this.#lc, replicaFile, {readonly: true});\n    try {\n      const {replicaVersion, watermark, backfills} = db.transaction(() => ({\n        ...getSubscriptionState(new StatementRunner(db)),\n        backfills: readReplicaBackfills(db),\n      }));\n      return this.#subscriberContext(replicaVersion, watermark, backfills);\n    } finally {\n      db.close();\n    }\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n    // Abort any polling loop that the write worker may be in.\n    this.#worker.abort();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA4CA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;AAOrC,IAAM,uCAAuC;;;;;;;;AAS7C,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAIA;CACA,iCAAiC;CAEjC,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,iCAAiE,CAAC,GAClE,mBACA,YACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BACnB,IACA,8BACF;EACA,KAAKI,qBAAqB;EAG1B,KAAKH,gBACH,SAAS,aAAa,aAClB,IAAI,aAAa,IAAI,gBAAgB,KAAKC,QAAQ,kBAChD,KAAKG,kBAAkB,WAAW,WAAW,CAC/C,IACA,KAAA;CACR;CAEA,QAAuB;EACrB,OAAO,KAAKL,UAAU,MAAM;CAC9B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKM,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKN,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKG,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,WAAW,cAChC,MAAM,KAAKN,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM,KAAKU,mBACf,gBACA,WACA,SACF;IAGA,MAAM,WAAW,KAAKL,eAAe,kBAAkB;IAKvD,MAAM,oBAAoB,KAAKG;IAC/B,KAAKA,qBAAqB,KAAA;IAC1B,aACE,aACC,oBACG,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAKT,gBAAgB,UAAU,GAAG;IAC9C,aAAa,KAAKO,OAAO,aAAa,UAAU;IAChD,KAAKJ,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKW,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAGA,IAAI,WAAW;IAMf,MAAM,cAAc,YAAiC;KACnD,MAAM,OAAO,QAAQ;KACrB,IAAI,SAAS,WAAW,QAAQ,GAAG,mBAAmB,WAAW;MAC/D,GAAG,QACD,kCAAkC,QAAQ,GAAG,iBAC/C;MACA,WAAW;KACb;KACA,IACE,aACC,SAAS,WACR,SAAS,UACT,SAAS,YACT,SAAS,aACX;MACA,WAAW,SAAS,YAAY,SAAS;MACzC,OAAO;KACT;KACA,OAAO;IACT;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KACpD,IAAI,WAAW,OAAO,GACpB;KAGF,KAAKL,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,WAAW,aAAY,QAAQ;OACtC,KAAKK,iCAAiC;OACtC,IAAI,UACF,KAAKP,eAAe,sBAAsB,UAAU;OAEtD,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKD,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OACZ,MAAM,EACJ,MACA,SAAS,KACT,gBAAgB,cACd,QAAQ;OACZ,IACE,SAAA,KACA,cAAc,KAAA,KACd,YAAY,kBACZ,KAAKQ,mCACH,sCACF;QAKA,GAAG,OACD,uDACM,UAAU,SAAS,eAAe,eAC1C;QACA;OACF;OAGA,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKV,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,IAAI,QAAQ,KAAA,KAAa,KAAKG,eAAe,mBAC3C;GAEF,KAAKA,eAAe,kBAAkB,GAAG,OAAO;GAChD,MAAM,KAAKC,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKJ,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,mBACE,gBACA,WACA,WACmB;EACnB,OAAO;GACL,iBAAA;GACA,QAAQ,KAAKN;GACb,IAAI,KAAKC;GACT,MAAM,KAAKG;GACX;GACA;GACA;GACA,GAAI,KAAKI,gBAAgB,EAAC,WAAW,KAAI,IAAI,CAAC;EAChD;CACF;;;;;;CAOA,kBAAkB,aAAwC;EACxD,MAAM,KAAK,IAAI,SAAS,KAAKT,KAAK,aAAa,EAAC,UAAU,KAAI,CAAC;EAC/D,IAAI;GACF,MAAM,EAAC,gBAAgB,WAAW,cAAa,GAAG,mBAAmB;IACnE,GAAG,qBAAqB,IAAI,gBAAgB,EAAE,CAAC;IAC/C,WAAW,qBAAqB,EAAE;GACpC,EAAE;GACF,OAAO,KAAKc,mBAAmB,gBAAgB,WAAW,SAAS;EACrE,UAAU;GACR,GAAG,MAAM;EACX;CACF;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKJ,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKN,QAAQ,MAAM;CACrB;AACF"}

package/out/zero-cache/src/services/replicator/replication-status.d.ts

@@ -1,7 +1,8 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { JSONObject } from '../../../../zero-events/src/json.ts';
-import type { ReplicationStage, ReplicationState, ReplicationStatusEvent, Status } from '../../../../zero-events/src/status.ts';
+import type { IndexingStatus, ReplicationStage, ReplicationState, ReplicationStatusEvent, Status } from '../../../../zero-events/src/status.ts';
 import { Database } from '../../../../zqlite/src/db.ts';
+import type { LiteIndexSpec } from '../../db/specs.ts';
 import { publishCriticalEvent } from '../../observability/events.ts';
 export declare class ReplicationStatusPublisher {
     #private;
@@ -10,9 +11,42 @@ export declare class ReplicationStatusPublisher {
     static forReplicaFile(file: string, publishFn?: typeof publishCriticalEvent): ReplicationStatusPublisher;
     constructor(dbRunner: <T>(lc: LogContext, fn: (db: Database) => T) => T, publishFn?: typeof publishCriticalEvent);
     publish(lc: LogContext, stage: ReplicationStage, description?: string, interval?: number, extraState?: () => Partial<ReplicationState>, now?: Date): this;
+    /**
+     * Like {@link publish}, but waits (for at most `maxWaitMs`) for the event
+     * to be sent. Use this before a long synchronous operation (e.g. creating
+     * an index) that blocks the event loop and would otherwise prevent the
+     * event from being sent until the operation completes.
+     */
+    publishAndFlush(lc: LogContext, stage: ReplicationStage, description?: string, interval?: number, extraState?: () => Partial<ReplicationState>, maxWaitMs?: number): Promise<void>;
     publishAndThrowError(lc: LogContext, stage: ReplicationStage, e: unknown): Promise<never>;
     stop(): this;
 }
+/**
+ * Tracks the progress of a sequence of index creations for reporting in
+ * the `indexingStatus` field of {@link ReplicationState}.
+ */
+export declare class IndexingProgress {
+    #private;
+    /**
+     * @param totalIndexes The total number of indexes to be created, if known.
+     */
+    constructor(totalIndexes?: number, now?: () => number);
+    /** Marks the start of creating the `index`. */
+    start(index: LiteIndexSpec): void;
+    /**
+     * Restarts the timer of the current index, e.g. to exclude time spent
+     * reporting its start from its creation time.
+     */
+    restartTimer(): void;
+    /**
+     * Marks the completion of the index passed to the last call to
+     * {@link start}, returning the milliseconds it took to create.
+     */
+    finish(): number;
+    /** The `extraState` for {@link ReplicationStatusPublisher.publish}. */
+    readonly state: () => Partial<ReplicationState>;
+    status(): IndexingStatus | undefined;
+}
 export declare function publishReplicationError(lc: LogContext, stage: ReplicationStage, description: string, errorDetails?: JSONObject, now?: Date): Promise<void>;
 export declare function replicationStatusError(lc: LogContext, stage: ReplicationStage, e: unknown, db?: Database, now?: Date): ReplicationStatusEvent;
 export declare function replicationStatusEvent(lc: LogContext, db: Database | undefined, stage: ReplicationStage, status: Status, description?: string, now?: Date): ReplicationStatusEvent;

package/out/zero-cache/src/services/replicator/replication-status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-status.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,qCAAqC,CAAC;AACpE,OAAO,KAAK,EAGV,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,MAAM,EACP,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAGtD,OAAO,EAEL,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAKvC,qBAAa,0BAA0B;;IAKrC,MAAM,CAAC,UAAU,+BAEhB;IAED,MAAM,CAAC,qBAAqB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,8BAAuB,8BAE1E;IAED,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,8BAAuB,8BASnE;IAED,YACE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,CAAC,KAAK,CAAC,EAC3D,SAAS,8BAAuB,EAIjC;IAED,OAAO,CACL,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,GAAG,OAAa,GACf,IAAI,CAqBN;IAEK,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,GACT,OAAO,CAAC,KAAK,CAAC,CAOhB;IAED,IAAI,IAAI,IAAI,CAGX;CACF;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,UAAU,EACzB,GAAG,OAAa,iBAYjB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,EACV,EAAE,CAAC,EAAE,QAAQ,EACb,GAAG,OAAa,0BAKjB;AAGD,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GAAG,SAAS,EACxB,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,GAAG,OAAa,GACf,sBAAsB,CAmCxB"}
+{"version":3,"file":"replication-status.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replication-status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,qCAAqC,CAAC;AACpE,OAAO,KAAK,EACV,cAAc,EAGd,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,MAAM,EACP,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAEtD,OAAO,KAAK,EAAC,aAAa,EAAgB,MAAM,mBAAmB,CAAC;AACpE,OAAO,EAEL,oBAAoB,EACrB,MAAM,+BAA+B,CAAC;AAKvC,qBAAa,0BAA0B;;IAKrC,MAAM,CAAC,UAAU,+BAEhB;IAED,MAAM,CAAC,qBAAqB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,8BAAuB,8BAE1E;IAED,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,8BAAuB,8BASnE;IAED,YACE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,QAAQ,KAAK,CAAC,KAAK,CAAC,EAC3D,SAAS,8BAAuB,EAIjC;IAED,OAAO,CACL,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,GAAG,OAAa,GACf,IAAI,CAGN;IAED;;;;;OAKG;IACG,eAAe,CACnB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,CAAC,EAAE,MAAM,EACpB,QAAQ,SAAI,EACZ,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,EAC5C,SAAS,SAAO,GACf,OAAO,CAAC,IAAI,CAAC,CAmBf;IAgCK,oBAAoB,CACxB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,GACT,OAAO,CAAC,KAAK,CAAC,CAOhB;IAED,IAAI,IAAI,IAAI,CAGX;CACF;AAED;;;GAGG;AACH,qBAAa,gBAAgB;;IAS3B;;OAEG;IACH,YAAY,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,eAA0B,EAG/D;IAED,+CAA+C;IAC/C,KAAK,CAAC,KAAK,EAAE,aAAa,QAGzB;IAED;;;OAGG;IACH,YAAY,SAIX;IAED;;;OAGG;IACH,MAAM,IAAI,MAAM,CAUf;IAED,uEAAuE;IACvE,QAAQ,CAAC,KAAK,QAAO,OAAO,CAAC,gBAAgB,CAAC,CAG5C;IAEF,MAAM,IAAI,cAAc,GAAG,SAAS,CAoBnC;CACF;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,WAAW,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,UAAU,EACzB,GAAG,OAAa,iBAYjB;AAED,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,gBAAgB,EACvB,CAAC,EAAE,OAAO,EACV,EAAE,CAAC,EAAE,QAAQ,EACb,GAAG,OAAa,0BAKjB;AAGD,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,GAAG,SAAS,EACxB,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,GAAG,OAAa,GACf,sBAAsB,CAmCxB"}

package/out/zero-cache/src/services/replicator/replication-status.js

@@ -6,7 +6,7 @@ import { createSilentLogContext } from "../../../../shared/src/logging-test-util
 var byKeys = (a, b) => a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
 var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 	#dbRunner;
-	#publish;
+	#publishFn;
 	#timer;
 	static forTesting() {
 		return ReplicationStatusPublisher.forReplicaFile(":memory:");
@@ -26,23 +26,44 @@ var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 	}
 	constructor(dbRunner, publishFn = publishCriticalEvent) {
 		this.#dbRunner = dbRunner;
-		this.#publish = publishFn;
+		this.#publishFn = publishFn;
 	}
 	publish(lc, stage, description, interval = 0, extraState, now = /* @__PURE__ */ new Date()) {
+		this.#publish(lc, stage, description, interval, extraState, now);
+		return this;
+	}
+	/**
+	* Like {@link publish}, but waits (for at most `maxWaitMs`) for the event
+	* to be sent. Use this before a long synchronous operation (e.g. creating
+	* an index) that blocks the event loop and would otherwise prevent the
+	* event from being sent until the operation completes.
+	*/
+	async publishAndFlush(lc, stage, description, interval = 0, extraState, maxWaitMs = 1e3) {
+		const published = this.#publish(lc, stage, description, interval, extraState);
+		let timer;
+		try {
+			await Promise.race([published, new Promise((resolve) => {
+				timer = setTimeout(resolve, maxWaitMs);
+			})]);
+		} finally {
+			clearTimeout(timer);
+		}
+	}
+	#publish(lc, stage, description, interval, extraState, now = /* @__PURE__ */ new Date()) {
 		this.stop();
 		const event = this.#dbRunner(lc, (db) => replicationStatusEvent(lc, db, stage, "OK", description, now));
 		if (event.state) event.state = {
 			...event.state,
 			...extraState?.()
 		};
-		this.#publish(lc, event);
+		const published = this.#publishFn(lc, event);
 		if (interval) this.#timer = setInterval(() => this.publish(lc, stage, description, interval, extraState), interval);
-		return this;
+		return published;
 	}
 	async publishAndThrowError(lc, stage, e) {
 		this.stop();
 		const event = this.#dbRunner(lc, (db) => replicationStatusError(lc, stage, e, db));
-		await this.#publish(lc, event);
+		await this.#publishFn(lc, event);
 		throw e;
 	}
 	stop() {
@@ -50,6 +71,75 @@ var ReplicationStatusPublisher = class ReplicationStatusPublisher {
 		return this;
 	}
 };
+/**
+* Tracks the progress of a sequence of index creations for reporting in
+* the `indexingStatus` field of {@link ReplicationState}.
+*/
+var IndexingProgress = class {
+	#totalIndexes;
+	#now;
+	#count = 0;
+	#completedMs = 0;
+	#current;
+	/**
+	* @param totalIndexes The total number of indexes to be created, if known.
+	*/
+	constructor(totalIndexes, now = () => performance.now()) {
+		this.#totalIndexes = totalIndexes;
+		this.#now = now;
+	}
+	/** Marks the start of creating the `index`. */
+	start(index) {
+		this.#count++;
+		this.#current = {
+			index,
+			start: this.#now(),
+			elapsedMs: void 0
+		};
+	}
+	/**
+	* Restarts the timer of the current index, e.g. to exclude time spent
+	* reporting its start from its creation time.
+	*/
+	restartTimer() {
+		if (this.#current) this.#current.start = this.#now();
+	}
+	/**
+	* Marks the completion of the index passed to the last call to
+	* {@link start}, returning the milliseconds it took to create.
+	*/
+	finish() {
+		const current = this.#current;
+		if (!current) return 0;
+		if (current.elapsedMs === void 0) {
+			current.elapsedMs = this.#now() - current.start;
+			this.#completedMs += current.elapsedMs;
+		}
+		return current.elapsedMs;
+	}
+	/** The `extraState` for {@link ReplicationStatusPublisher.publish}. */
+	state = () => {
+		const status = this.status();
+		return status ? { indexingStatus: status } : {};
+	};
+	status() {
+		const current = this.#current;
+		if (!current) return;
+		const { index, start, elapsedMs } = current;
+		const done = elapsedMs !== void 0;
+		return {
+			name: index.name,
+			table: index.tableName,
+			columns: Object.keys(index.columns),
+			unique: index.unique,
+			index: this.#count,
+			totalIndexes: this.#totalIndexes,
+			elapsedMs: Math.round(done ? elapsedMs : this.#now() - start),
+			completedMs: Math.round(done ? this.#completedMs - elapsedMs : this.#completedMs),
+			done
+		};
+	}
+};
 async function publishReplicationError(lc, stage, description, errorDetails, now = /* @__PURE__ */ new Date()) {
 	await publishCriticalEvent(lc, {
 		type: "zero/events/status/replication/v1",
@@ -115,10 +205,11 @@ function getReplicatedTables(db) {
 	}));
 }
 function getReplicatedIndexes(db) {
-	return listIndexes(db).map(({ tableName: table, columns, unique }) => ({
+	return listIndexes(db).map(({ name, tableName: table, columns, unique }) => ({
+		name,
 		table,
 		unique,
-		columns: Object.entries(columns).sort(byKeys).map(([column, dir]) => ({
+		columns: Object.entries(columns).map(([column, dir]) => ({
 			column,
 			dir
 		}))
@@ -130,6 +221,6 @@ function getReplicaSize(db) {
 	return pageCount * pageSize;
 }
 //#endregion
-export { ReplicationStatusPublisher, publishReplicationError, replicationStatusError };
+export { IndexingProgress, ReplicationStatusPublisher, publishReplicationError, replicationStatusError };
 
 //# sourceMappingURL=replication-status.js.map