@rocicorp/zero 1.11.0-canary.21 → 1.11.0-canary.22

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js

@@ -102,7 +102,7 @@ var ChangeStreamMultiplexer = class {
 	*/
 	pushStatus(message) {
 		this.#listeners.forEach((l) => l.onChange(message));
-		if (message[1].ack || message[1].lagReport) this.#sub.push(message);
+		this.#sub.push(message);
 	}
 	/**
 	* `push()` must only be called by a producer after it has

package/out/zero-cache/src/services/change-source/common/change-stream-multiplexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#cancel","#lastWatermark","#canceled"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    // Let listeners know about all status messages.\n    this.#listeners.forEach(l => l.onChange(message));\n    // The ChangeStreamer only cares about status messages requiring an ack\n    // or containing a lagReport. To reduce churn, avoid sending other status\n    // messages.\n    if (message[1].ack || message[1].lagReport) {\n      this.#sub.push(message);\n    }\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKI,QAAQ,EAC9B,CAAC;EACD,KAAKC,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAKC,YAAY;EACjB,KAAKL,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAKE,SAAS,OAAO,CAAC,GAAG;GACnE,KAAKJ,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKE,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKI,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAKD,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAKF,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKE,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKN,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAE3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAIhD,IAAI,QAAQ,GAAG,OAAO,QAAQ,GAAG,WAC/B,KAAKF,KAAK,KAAK,OAAO;CAE1B;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKK,mBAAmB,MACxB,gDACF;EACA,KAAKH,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}
+{"version":3,"file":"change-stream-multiplexer.js","names":["#lc","#sub","#producers","#listeners","#waiters","#cancel","#lastWatermark","#canceled"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/change-stream-multiplexer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Source} from '../../../types/streams.ts';\nimport {Subscription} from '../../../types/subscription.ts';\nimport type {\n  ChangeStreamMessage,\n  DownstreamStatusMessage,\n} from '../protocol/current.ts';\n\nexport type Cancelable = {\n  cancel(): void;\n};\n\nexport type Listener = {\n  onChange(change: ChangeStreamMessage): void;\n};\n\ntype Waiter = {\n  producer: string;\n  startTime: number;\n  grantReservation: (watermark: string) => void;\n  rejectReservation: (err: Error) => void;\n};\n\n/**\n * Facilitates cooperative multiplexing of transactions from\n * multiple producers into a single change stream.\n */\nexport class ChangeStreamMultiplexer {\n  readonly #lc: LogContext;\n  readonly #sub: Subscription<ChangeStreamMessage>;\n  readonly #producers: Cancelable[] = [];\n  readonly #listeners: Listener[] = [];\n\n  /**\n   * The `#lastWatermark` tracks the watermark of the last transaction\n   * committed to stream. It is set to null when a producer has reserved\n   * the stream, and set to the new watermark when the producer releases\n   * the reservation.\n   */\n  #lastWatermark: string | null;\n\n  /**\n   * Tracks the queue of producers waiting for a reservation.\n   */\n  readonly #waiters: Waiter[] = [];\n\n  #canceled = false;\n\n  constructor(lc: LogContext, lastWatermark: string) {\n    this.#lc = lc;\n    this.#sub = Subscription.create<ChangeStreamMessage>({\n      cleanup: () => this.#cancel(),\n    });\n    this.#lastWatermark = lastWatermark;\n  }\n\n  #cancel() {\n    this.#canceled = true;\n    this.#producers.forEach(p => p.cancel());\n\n    // A producer awaiting a reservation would otherwise remain suspended\n    // forever, since the reservation held when the stream was canceled is\n    // never released. Reject the pending reservations so that the producers\n    // unwind and release the resources (e.g. upstream connections and\n    // transactions) that they were holding for the stream.\n    for (const {producer, rejectReservation} of this.#waiters.splice(0)) {\n      this.#lc.info?.(\n        `rejecting reservation request from ${producer}: stream canceled`,\n      );\n      rejectReservation(new AbortError('change stream canceled'));\n    }\n  }\n\n  addProducers(...p: Cancelable[]): this {\n    this.#producers.push(...p);\n    return this;\n  }\n\n  addListeners(...l: Listener[]): this {\n    this.#listeners.push(...l);\n    return this;\n  }\n\n  /**\n   * Called by a producer to \"reserve\" the exclusive right to push data\n   * changes to the stream. If the stream is currently reserved,\n   * the resulting Promise must be awaited.\n   *\n   * The producer must then {@link release()} the reservation when it sees\n   * fit.\n   *\n   * @param producer The name of the producer, purely for debugging output\n   */\n  reserve(producer: string): string | Promise<string> {\n    if (this.#canceled) {\n      return Promise.reject(new AbortError('change stream canceled'));\n    }\n    if (this.#lastWatermark !== null) {\n      // If the stream is not reserved, reserve it and return the\n      // watermark.\n      const lastWatermark = this.#lastWatermark;\n      this.#lastWatermark = null;\n      return lastWatermark;\n    }\n\n    // Otherwise, wait for the current reservation to be released.\n    const startTime = performance.now();\n    const {\n      promise,\n      resolve: grantReservation,\n      reject: rejectReservation,\n    } = resolver<string>();\n    this.#waiters.push({\n      producer,\n      startTime,\n      grantReservation,\n      rejectReservation,\n    });\n\n    return promise;\n  }\n\n  /**\n   * If there are producers currently awaiting a reservation, returns\n   * the duration (in milliseconds) of the oldest reservation request.\n   * Returns a negative number if there are no waiters.\n   */\n  waiterDelay(): number {\n    if (this.#waiters.length === 0) {\n      return -1;\n    }\n    return performance.now() - this.#waiters[0].startTime;\n  }\n\n  /**\n   * Called by a producer to release its reservation after committing its\n   * last transaction.\n   */\n  release(newWatermark: string) {\n    const waiter = this.#waiters.shift();\n    if (!waiter) {\n      this.#lastWatermark = newWatermark;\n    } else {\n      const {producer, startTime, grantReservation} = waiter;\n      grantReservation(newWatermark);\n      const elapsed = performance.now() - startTime;\n      this.#lc.info?.(\n        `${producer} waited ${elapsed.toFixed(3)} ms for stream reservation`,\n      );\n    }\n  }\n\n  /**\n   * `pushStatus()` can be called without a reservation, as it\n   * does not constitute a data change and can appear anywhere\n   * in the stream.\n   */\n  pushStatus(message: DownstreamStatusMessage) {\n    this.#listeners.forEach(l => l.onChange(message));\n    this.#sub.push(message);\n  }\n\n  /**\n   * `push()` must only be called by a producer after it has\n   * {@link reserve}d the stream.\n   */\n  push(message: ChangeStreamMessage): Promise<unknown> {\n    assert(\n      this.#lastWatermark === null,\n      `push() called without reserve()-ing the stream`,\n    );\n    this.#listeners.forEach(l => l.onChange(message));\n    return this.#sub.push(message).result;\n  }\n\n  fail(err: Error) {\n    this.#sub.fail(err);\n  }\n\n  asSource(): Source<ChangeStreamMessage> {\n    return this.#sub;\n  }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA,aAAoC,CAAC;CACrC,aAAkC,CAAC;;;;;;;CAQnC;;;;CAKA,WAA8B,CAAC;CAE/B,YAAY;CAEZ,YAAY,IAAgB,eAAuB;EACjD,KAAKA,MAAM;EACX,KAAKC,OAAO,aAAa,OAA4B,EACnD,eAAe,KAAKI,QAAQ,EAC9B,CAAC;EACD,KAAKC,iBAAiB;CACxB;CAEA,UAAU;EACR,KAAKC,YAAY;EACjB,KAAKL,WAAW,SAAQ,MAAK,EAAE,OAAO,CAAC;EAOvC,KAAK,MAAM,EAAC,UAAU,uBAAsB,KAAKE,SAAS,OAAO,CAAC,GAAG;GACnE,KAAKJ,IAAI,OACP,sCAAsC,SAAS,kBACjD;GACA,kBAAkB,IAAI,WAAW,wBAAwB,CAAC;EAC5D;CACF;CAEA,aAAa,GAAG,GAAuB;EACrC,KAAKE,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;CAEA,aAAa,GAAG,GAAqB;EACnC,KAAKC,WAAW,KAAK,GAAG,CAAC;EACzB,OAAO;CACT;;;;;;;;;;;CAYA,QAAQ,UAA4C;EAClD,IAAI,KAAKI,WACP,OAAO,QAAQ,OAAO,IAAI,WAAW,wBAAwB,CAAC;EAEhE,IAAI,KAAKD,mBAAmB,MAAM;GAGhC,MAAM,gBAAgB,KAAKA;GAC3B,KAAKA,iBAAiB;GACtB,OAAO;EACT;EAGA,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,EACJ,SACA,SAAS,kBACT,QAAQ,sBACN,SAAiB;EACrB,KAAKF,SAAS,KAAK;GACjB;GACA;GACA;GACA;EACF,CAAC;EAED,OAAO;CACT;;;;;;CAOA,cAAsB;EACpB,IAAI,KAAKA,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO,YAAY,IAAI,IAAI,KAAKA,SAAS,GAAG;CAC9C;;;;;CAMA,QAAQ,cAAsB;EAC5B,MAAM,SAAS,KAAKA,SAAS,MAAM;EACnC,IAAI,CAAC,QACH,KAAKE,iBAAiB;OACjB;GACL,MAAM,EAAC,UAAU,WAAW,qBAAoB;GAChD,iBAAiB,YAAY;GAC7B,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,KAAKN,IAAI,OACP,GAAG,SAAS,UAAU,QAAQ,QAAQ,CAAC,EAAE,2BAC3C;EACF;CACF;;;;;;CAOA,WAAW,SAAkC;EAC3C,KAAKG,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,KAAKF,KAAK,KAAK,OAAO;CACxB;;;;;CAMA,KAAK,SAAgD;EACnD,OACE,KAAKK,mBAAmB,MACxB,gDACF;EACA,KAAKH,WAAW,SAAQ,MAAK,EAAE,SAAS,OAAO,CAAC;EAChD,OAAO,KAAKF,KAAK,KAAK,OAAO,EAAE;CACjC;CAEA,KAAK,KAAY;EACf,KAAKA,KAAK,KAAK,GAAG;CACpB;CAEA,WAAwC;EACtC,OAAO,KAAKA;CACd;AACF"}

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts

@@ -3,9 +3,32 @@ import type { LitestreamConfig } from '../../../config/normalize.ts';
 import { type ReplicaConstraints } from '../../litestream/commands.ts';
 import type { SubscriptionState } from '../../replicator/schema/replication-state.ts';
 import type { ChangeSource } from '../change-source.ts';
+export interface PurgeLock {
+    release(): Promise<void>;
+}
+/** A PurgeLock that also constrains the replica that can be restored. */
+export interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {
+}
+/**
+ * Acquires a purge lock on the PG change-log (if it is not empty).
+ *
+ * When resuming the change-log from a replication slot, `slotWatermark` is the
+ * slot's position (as a watermark). If the change-log's head is behind it, the
+ * change-log may be missing transactions that the slot will not stream, in
+ * which case nothing is locked and `'behind-slot'` is returned.
+ */
+export type PgChangeLogPurgeLocker = (slotWatermark?: string) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;
 export type RestoreOptions = {
     litestream?: LitestreamConfig;
-    constraints?: ReplicaConstraints | undefined;
+    /**
+     * With the PG change-log enabled, purge-locks the change-log before the
+     * replica is restored, which constrains the replica to one from which the
+     * change-log can be resumed. The lock is acquired as late as possible (i.e.
+     * after a replication slot is created, if applicable) because, if the
+     * change-db is the upstream db, a transaction holding the lock would block
+     * the creation of a replication slot.
+     */
+    acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;
 };
 export type InitializeResult = {
     subscriptionState: SubscriptionState;
@@ -30,6 +53,13 @@ export type InitializeResult = {
      * an abandoned replica is resumed.
      */
     waitForBackupBeforeServing: boolean;
+    /**
+     * Whether the PG change-log's head was behind the position of the
+     * replication slot from which replication resumes (see
+     * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from
+     * the restored replica if it does not contain the replica's changes.
+     */
+    pgChangeLogBehindSlot?: boolean | undefined;
 };
 export declare function restoreReplica(lc: LogContext, config: LitestreamConfig, replicaFile: string, replicaConstraints: ReplicaConstraints | undefined): Promise<void>;
 //# sourceMappingURL=replica-restore.d.ts.map

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEtD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;CACrC,CAAC;AAOF,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,GACjD,OAAO,CAAC,IAAI,CAAC,CAyCf"}
+{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEtD,MAAM,WAAW,SAAS;IACxB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAsB,SAAQ,SAAS,EAAE,kBAAkB;CAAG;AAE/E;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,CACnC,aAAa,CAAC,EAAE,MAAM,KACnB,OAAO,CAAC,qBAAqB,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,sBAAsB,GAAG,SAAS,CAAC;CACvD,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;IAEpC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7C,CAAC;AAOF,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,GACjD,OAAO,CAAC,IAAI,CAAC,CAyCf"}

package/out/zero-cache/src/services/change-source/common/replica-restore.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  constraints?: ReplicaConstraints | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n): Promise<void> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n        );\n        result = attempt.result;\n        return;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          return;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AAiDA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAE/B,eAAsB,eACpB,IACA,QACA,aACA,oBACe;CACf,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAC3D,IAAI;GAQF,UAAS,MAPa,WACpB,IACA,QACA,aACA,oBACA,qBACF,GACiB;GACjB;EACF,SAAS,GAAG;GACV,IAAI,eAAe,sBAAsB;IACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;IACA;GACF;GAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;GACA,MAAM,MAAM,sBAAsB;EACpC;CAEJ,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}
+{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\n\nexport interface PurgeLock {\n  release(): Promise<void>;\n}\n\n/** A PurgeLock that also constrains the replica that can be restored. */\nexport interface ConstrainingPurgeLock extends PurgeLock, ReplicaConstraints {}\n\n/**\n * Acquires a purge lock on the PG change-log (if it is not empty).\n *\n * When resuming the change-log from a replication slot, `slotWatermark` is the\n * slot's position (as a watermark). If the change-log's head is behind it, the\n * change-log may be missing transactions that the slot will not stream, in\n * which case nothing is locked and `'behind-slot'` is returned.\n */\nexport type PgChangeLogPurgeLocker = (\n  slotWatermark?: string,\n) => Promise<ConstrainingPurgeLock | null | 'behind-slot'>;\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  /**\n   * With the PG change-log enabled, purge-locks the change-log before the\n   * replica is restored, which constrains the replica to one from which the\n   * change-log can be resumed. The lock is acquired as late as possible (i.e.\n   * after a replication slot is created, if applicable) because, if the\n   * change-db is the upstream db, a transaction holding the lock would block\n   * the creation of a replication slot.\n   */\n  acquirePurgeLock?: PgChangeLogPurgeLocker | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n\n  /**\n   * Whether the PG change-log's head was behind the position of the\n   * replication slot from which replication resumes (see\n   * {@link PgChangeLogPurgeLocker}), in which case it is re-initialized from\n   * the restored replica if it does not contain the replica's changes.\n   */\n  pgChangeLogBehindSlot?: boolean | undefined;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n): Promise<void> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n        );\n        result = attempt.result;\n        return;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          return;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AAoFA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAE/B,eAAsB,eACpB,IACA,QACA,aACA,oBACe;CACf,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAC3D,IAAI;GAQF,UAAS,MAPa,WACpB,IACA,QACA,aACA,oBACA,qBACF,GACiB;GACjB;EACF,SAAS,GAAG;GACV,IAAI,eAAe,sBAAsB;IACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;IACA;GACF;GAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;GACA,MAAM,MAAM,sBAAsB;EACpC;CAEJ,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts

@@ -9,7 +9,7 @@ export type ServerContext = JSONObject;
  * Initializes a Custom change source before streaming changes from the
  * corresponding logical replication stream.
  */
-export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext, { litestream, constraints }?: RestoreOptions): Promise<InitializeResult>;
+export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext, { litestream, acquirePurgeLock }?: RestoreOptions): Promise<InitializeResult>;
 /**
  * Initial sync for a custom change source makes a request to the
  * change source endpoint with no `replicaVersion` or `lastWatermark`.

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAcnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAQtC,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,EACtB,EAAC,UAAU,EAAE,WAAW,EAAC,GAAE,cAAmB,GAC7C,OAAO,CAAC,gBAAgB,CAAC,CAwD3B;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAcnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAQtC,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,EACtB,EAAC,UAAU,EAAE,gBAAgB,EAAC,GAAE,cAAmB,GAClD,OAAO,CAAC,gBAAgB,CAAC,CA4D3B;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -19,8 +19,11 @@ import { WebSocket } from "ws";
 * Initializes a Custom change source before streaming changes from the
 * corresponding logical replication stream.
 */
-async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context, { litestream, constraints } = {}) {
-	if (litestream?.backupURL) await restoreReplica(lc, litestream, replicaDbFile, constraints);
+async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context, { litestream, acquirePurgeLock } = {}) {
+	if (litestream?.backupURL) {
+		const lock = await acquirePurgeLock?.();
+		await restoreReplica(lc, litestream, replicaDbFile, lock === "behind-slot" ? void 0 : lock ?? void 0);
+	}
 	let initialSynced = false;
 	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => {
 		initialSynced = true;

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, constraints}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,gBAA+B,CAAC,GAClB;CAS3B,IAAI,YAAY,WACd,MAAM,eAAe,IAAI,YAAY,eAAe,WAAW;CAGjE,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}
+{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n  stringify,\n  type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  createReplicationStateTables,\n  getSubscriptionState,\n  initReplicationState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n  type BackfillRequest,\n  type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  context: ServerContext,\n  {litestream, acquirePurgeLock}: RestoreOptions = {},\n): Promise<InitializeResult> {\n  // At the moment, the custom change-source implementation does not support\n  // per-RM backups (and thus does not support litestream v5 or RMv2). The\n  // current proposal to support this without requiring additional upstream\n  // state tracking is to:\n  // - use litestream to (ltx) query a fixed pool of subfolders within the\n  //   backupURL (e.g. /a/, /b/, /c/, /d/, /e/)\n  // - restore from the subfolder with the latest ltx file\n  // - backup to the subfolder with the oldest (or non-existent) ltx file\n  if (litestream?.backupURL) {\n    // Without a replication slot, the change-log is never behind one.\n    const lock = await acquirePurgeLock?.();\n    const constraints =\n      lock === 'behind-slot' ? undefined : (lock ?? undefined);\n    await restoreReplica(lc, litestream, replicaDbFile, constraints);\n  }\n\n  let initialSynced = false;\n  await initReplica(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    replicaDbFile,\n    (log, tx) => {\n      initialSynced = true;\n      return initialSync(log, shard, tx, upstreamURI, context);\n    },\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = shard.publications.toSorted();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {\n    subscriptionState,\n    changeSource,\n    destinationBackupURL: litestream?.backupURL,\n    // A custom change source has no upstream `replicas` table to identify\n    // itself from, so the generation is the whole of the SQLite change log's\n    // identity here.\n    replicaID: null,\n    waitForBackupBeforeServing: initialSynced,\n  };\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startLagReporter() {\n    return null; // Not supported for custom sources\n  }\n\n  stop(): Promise<void> {\n    return Promise.resolve();\n  }\n\n  startStream(\n    clientWatermark: string,\n    backfillRequests: BackfillRequest[] = [],\n  ): Promise<ChangeStream> {\n    if (backfillRequests?.length) {\n      throw new Error(\n        'backfill is yet not supported for custom change sources',\n      );\n    }\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n  context: ServerContext,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  createReplicationStateTables(tx);\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(\n            tx,\n            publications.toSorted(),\n            commitWatermark,\n            context,\n            false,\n          );\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6CA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SACA,EAAC,YAAY,qBAAoC,CAAC,GACvB;CAS3B,IAAI,YAAY,WAAW;EAEzB,MAAM,OAAO,MAAM,mBAAmB;EAGtC,MAAM,eAAe,IAAI,YAAY,eADnC,SAAS,gBAAgB,KAAA,IAAa,QAAQ,KAAA,CACe;CACjE;CAEA,IAAI,gBAAgB;CACpB,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO;EACX,gBAAgB;EAChB,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,OAAO;CACzD,CACF;CAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;CAC9C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;CAC3E,QAAQ,MAAM;CAEd,IAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,SAAS;EAC9C,MAAM,aAAa,kBAAkB,aAAa,KAAK;EACvD,IAAI,CAAC,UAAU,WAAW,UAAU,GAClC,MAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,EAC9G;CAEJ;CASA,OAAO;EACL;EACA,cAAA,IATuB,mBACvB,IACA,aACA,OACA,iBAKA;EACA,sBAAsB,YAAY;EAIlC,WAAW;EACX,4BAA4B;CAC9B;AACF;AAEA,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,eAAe;EACtD,KAAKC,eAAe;EACpB,KAAKC,SAAS;EACd,KAAKC,qBAAqB;CAC5B;CAEA,cAA4B;EAC1B,OAAO,KAAKC,aAAa;CAC3B;CAEA,mBAAmB;EACjB,OAAO;CACT;CAEA,OAAsB;EACpB,OAAO,QAAQ,QAAQ;CACzB;CAEA,YACE,iBACA,mBAAsC,CAAC,GAChB;EACvB,IAAI,kBAAkB,QACpB,MAAM,IAAI,MACR,yDACF;EAEF,OAAO,QAAQ,QAAQ,KAAKA,aAAa,eAAe,CAAC;CAC3D;CAEA,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,KAAKD;EAC5C,MAAM,EAAC,OAAO,aAAY,KAAKD;EAC/B,MAAM,MAAM,IAAI,IAAI,KAAKD,YAAY;EACrC,IAAI,aAAa,IAAI,SAAS,KAAK;EACnC,IAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;EACjD,KAAK,MAAM,OAAO,cAChB,IAAI,aAAa,OAAO,gBAAgB,GAAG;EAE7C,IAAI,iBAAiB;GACnB,OACE,eAAe,QACf,wDACF;GACA,IAAI,aAAa,IAAI,iBAAiB,eAAe;GACrD,IAAI,aAAa,IAAI,kBAAkB,cAAc;EACvD;EAEA,MAAM,KAAK,IAAI,UAAU,GAAG;EAC5B,MAAM,EAAC,UAAU,cAAa,OAC5B,KAAKD,KACL,IACA,2BAGA,EAAC,WAAW,SAA+B,KAAI,CACjD;EACA,OAAO;GAAC,SAAS;GAAU,MAAM;EAAS;CAC5C;AACF;;;;;;;;;;;;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAAW,IAJO,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;CACF,CACkB,EAAa,YAAY;CAE3C,6BAA6B,EAAE;CAC/B,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,EAAE,GACtB,iBACC,GAAG,QAAQ;EACV,MAAM;CACR,CACF;CAEA,MAAM,kBAAkB,2BAA2B,sBAAsB,EAAE;CAC3E,IAAI;EACF,IAAI,MAAM;EACV,WAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;GACd,QAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;KACjC,GAAG,OACD,yBAAyB,GAAG,qBAAqB,iBACnD;KACA,gBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,GACF;KACA,qBACE,IACA,aAAa,SAAS,GACtB,iBACA,SACA,KACF;KACA,UAAU,eAAe,IAAI,MAAM;KACnC;IACF;IACA,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,IAAI,EAAE,MAAM,QAAS,GACnB,GAAG,QAAQ,aAAa,IAAI,SAAS;KAEvC;IACF,KAAK;KACH,UAAU,eAAe,IAAI,MAAM;KACnC,4BAA4B,IAAI,IAAI,KAAK;KACzC,GAAG,OAAO,4BAA4B,IAAI,SAAS;KACnD;IAEF,KAAK,UACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;KAC9B,IAAI,QAAQ,kBACV,MAAM,IAAI,gBACR,WAAW,sCACb;IAEJ;IAEA,KAAK,YACH,MAAM,IAAI,MACR,2CAA2C,UAAU,MAAM,GAC7D;IACF,SACE,YAAY,MAAM;GACtB;EACF;EACA,MAAM,IAAI,MACR,iBAAiB,YAAY,sCAC/B;CACF,SAAS,GAAG;EACV,MAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;CAClE,UAAU;EACR,gBAAgB,KAAK;CACvB;AACF;AAIA,SAAS,kBAAkB,EACzB,OACA,YACuD;CACvD,OAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,gBAAgB,EAAC,MAAM,SAAQ;GAC/B,QAAQ,EAAC,MAAM,SAAQ;EACzB;GACC,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,SAAQ;GAC9B,UAAU,EAAC,MAAM,SAAQ;GACzB,YAAY,EAAC,MAAM,SAAQ;GAC3B,UAAU,EAAC,MAAM,OAAM;EACzB;GACC,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,OAAM;GAC1B,MAAM,EAAC,MAAM,SAAQ;EACvB;CACF;AACF;AAEA,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,KAAI,CAAC;CACxE,MAAM,WAAW,kBAAkB,KAAK;CACxC,KAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,QAAQ,GAAG;EACtD,MAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;EAChC,IAAI,CAAC,OACH,MAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,KAAK,CACjB,EAAE,qFAGJ;EAEF,KAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,OAAO,GAAG;GACnD,MAAM,QAAQ,MAAM;GACpB,IAAI,CAAC,OACH,MAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,SACnD;GAEF,IAAI,MAAM,SAAS,MACjB,MAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,OAClF;EAEJ;CACF;AACF"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts

@@ -3,10 +3,8 @@ import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardConfig, type ShardID } from '../../../types/shards.ts';
 import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 import { type InitialSyncOptions, type ReplicaOptions, type ServerContext } from './initial-sync.ts';
+import { type LSN } from './lsn.ts';
 import { type ReplicaState } from './schema/shard.ts';
-interface PurgeLock {
-    release(): Promise<void>;
-}
 export type InitializeOptions = ReplicaOptions & {
     slotPerReplica: boolean | undefined;
     inactiveReplicaGracePeriodMs: number;
@@ -16,10 +14,11 @@ export type InitializeOptions = ReplicaOptions & {
  * replica, before streaming changes from the corresponding logical replication
  * stream.
  */
-export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, purgeLock?: PurgeLock | null, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
+export declare function initializePostgresChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, syncOptions: InitialSyncOptions, context: ServerContext, lagReportIntervalMs?: number, restoreOptions?: RestoreOptions, { epoch, slotPerReplica, backupV5, inactiveReplicaGracePeriodMs, }?: InitializeOptions, streamInboundTimeoutMs?: number | undefined): Promise<InitializeResult>;
 export declare function getSourceAndDestinationReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, epoch: number, slotFailover: boolean, gracePeriodMs: number, pollIntervalMs?: number): Promise<{
     restoreFrom: ReplicaState;
     replicateTo: ReplicaState;
+    /** The position of the slot from which replication resumes. */
+    slotLSN: LSN;
 } | undefined>;
-export {};
 //# sourceMappingURL=change-source-init.d.ts.map

package/out/zero-cache/src/services/change-source/pg/change-source-init.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAMxE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAU3B,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,EAC5B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAyFD,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CAAC;IAAC,WAAW,EAAE,YAAY,CAAC;IAAC,WAAW,EAAE,YAAY,CAAA;CAAC,GAAG,SAAS,CAAC,CAoI7E"}
+{"version":3,"file":"change-source-init.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAC,KAAK,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAMxE,OAAO,EAEL,KAAK,gBAAgB,EAErB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAiC,KAAK,GAAG,EAAC,MAAM,UAAU,CAAC;AASlE,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,iBAAiB,GAAG,cAAc,GAAG;IAG/C,cAAc,EAAE,OAAO,GAAG,SAAS,CAAC;IAEpC,4BAA4B,EAAE,MAAM,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,EACvB,cAAc,GAAE,cAAmB,EACnC,EACE,KAAK,EACL,cAAc,EACd,QAAQ,EACR,4BAA4B,GAC7B,GAAE,iBAKF,EACD,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1C,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAqID,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,OAAO,EACrB,aAAa,EAAE,MAAM,EACrB,cAAc,SAA2B,GACxC,OAAO,CACN;IACE,WAAW,EAAE,YAAY,CAAC;IAC1B,WAAW,EAAE,YAAY,CAAC;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,GAAG,CAAC;CACd,GACD,SAAS,CACZ,CAiJA"}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js

@@ -7,7 +7,7 @@ import { Database } from "../../../../../zqlite/src/db.js";
 import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../../db/statements.js";
 import { connectPgClient } from "../../../types/pg.js";
-import { toBigInt } from "./lsn.js";
+import { toBigInt, toStateVersionString } from "./lsn.js";
 import "./schema/replica-stage-enum.js";
 import { dropShard, getActiveReplicas, getReplicaAtVersion, getRestoreCandidates, initRestoreReplica, internalPublicationPrefix } from "./schema/shard.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
@@ -28,12 +28,11 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
 	slotPerReplica: false,
 	backupV5: true,
 	inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS
-}, purgeLock, streamInboundTimeoutMs) {
+}, streamInboundTimeoutMs) {
 	const db = await connectPgClient(lc, upstreamURI, "change-source-init");
 	try {
 		await ensureShardSchema(lc, db, shard, syncOptions.installPartialIndexTriggers);
-		if (slotPerReplica) assert(purgeLock === null, `There should be no purgeLock for RMv2`);
-		const restoredReplica = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
+		const { replica: restoredReplica, purgeLock, pgChangeLogBehindSlot } = slotPerReplica ? await forkOrResumeReplica(lc, db, shard, epoch, syncOptions.replicationSlotFailover ?? false, replicaDbFile, restoreOptions, inactiveReplicaGracePeriodMs) : await selectAndRestoreReplica(lc, db, shard, replicaDbFile, restoreOptions);
 		let initialSyncedReplica;
 		await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, async (log, tx) => {
 			purgeLock?.release();
@@ -55,17 +54,25 @@ async function initializePostgresChangeSource(lc, upstreamURI, shard, replicaDbF
 			}, context, lagReportIntervalMs, syncOptions.textCopy, streamInboundTimeoutMs),
 			destinationBackupURL: backupPath && restoreOptions.litestream?.backupURL ? new URL(backupPath, restoreOptions.litestream.backupURL).toString() : restoreOptions.litestream?.backupURL,
 			replicaID: upstreamReplica.id,
-			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null)
+			waitForBackupBeforeServing: initialSyncedReplica !== void 0 || backupPath !== (restoredReplica?.backupPath ?? null),
+			pgChangeLogBehindSlot
 		};
 	} finally {
 		await db.end();
 	}
 }
-async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, constraints }) {
+async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream, acquirePurgeLock }) {
+	const lock = await acquirePurgeLock?.();
+	assert(lock !== "behind-slot", "unexpected purge-lock result");
+	const purgeLock = lock ?? null;
+	const constraints = purgeLock ?? void 0;
 	const replicas = (await getActiveReplicas(lc, sql, shard)).filter(({ generation }) => generation === (constraints?.replicaVersion ?? generation));
 	if (replicas.length === 0) {
 		lc.info?.(`no suitable replicas to restore from`, { replicas });
-		return;
+		return {
+			replica: void 0,
+			purgeLock
+		};
 	}
 	const [replica] = replicas;
 	if (litestream?.backupURL) {
@@ -78,12 +85,22 @@ async function selectAndRestoreReplica(lc, sql, shard, replicaFile, { litestream
 			backupURL
 		}, replicaFile, constraints);
 	}
-	return replica;
+	return {
+		replica,
+		purgeLock
+	};
 }
-async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, constraints }, gracePeriodMs) {
+async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaFile, { litestream, acquirePurgeLock }, gracePeriodMs) {
 	const result = await getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs);
-	if (!result) return;
-	const { restoreFrom, replicateTo } = result;
+	if (!result) return {
+		replica: void 0,
+		purgeLock: null
+	};
+	const { restoreFrom, replicateTo, slotLSN } = result;
+	const lock = acquirePurgeLock ? await acquirePurgeLock(toStateVersionString(slotLSN)) : null;
+	const pgChangeLogBehindSlot = lock === "behind-slot";
+	const purgeLock = lock === "behind-slot" ? null : lock;
+	const constraints = purgeLock ?? void 0;
 	if (litestream?.backupURL) {
 		const { backupURL: backupBaseURL } = litestream;
 		const { slot, backupPath, confirmedFlushLsn } = restoreFrom;
@@ -97,13 +114,22 @@ async function forkOrResumeReplica(lc, sql, shard, epoch, slotFailover, replicaF
 			backupURL
 		}, replicaFile, constraints);
 	}
-	return replicateTo;
+	return {
+		replica: replicateTo,
+		purgeLock,
+		pgChangeLogBehindSlot
+	};
+}
+async function currentWalLSN(sql) {
+	const [{ lsn }] = await sql`SELECT pg_current_wal_lsn() AS lsn`;
+	return lsn;
 }
 var REPLICA_POLL_INTERVAL_MS = 5e3;
 var DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 2e4;
 async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailover, gracePeriodMs, pollIntervalMs = REPLICA_POLL_INTERVAL_MS) {
 	const inactiveSince = /* @__PURE__ */ new Map();
 	let destination;
+	let forkLSN;
 	try {
 		for (let i = 0;; i++) {
 			if (i > 0) await sleep(pollIntervalMs);
@@ -127,19 +153,21 @@ async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailov
 							backupPath: null,
 							backupV5: true
 						}, () => promiseVoid, 2);
-						if (toBigInt(confirmedFlushLsn) < toBigInt(destination.slot.consistent_point)) {
-							lc.info?.(`waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`, { replica });
+						forkLSN ??= await currentWalLSN(sql);
+						if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {
+							lc.info?.(`waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`, { replica });
 							break;
 						}
 					}
-					if (destination && toBigInt(confirmedFlushLsn) >= toBigInt(destination.slot.consistent_point)) {
+					if (destination && forkLSN && toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)) {
 						lc.info?.(`forking replica ${replica.id}@${replica.slot}`, { replica });
 						return {
 							restoreFrom: replica,
 							replicateTo: must(await initRestoreReplica(sql, shard, {
 								sourceID: replica.id,
 								destID: destination.replica.id
-							}), `replica ${destination.replica.id} was deleted`)
+							}), `replica ${destination.replica.id} was deleted`),
+							slotLSN: destination.slot.consistent_point
 						};
 					}
 				}
@@ -155,7 +183,8 @@ async function getSourceAndDestinationReplicas(lc, sql, shard, epoch, slotFailov
 							destination?.initialSession.destroy();
 							return {
 								restoreFrom: replica,
-								replicateTo: reserved.replica
+								replicateTo: reserved.replica,
+								slotLSN: replica.confirmedFlushLsn
 							};
 						}
 					}

package/out/zero-cache/src/services/change-source/pg/change-source-init.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  purgeLock?: PurgeLock | null,\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    if (slotPerReplica) {\n      // Sanity check: This should be disabled via pgChangeLogEnabled=false.\n      assert(purgeLock === null, `There should be no purgeLock for RMv2`);\n    }\n\n    const restoredReplica = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // In RMv1, the purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2.\n        void purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n        // recorded in the replicas table by the PostgresChangeSource.\n        backupV5\n        ? String(Date.now())\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n    };\n  } finally {\n    await db.end();\n  }\n}\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, constraints}: RestoreOptions,\n): Promise<ReplicaState | undefined> {\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the constraints, if present\n    ({generation}) =>\n      generation === (constraints?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return undefined;\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return replica;\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, constraints}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<ReplicaState | undefined> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    return undefined; // can't restore, must initial-sync\n  }\n  const {restoreFrom, replicateTo} = result;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return replicateTo;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<{restoreFrom: ReplicaState; replicateTo: ReplicaState} | undefined> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            if (\n              toBigInt(confirmedFlushLsn) <\n              toBigInt(destination.slot.consistent_point)\n            ) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${destination.slot.slot_name}@${destination.slot.consistent_point}`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            toBigInt(confirmedFlushLsn) >=\n              toBigInt(destination.slot.consistent_point)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {restoreFrom: replica, replicateTo};\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,WACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,IAAI,gBAEF,OAAO,cAAc,MAAM,uCAAuC;EAGpE,MAAM,kBAAkB,iBACpB,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,WAAgB,QAAQ;GACxB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,CAClB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAEA,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,WACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;EACnD;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAIA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,eACsB;CACnC,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBACA,gBAAgB,aAAa,kBAAkB,WACnD;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D;CACF;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,eACb,eACmC;CACnC,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QACH;CAEF,MAAM,EAAC,aAAa,gBAAe;CAEnC,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BAC4D;CAC7E,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MACA,IACE,SAAS,iBAAiB,IAC1B,SAAS,YAAY,KAAK,gBAAgB,GAC1C;OACA,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,oBAC1G,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,SAAS,iBAAiB,KACxB,SAAS,YAAY,KAAK,gBAAgB,GAC5C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OAAC,aAAa;OAAS,aAPV,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAEN;MAAW;KAC3C;IACF;IAEA,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;OACxB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}
+{"version":3,"file":"change-source-init.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source-init.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {connectPgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {type ShardConfig, type ShardID} from '../../../types/shards.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  getSubscriptionStateAndContext,\n  type SubscriptionStateAndContext,\n} from '../../replicator/schema/replication-state.ts';\nimport {\n  restoreReplica,\n  type InitializeResult,\n  type ConstrainingPurgeLock,\n  type RestoreOptions,\n} from '../common/replica-restore.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {PostgresChangeSource} from './change-source.ts';\nimport {\n  initialSync,\n  type InitialSyncOptions,\n  type ReplicaOptions,\n  type ServerContext,\n} from './initial-sync.ts';\nimport {toBigInt, toStateVersionString, type LSN} from './lsn.ts';\nimport {\n  claimSlotForResumption,\n  createReplicaAndSlot,\n  type ReplicationSlotResult,\n} from './replication-slots.ts';\nimport {ensureShardSchema} from './schema/init.ts';\nimport * as ReplicaStage from './schema/replica-stage-enum.ts';\nimport {InitialSync, Replicate} from './schema/replica-stage-enum.ts';\nimport {\n  dropShard,\n  getActiveReplicas,\n  getReplicaAtVersion,\n  getRestoreCandidates,\n  initRestoreReplica,\n  internalPublicationPrefix,\n  type ReplicaState,\n} from './schema/shard.ts';\n\nexport type InitializeOptions = ReplicaOptions & {\n  // Create a new slot for the replica rather than taking over\n  // an existing one (i.e. high-availability mode).\n  slotPerReplica: boolean | undefined;\n\n  inactiveReplicaGracePeriodMs: number;\n};\n\n/**\n * Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n */\nexport async function initializePostgresChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n  syncOptions: InitialSyncOptions,\n  context: ServerContext,\n  lagReportIntervalMs = 0,\n  restoreOptions: RestoreOptions = {},\n  {\n    epoch,\n    slotPerReplica,\n    backupV5,\n    inactiveReplicaGracePeriodMs,\n  }: InitializeOptions = {\n    epoch: 0,\n    slotPerReplica: false,\n    backupV5: true,\n    inactiveReplicaGracePeriodMs: DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS,\n  },\n  streamInboundTimeoutMs?: number | undefined,\n): Promise<InitializeResult> {\n  const db = await connectPgClient(lc, upstreamURI, 'change-source-init');\n  try {\n    await ensureShardSchema(\n      lc,\n      db,\n      shard,\n      syncOptions.installPartialIndexTriggers,\n    );\n\n    const {\n      replica: restoredReplica,\n      purgeLock,\n      pgChangeLogBehindSlot,\n    } = slotPerReplica\n      ? await forkOrResumeReplica(\n          lc,\n          db,\n          shard,\n          epoch,\n          syncOptions.replicationSlotFailover ?? false,\n          replicaDbFile,\n          restoreOptions,\n          inactiveReplicaGracePeriodMs,\n        )\n      : await selectAndRestoreReplica(\n          lc,\n          db,\n          shard,\n          replicaDbFile,\n          restoreOptions,\n        );\n\n    let initialSyncedReplica: ReplicaState | undefined;\n    await initReplica(\n      lc,\n      `replica-${shard.appID}-${shard.shardNum}`,\n      replicaDbFile,\n      async (log, tx) => {\n        // The purge lock on the change-db must be released before performing\n        // initial sync; if the change-db and upstream are the same db, a lock-holding\n        // transaction will prevent a replication slot from being created. This awkward\n        // dependency can go away with RMv2.\n        void purgeLock?.release();\n        initialSyncedReplica = await initialSync(\n          log,\n          shard,\n          tx,\n          upstreamURI,\n          syncOptions,\n          context,\n          {epoch, backupV5},\n        );\n      },\n    );\n\n    const replica = new Database(lc, replicaDbFile);\n    const subscriptionState = getSubscriptionStateAndContext(\n      new StatementRunner(replica),\n    );\n    replica.close();\n\n    // Check that upstream is properly setup, and throw an AutoReset to re-run\n    // initial sync if not.\n    const {upstreamReplica, pgVersion} = await checkAndUpdateUpstream(\n      lc,\n      db,\n      shard,\n      subscriptionState,\n      (initialSyncedReplica ?? restoredReplica)?.id,\n    );\n\n    const backupPath = initialSyncedReplica\n      ? // If initial sync was performed, use that initial backupPath.\n        initialSyncedReplica.backupPath\n      : // Otherwise, use a new, unique path when backing up with litestream v5. This will be\n        // recorded in the replicas table by the PostgresChangeSource.\n        backupV5\n        ? String(Date.now())\n        : (restoredReplica?.backupPath ?? null);\n\n    const changeSource = new PostgresChangeSource(\n      lc,\n      upstreamURI,\n      shard,\n      upstreamReplica,\n      pgVersion,\n      {backupPath, backupV5},\n      context,\n      lagReportIntervalMs,\n      syncOptions.textCopy,\n      streamInboundTimeoutMs,\n    );\n\n    const destinationBackupURL =\n      backupPath && restoreOptions.litestream?.backupURL\n        ? new URL(backupPath, restoreOptions.litestream.backupURL).toString()\n        : // For legacy RMv1 replicas (on litestream-v3), backup to the same location\n          restoreOptions.litestream?.backupURL;\n\n    return {\n      subscriptionState,\n      changeSource,\n      destinationBackupURL,\n      // The replica this change stream belongs to. It is part of the identity\n      // the SQLite change log records, because a generation (i.e.\n      // `replicaVersion`) is shared by every sibling of a forked replica and so\n      // cannot distinguish two siblings' logs.\n      replicaID: upstreamReplica.id,\n      waitForBackupBeforeServing:\n        // Wait for the first backup if there was an initial sync,\n        initialSyncedReplica !== undefined ||\n        // or if the destination differs from where it was restored\n        // (i.e. backupV5).\n        backupPath !== (restoredReplica?.backupPath ?? null),\n      pgChangeLogBehindSlot,\n    };\n  } finally {\n    await db.end();\n  }\n}\n\ntype RestoredReplica = {\n  replica: ReplicaState | undefined;\n  purgeLock: ConstrainingPurgeLock | null;\n  pgChangeLogBehindSlot?: boolean | undefined;\n};\n\n// RMv1: Selects a replica to restore from and returns it, with the\n//       intention of taking over the slot (in the ChangeSource).\nasync function selectAndRestoreReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n): Promise<RestoredReplica> {\n  // The purge lock constrains the generation of the replica to restore. With\n  // a shared replication slot, the change-log is resumed from its head\n  // without being checked against the slot's position: the slot is still\n  // being acked by the previous replication-manager, on an idle upstream past\n  // the change-log's head (see PgChangeLogPurgeLocker).\n  const lock = await acquirePurgeLock?.();\n  assert(lock !== 'behind-slot', 'unexpected purge-lock result');\n  const purgeLock = lock ?? null;\n  const constraints = purgeLock ?? undefined;\n  const replicas = (await getActiveReplicas(lc, sql, shard)).filter(\n    // filter to the generation specified by the constraints, if present\n    ({generation}) =>\n      generation === (constraints?.replicaVersion ?? generation),\n  );\n  if (replicas.length === 0) {\n    lc.info?.(`no suitable replicas to restore from`, {replicas});\n    return {replica: undefined, purgeLock};\n  }\n  const [replica] = replicas;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = replica;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {replicas},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica, purgeLock};\n}\n\n// RMv2: Restores from an active replica and creates a new replica / slot\n// to continue replication (i.e. \"fork\"). If only orphaned replicas remain,\n// claims a slot, restores the backup, and \"resumes\" replication for that\n// slot.\nasync function forkOrResumeReplica(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  replicaFile: string,\n  {litestream, acquirePurgeLock}: RestoreOptions,\n  gracePeriodMs: number,\n): Promise<RestoredReplica> {\n  const result = await getSourceAndDestinationReplicas(\n    lc,\n    sql,\n    shard,\n    epoch,\n    slotFailover,\n    gracePeriodMs,\n  );\n  if (!result) {\n    // can't restore, must initial-sync\n    return {replica: undefined, purgeLock: null};\n  }\n  const {restoreFrom, replicateTo, slotLSN} = result;\n\n  // Acquired only after the replication slot has been created (or claimed),\n  // and only if the change-log can be resumed from the slot's position.\n  //\n  // For a fork, this is the case whenever the change-log is maintained (i.e.\n  // by a replication-manager that acks the slot only once a transaction has\n  // been stored in the change-log): the fork waits for the source replica's\n  // slot to pass the position after the creation of the new replica, which is\n  // a transaction (in the publication) after the new slot's position.\n  //\n  // For a resumed slot, the slot can also have been acked past the head of a\n  // maintained change-log, for changes outside of the publication (e.g. on an\n  // idle upstream). The change-log is then re-initialized unnecessarily, which\n  // resets subscribers behind the replica. (This could be avoided by recording\n  // the positions acked for such changes in the change-log, e.g. an\n  // `ackedThrough` column, and checking the slot's position against that.\n  // Resumption is uncommon, however, and involves downtime anyway.)\n  const lock = acquirePurgeLock\n    ? await acquirePurgeLock(toStateVersionString(slotLSN))\n    : null;\n  const pgChangeLogBehindSlot = lock === 'behind-slot';\n  const purgeLock = lock === 'behind-slot' ? null : lock;\n  const constraints = purgeLock ?? undefined;\n\n  if (litestream?.backupURL) {\n    const {backupURL: backupBaseURL} = litestream;\n    const {slot, backupPath, confirmedFlushLsn} = restoreFrom;\n    const backupURL = new URL(backupPath ?? '', backupBaseURL).toString();\n    lc.info?.(\n      `restoring replica from ${backupURL} (${slot}@${confirmedFlushLsn})`,\n      {restoreFrom, replicateTo},\n    );\n    await restoreReplica(\n      lc,\n      {...litestream, backupURL}, // includes the replica's backup sub-path\n      replicaFile,\n      constraints,\n    );\n  }\n  return {replica: replicateTo, purgeLock, pgChangeLogBehindSlot};\n}\n\nasync function currentWalLSN(sql: PostgresDB): Promise<LSN> {\n  const [{lsn}] = await sql<{lsn: LSN}[]>`SELECT pg_current_wal_lsn() AS lsn`;\n  return lsn;\n}\n\nconst REPLICA_POLL_INTERVAL_MS = 5_000;\nconst DEFAULT_INACTIVE_REPLICA_GRACE_PERIOD_MS = 20_000;\n\n// Exported for testing.\nexport async function getSourceAndDestinationReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  epoch: number,\n  slotFailover: boolean,\n  gracePeriodMs: number,\n  pollIntervalMs = REPLICA_POLL_INTERVAL_MS,\n): Promise<\n  | {\n      restoreFrom: ReplicaState;\n      replicateTo: ReplicaState;\n      /** The position of the slot from which replication resumes. */\n      slotLSN: LSN;\n    }\n  | undefined\n> {\n  const inactiveSince = new Map<string, number>(); // tracks replica inactivity\n  let destination: ReplicationSlotResult<void> | undefined;\n  // The position after the creation of the destination replica.\n  let forkLSN: LSN | undefined;\n\n  try {\n    for (let i = 0; ; i++) {\n      if (i > 0) {\n        await sleep(pollIntervalMs);\n      }\n      const replicas = await getRestoreCandidates(lc, sql, shard, epoch);\n      if (replicas.length === 0) {\n        lc.info?.(`no suitable replicas to restore from`, {replicas});\n        destination?.initialSession.destroy();\n        return undefined;\n      }\n      for (const replica of replicas) {\n        // Track inactivity to resume replicas after a grace period.\n        if (replica.active) {\n          inactiveSince.delete(replica.id);\n        } else if (!inactiveSince.has(replica.id)) {\n          inactiveSince.set(replica.id, Date.now());\n        }\n\n        // Note: Only `active` InitialSync replicas are returned from\n        // getRestoreCandidates().\n        if (replica.stage === InitialSync) {\n          // Log periodically; initial-sync can be long\n          if (i % 12 === 0) {\n            lc.info?.(`waiting for initial sync of ${replica.id}`, {replica});\n          }\n          break;\n        }\n\n        if (replica.stage === Replicate) {\n          const {active, confirmedFlushLsn} = replica;\n          if (active) {\n            // Create a replication slot to fork the active replica.\n            destination ??= await createReplicaAndSlot(\n              lc,\n              sql,\n              'fork-replica-session',\n              shard,\n              epoch,\n              Date.now().toString(), // replicaID\n              slotFailover,\n              {\n                backupPath: null, // set only after the backup has been confirmed\n                backupV5: true, // RMv2 requires backupV5\n              },\n              () => promiseVoid,\n              ReplicaStage.Restore,\n            );\n            // The replica row of the destination is inserted in a transaction\n            // (in the publication) after the destination slot's position.\n            // Waiting for the source replica's slot to pass it (rather than\n            // just the destination slot's position) ensures that a source\n            // replica that maintains the PG change-log has stored a\n            // transaction past the destination slot's position, i.e. that the\n            // change-log can be resumed from the destination slot.\n            forkLSN ??= await currentWalLSN(sql);\n            if (toBigInt(confirmedFlushLsn) < toBigInt(forkLSN)) {\n              lc.info?.(\n                `waiting for ${replica.id}@${confirmedFlushLsn} to reach ${forkLSN} (after ${destination.slot.slot_name}@${destination.slot.consistent_point})`,\n                {replica},\n              );\n              break;\n            }\n          }\n\n          // If the replica is past the destination LSN, fork it, regardless of\n          // whether its slot is still active. Forking is preferable to resuming,\n          // as resuming removes the replica from being a candidate for\n          // subsequent forks, and carries the risk of stealing the slot from a\n          // task attempting to reconnect.\n          if (\n            destination &&\n            forkLSN &&\n            toBigInt(confirmedFlushLsn) >= toBigInt(forkLSN)\n          ) {\n            lc.info?.(`forking replica ${replica.id}@${replica.slot}`, {\n              replica,\n            });\n            const replicateTo = must(\n              await initRestoreReplica(sql, shard, {\n                sourceID: replica.id,\n                destID: destination.replica.id,\n              }),\n              `replica ${destination.replica.id} was deleted`,\n            );\n            return {\n              restoreFrom: replica,\n              replicateTo,\n              slotLSN: destination.slot.consistent_point,\n            };\n          }\n        }\n\n        if (replica.backupPath && !replica.active) {\n          // An inactive replica that has a backupPath may be orphaned, or it\n          // may be an active task that was temporarily disconnected and\n          // attempting to reestablish a session.\n          //\n          // Resume the replica after a grace period to avoid stealing the slot\n          // from an active task that was temporarily disconnected. Resumption\n          // is a last resort only taken in the absence of alternatives.\n          const now = Date.now();\n          const inactiveMs = now - (inactiveSince.get(replica.id) ?? now);\n          lc.info?.(\n            `replica ${replica.id}@${replica.slot} as been inactive for ${inactiveMs}ms`,\n            {replica},\n          );\n          if (inactiveMs >= gracePeriodMs) {\n            const reserved = await claimSlotForResumption(\n              lc,\n              sql,\n              shard,\n              'resume-replica',\n              replica.slot,\n            );\n            if (!reserved) {\n              lc.warn?.(\n                `unable to resume replica ${replica.id}@${replica.slot}`,\n              );\n            } else {\n              lc.info?.(`resuming replica ${replica.id}@${replica.slot}`);\n              // If a new replication slot was created in anticipation of forking,\n              // cancel it\n              destination?.initialSession.destroy();\n              return {\n                restoreFrom: replica,\n                replicateTo: reserved.replica,\n                // The slot is inactive, so its position is fixed.\n                slotLSN: replica.confirmedFlushLsn,\n              };\n            }\n          }\n        }\n      }\n    }\n  } catch (e) {\n    destination?.initialSession.destroy();\n    throw e;\n  }\n}\n\nasync function checkAndUpdateUpstream(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardConfig,\n  {\n    replicaVersion,\n    publications: subscribed,\n    initialSyncContext,\n  }: SubscriptionStateAndContext,\n  replicaID: string | undefined,\n) {\n  const upstreamReplica = await getReplicaAtVersion(\n    lc,\n    sql,\n    shard,\n    replicaVersion,\n    replicaID,\n    initialSyncContext,\n  );\n  if (!upstreamReplica) {\n    throw new AutoResetSignal(\n      `No replication slot for replica at version ${replicaVersion} and id ${replicaID}}`,\n    );\n  }\n\n  // Verify that the publications match what is being replicated.\n  const requested = shard.publications.toSorted();\n  const replicated = upstreamReplica.publications\n    .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n    .sort();\n  if (!deepEqual(requested, replicated)) {\n    lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n    await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n    throw new AutoResetSignal(\n      `Requested publications [${requested}] do not match configured ` +\n        `publications: [${replicated}]`,\n    );\n  }\n\n  // Sanity check: The subscription state on the replica should have the\n  // same publications. This should be guaranteed by the equivalence of the\n  // replicaVersion, but it doesn't hurt to verify.\n  if (!deepEqual(upstreamReplica.publications, subscribed)) {\n    throw new AutoResetSignal(\n      `Upstream publications [${upstreamReplica.publications}] do not ` +\n        `match subscribed publications [${subscribed}]`,\n    );\n  }\n\n  // Verify that the publications exist.\n  const exists = await sql`\n    SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n  `.values();\n  if (exists.length !== subscribed.length) {\n    throw new AutoResetSignal(\n      `Upstream publications [${exists.flat()}] do not contain ` +\n        `all subscribed publications [${subscribed}]`,\n    );\n  }\n\n  const {slot} = upstreamReplica;\n  const result = await sql<{restartLSN: LSN | null; walStatus: string | null}[]>\n  /*sql*/ `\n    SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n      WHERE slot_name = ${slot}`;\n  if (result.length === 0) {\n    throw new AutoResetSignal(`replication slot ${slot} is missing`);\n  }\n  const [{restartLSN, walStatus}] = result;\n  if (restartLSN === null || walStatus === 'lost') {\n    throw new AutoResetSignal(\n      `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n    );\n  }\n  const [{pgVersion}] = await sql<{pgVersion: number}[]> /*sql*/ `\n    SELECT current_setting('server_version_num')::int as \"pgVersion\"`;\n  return {upstreamReplica, pgVersion};\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA6DA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aACA,SACA,sBAAsB,GACtB,iBAAiC,CAAC,GAClC,EACE,OACA,gBACA,UACA,iCACqB;CACrB,OAAO;CACP,gBAAgB;CAChB,UAAU;CACV,8BAA8B;AAChC,GACA,wBAC2B;CAC3B,MAAM,KAAK,MAAM,gBAAgB,IAAI,aAAa,oBAAoB;CACtE,IAAI;EACF,MAAM,kBACJ,IACA,IACA,OACA,YAAY,2BACd;EAEA,MAAM,EACJ,SAAS,iBACT,WACA,0BACE,iBACA,MAAM,oBACJ,IACA,IACA,OACA,OACA,YAAY,2BAA2B,OACvC,eACA,gBACA,4BACF,IACA,MAAM,wBACJ,IACA,IACA,OACA,eACA,cACF;EAEJ,IAAI;EACJ,MAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,eACA,OAAO,KAAK,OAAO;GAKjB,WAAgB,QAAQ;GACxB,uBAAuB,MAAM,YAC3B,KACA,OACA,IACA,aACA,aACA,SACA;IAAC;IAAO;GAAQ,CAClB;EACF,CACF;EAEA,MAAM,UAAU,IAAI,SAAS,IAAI,aAAa;EAC9C,MAAM,oBAAoB,+BACxB,IAAI,gBAAgB,OAAO,CAC7B;EACA,QAAQ,MAAM;EAId,MAAM,EAAC,iBAAiB,cAAa,MAAM,uBACzC,IACA,IACA,OACA,oBACC,wBAAwB,kBAAkB,EAC7C;EAEA,MAAM,aAAa,uBAEf,qBAAqB,aAGrB,WACE,OAAO,KAAK,IAAI,CAAC,IAChB,iBAAiB,cAAc;EAqBtC,OAAO;GACL;GACA,cAAA,IArBuB,qBACvB,IACA,aACA,OACA,iBACA,WACA;IAAC;IAAY;GAAQ,GACrB,SACA,qBACA,YAAY,UACZ,sBAWA;GACA,sBARA,cAAc,eAAe,YAAY,YACrC,IAAI,IAAI,YAAY,eAAe,WAAW,SAAS,EAAE,SAAS,IAElE,eAAe,YAAY;GAU/B,WAAW,gBAAgB;GAC3B,4BAEE,yBAAyB,KAAA,KAGzB,gBAAgB,iBAAiB,cAAc;GACjD;EACF;CACF,UAAU;EACR,MAAM,GAAG,IAAI;CACf;AACF;AAUA,eAAe,wBACb,IACA,KACA,OACA,aACA,EAAC,YAAY,oBACa;CAM1B,MAAM,OAAO,MAAM,mBAAmB;CACtC,OAAO,SAAS,eAAe,8BAA8B;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,cAAc,aAAa,KAAA;CACjC,MAAM,YAAY,MAAM,kBAAkB,IAAI,KAAK,KAAK,GAAG,QAExD,EAAC,iBACA,gBAAgB,aAAa,kBAAkB,WACnD;CACA,IAAI,SAAS,WAAW,GAAG;EACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;EAC5D,OAAO;GAAC,SAAS,KAAA;GAAW;EAAS;CACvC;CACA,MAAM,CAAC,WAAW;CAElB,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE,EAAC,SAAQ,CACX;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC;EAAS;CAAS;AAC5B;AAMA,eAAe,oBACb,IACA,KACA,OACA,OACA,cACA,aACA,EAAC,YAAY,oBACb,eAC0B;CAC1B,MAAM,SAAS,MAAM,gCACnB,IACA,KACA,OACA,OACA,cACA,aACF;CACA,IAAI,CAAC,QAEH,OAAO;EAAC,SAAS,KAAA;EAAW,WAAW;CAAI;CAE7C,MAAM,EAAC,aAAa,aAAa,YAAW;CAkB5C,MAAM,OAAO,mBACT,MAAM,iBAAiB,qBAAqB,OAAO,CAAC,IACpD;CACJ,MAAM,wBAAwB,SAAS;CACvC,MAAM,YAAY,SAAS,gBAAgB,OAAO;CAClD,MAAM,cAAc,aAAa,KAAA;CAEjC,IAAI,YAAY,WAAW;EACzB,MAAM,EAAC,WAAW,kBAAiB;EACnC,MAAM,EAAC,MAAM,YAAY,sBAAqB;EAC9C,MAAM,YAAY,IAAI,IAAI,cAAc,IAAI,aAAa,EAAE,SAAS;EACpE,GAAG,OACD,0BAA0B,UAAU,IAAI,KAAK,GAAG,kBAAkB,IAClE;GAAC;GAAa;EAAW,CAC3B;EACA,MAAM,eACJ,IACA;GAAC,GAAG;GAAY;EAAS,GACzB,aACA,WACF;CACF;CACA,OAAO;EAAC,SAAS;EAAa;EAAW;CAAqB;AAChE;AAEA,eAAe,cAAc,KAA+B;CAC1D,MAAM,CAAC,EAAC,SAAQ,MAAM,GAAiB;CACvC,OAAO;AACT;AAEA,IAAM,2BAA2B;AACjC,IAAM,2CAA2C;AAGjD,eAAsB,gCACpB,IACA,KACA,OACA,OACA,cACA,eACA,iBAAiB,0BASjB;CACA,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,IAAI;CAEJ,IAAI;CAEJ,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,IAAI,GACN,MAAM,MAAM,cAAc;GAE5B,MAAM,WAAW,MAAM,qBAAqB,IAAI,KAAK,OAAO,KAAK;GACjE,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,wCAAwC,EAAC,SAAQ,CAAC;IAC5D,aAAa,eAAe,QAAQ;IACpC;GACF;GACA,KAAK,MAAM,WAAW,UAAU;IAE9B,IAAI,QAAQ,QACV,cAAc,OAAO,QAAQ,EAAE;SAC1B,IAAI,CAAC,cAAc,IAAI,QAAQ,EAAE,GACtC,cAAc,IAAI,QAAQ,IAAI,KAAK,IAAI,CAAC;IAK1C,IAAI,QAAQ,UAAA,GAAuB;KAEjC,IAAI,IAAI,OAAO,GACb,GAAG,OAAO,+BAA+B,QAAQ,MAAM,EAAC,QAAO,CAAC;KAElE;IACF;IAEA,IAAI,QAAQ,UAAA,GAAqB;KAC/B,MAAM,EAAC,QAAQ,sBAAqB;KACpC,IAAI,QAAQ;MAEV,gBAAgB,MAAM,qBACpB,IACA,KACA,wBACA,OACA,OACA,KAAK,IAAI,EAAE,SAAS,GACpB,cACA;OACE,YAAY;OACZ,UAAU;MACZ,SACM,aACN,CACF;MAQA,YAAY,MAAM,cAAc,GAAG;MACnC,IAAI,SAAS,iBAAiB,IAAI,SAAS,OAAO,GAAG;OACnD,GAAG,OACD,eAAe,QAAQ,GAAG,GAAG,kBAAkB,YAAY,QAAQ,UAAU,YAAY,KAAK,UAAU,GAAG,YAAY,KAAK,iBAAiB,IAC7I,EAAC,QAAO,CACV;OACA;MACF;KACF;KAOA,IACE,eACA,WACA,SAAS,iBAAiB,KAAK,SAAS,OAAO,GAC/C;MACA,GAAG,OAAO,mBAAmB,QAAQ,GAAG,GAAG,QAAQ,QAAQ,EACzD,QACF,CAAC;MAQD,OAAO;OACL,aAAa;OACb,aATkB,KAClB,MAAM,mBAAmB,KAAK,OAAO;QACnC,UAAU,QAAQ;QAClB,QAAQ,YAAY,QAAQ;OAC9B,CAAC,GACD,WAAW,YAAY,QAAQ,GAAG,aAIlC;OACA,SAAS,YAAY,KAAK;MAC5B;KACF;IACF;IAEA,IAAI,QAAQ,cAAc,CAAC,QAAQ,QAAQ;KAQzC,MAAM,MAAM,KAAK,IAAI;KACrB,MAAM,aAAa,OAAO,cAAc,IAAI,QAAQ,EAAE,KAAK;KAC3D,GAAG,OACD,WAAW,QAAQ,GAAG,GAAG,QAAQ,KAAK,wBAAwB,WAAW,KACzE,EAAC,QAAO,CACV;KACA,IAAI,cAAc,eAAe;MAC/B,MAAM,WAAW,MAAM,uBACrB,IACA,KACA,OACA,kBACA,QAAQ,IACV;MACA,IAAI,CAAC,UACH,GAAG,OACD,4BAA4B,QAAQ,GAAG,GAAG,QAAQ,MACpD;WACK;OACL,GAAG,OAAO,oBAAoB,QAAQ,GAAG,GAAG,QAAQ,MAAM;OAG1D,aAAa,eAAe,QAAQ;OACpC,OAAO;QACL,aAAa;QACb,aAAa,SAAS;QAEtB,SAAS,QAAQ;OACnB;MACF;KACF;IACF;GACF;EACF;CACF,SAAS,GAAG;EACV,aAAa,eAAe,QAAQ;EACpC,MAAM;CACR;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EACE,gBACA,cAAc,YACd,sBAEF,WACA;CACA,MAAM,kBAAkB,MAAM,oBAC5B,IACA,KACA,OACA,gBACA,WACA,kBACF;CACA,IAAI,CAAC,iBACH,MAAM,IAAI,gBACR,8CAA8C,eAAe,UAAU,UAAU,EACnF;CAIF,MAAM,YAAY,MAAM,aAAa,SAAS;CAC9C,MAAM,aAAa,gBAAgB,aAChC,QAAO,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAK;CACR,IAAI,CAAC,UAAU,WAAW,UAAU,GAAG;EACrC,GAAG,OAAO,8CAA8C,UAAU,EAAE;EACpE,MAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;EACvD,MAAM,IAAI,gBACR,2BAA2B,UAAU,2CACjB,WAAW,EACjC;CACF;CAKA,IAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GACrD,MAAM,IAAI,gBACR,0BAA0B,gBAAgB,aAAa,0CACnB,WAAW,EACjD;CAIF,MAAM,SAAS,MAAM,GAAG;0DACgC,IAAI,UAAU,EAAE;IACtE,OAAO;CACT,IAAI,OAAO,WAAW,WAAW,QAC/B,MAAM,IAAI,gBACR,0BAA0B,OAAO,KAAK,EAAE,gDACN,WAAW,EAC/C;CAGF,MAAM,EAAC,SAAQ;CACf,MAAM,SAAS,MAAM,GACb;;0BAEgB;CACxB,IAAI,OAAO,WAAW,GACpB,MAAM,IAAI,gBAAgB,oBAAoB,KAAK,YAAY;CAEjE,MAAM,CAAC,EAAC,YAAY,eAAc;CAClC,IAAI,eAAe,QAAQ,cAAc,QACvC,MAAM,IAAI,gBACR,oBAAoB,KAAK,+DAC3B;CAEF,MAAM,CAAC,EAAC,eAAc,MAAM,GAAmC;;CAE/D,OAAO;EAAC;EAAiB;CAAS;AACpC"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CA0B1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAsCF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAMzD,OAAO,KAAK,EAEV,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAA4B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAChF,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAKtE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,EAAC,YAAY,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EAExB,MAAM,wBAAwB,CAAC;AAQhC,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAE3C,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAC,KAAK,aAAa,EAAC,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAEV,cAAc,EACd,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAC,SAAS,EAAqB,MAAM,iCAAiC,CAAC;AAK9E,OAAO,EAIL,KAAK,iCAAiC,EACvC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,OAAO,EACb,MAAM,mBAAmB,CAAC;AAU3B;;;GAGG;AACH,qBAAa,oBAAqB,YAAW,YAAY;;IAgBvD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE,aAAa,EACtB,mBAAmB,EAAE,MAAM,EAC3B,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,EAC9B,sBAAsB,CAAC,EAAE,MAAM,GAAG,SAAS,EAG3C,IAAI,GAAE;QACJ,SAAS,CAAC,EAAE,OAAO,SAAS,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;KACnC,EAwCP;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAI1B;IAEK,gBAAgB;;;cAyBrB;IAEK,WAAW,CACf,eAAe,EAAE,MAAM,EACvB,gBAAgB,GAAE,eAAe,EAAO,GACvC,OAAO,CAAC,YAAY,CAAC,CAqBvB;IAGK,mBAAmB,CACvB,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,WAAW,EAAE,mBAAmB,EAChC,gBAAgB,EAAE,eAAe,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAgKvB;CAiFF;AAGD,qBAAa,KAAM,YAAW,QAAQ;;IAIpC;;;;;;;OAOG;IACH,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI,EAG7D;IAED,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAyB1C;IAMD,GAAG,CAAC,SAAS,EAAE,WAAW,QAQzB;CAYF;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAIxD,qBAAa,WAAW;;IACtB,MAAM,CAAC,QAAQ,CAAC,cAAc,oBAAoB;IAGlD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAoB/B,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAE,EAAE,UAAU,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAED,IAAI,SAAS,WAEZ;IAEK,iBAAiB,CAAC,GAAG,UAAQ;;;OAuElC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAYhE;IAED,IAAI,SAGH;IAwCD,uBAAuB,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAUpE;CAsCF;AA2rBD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E;AAsBD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,kBAAkB,EACxB,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,EAC1D,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,iCAAiC,CAAC,GACzD,OAAO,CAwET"}