@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/restore-admission.js

@@ -0,0 +1,317 @@
+import { getOrCreateGauge } from "../../observability/metrics.js";
+/**
+* The default time for which a replacement continues to be counted after its
+* subscriber has caught up, which covers the time for the subscriber to start
+* serving (i.e. become ready) after catching up. See
+* {@link RestoreAdmissionOptions.readinessGracePeriodMs}.
+*/
+var DEFAULT_READINESS_GRACE_PERIOD_MS = 5e3;
+/**
+* Limits the number of view-syncers that replace their replica at the same
+* time, so that view-syncers switching over from an incompatible
+* replication-manager (e.g. after a resync) do not all stop serving at once.
+*
+* Capacity is consumed by:
+* * Replacement restores: snapshot reservations of subscribers that are
+*   replacing an (incompatible or outdated) local replica, i.e. subscribers
+*   that were serving. Restores onto an empty disk (e.g. a new pod) do not
+*   take a serving subscriber out of service, and are not counted. The
+*   reservation's connection is held open for the duration of the restore,
+*   and the subscription that follows it is held open until the restored
+*   subscriber has caught up, so their liveness tracks that of the
+*   replacement without a timeout.
+* * Outstanding grants: permission to restore that has not yet been
+*   followed by a replacement reservation (e.g. while the subscriber drains
+*   and restarts). Grants are not matched to reservations by identity:
+*   any replacement reservation consumes an outstanding grant, and a grant
+*   that is not consumed expires a short time after the granted connection
+*   closes (e.g. if the subscriber reconnected to a different
+*   replication-manager or was deleted).
+*
+* ## Life cycle of a switchover restore
+*
+* The following traces a view-syncer (VS) with a replica at version `R1`,
+* served by an old replication-manager (RM) at `R1`, while a new RM (e.g.
+* after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new
+* RM. Terms in `code` are those used in the code.
+*
+* 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends
+*    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.
+*    It continues to stream changes to them, and exits once they have all
+*    disconnected.
+*
+* 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`
+*    with `switchover: true` at the watermark of its last committed
+*    transaction, which is routed to some RM. The VS continues to consume
+*    the old RM's stream. If the RM is compatible (i.e. another old RM),
+*    the subscription is accepted and the VS simply moves to it. If the RM
+*    is the new one, the subscriber's replica version is older, and the
+*    subscription proceeds to step 3.
+*
+* 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription
+*    (it is not added to the forwarder) and calls {@link request}. The
+*    request is `queued` and `onQueued(position)` sends
+*    `status {restore: {state: 'queued', position}}`, again whenever the
+*    position changes. If the held connection closes while queued (e.g.
+*    the VS's old stream ended and it gave up on the switchover), the
+*    ticket is `release()`d and the request leaves the queue.
+*
+* 4. **Granted.** When the number in flight (`replacing` + `granted`) is
+*    below `maxConcurrentRestores`, the head of the queue is `granted`: a
+*    {@link Grant} is added (counting toward capacity), and `onGranted()`
+*    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted
+*    connection open, and initiates a self-drain (`drainRequested`): it
+*    fails its health checks, drains its clients, and keeps replicating
+*    from the old RM in the meantime.
+*
+* 5. **Restart.** The VS process exits, closing the granted connection.
+*    The ticket is `release()`d, marking the grant as `released` and
+*    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),
+*    within which a replacement reservation is expected. The drain itself
+*    does not count against the TTL.
+*
+* 6. **Reservation opened.** The restarted VS sends
+*    `reserve-snapshot {taskID, localReplica}`, and
+*    `ChangeStreamerImpl.startSnapshotReservation()` calls
+*    {@link reservationOpened}:
+*    * A `localReplica` of a different replica version (e.g. `R1`) is
+*      `replacing`: the reservation is counted, and consumes an outstanding
+*      grant (preferring a `released` one), so the number in flight is
+*      unchanged.
+*    * A `localReplica` of the same replica version is `unclassified` until
+*      the reservation is confirmed ({@link reservationConfirmed}) with the
+*      backup's minimum watermark. It is `replacing` if its watermark is
+*      older (i.e. it cannot catch up from the change log).
+*    * No `localReplica` (an empty disk) is not counted.
+*
+*    If the VS reached a different RM instead, the grant here expires after
+*    the TTL and frees its capacity, while the replacement is counted (as
+*    an ungranted one) by the RM that it reached.
+*
+* 7. **Restoring.** The VS restores the backup while holding the
+*    reservation's connection open. A retried reservation from the same
+*    task that arrives while the previous one is still open replaces it
+*    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it
+*    remains `replacing`. If the reservation instead closes without
+*    subscribing (e.g. a failed or disconnected restore, after which the VS
+*    retries), {@link reservationClosed} converts its capacity into a
+*    `released` grant, which the retry's replacement reservation consumes
+*    (step 6), or which expires after the pending grant TTL (e.g. if the VS
+*    was deleted).
+*
+* 8. **Subscribed.** Once restored, the VS subscribes on the same
+*    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`
+*    calls {@link reservationSubscribed}, which moves the replacement from
+*    its reservation (the key leaves `replacing`) to the subscription
+*    (`catchingUp`), and then closes the task's reservation. Its `onEnd`
+*    calls {@link reservationClosed}, which is a no-op for the moved key.
+*
+* 9. **Caught up.** The VS catches up on the change stream from the
+*    backup's watermark. Once it has acked all changes up to the head of
+*    the stream (`Subscriber.caughtUp`), the replacement is released after
+*    the readiness grace period, which covers the time for the VS to start
+*    serving, and the next queued request is granted. If the VS disconnects
+*    first, the replacement is released immediately.
+*
+* A VS that restarts outside of a switchover with an incompatible replica
+* (e.g. after a crash) enters at step 6 without a grant, and its restore
+* consumes capacity all the same, delaying queued requests.
+*
+* ## States
+*
+* | Term          | Where                    | Meaning                          |
+* | ------------- | ------------------------ | -------------------------------- |
+* | queued        | `Entry.state`            | Held, waiting for capacity       |
+* | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |
+* |               |                          | replacement reservation          |
+* | released      | `Entry.state`            | The held connection closed       |
+* | released      | `Grant.released`         | The granted connection, or an    |
+* |               |                          | unsubscribed replacement         |
+* |               |                          | reservation, closed; the TTL is  |
+* |               |                          | running                          |
+* | unclassified  | `#unclassified`          | Same-version reservation, not    |
+* |               |                          | yet confirmed                    |
+* | replacing     | `#replacing`             | Open replacement reservation     |
+* | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |
+* |               |                          | caught up (or within the         |
+* |               |                          | readiness grace period)          |
+* | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |
+* |               |                          | `granted`, bounded by            |
+* |               |                          | `maxConcurrentRestores`          |
+*/
+var RestoreAdmission = class {
+	#lc;
+	#maxConcurrentRestores;
+	#pendingGrantTTLMs;
+	#readinessGracePeriodMs;
+	#setTimeout;
+	#clearTimeout;
+	#replicaVersion;
+	/** Keys (task IDs) of open replacement reservations. */
+	#replacing = /* @__PURE__ */ new Set();
+	/**
+	* Replacements whose subscribers have subscribed, but have not caught up
+	* (or are within the readiness grace period).
+	*/
+	#catchingUp = /* @__PURE__ */ new Set();
+	/**
+	* The local replicas of open reservations with this replica version, which
+	* are classified upon confirmation of the reservation's minimum watermark.
+	*/
+	#unclassified = /* @__PURE__ */ new Map();
+	#grants = /* @__PURE__ */ new Set();
+	#queue = [];
+	constructor(lc, opts) {
+		this.#lc = lc.withContext("component", "restore-admission");
+		this.#replicaVersion = opts.replicaVersion;
+		this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);
+		this.#pendingGrantTTLMs = opts.pendingGrantTTLMs ?? 9e4;
+		this.#readinessGracePeriodMs = opts.readinessGracePeriodMs ?? 5e3;
+		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
+		getOrCreateGauge("replication", "restore_admission.queued", "Switchover subscribers waiting for permission to restore a new replica.").addCallback((result) => result.observe(this.#queue.length));
+		getOrCreateGauge("replication", "restore_admission.in_flight", "Subscribers replacing, or granted permission to replace, their replica.").addCallback((result) => result.observe(this.#inFlight));
+	}
+	get #inFlight() {
+		return this.#replacing.size + this.#catchingUp.size + this.#grants.size;
+	}
+	/** For testing and logging. */
+	get stats() {
+		return {
+			replacing: this.#replacing.size + this.#catchingUp.size,
+			granted: this.#grants.size,
+			queued: this.#queue.length
+		};
+	}
+	/**
+	* Called when a snapshot reservation is opened by a subscriber with the
+	* given `localReplica` (if any). A subscriber with a local replica of a
+	* different replica version is replacing it. One with the same replica
+	* version is replacing it if it is outdated, which is determined by
+	* {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a
+	* retried reservation).
+	*/
+	reservationOpened(key, localReplica) {
+		this.#unclassified.delete(key);
+		if (!localReplica) return;
+		if (localReplica.replicaVersion !== this.#replicaVersion) this.#replacementOpened(key);
+		else this.#unclassified.set(key, localReplica);
+	}
+	/**
+	* Called when a reservation is confirmed with the minimum watermark of the
+	* replica to restore.
+	*/
+	reservationConfirmed(key, minWatermark) {
+		const localReplica = this.#unclassified.get(key);
+		this.#unclassified.delete(key);
+		if (localReplica && localReplica.watermark < minWatermark) this.#replacementOpened(key);
+	}
+	/**
+	* Called when a reservation ends. A replacement reservation normally ends
+	* after its task subscribes ({@link reservationSubscribed}), in which case
+	* this is a no-op. A replacement reservation that ends without subscribing
+	* (e.g. a failed or disconnected restore) is most likely followed by a
+	* retry, so its capacity is retained as a released grant (which the retry
+	* consumes) rather than released immediately.
+	*/
+	reservationClosed(key) {
+		this.#unclassified.delete(key);
+		if (this.#replacing.delete(key)) {
+			const grant = {
+				released: true,
+				timer: void 0
+			};
+			this.#grants.add(grant);
+			this.#startGrantTTL(grant);
+		}
+	}
+	/**
+	* Called when the task of a reservation subscribes (before the reservation
+	* is closed). If the reservation is a replacement, it continues to be
+	* counted until the readiness grace period after the subscriber has caught
+	* up, or until the subscription closes.
+	*/
+	reservationSubscribed(key) {
+		this.#unclassified.delete(key);
+		if (!this.#replacing.delete(key)) return;
+		const replacement = {};
+		this.#catchingUp.add(replacement);
+		let timer;
+		const release = () => {
+			if (timer) this.#clearTimeout(timer);
+			if (this.#catchingUp.delete(replacement)) this.#processQueue();
+		};
+		return {
+			caughtUp: () => {
+				if (this.#catchingUp.has(replacement) && !timer) timer = this.#setTimeout(release, this.#readinessGracePeriodMs);
+			},
+			closed: release
+		};
+	}
+	#replacementOpened(key) {
+		if (this.#replacing.has(key)) return;
+		this.#replacing.add(key);
+		const grants = [...this.#grants];
+		const grant = grants.find((g) => g.released) ?? grants[0];
+		if (grant) this.#deleteGrant(grant);
+	}
+	request(request) {
+		const entry = {
+			request,
+			state: "queued",
+			position: 0
+		};
+		this.#queue.push(entry);
+		this.#processQueue();
+		return { release: () => this.#release(entry) };
+	}
+	#release(entry) {
+		const { state, grant } = entry;
+		entry.state = "released";
+		if (state === "queued") {
+			const i = this.#queue.indexOf(entry);
+			if (i >= 0) this.#queue.splice(i, 1);
+			this.#processQueue();
+		} else if (state === "granted" && grant && this.#grants.has(grant)) {
+			grant.released = true;
+			this.#startGrantTTL(grant);
+		}
+	}
+	#startGrantTTL(grant) {
+		grant.timer = this.#setTimeout(() => {
+			if (this.#grants.has(grant)) {
+				this.#lc.info?.(`grant expired without a replacement reservation`);
+				this.#deleteGrant(grant);
+				this.#processQueue();
+			}
+		}, this.#pendingGrantTTLMs);
+	}
+	#deleteGrant(grant) {
+		if (grant.timer) this.#clearTimeout(grant.timer);
+		this.#grants.delete(grant);
+	}
+	#processQueue() {
+		while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {
+			const entry = this.#queue.shift();
+			entry.state = "granted";
+			entry.grant = {
+				released: false,
+				timer: void 0
+			};
+			this.#grants.add(entry.grant);
+			this.#lc.info?.(`granting permission to restore`, { ...this.stats });
+			entry.request.onGranted();
+		}
+		this.#queue.forEach((entry, i) => {
+			const position = i + 1;
+			if (entry.position !== position) {
+				entry.position = position;
+				entry.request.onQueued(position);
+			}
+		});
+	}
+};
+//#endregion
+export { DEFAULT_READINESS_GRACE_PERIOD_MS, RestoreAdmission };
+
+//# sourceMappingURL=restore-admission.js.map

package/out/zero-cache/src/services/change-streamer/restore-admission.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"restore-admission.js","names":["#lc","#maxConcurrentRestores","#pendingGrantTTLMs","#readinessGracePeriodMs","#setTimeout","#clearTimeout","#replicaVersion","#replacing","#catchingUp","#unclassified","#grants","#queue","#inFlight","#replacementOpened","#startGrantTTL","#processQueue","#deleteGrant","#release"],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nexport const DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\n/**\n * The default time for which a replacement continues to be counted after its\n * subscriber has caught up, which covers the time for the subscriber to start\n * serving (i.e. become ready) after catching up. See\n * {@link RestoreAdmissionOptions.readinessGracePeriodMs}.\n */\nexport const DEFAULT_READINESS_GRACE_PERIOD_MS = 5_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  /**\n   * The time for which a replacement continues to be counted after its\n   * subscriber has caught up, so that the next subscriber is not granted\n   * permission to restore (and stop serving) before the restored one has\n   * started serving. Catching up is followed by the subscriber's readiness\n   * checks (e.g. waiting for the next replication lag report), and by the\n   * container runner's detection of its readiness.\n   */\n  readinessGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\n/** A subscribed replacement, returned by {@link RestoreAdmission.reservationSubscribed}. */\nexport type Replacement = {\n  /**\n   * Called when the subscriber has caught up. The replacement is released\n   * after the readiness grace period.\n   */\n  caughtUp: () => void;\n\n  /** Called when the subscription closes. The replacement is released. */\n  closed: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), the replacement is released after\n *    the readiness grace period, which covers the time for the VS to start\n *    serving, and the next queued request is granted. If the VS disconnects\n *    first, the replacement is released immediately.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up (or within the         |\n * |               |                          | readiness grace period)          |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #readinessGracePeriodMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /**\n   * Replacements whose subscribers have subscribed, but have not caught up\n   * (or are within the readiness grace period).\n   */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#readinessGracePeriodMs =\n      opts.readinessGracePeriodMs ?? DEFAULT_READINESS_GRACE_PERIOD_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the readiness grace period after the subscriber has caught\n   * up, or until the subscription closes.\n   */\n  reservationSubscribed(key: string): Replacement | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const release = () => {\n      if (timer) {\n        this.#clearTimeout(timer);\n      }\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n    return {\n      caughtUp: () => {\n        if (this.#catchingUp.has(replacement) && !timer) {\n          timer = this.#setTimeout(release, this.#readinessGracePeriodMs);\n        }\n      },\n      closed: release,\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAqBA,IAAa,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4MjD,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;;;;CAKtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKM,kBAAkB,KAAK;EAC5B,KAAKL,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAKC,qBACH,KAAK,qBAAA;EACP,KAAKC,0BACH,KAAK,0BAAA;EACP,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKM,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,SAAS,CAAC;CACxD;CAEA,IAAIA,YAAY;EACd,OAAO,KAAKL,WAAW,OAAO,KAAKC,YAAY,OAAO,KAAKE,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAKH,WAAW,OAAO,KAAKC,YAAY;GACnD,SAAS,KAAKE,QAAQ;GACtB,QAAQ,KAAKC,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAKF,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAKH,iBACvC,KAAKO,mBAAmB,GAAG;OAE3B,KAAKJ,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAKA,cAAc,IAAI,GAAG;EAC/C,KAAKA,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAKI,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAKJ,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAKF,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAKG,QAAQ,IAAI,KAAK;GACtB,KAAKI,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAsC;EAC1D,KAAKL,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAKF,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAKC,YAAY,IAAI,WAAW;EAChC,IAAI;EACJ,MAAM,gBAAgB;GACpB,IAAI,OACF,KAAKH,cAAc,KAAK;GAE1B,IAAI,KAAKG,YAAY,OAAO,WAAW,GACrC,KAAKO,cAAc;EAEvB;EACA,OAAO;GACL,gBAAgB;IACd,IAAI,KAAKP,YAAY,IAAI,WAAW,KAAK,CAAC,OACxC,QAAQ,KAAKJ,YAAY,SAAS,KAAKD,uBAAuB;GAElE;GACA,QAAQ;EACV;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAKI,WAAW,IAAI,GAAG,GACzB;EAEF,KAAKA,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAKG,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAKM,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAKL,OAAO,KAAK,KAAK;EACtB,KAAKI,cAAc;EACnB,OAAO,EAAC,eAAe,KAAKE,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAKN,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAKA,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAKI,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAKL,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAKI,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAKV,kBAAkB;GACnC,IAAI,KAAKM,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAKV,IAAI,OAAO,iDAAiD;IACjE,KAAKgB,aAAa,KAAK;IACvB,KAAKD,cAAc;GACrB;EACF,GAAG,KAAKb,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAKG,cAAc,MAAM,KAAK;EAEhC,KAAKK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAKE,YAAY,KAAKX,0BAA0B,KAAKU,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAKA,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAKD,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAKV,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAKW,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts

@@ -48,7 +48,64 @@ interface PurgeLock {
 }
 export declare function setupCDCTables(lc: LogContext, db: postgres.TransactionSql, shard: ShardID): Promise<void>;
 export declare function markResetRequired(sql: PostgresDB, shard: ShardID): Promise<void>;
-export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<void>;
+export declare function ensureReplicationConfig(lc: LogContext, db: PostgresDB, subscriptionState: Pick<SubscriptionState, 'publications' | 'replicaVersion' | 'watermark'>, shard: ShardID, autoReset: boolean, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout, pgChangeLogEnabled?: boolean): Promise<void>;
+export type ChangeLogReinitialization = {
+    /**
+     * The watermark of the replica from which the change log is re-initialized,
+     * i.e. the watermark from which the change stream will resume.
+     */
+    watermark: string;
+    cookies: {
+        tableMetadata: readonly {
+            schema: string;
+            table: string;
+            metadata: TableMetadata;
+        }[];
+        backfilling: readonly {
+            schema: string;
+            table: string;
+            column: string;
+            backfill: BackfillID;
+        }[];
+    };
+    owner: string;
+    ownerAddress: string;
+};
+/**
+ * Re-initializes the change log from the state of the replica if it does not
+ * contain all of the replica's changes, i.e. if its `lastWatermark` is before
+ * the replica's watermark. This is the case if the change log is no longer
+ * maintained (e.g. after running in high-availability mode), or if its last
+ * owner stopped before persisting changes that it had already forwarded
+ * (and backed up). Ownership of the change log is taken in the same
+ * transaction, in which:
+ *
+ * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`
+ *   tables are truncated. (The `replicationConfig` is kept;
+ *   {@link ensureReplicationConfig} handles changes of the replica version
+ *   or publications.)
+ * * The change log is seeded with an empty transaction at the replica's
+ *   watermark, so that subscribers at that watermark can be caught up, while
+ *   subscribers at earlier watermarks are rejected as being too old.
+ * * The `tableMetadata` and `backfilling` cookies are set from the replica.
+ *
+ * Returns `'maintained'` without modifying anything if the change log's
+ * `lastWatermark` is at or past the replica's watermark, i.e. the change log
+ * contains all of the replica's changes. Because a replication slot is only
+ * acked past a transaction once it has been backed up, this means that the
+ * change log contains every transaction up to the position of the replica's
+ * replication slot, and it can be taken over (and resumed from its head).
+ *
+ * This is called when the change log's head is behind the position of the
+ * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be
+ * compared with the watermarks of transactions to determine whether the
+ * change log is maintained, as it is also advanced past positions of changes
+ * outside of the publication.
+ *
+ * @param purgeLock A purge lock held by the caller, if any, which is released
+ *     before the tables are truncated.
+ */
+export declare function reinitializeChangeLog(lc: LogContext, db: PostgresDB, shard: ShardID, { watermark, cookies, owner, ownerAddress }: ChangeLogReinitialization, purgeLock?: PurgeLock, setTimeoutFn?: typeof setTimeout): Promise<'reinitialized' | 'maintained'>;
 export declare const CHANGE_STREAMER_APP_NAME = "zero-change-streamer";
 export declare class AutoResetSignal extends AbortError {
     readonly name = "AutoResetSignal";

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,iBA4H7C;AAMD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}
+{"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGpE,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EACV,UAAU,EACV,MAAM,EACN,aAAa,EACd,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAaF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,UAiBlD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAYF,UAAU,SAAS;IACjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,IAAI,CACrB,iBAAiB,EACjB,cAAc,GAAG,gBAAgB,GAAG,WAAW,CAChD,EACD,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,EAClB,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,EAC5C,kBAAkB,UAAO,iBAyH1B;AA6CD,MAAM,MAAM,yBAAyB,GAAG;IACtC;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,aAAa,EAAE,SAAS;YACtB,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,aAAa,CAAC;SACzB,EAAE,CAAC;QACJ,WAAW,EAAE,SAAS;YACpB,MAAM,EAAE,MAAM,CAAC;YACf,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;YACf,QAAQ,EAAE,UAAU,CAAC;SACtB,EAAE,CAAC;KACL,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,EAAC,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAC,EAAE,yBAAyB,EACpE,SAAS,CAAC,EAAE,SAAS,EACrB,YAAY,GAAE,OAAO,UAAuB,GAC3C,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAuDzC;AAED,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAE/D,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,iBA4Cf"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -77,7 +77,7 @@ async function markResetRequired(sql, shard) {
   UPDATE ${sql(cdcSchema(shard))}."replicationConfig"
     SET "resetRequired" = true`;
 }
-async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout) {
+async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout, pgChangeLogEnabled = true) {
 	const { publications, replicaVersion, watermark } = subscriptionState;
 	const replicaConfig = {
 		publications,
@@ -102,12 +102,12 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				lc.info?.(`Data in cdc tables @${replicaVersion} is incompatible with replica @${replicaConfig.replicaVersion}. Clearing tables.`);
 				purgeLock?.release();
 				needsTruncate = true;
-				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`, sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
+				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, ...pgChangeLogEnabled ? [sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`] : [], sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
 			}
 		}
 		if (results.length === 0 || needsTruncate) {
 			const watermark = replicaConfig.replicaVersion;
-			const initialTx = [{
+			const initialTx = pgChangeLogEnabled ? [{
 				watermark,
 				pos: 0,
 				change: { tag: "begin" }
@@ -115,20 +115,10 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 				watermark,
 				pos: 1,
 				change: { tag: "commit" }
-			}];
+			}] : [];
 			stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
               ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
-			if (needsTruncate) {
-				const timer = setTimeoutFn(async () => {
-					lc.info?.("ensureReplicationConfig blocked, terminating lock holders");
-					await terminateChangeDBLockHolders(lc, db, shard);
-				}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
-				try {
-					return await Promise.all(stmts);
-				} finally {
-					clearTimeout(timer);
-				}
-			}
+			if (needsTruncate) return runStatementsAndTerminateLockHolders(lc, db, shard, "ensureReplicationConfig", stmts, setTimeoutFn);
 			return Promise.all(stmts);
 		}
 		const { resetRequired } = results[0];
@@ -140,6 +130,107 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 	});
 }
 var LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5e3;
+/**
+* Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),
+* which may be blocked by old storer catchup reads. Races against a
+* timeout that terminates the blocking backends if the `stmts` take too
+* long. The check is repeated until the statements complete, since the
+* `stmts` may not yet be waiting on a lock when the timer fires.
+*/
+async function runStatementsAndTerminateLockHolders(lc, db, shard, name, stmts, setTimeoutFn) {
+	let done = false;
+	let timer;
+	const scheduleTerminate = () => {
+		timer = setTimeoutFn(async () => {
+			lc.info?.(`${name} blocked, terminating lock holders`);
+			try {
+				await terminateChangeDBLockHolders(lc, db, shard);
+			} catch (e) {
+				lc.warn?.("error terminating lock holders", e);
+			}
+			if (!done) scheduleTerminate();
+		}, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);
+	};
+	scheduleTerminate();
+	try {
+		return await Promise.all(stmts);
+	} finally {
+		done = true;
+		clearTimeout(timer);
+	}
+}
+/**
+* Re-initializes the change log from the state of the replica if it does not
+* contain all of the replica's changes, i.e. if its `lastWatermark` is before
+* the replica's watermark. This is the case if the change log is no longer
+* maintained (e.g. after running in high-availability mode), or if its last
+* owner stopped before persisting changes that it had already forwarded
+* (and backed up). Ownership of the change log is taken in the same
+* transaction, in which:
+*
+* * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`
+*   tables are truncated. (The `replicationConfig` is kept;
+*   {@link ensureReplicationConfig} handles changes of the replica version
+*   or publications.)
+* * The change log is seeded with an empty transaction at the replica's
+*   watermark, so that subscribers at that watermark can be caught up, while
+*   subscribers at earlier watermarks are rejected as being too old.
+* * The `tableMetadata` and `backfilling` cookies are set from the replica.
+*
+* Returns `'maintained'` without modifying anything if the change log's
+* `lastWatermark` is at or past the replica's watermark, i.e. the change log
+* contains all of the replica's changes. Because a replication slot is only
+* acked past a transaction once it has been backed up, this means that the
+* change log contains every transaction up to the position of the replica's
+* replication slot, and it can be taken over (and resumed from its head).
+*
+* This is called when the change log's head is behind the position of the
+* replication slot (see `PgChangeLogPurgeLocker`). That position cannot be
+* compared with the watermarks of transactions to determine whether the
+* change log is maintained, as it is also advanced past positions of changes
+* outside of the publication.
+*
+* @param purgeLock A purge lock held by the caller, if any, which is released
+*     before the tables are truncated.
+*/
+function reinitializeChangeLog(lc, db, shard, { watermark, cookies, owner, ownerAddress }, purgeLock, setTimeoutFn = setTimeout) {
+	const schema = cdcSchema(shard);
+	return runTx(db, async (sql) => {
+		const lastWatermark = (await sql`
+      SELECT "lastWatermark" FROM ${sql(schema)}."replicationState" FOR UPDATE`)[0]?.lastWatermark;
+		if (lastWatermark !== void 0 && lastWatermark >= watermark) {
+			lc.info?.(`changeLog@${lastWatermark} contains the replica's changes (@${watermark})`);
+			return "maintained";
+		}
+		lc.info?.(`re-initializing changeLog@${lastWatermark} from the replica @${watermark}`);
+		const replicationState = {
+			lastWatermark: watermark,
+			owner,
+			ownerAddress
+		};
+		const initialTx = [{
+			watermark,
+			pos: 0,
+			change: { tag: "begin" }
+		}, {
+			watermark,
+			pos: 1,
+			change: { tag: "commit" }
+		}];
+		await purgeLock?.release();
+		await runStatementsAndTerminateLockHolders(lc, db, shard, "reinitializeChangeLog", [
+			sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`,
+			sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`,
+			sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`,
+			sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`
+		], setTimeoutFn);
+		await sql`INSERT INTO ${sql(schema)}."replicationState" ${sql(replicationState)}`;
+		for (const entry of initialTx) await sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(entry)}`;
+		for (const row of cookies.tableMetadata) await sql`INSERT INTO ${sql(schema)}."tableMetadata" ${sql({ ...row })}`;
+		for (const row of cookies.backfilling) await sql`INSERT INTO ${sql(schema)}."backfilling" ${sql({ ...row })}`;
+		return "reinitialized";
+	});
+}
 var CHANGE_STREAMER_APP_NAME = "zero-change-streamer";
 var AutoResetSignal = class extends AbortError {
 	name = "AutoResetSignal";
@@ -187,6 +278,6 @@ async function terminateChangeDBLockHolders(lc, db, shard) {
 	else for (const { pid, applicationName, query, terminated: ok } of terminated) lc.info?.(`terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`);
 }
 //#endregion
-export { AutoResetSignal, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, setupCDCTables };
+export { AutoResetSignal, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, reinitializeChangeLog, setupCDCTables };
 
 //# sourceMappingURL=tables.js.map