@rocicorp/zero 1.11.0-canary.36 → 1.11.0-canary.37

package/out/zero-cache/src/services/change-streamer/restore-admission.js

@@ -0,0 +1,294 @@
+import { getOrCreateGauge } from "../../observability/metrics.js";
+/**
+* Limits the number of view-syncers that replace their replica at the same
+* time, so that view-syncers switching over from an incompatible
+* replication-manager (e.g. after a resync) do not all stop serving at once.
+*
+* Capacity is consumed by:
+* * Replacement restores: snapshot reservations of subscribers that are
+*   replacing an (incompatible or outdated) local replica, i.e. subscribers
+*   that were serving. Restores onto an empty disk (e.g. a new pod) do not
+*   take a serving subscriber out of service, and are not counted. The
+*   reservation's connection is held open for the duration of the restore,
+*   and the subscription that follows it is held open until the restored
+*   subscriber has caught up, so their liveness tracks that of the
+*   replacement without a timeout.
+* * Outstanding grants: permission to restore that has not yet been
+*   followed by a replacement reservation (e.g. while the subscriber drains
+*   and restarts). Grants are not matched to reservations by identity:
+*   any replacement reservation consumes an outstanding grant, and a grant
+*   that is not consumed expires a short time after the granted connection
+*   closes (e.g. if the subscriber reconnected to a different
+*   replication-manager or was deleted).
+*
+* ## Life cycle of a switchover restore
+*
+* The following traces a view-syncer (VS) with a replica at version `R1`,
+* served by an old replication-manager (RM) at `R1`, while a new RM (e.g.
+* after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new
+* RM. Terms in `code` are those used in the code.
+*
+* 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends
+*    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.
+*    It continues to stream changes to them, and exits once they have all
+*    disconnected.
+*
+* 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`
+*    with `switchover: true` at the watermark of its last committed
+*    transaction, which is routed to some RM. The VS continues to consume
+*    the old RM's stream. If the RM is compatible (i.e. another old RM),
+*    the subscription is accepted and the VS simply moves to it. If the RM
+*    is the new one, the subscriber's replica version is older, and the
+*    subscription proceeds to step 3.
+*
+* 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription
+*    (it is not added to the forwarder) and calls {@link request}. The
+*    request is `queued` and `onQueued(position)` sends
+*    `status {restore: {state: 'queued', position}}`, again whenever the
+*    position changes. If the held connection closes while queued (e.g.
+*    the VS's old stream ended and it gave up on the switchover), the
+*    ticket is `release()`d and the request leaves the queue.
+*
+* 4. **Granted.** When the number in flight (`replacing` + `granted`) is
+*    below `maxConcurrentRestores`, the head of the queue is `granted`: a
+*    {@link Grant} is added (counting toward capacity), and `onGranted()`
+*    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted
+*    connection open, and initiates a self-drain (`drainRequested`): it
+*    fails its health checks, drains its clients, and keeps replicating
+*    from the old RM in the meantime.
+*
+* 5. **Restart.** The VS process exits, closing the granted connection.
+*    The ticket is `release()`d, marking the grant as `released` and
+*    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),
+*    within which a replacement reservation is expected. The drain itself
+*    does not count against the TTL.
+*
+* 6. **Reservation opened.** The restarted VS sends
+*    `reserve-snapshot {taskID, localReplica}`, and
+*    `ChangeStreamerImpl.startSnapshotReservation()` calls
+*    {@link reservationOpened}:
+*    * A `localReplica` of a different replica version (e.g. `R1`) is
+*      `replacing`: the reservation is counted, and consumes an outstanding
+*      grant (preferring a `released` one), so the number in flight is
+*      unchanged.
+*    * A `localReplica` of the same replica version is `unclassified` until
+*      the reservation is confirmed ({@link reservationConfirmed}) with the
+*      backup's minimum watermark. It is `replacing` if its watermark is
+*      older (i.e. it cannot catch up from the change log).
+*    * No `localReplica` (an empty disk) is not counted.
+*
+*    If the VS reached a different RM instead, the grant here expires after
+*    the TTL and frees its capacity, while the replacement is counted (as
+*    an ungranted one) by the RM that it reached.
+*
+* 7. **Restoring.** The VS restores the backup while holding the
+*    reservation's connection open. A retried reservation from the same
+*    task that arrives while the previous one is still open replaces it
+*    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it
+*    remains `replacing`. If the reservation instead closes without
+*    subscribing (e.g. a failed or disconnected restore, after which the VS
+*    retries), {@link reservationClosed} converts its capacity into a
+*    `released` grant, which the retry's replacement reservation consumes
+*    (step 6), or which expires after the pending grant TTL (e.g. if the VS
+*    was deleted).
+*
+* 8. **Subscribed.** Once restored, the VS subscribes on the same
+*    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`
+*    calls {@link reservationSubscribed}, which moves the replacement from
+*    its reservation (the key leaves `replacing`) to the subscription
+*    (`catchingUp`), and then closes the task's reservation. Its `onEnd`
+*    calls {@link reservationClosed}, which is a no-op for the moved key.
+*
+* 9. **Caught up.** The VS catches up on the change stream from the
+*    backup's watermark. Once it has acked all changes up to the head of
+*    the stream (`Subscriber.caughtUp`), or if it disconnects first, the
+*    replacement is released and the next queued request is granted.
+*
+* A VS that restarts outside of a switchover with an incompatible replica
+* (e.g. after a crash) enters at step 6 without a grant, and its restore
+* consumes capacity all the same, delaying queued requests.
+*
+* ## States
+*
+* | Term          | Where                    | Meaning                          |
+* | ------------- | ------------------------ | -------------------------------- |
+* | queued        | `Entry.state`            | Held, waiting for capacity       |
+* | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |
+* |               |                          | replacement reservation          |
+* | released      | `Entry.state`            | The held connection closed       |
+* | released      | `Grant.released`         | The granted connection, or an    |
+* |               |                          | unsubscribed replacement         |
+* |               |                          | reservation, closed; the TTL is  |
+* |               |                          | running                          |
+* | unclassified  | `#unclassified`          | Same-version reservation, not    |
+* |               |                          | yet confirmed                    |
+* | replacing     | `#replacing`             | Open replacement reservation     |
+* | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |
+* |               |                          | caught up                        |
+* | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |
+* |               |                          | `granted`, bounded by            |
+* |               |                          | `maxConcurrentRestores`          |
+*/
+var RestoreAdmission = class {
+	#lc;
+	#maxConcurrentRestores;
+	#pendingGrantTTLMs;
+	#setTimeout;
+	#clearTimeout;
+	#replicaVersion;
+	/** Keys (task IDs) of open replacement reservations. */
+	#replacing = /* @__PURE__ */ new Set();
+	/** Replacements whose subscribers have subscribed but not caught up. */
+	#catchingUp = /* @__PURE__ */ new Set();
+	/**
+	* The local replicas of open reservations with this replica version, which
+	* are classified upon confirmation of the reservation's minimum watermark.
+	*/
+	#unclassified = /* @__PURE__ */ new Map();
+	#grants = /* @__PURE__ */ new Set();
+	#queue = [];
+	constructor(lc, opts) {
+		this.#lc = lc.withContext("component", "restore-admission");
+		this.#replicaVersion = opts.replicaVersion;
+		this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);
+		this.#pendingGrantTTLMs = opts.pendingGrantTTLMs ?? 9e4;
+		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
+		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
+		getOrCreateGauge("replication", "restore_admission.queued", "Switchover subscribers waiting for permission to restore a new replica.").addCallback((result) => result.observe(this.#queue.length));
+		getOrCreateGauge("replication", "restore_admission.in_flight", "Subscribers replacing, or granted permission to replace, their replica.").addCallback((result) => result.observe(this.#inFlight));
+	}
+	get #inFlight() {
+		return this.#replacing.size + this.#catchingUp.size + this.#grants.size;
+	}
+	/** For testing and logging. */
+	get stats() {
+		return {
+			replacing: this.#replacing.size + this.#catchingUp.size,
+			granted: this.#grants.size,
+			queued: this.#queue.length
+		};
+	}
+	/**
+	* Called when a snapshot reservation is opened by a subscriber with the
+	* given `localReplica` (if any). A subscriber with a local replica of a
+	* different replica version is replacing it. One with the same replica
+	* version is replacing it if it is outdated, which is determined by
+	* {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a
+	* retried reservation).
+	*/
+	reservationOpened(key, localReplica) {
+		this.#unclassified.delete(key);
+		if (!localReplica) return;
+		if (localReplica.replicaVersion !== this.#replicaVersion) this.#replacementOpened(key);
+		else this.#unclassified.set(key, localReplica);
+	}
+	/**
+	* Called when a reservation is confirmed with the minimum watermark of the
+	* replica to restore.
+	*/
+	reservationConfirmed(key, minWatermark) {
+		const localReplica = this.#unclassified.get(key);
+		this.#unclassified.delete(key);
+		if (localReplica && localReplica.watermark < minWatermark) this.#replacementOpened(key);
+	}
+	/**
+	* Called when a reservation ends. A replacement reservation normally ends
+	* after its task subscribes ({@link reservationSubscribed}), in which case
+	* this is a no-op. A replacement reservation that ends without subscribing
+	* (e.g. a failed or disconnected restore) is most likely followed by a
+	* retry, so its capacity is retained as a released grant (which the retry
+	* consumes) rather than released immediately.
+	*/
+	reservationClosed(key) {
+		this.#unclassified.delete(key);
+		if (this.#replacing.delete(key)) {
+			const grant = {
+				released: true,
+				timer: void 0
+			};
+			this.#grants.add(grant);
+			this.#startGrantTTL(grant);
+		}
+	}
+	/**
+	* Called when the task of a reservation subscribes (before the reservation
+	* is closed). If the reservation is a replacement, it continues to be
+	* counted until the returned function is called, i.e. when the subscriber
+	* has caught up or disconnected.
+	*/
+	reservationSubscribed(key) {
+		this.#unclassified.delete(key);
+		if (!this.#replacing.delete(key)) return;
+		const replacement = {};
+		this.#catchingUp.add(replacement);
+		return () => {
+			if (this.#catchingUp.delete(replacement)) this.#processQueue();
+		};
+	}
+	#replacementOpened(key) {
+		if (this.#replacing.has(key)) return;
+		this.#replacing.add(key);
+		const grants = [...this.#grants];
+		const grant = grants.find((g) => g.released) ?? grants[0];
+		if (grant) this.#deleteGrant(grant);
+	}
+	request(request) {
+		const entry = {
+			request,
+			state: "queued",
+			position: 0
+		};
+		this.#queue.push(entry);
+		this.#processQueue();
+		return { release: () => this.#release(entry) };
+	}
+	#release(entry) {
+		const { state, grant } = entry;
+		entry.state = "released";
+		if (state === "queued") {
+			const i = this.#queue.indexOf(entry);
+			if (i >= 0) this.#queue.splice(i, 1);
+			this.#processQueue();
+		} else if (state === "granted" && grant && this.#grants.has(grant)) {
+			grant.released = true;
+			this.#startGrantTTL(grant);
+		}
+	}
+	#startGrantTTL(grant) {
+		grant.timer = this.#setTimeout(() => {
+			if (this.#grants.has(grant)) {
+				this.#lc.info?.(`grant expired without a replacement reservation`);
+				this.#deleteGrant(grant);
+				this.#processQueue();
+			}
+		}, this.#pendingGrantTTLMs);
+	}
+	#deleteGrant(grant) {
+		if (grant.timer) this.#clearTimeout(grant.timer);
+		this.#grants.delete(grant);
+	}
+	#processQueue() {
+		while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {
+			const entry = this.#queue.shift();
+			entry.state = "granted";
+			entry.grant = {
+				released: false,
+				timer: void 0
+			};
+			this.#grants.add(entry.grant);
+			this.#lc.info?.(`granting permission to restore`, { ...this.stats });
+			entry.request.onGranted();
+		}
+		this.#queue.forEach((entry, i) => {
+			const position = i + 1;
+			if (entry.position !== position) {
+				entry.position = position;
+				entry.request.onQueued(position);
+			}
+		});
+	}
+};
+//#endregion
+export { RestoreAdmission };
+
+//# sourceMappingURL=restore-admission.js.map

package/out/zero-cache/src/services/change-streamer/restore-admission.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"restore-admission.js","names":["#lc","#maxConcurrentRestores","#pendingGrantTTLMs","#setTimeout","#clearTimeout","#replicaVersion","#replacing","#catchingUp","#unclassified","#grants","#queue","#inFlight","#replacementOpened","#startGrantTTL","#processQueue","#deleteGrant","#release"],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nexport const DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), or if it disconnects first, the\n *    replacement is released and the next queued request is granted.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up                        |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /** Replacements whose subscribers have subscribed but not caught up. */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the returned function is called, i.e. when the subscriber\n   * has caught up or disconnected.\n   */\n  reservationSubscribed(key: string): (() => void) | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    return () => {\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;CAEtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKK,kBAAkB,KAAK;EAC5B,KAAKJ,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAKC,qBACH,KAAK,qBAAA;EACP,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKM,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,SAAS,CAAC;CACxD;CAEA,IAAIA,YAAY;EACd,OAAO,KAAKL,WAAW,OAAO,KAAKC,YAAY,OAAO,KAAKE,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAKH,WAAW,OAAO,KAAKC,YAAY;GACnD,SAAS,KAAKE,QAAQ;GACtB,QAAQ,KAAKC,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAKF,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAKH,iBACvC,KAAKO,mBAAmB,GAAG;OAE3B,KAAKJ,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAKA,cAAc,IAAI,GAAG;EAC/C,KAAKA,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAKI,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAKJ,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAKF,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAKG,QAAQ,IAAI,KAAK;GACtB,KAAKI,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAuC;EAC3D,KAAKL,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAKF,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAKC,YAAY,IAAI,WAAW;EAChC,aAAa;GACX,IAAI,KAAKA,YAAY,OAAO,WAAW,GACrC,KAAKO,cAAc;EAEvB;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAKR,WAAW,IAAI,GAAG,GACzB;EAEF,KAAKA,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAKG,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAKM,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAKL,OAAO,KAAK,KAAK;EACtB,KAAKI,cAAc;EACnB,OAAO,EAAC,eAAe,KAAKE,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAKN,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAKA,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAKI,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAKL,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAKI,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAKV,kBAAkB;GACnC,IAAI,KAAKM,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAKT,IAAI,OAAO,iDAAiD;IACjE,KAAKe,aAAa,KAAK;IACvB,KAAKD,cAAc;GACrB;EACF,GAAG,KAAKZ,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAKE,cAAc,MAAM,KAAK;EAEhC,KAAKK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAKE,YAAY,KAAKV,0BAA0B,KAAKS,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAKA,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAKD,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAKT,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAKU,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts

@@ -3,9 +3,21 @@ import type { Source } from '../../types/streams.ts';
 import type { BackupConfig } from './change-streamer-service.ts';
 import type { SnapshotMessage } from './snapshot-message.ts';
 import type { ChangeLogReadSource } from './sqlite-change-log-read-router.ts';
+export type ReservationCallbacks = {
+    /**
+     * Called whenever a reservation is closed, including when it is replaced
+     * by a new reservation (e.g. a retry) for the same task.
+     */
+    onClose?: ((taskID: string) => void) | undefined;
+    /**
+     * Called when a task's reservation is closed without being replaced,
+     * i.e. when the task no longer holds a reservation.
+     */
+    onEnd?: ((taskID: string) => void) | undefined;
+};
 export declare class SnapshotReservations {
     #private;
-    constructor(lc: LogContext, backupConfig: BackupConfig, onClose?: ((taskID: string) => void) | undefined);
+    constructor(lc: LogContext, backupConfig: BackupConfig, callbacks?: ReservationCallbacks);
     open(taskID: string): Source<SnapshotMessage>;
     close(taskID: string): void;
     isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean;

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAE5E,qBAAa,oBAAoB;;IAM/B,YACE,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,EAKjD;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,CAU5C;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAEnB;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,OAAO,CAElE;IAsCD,qBAAqB,YAOpB;IAED,kBAAkB,IAAI,MAAM,EAAE,CAI7B;IAED,0EAA0E;IAC1E,UAAU,CACR,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,mBAAmB,GAC1B,IAAI,CAmBN;IAeD;;;;OAIG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE/C;IAED,qBAAqB,aAKpB;CACF"}
+{"version":3,"file":"snapshot-reservations.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAC/D,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAE5E,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAEjD;;;OAGG;IACH,KAAK,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CAChD,CAAC;AAEF,qBAAa,oBAAoB;;IAM/B,YACE,EAAE,EAAE,UAAU,EACd,YAAY,EAAE,YAAY,EAC1B,SAAS,GAAE,oBAAyB,EAKrC;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,CAU5C;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAEnB;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,OAAO,CAElE;IA6CD,qBAAqB,YAOpB;IAED,kBAAkB,IAAI,MAAM,EAAE,CAI7B;IAED,0EAA0E;IAC1E,UAAU,CACR,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,mBAAmB,GAC1B,IAAI,CAmBN;IAeD;;;;OAIG;IACH,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAE/C;IAED,qBAAqB,aAKpB;CACF"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js

@@ -5,15 +5,15 @@ import { statSync } from "node:fs";
 var SnapshotReservations = class {
 	#lc;
 	#backupConfig;
-	#onClose;
+	#callbacks;
 	#reservations = /* @__PURE__ */ new Map();
-	constructor(lc, backupConfig, onClose) {
+	constructor(lc, backupConfig, callbacks = {}) {
 		this.#lc = lc.withContext("component", "snapshot-reserver");
 		this.#backupConfig = backupConfig;
-		this.#onClose = onClose;
+		this.#callbacks = callbacks;
 	}
 	open(taskID) {
-		this.close(taskID);
+		this.#close(taskID, void 0, true);
 		const instanceID = {};
 		const downstream = Subscription.create({ cleanup: () => this.#close(taskID, instanceID) });
 		this.#reservations.set(taskID, new Reservation(instanceID, downstream));
@@ -29,11 +29,12 @@ var SnapshotReservations = class {
 	#metricAttrs() {
 		return litestreamMonitorMetricAttrs(this.#backupConfig.backupURL, this.#backupConfig.litestreamVersion, "view_syncer");
 	}
-	#close(taskID, cancelledInstanceID) {
+	#close(taskID, cancelledInstanceID, replaced = false) {
 		const res = this.#reservations.get(taskID);
 		if (res && (!cancelledInstanceID || res.instanceID === cancelledInstanceID)) {
 			this.#reservations.delete(taskID);
-			this.#onClose?.(taskID);
+			this.#callbacks.onClose?.(taskID);
+			if (!replaced) this.#callbacks.onEnd?.(taskID);
 			res.close();
 			const duration = Date.now() - res.startTime.getTime();
 			this.#lc.info?.(`ended snapshot reservation for ${taskID} (${duration} ms)`);

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#onClose","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #onClose: ((taskID: string) => void) | undefined;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    onClose?: ((taskID: string) => void) | undefined,\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#onClose = onClose;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.close(taskID);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(taskID: string, cancelledInstanceID: InstanceID | undefined) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#onClose?.(taskID);\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AAaA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,SACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,WAAW;CAClB;CAEA,KAAK,QAAyC;EAC5C,KAAK,MAAM,MAAM;EAEjB,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKE,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OAAO,QAAgB,qBAA6C;EAClE,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,MAAM;GACtB,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#callbacks","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport type ReservationCallbacks = {\n  /**\n   * Called whenever a reservation is closed, including when it is replaced\n   * by a new reservation (e.g. a retry) for the same task.\n   */\n  onClose?: ((taskID: string) => void) | undefined;\n\n  /**\n   * Called when a task's reservation is closed without being replaced,\n   * i.e. when the task no longer holds a reservation.\n   */\n  onEnd?: ((taskID: string) => void) | undefined;\n};\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #callbacks: ReservationCallbacks;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    callbacks: ReservationCallbacks = {},\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#callbacks = callbacks;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.#close(taskID, undefined, true);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(\n    taskID: string,\n    cancelledInstanceID: InstanceID | undefined,\n    replaced = false,\n  ) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#callbacks.onClose?.(taskID);\n      if (!replaced) {\n        this.#callbacks.onEnd?.(taskID);\n      }\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AA2BA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,YAAkC,CAAC,GACnC;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,aAAa;CACpB;CAEA,KAAK,QAAyC;EAC5C,KAAKE,OAAO,QAAQ,KAAA,GAAW,IAAI;EAEnC,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKA,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OACE,QACA,qBACA,WAAW,OACX;EACA,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,UAAU,MAAM;GAChC,IAAI,CAAC,UACH,KAAKA,WAAW,QAAQ,MAAM;GAEhC,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts

@@ -13,9 +13,9 @@
  */
 import type { LogContext } from '@rocicorp/logger';
 import type { ReservationFollowup, SnapshotReserver } from './change-streamer-http.ts';
-import type { SnapshotStatus } from './subscribe.ts';
+import type { LocalReplica, SnapshotStatus } from './subscribe.ts';
 export type { SnapshotStatus };
-export declare function reserveAndGetSnapshotStatus(lc: LogContext, taskID: string, changeStreamer: SnapshotReserver): Promise<{
+export declare function reserveAndGetSnapshotStatus(lc: LogContext, taskID: string, changeStreamer: SnapshotReserver, localReplica?: LocalReplica | undefined): Promise<{
     reserved: SnapshotStatus;
     followup: ReservationFollowup;
 }>;

package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAC,cAAc,EAAC,CAAC;AAE7B,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,gBAAgB,GAC/B,OAAO,CAAC;IAAC,QAAQ,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,mBAAmB,CAAA;CAAC,CAAC,CAmDpE"}
+{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,KAAK,EACV,mBAAmB,EACnB,gBAAgB,EACjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAEjE,YAAY,EAAC,cAAc,EAAC,CAAC;AAE7B,wBAAsB,2BAA2B,CAC/C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,gBAAgB,EAChC,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,GACtC,OAAO,CAAC;IAAC,QAAQ,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,mBAAmB,CAAA;CAAC,CAAC,CAmDpE"}

package/out/zero-cache/src/services/change-streamer/snapshot.js

@@ -2,7 +2,7 @@ import { AbortError } from "../../../../shared/src/abort-error.js";
 import { sleep } from "../../../../shared/src/sleep.js";
 import { promiseOrAbort } from "../../../../shared/src/promise-race.js";
 //#region ../zero-cache/src/services/change-streamer/snapshot.ts
-async function reserveAndGetSnapshotStatus(lc, taskID, changeStreamer) {
+async function reserveAndGetSnapshotStatus(lc, taskID, changeStreamer, localReplica) {
 	const abort = new AbortController();
 	const { signal } = abort;
 	const onSignal = () => abort.abort(new AbortError("SIGTERM/SIGINT"));
@@ -11,7 +11,7 @@ async function reserveAndGetSnapshotStatus(lc, taskID, changeStreamer) {
 	try {
 		for (let i = 0;; i++) {
 			let err;
-			const reservation = changeStreamer.reserveSnapshot(taskID);
+			const reservation = changeStreamer.reserveSnapshot(taskID, localReplica);
 			try {
 				return await promiseOrAbort(reservation, signal);
 			} catch (e) {

package/out/zero-cache/src/services/change-streamer/snapshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {promiseOrAbort} from '../../../../shared/src/promise-race.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\n\n// The schema definitions live in the pure schema module (with no dependency on\n// the HTTP client) so that subscribe.ts can import them without forming an\n// import cycle through change-streamer-http.ts. Imported here for local use and\n// re-exported for existing consumers.\nimport type {\n  ReservationFollowup,\n  SnapshotReserver,\n} from './change-streamer-http.ts';\nimport type {SnapshotStatus} from './subscribe.ts';\n\nexport type {SnapshotStatus};\n\nexport async function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  taskID: string,\n  changeStreamer: SnapshotReserver,\n): Promise<{reserved: SnapshotStatus; followup: ReservationFollowup}> {\n  const abort = new AbortController();\n  const {signal} = abort;\n  // Use our own AbortError as the abort reason (rather than the native\n  // DOMException that AbortController.abort() defaults to) so that\n  // promiseOrAbort()'s rejection, which propagates `signal.reason` verbatim,\n  // is always an instance of the shared AbortError type.\n  const onSignal = () => abort.abort(new AbortError('SIGTERM/SIGINT'));\n  process.on('SIGINT', onSignal);\n  process.on('SIGTERM', onSignal);\n\n  try {\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      const reservation = changeStreamer.reserveSnapshot(taskID);\n      try {\n        return await promiseOrAbort(reservation, signal);\n      } catch (e) {\n        err = e;\n      }\n      if (signal.aborted) {\n        // promiseOrAbort() does not cancel the loser of the race: the\n        // reservation may still resolve later. If it does, cancel its\n        // connection rather than leaking it, since this function has already\n        // given up and nobody will call followup.subscribe().\n        reservation.then(\n          ({followup}) => followup.cancel(),\n          () => {},\n        );\n        throw err;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      await sleep(5000, signal);\n    }\n  } finally {\n    // This function is called repeatedly (e.g. by the replicator's restore\n    // loop), so the signal handlers must not outlive the reservation.\n    process.off('SIGINT', onSignal);\n    process.off('SIGTERM', onSignal);\n  }\n}\n"],"mappings":";;;;AA+BA,eAAsB,4BACpB,IACA,QACA,gBACoE;CACpE,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,EAAC,WAAU;CAKjB,MAAM,iBAAiB,MAAM,MAAM,IAAI,WAAW,gBAAgB,CAAC;CACnE,QAAQ,GAAG,UAAU,QAAQ;CAC7B,QAAQ,GAAG,WAAW,QAAQ;CAE9B,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,MAAM,cAAc,eAAe,gBAAgB,MAAM;GACzD,IAAI;IACF,OAAO,MAAM,eAAe,aAAa,MAAM;GACjD,SAAS,GAAG;IACV,MAAM;GACR;GACA,IAAI,OAAO,SAAS;IAKlB,YAAY,MACT,EAAC,eAAc,SAAS,OAAO,SAC1B,CAAC,CACT;IACA,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,MAAM,MAAM,KAAM,MAAM;EAC1B;CACF,UAAU;EAGR,QAAQ,IAAI,UAAU,QAAQ;EAC9B,QAAQ,IAAI,WAAW,QAAQ;CACjC;AACF"}
+{"version":3,"file":"snapshot.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot.ts"],"sourcesContent":["/**\n * The `snapshot` API serves the purpose of:\n * - informing subscribers (i.e. view-syncers) of the (litestream)\n *   backup location from which to restore a replica snapshot\n * - checking whether a restored backup or existing replica is\n *   compatible with the change-streamer\n * - preventing change-log cleanup while a snapshot restore is in\n *   progress\n * - tracking the approximate time it takes from the beginning of\n *   snapshot \"reservation\" to the subsequent subscription, which\n *   serves as the minimum interval to wait before cleaning up\n *   backed up changes.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {promiseOrAbort} from '../../../../shared/src/promise-race.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\n\n// The schema definitions live in the pure schema module (with no dependency on\n// the HTTP client) so that subscribe.ts can import them without forming an\n// import cycle through change-streamer-http.ts. Imported here for local use and\n// re-exported for existing consumers.\nimport type {\n  ReservationFollowup,\n  SnapshotReserver,\n} from './change-streamer-http.ts';\nimport type {LocalReplica, SnapshotStatus} from './subscribe.ts';\n\nexport type {SnapshotStatus};\n\nexport async function reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  taskID: string,\n  changeStreamer: SnapshotReserver,\n  localReplica?: LocalReplica | undefined,\n): Promise<{reserved: SnapshotStatus; followup: ReservationFollowup}> {\n  const abort = new AbortController();\n  const {signal} = abort;\n  // Use our own AbortError as the abort reason (rather than the native\n  // DOMException that AbortController.abort() defaults to) so that\n  // promiseOrAbort()'s rejection, which propagates `signal.reason` verbatim,\n  // is always an instance of the shared AbortError type.\n  const onSignal = () => abort.abort(new AbortError('SIGTERM/SIGINT'));\n  process.on('SIGINT', onSignal);\n  process.on('SIGTERM', onSignal);\n\n  try {\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      const reservation = changeStreamer.reserveSnapshot(taskID, localReplica);\n      try {\n        return await promiseOrAbort(reservation, signal);\n      } catch (e) {\n        err = e;\n      }\n      if (signal.aborted) {\n        // promiseOrAbort() does not cancel the loser of the race: the\n        // reservation may still resolve later. If it does, cancel its\n        // connection rather than leaking it, since this function has already\n        // given up and nobody will call followup.subscribe().\n        reservation.then(\n          ({followup}) => followup.cancel(),\n          () => {},\n        );\n        throw err;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      await sleep(5000, signal);\n    }\n  } finally {\n    // This function is called repeatedly (e.g. by the replicator's restore\n    // loop), so the signal handlers must not outlive the reservation.\n    process.off('SIGINT', onSignal);\n    process.off('SIGTERM', onSignal);\n  }\n}\n"],"mappings":";;;;AA+BA,eAAsB,4BACpB,IACA,QACA,gBACA,cACoE;CACpE,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,EAAC,WAAU;CAKjB,MAAM,iBAAiB,MAAM,MAAM,IAAI,WAAW,gBAAgB,CAAC;CACnE,QAAQ,GAAG,UAAU,QAAQ;CAC7B,QAAQ,GAAG,WAAW,QAAQ;CAE9B,IAAI;EACF,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,MAAM,cAAc,eAAe,gBAAgB,QAAQ,YAAY;GACvE,IAAI;IACF,OAAO,MAAM,eAAe,aAAa,MAAM;GACjD,SAAS,GAAG;IACV,MAAM;GACR;GACA,IAAI,OAAO,SAAS;IAKlB,YAAY,MACT,EAAC,eAAc,SAAS,OAAO,SAC1B,CAAC,CACT;IACA,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,MAAM,MAAM,KAAM,MAAM;EAC1B;CACF,UAAU;EAGR,QAAQ,IAAI,UAAU,QAAQ;EAC9B,QAAQ,IAAI,WAAW,QAAQ;CACjC;AACF"}

package/out/zero-cache/src/services/change-streamer/subscribe.d.ts

@@ -35,8 +35,23 @@ declare const snapshotStatusSchema: v.ObjectType<Omit<{
     tag: v.Type<"snapshot">;
 }, undefined>;
 export type SnapshotStatus = v.Infer<typeof snapshotStatusSchema>;
+/**
+ * The state of the replica on the subscriber's disk, if any. This allows the
+ * replication-manager to distinguish a subscriber that is replacing its
+ * replica (i.e. one that was serving, see `RestoreAdmission`) from one that
+ * is restoring onto an empty disk.
+ */
+export declare const localReplicaSchema: v.ObjectType<{
+    replicaVersion: v.Type<string>;
+    watermark: v.Type<string>;
+}, undefined>;
+export type LocalReplica = v.Infer<typeof localReplicaSchema>;
 export declare const reserveSnapshotMessageSchema: v.TupleType<[v.Type<"reserve-snapshot">, v.ObjectType<{
     taskID: v.Type<string>;
+    localReplica: v.Optional<{
+        replicaVersion: string;
+        watermark: string;
+    }>;
 }, undefined>]>;
 /**
  * The subscription-initialization context, sent in the `start-subscription`
@@ -82,6 +97,10 @@ export declare const subscribeContextSchema: v.ObjectType<{
             } | undefined;
         }>;
     }[]>;
+    /** See `SubscriberContext.drainable`. */
+    drainable: v.Optional<boolean>;
+    /** See `SubscriberContext.switchover`. */
+    switchover: v.Optional<boolean>;
 }, undefined>;
 export type SubscribeContext = v.Infer<typeof subscribeContextSchema>;
 export declare const startSubscriptionMessageSchema: v.TupleType<[v.Type<"start-subscription">, v.ObjectType<{
@@ -118,9 +137,17 @@ export declare const startSubscriptionMessageSchema: v.TupleType<[v.Type<"start-
             } | undefined;
         }>;
     }[]>;
+    /** See `SubscriberContext.drainable`. */
+    drainable: v.Optional<boolean>;
+    /** See `SubscriberContext.switchover`. */
+    switchover: v.Optional<boolean>;
 }, undefined>]>;
 export declare const subscribeUpstreamSchema: v.UnionType<[v.TupleType<[v.Type<"reserve-snapshot">, v.ObjectType<{
     taskID: v.Type<string>;
+    localReplica: v.Optional<{
+        replicaVersion: string;
+        watermark: string;
+    }>;
 }, undefined>]>, v.TupleType<[v.Type<"start-subscription">, v.ObjectType<{
     /** Task ID, links the subscription to a preceding snapshot reservation. */
     taskID: v.Type<string>;
@@ -155,6 +182,10 @@ export declare const subscribeUpstreamSchema: v.UnionType<[v.TupleType<[v.Type<"
             } | undefined;
         }>;
     }[]>;
+    /** See `SubscriberContext.drainable`. */
+    drainable: v.Optional<boolean>;
+    /** See `SubscriberContext.switchover`. */
+    switchover: v.Optional<boolean>;
 }, undefined>]>]>;
 export type SubscribeUpstream = v.Infer<typeof subscribeUpstreamSchema>;
 /**
@@ -191,6 +222,13 @@ export declare const subscribeDownstreamSchema: v.UnionType<[v.TupleType<[v.Type
         } | undefined;
         nextSendTimeMs: number;
     }>;
+    draining: v.Optional<boolean>;
+    restore: v.Optional<{
+        state: "queued";
+        position: number;
+    } | {
+        state: "granted";
+    }>;
 }, undefined>]>, v.UnionType<[v.TupleType<[v.Type<"begin">, v.ObjectType<{
     tag: v.Type<"begin">;
     json: v.Optional<"p" | "s">;
@@ -440,6 +478,7 @@ export declare const subscribeDownstreamSchema: v.UnionType<[v.TupleType<[v.Type
 }, undefined>]>]>, v.TupleType<[v.Type<"error">, v.ObjectType<{
     type: v.Type<number>;
     message: v.Optional<string>;
+    replicaVersion: v.Optional<string>;
 }, undefined>]>]>]>;
 export type SubscribeDownstream = v.Infer<typeof subscribeDownstreamSchema>;
 export {};

package/out/zero-cache/src/services/change-streamer/subscribe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"subscribe.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscribe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AAKtD,QAAA,MAAM,oBAAoB;;;;;;;;aAExB,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAMlE,eAAO,MAAM,4BAA4B;;eAGvC,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,sBAAsB;IACjC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;aAEH,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,8BAA8B;IApBzC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;eASH,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;IAzBlC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;iBAcJ,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAMxE;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;eAGhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"subscribe.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/subscribe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,CAAC,MAAM,kCAAkC,CAAC;AAKtD,QAAA,MAAM,oBAAoB;;;;;;;;aAExB,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAMlE;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB;;;aAG7B,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,eAAO,MAAM,4BAA4B;;;;;;eAMvC,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,sBAAsB;IACjC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;IAEH,yCAAyC;;IAEzC,0CAA0C;;aAE1C,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,8BAA8B;IAxBzC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;IAEH,yCAAyC;;IAEzC,0CAA0C;;eAS1C,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;IA7BlC,2EAA2E;;IAE3E,sCAAsC;;IAEtC,8DAA8D;;IAE9D,gEAAgE;;IAEhE,8DAA8D;;IAE9D;;;;OAIG;;;;;;;;;;;;;;;;;;;IAEH,yCAAyC;;IAEzC,0CAA0C;;iBAc3C,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAMxE;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;eAGhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}