@rocicorp/zero 1.11.0-canary.39 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/restore-admission.js

@@ -1,5 +1,12 @@
 import { getOrCreateGauge } from "../../observability/metrics.js";
 /**
+* The default time for which a replacement continues to be counted after its
+* subscriber has caught up, which covers the time for the subscriber to start
+* serving (i.e. become ready) after catching up. See
+* {@link RestoreAdmissionOptions.readinessGracePeriodMs}.
+*/
+var DEFAULT_READINESS_GRACE_PERIOD_MS = 5e3;
+/**
 * Limits the number of view-syncers that replace their replica at the same
 * time, so that view-syncers switching over from an incompatible
 * replication-manager (e.g. after a resync) do not all stop serving at once.
@@ -101,8 +108,10 @@ import { getOrCreateGauge } from "../../observability/metrics.js";
 *
 * 9. **Caught up.** The VS catches up on the change stream from the
 *    backup's watermark. Once it has acked all changes up to the head of
-*    the stream (`Subscriber.caughtUp`), or if it disconnects first, the
-*    replacement is released and the next queued request is granted.
+*    the stream (`Subscriber.caughtUp`), the replacement is released after
+*    the readiness grace period, which covers the time for the VS to start
+*    serving, and the next queued request is granted. If the VS disconnects
+*    first, the replacement is released immediately.
 *
 * A VS that restarts outside of a switchover with an incompatible replica
 * (e.g. after a crash) enters at step 6 without a grant, and its restore
@@ -124,7 +133,8 @@ import { getOrCreateGauge } from "../../observability/metrics.js";
 * |               |                          | yet confirmed                    |
 * | replacing     | `#replacing`             | Open replacement reservation     |
 * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |
-* |               |                          | caught up                        |
+* |               |                          | caught up (or within the         |
+* |               |                          | readiness grace period)          |
 * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |
 * |               |                          | `granted`, bounded by            |
 * |               |                          | `maxConcurrentRestores`          |
@@ -133,12 +143,16 @@ var RestoreAdmission = class {
 	#lc;
 	#maxConcurrentRestores;
 	#pendingGrantTTLMs;
+	#readinessGracePeriodMs;
 	#setTimeout;
 	#clearTimeout;
 	#replicaVersion;
 	/** Keys (task IDs) of open replacement reservations. */
 	#replacing = /* @__PURE__ */ new Set();
-	/** Replacements whose subscribers have subscribed but not caught up. */
+	/**
+	* Replacements whose subscribers have subscribed, but have not caught up
+	* (or are within the readiness grace period).
+	*/
 	#catchingUp = /* @__PURE__ */ new Set();
 	/**
 	* The local replicas of open reservations with this replica version, which
@@ -152,6 +166,7 @@ var RestoreAdmission = class {
 		this.#replicaVersion = opts.replicaVersion;
 		this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);
 		this.#pendingGrantTTLMs = opts.pendingGrantTTLMs ?? 9e4;
+		this.#readinessGracePeriodMs = opts.readinessGracePeriodMs ?? 5e3;
 		this.#setTimeout = opts.setTimeoutFn ?? setTimeout;
 		this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;
 		getOrCreateGauge("replication", "restore_admission.queued", "Switchover subscribers waiting for permission to restore a new replica.").addCallback((result) => result.observe(this.#queue.length));
@@ -213,17 +228,25 @@ var RestoreAdmission = class {
 	/**
 	* Called when the task of a reservation subscribes (before the reservation
 	* is closed). If the reservation is a replacement, it continues to be
-	* counted until the returned function is called, i.e. when the subscriber
-	* has caught up or disconnected.
+	* counted until the readiness grace period after the subscriber has caught
+	* up, or until the subscription closes.
 	*/
 	reservationSubscribed(key) {
 		this.#unclassified.delete(key);
 		if (!this.#replacing.delete(key)) return;
 		const replacement = {};
 		this.#catchingUp.add(replacement);
-		return () => {
+		let timer;
+		const release = () => {
+			if (timer) this.#clearTimeout(timer);
 			if (this.#catchingUp.delete(replacement)) this.#processQueue();
 		};
+		return {
+			caughtUp: () => {
+				if (this.#catchingUp.has(replacement) && !timer) timer = this.#setTimeout(release, this.#readinessGracePeriodMs);
+			},
+			closed: release
+		};
 	}
 	#replacementOpened(key) {
 		if (this.#replacing.has(key)) return;
@@ -289,6 +312,6 @@ var RestoreAdmission = class {
 	}
 };
 //#endregion
-export { RestoreAdmission };
+export { DEFAULT_READINESS_GRACE_PERIOD_MS, RestoreAdmission };
 
 //# sourceMappingURL=restore-admission.js.map

package/out/zero-cache/src/services/change-streamer/restore-admission.js.map

@@ -1 +1 @@
-{"version":3,"file":"restore-admission.js","names":["#lc","#maxConcurrentRestores","#pendingGrantTTLMs","#setTimeout","#clearTimeout","#replicaVersion","#replacing","#catchingUp","#unclassified","#grants","#queue","#inFlight","#replacementOpened","#startGrantTTL","#processQueue","#deleteGrant","#release"],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nexport const DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), or if it disconnects first, the\n *    replacement is released and the next queued request is granted.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up                        |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /** Replacements whose subscribers have subscribed but not caught up. */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the returned function is called, i.e. when the subscriber\n   * has caught up or disconnected.\n   */\n  reservationSubscribed(key: string): (() => void) | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    return () => {\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiMA,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;CAEtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKK,kBAAkB,KAAK;EAC5B,KAAKJ,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAKC,qBACH,KAAK,qBAAA;EACP,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKM,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,SAAS,CAAC;CACxD;CAEA,IAAIA,YAAY;EACd,OAAO,KAAKL,WAAW,OAAO,KAAKC,YAAY,OAAO,KAAKE,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAKH,WAAW,OAAO,KAAKC,YAAY;GACnD,SAAS,KAAKE,QAAQ;GACtB,QAAQ,KAAKC,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAKF,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAKH,iBACvC,KAAKO,mBAAmB,GAAG;OAE3B,KAAKJ,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAKA,cAAc,IAAI,GAAG;EAC/C,KAAKA,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAKI,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAKJ,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAKF,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAKG,QAAQ,IAAI,KAAK;GACtB,KAAKI,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAuC;EAC3D,KAAKL,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAKF,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAKC,YAAY,IAAI,WAAW;EAChC,aAAa;GACX,IAAI,KAAKA,YAAY,OAAO,WAAW,GACrC,KAAKO,cAAc;EAEvB;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAKR,WAAW,IAAI,GAAG,GACzB;EAEF,KAAKA,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAKG,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAKM,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAKL,OAAO,KAAK,KAAK;EACtB,KAAKI,cAAc;EACnB,OAAO,EAAC,eAAe,KAAKE,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAKN,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAKA,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAKI,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAKL,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAKI,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAKV,kBAAkB;GACnC,IAAI,KAAKM,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAKT,IAAI,OAAO,iDAAiD;IACjE,KAAKe,aAAa,KAAK;IACvB,KAAKD,cAAc;GACrB;EACF,GAAG,KAAKZ,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAKE,cAAc,MAAM,KAAK;EAEhC,KAAKK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAKE,YAAY,KAAKV,0BAA0B,KAAKS,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAKA,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAKD,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAKT,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAKU,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"restore-admission.js","names":["#lc","#maxConcurrentRestores","#pendingGrantTTLMs","#readinessGracePeriodMs","#setTimeout","#clearTimeout","#replicaVersion","#replacing","#catchingUp","#unclassified","#grants","#queue","#inFlight","#replacementOpened","#startGrantTTL","#processQueue","#deleteGrant","#release"],"sources":["../../../../../../zero-cache/src/services/change-streamer/restore-admission.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {LocalReplica} from './subscribe.ts';\n\n/**\n * The default time to wait for a subscriber that was granted permission to\n * restore to reconnect (i.e. to open a replacement reservation) after its\n * held connection closes, or after its replacement reservation closes\n * without subscribing (e.g. to retry a failed restore). This covers the\n * subscriber's process exit and restart (e.g. a container restart, the\n * backoff of which starts at 10 seconds in Kubernetes), but not the restore\n * itself, which is tracked by the liveness of its snapshot reservation.\n */\nexport const DEFAULT_PENDING_GRANT_TTL_MS = 90_000;\n\n/**\n * The default time for which a replacement continues to be counted after its\n * subscriber has caught up, which covers the time for the subscriber to start\n * serving (i.e. become ready) after catching up. See\n * {@link RestoreAdmissionOptions.readinessGracePeriodMs}.\n */\nexport const DEFAULT_READINESS_GRACE_PERIOD_MS = 5_000;\n\nexport type RestoreAdmissionOptions = {\n  /** The replica version of the replication-manager. */\n  replicaVersion: string;\n  /**\n   * The maximum number of subscribers that may be restoring (or granted\n   * permission to restore) at the same time.\n   */\n  maxConcurrentRestores: number;\n  pendingGrantTTLMs?: number | undefined;\n  /**\n   * The time for which a replacement continues to be counted after its\n   * subscriber has caught up, so that the next subscriber is not granted\n   * permission to restore (and stop serving) before the restored one has\n   * started serving. Catching up is followed by the subscriber's readiness\n   * checks (e.g. waiting for the next replication lag report), and by the\n   * container runner's detection of its readiness.\n   */\n  readinessGracePeriodMs?: number | undefined;\n  // Injectable timers, for deterministic testing.\n  setTimeoutFn?: typeof setTimeout | undefined;\n  clearTimeoutFn?: typeof clearTimeout | undefined;\n};\n\nexport type AdmissionRequest = {\n  /**\n   * Called when the request is queued, and again whenever its (1-based)\n   * position in the queue changes.\n   */\n  onQueued: (position: number) => void;\n\n  /** Called (once) when the request is granted. */\n  onGranted: () => void;\n};\n\n/** A subscribed replacement, returned by {@link RestoreAdmission.reservationSubscribed}. */\nexport type Replacement = {\n  /**\n   * Called when the subscriber has caught up. The replacement is released\n   * after the readiness grace period.\n   */\n  caughtUp: () => void;\n\n  /** Called when the subscription closes. The replacement is released. */\n  closed: () => void;\n};\n\nexport type AdmissionTicket = {\n  /**\n   * Must be called when the requesting connection closes. A queued request\n   * is removed from the queue. For a granted request, this starts the\n   * timer within which the subscriber is expected to reconnect and open a\n   * replacement reservation.\n   */\n  release(): void;\n};\n\ntype Entry = {\n  readonly request: AdmissionRequest;\n  state: 'queued' | 'granted' | 'released';\n  position: number;\n  grant?: Grant | undefined;\n};\n\ntype Grant = {\n  /** Whether the granted subscriber's connection has closed. */\n  released: boolean;\n  timer: ReturnType<typeof setTimeout> | undefined;\n};\n\n/**\n * Limits the number of view-syncers that replace their replica at the same\n * time, so that view-syncers switching over from an incompatible\n * replication-manager (e.g. after a resync) do not all stop serving at once.\n *\n * Capacity is consumed by:\n * * Replacement restores: snapshot reservations of subscribers that are\n *   replacing an (incompatible or outdated) local replica, i.e. subscribers\n *   that were serving. Restores onto an empty disk (e.g. a new pod) do not\n *   take a serving subscriber out of service, and are not counted. The\n *   reservation's connection is held open for the duration of the restore,\n *   and the subscription that follows it is held open until the restored\n *   subscriber has caught up, so their liveness tracks that of the\n *   replacement without a timeout.\n * * Outstanding grants: permission to restore that has not yet been\n *   followed by a replacement reservation (e.g. while the subscriber drains\n *   and restarts). Grants are not matched to reservations by identity:\n *   any replacement reservation consumes an outstanding grant, and a grant\n *   that is not consumed expires a short time after the granted connection\n *   closes (e.g. if the subscriber reconnected to a different\n *   replication-manager or was deleted).\n *\n * ## Life cycle of a switchover restore\n *\n * The following traces a view-syncer (VS) with a replica at version `R1`,\n * served by an old replication-manager (RM) at `R1`, while a new RM (e.g.\n * after a resync) is at `R2 > R1`. The RestoreAdmission is that of the new\n * RM. Terms in `code` are those used in the code.\n *\n * 1. **Drain.** The old RM receives a SIGTERM, and its `DrainTracker` sends\n *    `status {draining: true}` to its `drainable` (i.e. serving) subscribers.\n *    It continues to stream changes to them, and exits once they have all\n *    disconnected.\n *\n * 2. **Switchover.** The VS's `DrainHandler` opens a second `/subscribe`\n *    with `switchover: true` at the watermark of its last committed\n *    transaction, which is routed to some RM. The VS continues to consume\n *    the old RM's stream. If the RM is compatible (i.e. another old RM),\n *    the subscription is accepted and the VS simply moves to it. If the RM\n *    is the new one, the subscriber's replica version is older, and the\n *    subscription proceeds to step 3.\n *\n * 3. **Queued.** `ChangeStreamerImpl.subscribe()` holds the subscription\n *    (it is not added to the forwarder) and calls {@link request}. The\n *    request is `queued` and `onQueued(position)` sends\n *    `status {restore: {state: 'queued', position}}`, again whenever the\n *    position changes. If the held connection closes while queued (e.g.\n *    the VS's old stream ended and it gave up on the switchover), the\n *    ticket is `release()`d and the request leaves the queue.\n *\n * 4. **Granted.** When the number in flight (`replacing` + `granted`) is\n *    below `maxConcurrentRestores`, the head of the queue is `granted`: a\n *    {@link Grant} is added (counting toward capacity), and `onGranted()`\n *    sends `status {restore: {state: 'granted'}}`. The VS keeps the granted\n *    connection open, and initiates a self-drain (`drainRequested`): it\n *    fails its health checks, drains its clients, and keeps replicating\n *    from the old RM in the meantime.\n *\n * 5. **Restart.** The VS process exits, closing the granted connection.\n *    The ticket is `release()`d, marking the grant as `released` and\n *    starting the pending grant TTL ({@link DEFAULT_PENDING_GRANT_TTL_MS}),\n *    within which a replacement reservation is expected. The drain itself\n *    does not count against the TTL.\n *\n * 6. **Reservation opened.** The restarted VS sends\n *    `reserve-snapshot {taskID, localReplica}`, and\n *    `ChangeStreamerImpl.startSnapshotReservation()` calls\n *    {@link reservationOpened}:\n *    * A `localReplica` of a different replica version (e.g. `R1`) is\n *      `replacing`: the reservation is counted, and consumes an outstanding\n *      grant (preferring a `released` one), so the number in flight is\n *      unchanged.\n *    * A `localReplica` of the same replica version is `unclassified` until\n *      the reservation is confirmed ({@link reservationConfirmed}) with the\n *      backup's minimum watermark. It is `replacing` if its watermark is\n *      older (i.e. it cannot catch up from the change log).\n *    * No `localReplica` (an empty disk) is not counted.\n *\n *    If the VS reached a different RM instead, the grant here expires after\n *    the TTL and frees its capacity, while the replacement is counted (as\n *    an ungranted one) by the RM that it reached.\n *\n * 7. **Restoring.** The VS restores the backup while holding the\n *    reservation's connection open. A retried reservation from the same\n *    task that arrives while the previous one is still open replaces it\n *    (`SnapshotReservations` `onClose`) but does not end it (`onEnd`), so it\n *    remains `replacing`. If the reservation instead closes without\n *    subscribing (e.g. a failed or disconnected restore, after which the VS\n *    retries), {@link reservationClosed} converts its capacity into a\n *    `released` grant, which the retry's replacement reservation consumes\n *    (step 6), or which expires after the pending grant TTL (e.g. if the VS\n *    was deleted).\n *\n * 8. **Subscribed.** Once restored, the VS subscribes on the same\n *    connection at `R2`, which is accepted. `ChangeStreamerImpl.subscribe()`\n *    calls {@link reservationSubscribed}, which moves the replacement from\n *    its reservation (the key leaves `replacing`) to the subscription\n *    (`catchingUp`), and then closes the task's reservation. Its `onEnd`\n *    calls {@link reservationClosed}, which is a no-op for the moved key.\n *\n * 9. **Caught up.** The VS catches up on the change stream from the\n *    backup's watermark. Once it has acked all changes up to the head of\n *    the stream (`Subscriber.caughtUp`), the replacement is released after\n *    the readiness grace period, which covers the time for the VS to start\n *    serving, and the next queued request is granted. If the VS disconnects\n *    first, the replacement is released immediately.\n *\n * A VS that restarts outside of a switchover with an incompatible replica\n * (e.g. after a crash) enters at step 6 without a grant, and its restore\n * consumes capacity all the same, delaying queued requests.\n *\n * ## States\n *\n * | Term          | Where                    | Meaning                          |\n * | ------------- | ------------------------ | -------------------------------- |\n * | queued        | `Entry.state`            | Held, waiting for capacity       |\n * | granted       | `Entry.state`, `#grants` | Permitted to restore, pending a  |\n * |               |                          | replacement reservation          |\n * | released      | `Entry.state`            | The held connection closed       |\n * | released      | `Grant.released`         | The granted connection, or an    |\n * |               |                          | unsubscribed replacement         |\n * |               |                          | reservation, closed; the TTL is  |\n * |               |                          | running                          |\n * | unclassified  | `#unclassified`          | Same-version reservation, not    |\n * |               |                          | yet confirmed                    |\n * | replacing     | `#replacing`             | Open replacement reservation     |\n * | catching up   | `#catchingUp`            | Subscribed replacement, not yet  |\n * |               |                          | caught up (or within the         |\n * |               |                          | readiness grace period)          |\n * | in flight     | `#inFlight`              | `replacing` + `catchingUp` +     |\n * |               |                          | `granted`, bounded by            |\n * |               |                          | `maxConcurrentRestores`          |\n */\nexport class RestoreAdmission {\n  readonly #lc: LogContext;\n  readonly #maxConcurrentRestores: number;\n  readonly #pendingGrantTTLMs: number;\n  readonly #readinessGracePeriodMs: number;\n  readonly #setTimeout: typeof setTimeout;\n  readonly #clearTimeout: typeof clearTimeout;\n\n  readonly #replicaVersion: string;\n  /** Keys (task IDs) of open replacement reservations. */\n  readonly #replacing = new Set<string>();\n  /**\n   * Replacements whose subscribers have subscribed, but have not caught up\n   * (or are within the readiness grace period).\n   */\n  readonly #catchingUp = new Set<object>();\n  /**\n   * The local replicas of open reservations with this replica version, which\n   * are classified upon confirmation of the reservation's minimum watermark.\n   */\n  readonly #unclassified = new Map<string, LocalReplica>();\n  readonly #grants = new Set<Grant>();\n  readonly #queue: Entry[] = [];\n\n  constructor(lc: LogContext, opts: RestoreAdmissionOptions) {\n    this.#lc = lc.withContext('component', 'restore-admission');\n    this.#replicaVersion = opts.replicaVersion;\n    this.#maxConcurrentRestores = Math.max(1, opts.maxConcurrentRestores);\n    this.#pendingGrantTTLMs =\n      opts.pendingGrantTTLMs ?? DEFAULT_PENDING_GRANT_TTL_MS;\n    this.#readinessGracePeriodMs =\n      opts.readinessGracePeriodMs ?? DEFAULT_READINESS_GRACE_PERIOD_MS;\n    this.#setTimeout = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeout = opts.clearTimeoutFn ?? clearTimeout;\n\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.queued',\n      'Switchover subscribers waiting for permission to restore a new replica.',\n    ).addCallback(result => result.observe(this.#queue.length));\n    getOrCreateGauge(\n      'replication',\n      'restore_admission.in_flight',\n      'Subscribers replacing, or granted permission to replace, their replica.',\n    ).addCallback(result => result.observe(this.#inFlight));\n  }\n\n  get #inFlight() {\n    return this.#replacing.size + this.#catchingUp.size + this.#grants.size;\n  }\n\n  /** For testing and logging. */\n  get stats() {\n    return {\n      replacing: this.#replacing.size + this.#catchingUp.size,\n      granted: this.#grants.size,\n      queued: this.#queue.length,\n    };\n  }\n\n  /**\n   * Called when a snapshot reservation is opened by a subscriber with the\n   * given `localReplica` (if any). A subscriber with a local replica of a\n   * different replica version is replacing it. One with the same replica\n   * version is replacing it if it is outdated, which is determined by\n   * {@link reservationConfirmed}. Idempotent for the same `key` (e.g. a\n   * retried reservation).\n   */\n  reservationOpened(key: string, localReplica: LocalReplica | undefined) {\n    this.#unclassified.delete(key);\n    if (!localReplica) {\n      return; // Restoring onto an empty disk.\n    }\n    if (localReplica.replicaVersion !== this.#replicaVersion) {\n      this.#replacementOpened(key);\n    } else {\n      this.#unclassified.set(key, localReplica);\n    }\n  }\n\n  /**\n   * Called when a reservation is confirmed with the minimum watermark of the\n   * replica to restore.\n   */\n  reservationConfirmed(key: string, minWatermark: string) {\n    const localReplica = this.#unclassified.get(key);\n    this.#unclassified.delete(key);\n    if (localReplica && localReplica.watermark < minWatermark) {\n      this.#replacementOpened(key); // The local replica is outdated.\n    }\n  }\n\n  /**\n   * Called when a reservation ends. A replacement reservation normally ends\n   * after its task subscribes ({@link reservationSubscribed}), in which case\n   * this is a no-op. A replacement reservation that ends without subscribing\n   * (e.g. a failed or disconnected restore) is most likely followed by a\n   * retry, so its capacity is retained as a released grant (which the retry\n   * consumes) rather than released immediately.\n   */\n  reservationClosed(key: string) {\n    this.#unclassified.delete(key);\n    if (this.#replacing.delete(key)) {\n      const grant: Grant = {released: true, timer: undefined};\n      this.#grants.add(grant);\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  /**\n   * Called when the task of a reservation subscribes (before the reservation\n   * is closed). If the reservation is a replacement, it continues to be\n   * counted until the readiness grace period after the subscriber has caught\n   * up, or until the subscription closes.\n   */\n  reservationSubscribed(key: string): Replacement | undefined {\n    this.#unclassified.delete(key);\n    if (!this.#replacing.delete(key)) {\n      return undefined;\n    }\n    const replacement = {};\n    this.#catchingUp.add(replacement);\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const release = () => {\n      if (timer) {\n        this.#clearTimeout(timer);\n      }\n      if (this.#catchingUp.delete(replacement)) {\n        this.#processQueue();\n      }\n    };\n    return {\n      caughtUp: () => {\n        if (this.#catchingUp.has(replacement) && !timer) {\n          timer = this.#setTimeout(release, this.#readinessGracePeriodMs);\n        }\n      },\n      closed: release,\n    };\n  }\n\n  #replacementOpened(key: string) {\n    if (this.#replacing.has(key)) {\n      return;\n    }\n    this.#replacing.add(key);\n    // The restore is now tracked by its reservation. Prefer consuming a grant\n    // whose subscriber has already disconnected, as it is the most likely\n    // origin of the reservation.\n    const grants = [...this.#grants];\n    const grant = grants.find(g => g.released) ?? grants[0];\n    if (grant) {\n      this.#deleteGrant(grant);\n    }\n  }\n\n  request(request: AdmissionRequest): AdmissionTicket {\n    const entry: Entry = {request, state: 'queued', position: 0};\n    this.#queue.push(entry);\n    this.#processQueue();\n    return {release: () => this.#release(entry)};\n  }\n\n  #release(entry: Entry) {\n    const {state, grant} = entry;\n    entry.state = 'released';\n    if (state === 'queued') {\n      const i = this.#queue.indexOf(entry);\n      if (i >= 0) {\n        this.#queue.splice(i, 1);\n      }\n      this.#processQueue();\n    } else if (state === 'granted' && grant && this.#grants.has(grant)) {\n      // Wait for the subscriber to reconnect with a replacement reservation.\n      grant.released = true;\n      this.#startGrantTTL(grant);\n    }\n  }\n\n  #startGrantTTL(grant: Grant) {\n    grant.timer = this.#setTimeout(() => {\n      if (this.#grants.has(grant)) {\n        this.#lc.info?.(`grant expired without a replacement reservation`);\n        this.#deleteGrant(grant);\n        this.#processQueue();\n      }\n    }, this.#pendingGrantTTLMs);\n  }\n\n  #deleteGrant(grant: Grant) {\n    if (grant.timer) {\n      this.#clearTimeout(grant.timer);\n    }\n    this.#grants.delete(grant);\n  }\n\n  #processQueue() {\n    while (this.#inFlight < this.#maxConcurrentRestores && this.#queue.length) {\n      const entry = this.#queue.shift()!;\n      entry.state = 'granted';\n      entry.grant = {released: false, timer: undefined};\n      this.#grants.add(entry.grant);\n      this.#lc.info?.(`granting permission to restore`, {...this.stats});\n      entry.request.onGranted();\n    }\n    this.#queue.forEach((entry, i) => {\n      const position = i + 1;\n      if (entry.position !== position) {\n        entry.position = position;\n        entry.request.onQueued(position);\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;AAqBA,IAAa,oCAAoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4MjD,IAAa,mBAAb,MAA8B;CAC5B;CACA;CACA;CACA;CACA;CACA;CAEA;;CAEA,6BAAsB,IAAI,IAAY;;;;;CAKtC,8BAAuB,IAAI,IAAY;;;;;CAKvC,gCAAyB,IAAI,IAA0B;CACvD,0BAAmB,IAAI,IAAW;CAClC,SAA2B,CAAC;CAE5B,YAAY,IAAgB,MAA+B;EACzD,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKM,kBAAkB,KAAK;EAC5B,KAAKL,yBAAyB,KAAK,IAAI,GAAG,KAAK,qBAAqB;EACpE,KAAKC,qBACH,KAAK,qBAAA;EACP,KAAKC,0BACH,KAAK,0BAAA;EACP,KAAKC,cAAc,KAAK,gBAAgB;EACxC,KAAKC,gBAAgB,KAAK,kBAAkB;EAE5C,iBACE,eACA,4BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKM,OAAO,MAAM,CAAC;EAC1D,iBACE,eACA,+BACA,yEACF,EAAE,aAAY,WAAU,OAAO,QAAQ,KAAKC,SAAS,CAAC;CACxD;CAEA,IAAIA,YAAY;EACd,OAAO,KAAKL,WAAW,OAAO,KAAKC,YAAY,OAAO,KAAKE,QAAQ;CACrE;;CAGA,IAAI,QAAQ;EACV,OAAO;GACL,WAAW,KAAKH,WAAW,OAAO,KAAKC,YAAY;GACnD,SAAS,KAAKE,QAAQ;GACtB,QAAQ,KAAKC,OAAO;EACtB;CACF;;;;;;;;;CAUA,kBAAkB,KAAa,cAAwC;EACrE,KAAKF,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,cACH;EAEF,IAAI,aAAa,mBAAmB,KAAKH,iBACvC,KAAKO,mBAAmB,GAAG;OAE3B,KAAKJ,cAAc,IAAI,KAAK,YAAY;CAE5C;;;;;CAMA,qBAAqB,KAAa,cAAsB;EACtD,MAAM,eAAe,KAAKA,cAAc,IAAI,GAAG;EAC/C,KAAKA,cAAc,OAAO,GAAG;EAC7B,IAAI,gBAAgB,aAAa,YAAY,cAC3C,KAAKI,mBAAmB,GAAG;CAE/B;;;;;;;;;CAUA,kBAAkB,KAAa;EAC7B,KAAKJ,cAAc,OAAO,GAAG;EAC7B,IAAI,KAAKF,WAAW,OAAO,GAAG,GAAG;GAC/B,MAAM,QAAe;IAAC,UAAU;IAAM,OAAO,KAAA;GAAS;GACtD,KAAKG,QAAQ,IAAI,KAAK;GACtB,KAAKI,eAAe,KAAK;EAC3B;CACF;;;;;;;CAQA,sBAAsB,KAAsC;EAC1D,KAAKL,cAAc,OAAO,GAAG;EAC7B,IAAI,CAAC,KAAKF,WAAW,OAAO,GAAG,GAC7B;EAEF,MAAM,cAAc,CAAC;EACrB,KAAKC,YAAY,IAAI,WAAW;EAChC,IAAI;EACJ,MAAM,gBAAgB;GACpB,IAAI,OACF,KAAKH,cAAc,KAAK;GAE1B,IAAI,KAAKG,YAAY,OAAO,WAAW,GACrC,KAAKO,cAAc;EAEvB;EACA,OAAO;GACL,gBAAgB;IACd,IAAI,KAAKP,YAAY,IAAI,WAAW,KAAK,CAAC,OACxC,QAAQ,KAAKJ,YAAY,SAAS,KAAKD,uBAAuB;GAElE;GACA,QAAQ;EACV;CACF;CAEA,mBAAmB,KAAa;EAC9B,IAAI,KAAKI,WAAW,IAAI,GAAG,GACzB;EAEF,KAAKA,WAAW,IAAI,GAAG;EAIvB,MAAM,SAAS,CAAC,GAAG,KAAKG,OAAO;EAC/B,MAAM,QAAQ,OAAO,MAAK,MAAK,EAAE,QAAQ,KAAK,OAAO;EACrD,IAAI,OACF,KAAKM,aAAa,KAAK;CAE3B;CAEA,QAAQ,SAA4C;EAClD,MAAM,QAAe;GAAC;GAAS,OAAO;GAAU,UAAU;EAAC;EAC3D,KAAKL,OAAO,KAAK,KAAK;EACtB,KAAKI,cAAc;EACnB,OAAO,EAAC,eAAe,KAAKE,SAAS,KAAK,EAAC;CAC7C;CAEA,SAAS,OAAc;EACrB,MAAM,EAAC,OAAO,UAAS;EACvB,MAAM,QAAQ;EACd,IAAI,UAAU,UAAU;GACtB,MAAM,IAAI,KAAKN,OAAO,QAAQ,KAAK;GACnC,IAAI,KAAK,GACP,KAAKA,OAAO,OAAO,GAAG,CAAC;GAEzB,KAAKI,cAAc;EACrB,OAAO,IAAI,UAAU,aAAa,SAAS,KAAKL,QAAQ,IAAI,KAAK,GAAG;GAElE,MAAM,WAAW;GACjB,KAAKI,eAAe,KAAK;EAC3B;CACF;CAEA,eAAe,OAAc;EAC3B,MAAM,QAAQ,KAAKV,kBAAkB;GACnC,IAAI,KAAKM,QAAQ,IAAI,KAAK,GAAG;IAC3B,KAAKV,IAAI,OAAO,iDAAiD;IACjE,KAAKgB,aAAa,KAAK;IACvB,KAAKD,cAAc;GACrB;EACF,GAAG,KAAKb,kBAAkB;CAC5B;CAEA,aAAa,OAAc;EACzB,IAAI,MAAM,OACR,KAAKG,cAAc,MAAM,KAAK;EAEhC,KAAKK,QAAQ,OAAO,KAAK;CAC3B;CAEA,gBAAgB;EACd,OAAO,KAAKE,YAAY,KAAKX,0BAA0B,KAAKU,OAAO,QAAQ;GACzE,MAAM,QAAQ,KAAKA,OAAO,MAAM;GAChC,MAAM,QAAQ;GACd,MAAM,QAAQ;IAAC,UAAU;IAAO,OAAO,KAAA;GAAS;GAChD,KAAKD,QAAQ,IAAI,MAAM,KAAK;GAC5B,KAAKV,IAAI,OAAO,kCAAkC,EAAC,GAAG,KAAK,MAAK,CAAC;GACjE,MAAM,QAAQ,UAAU;EAC1B;EACA,KAAKW,OAAO,SAAS,OAAO,MAAM;GAChC,MAAM,WAAW,IAAI;GACrB,IAAI,MAAM,aAAa,UAAU;IAC/B,MAAM,WAAW;IACjB,MAAM,QAAQ,SAAS,QAAQ;GACjC;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -11,12 +11,12 @@ import { backfillRequestSchema } from "../change-source/protocol/current/upstrea
 import "../change-source/protocol/current.js";
 import { cookieOps } from "../replicator/change-log-cookies.js";
 import "../../types/pg.js";
+import { orTimeout } from "../../types/timeout.js";
 import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
 import { Queue } from "../../../../shared/src/queue.js";
 import { AutoResetSignal, markResetRequired, reinitializeChangeLog } from "./schema/tables.js";
 import { TransactionPool, sharedSnapshot } from "../../db/transaction-pool.js";
-import { orTimeout } from "../../types/timeout.js";
 import { reconstructWatermarkedChange, serializeChangeStreamDataWithChange } from "./change-log-codec.js";
 import { resolver } from "@rocicorp/resolver";
 import "postgres";

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -39,6 +39,31 @@ export declare class ProcessManager {
     allWorkersReady(): Promise<void>;
     logErrorAndExit(err: unknown, name: string): void;
 }
+/**
+ * Runs the supporting subprocesses of a worker (e.g. the litestream process
+ * of the backup-replicator) as a {@link SingletonService}.
+ *
+ * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`
+ * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the
+ * worker's other services, i.e. after they have drained, so that the
+ * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`
+ * and `SIGABRT`), on the other hand, are forwarded immediately once the
+ * service is running, and the subprocesses are killed if the worker exits
+ * before they are stopped.
+ *
+ * `run()` settles when a subprocess exits before {@link stop()} is called,
+ * which stops the worker: a clean exit resolves (and the parent in turn treats
+ * the worker's clean exit outside of a drain as an intentional shutdown),
+ * while any other exit rejects.
+ */
+export declare class SupportingProcesses implements SingletonService {
+    #private;
+    readonly id = "supporting-processes";
+    constructor(lc: LogContext, parent: EventEmitter);
+    add<P extends Subprocess>(proc: P, name: string): P;
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
 export type SelfDrainOptions = {
     /**
      * The time after a self drain is requested at which the server sends itself
@@ -69,12 +94,20 @@ export declare class SelfDrain {
     constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions);
     request(): void;
 }
+/**
+ * The maximum time that {@link runUntilKilled} waits for its services to stop
+ * (i.e. for their `run()` to settle) after stopping them.
+ */
+export declare const STOP_TIMEOUT_MS = 60000;
 /**
  * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
  * an optional {@link SingletonService.drain drain()}, or stopping them
- * without draining for `SIGQUIT`.
+ * without draining for `SIGQUIT`. If a service stops (or throws) on its own,
+ * the remaining services are stopped without draining.
  *
- * @returns a Promise that resolves/rejects when any of the services stops/throws.
+ * @returns a Promise that settles once all services have stopped (or
+ *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting
+ *          with the outcome of the first service to stop.
  */
 export declare function runUntilKilled(lc: LogContext, parent: EventEmitter, ...services: SingletonService[]): Promise<void>;
 export declare function exitAfter(logContext: LogContext | (() => LogContext), run: () => Promise<void>): Promise<void>;

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Cf;AAED,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA0Df;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -3,6 +3,7 @@ import { singleProcessMode } from "../types/processes.js";
 import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { ConfigurationError } from "../types/configuration-error.js";
 import { logLastChanceSQLiteCorruptionDiagnostics } from "../db/sqlite-corruption.js";
+import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
 import { resolver } from "@rocicorp/resolver";
 import { pid } from "node:process";
@@ -153,6 +154,71 @@ var ProcessManager = class {
 	}
 };
 /**
+* Runs the supporting subprocesses of a worker (e.g. the litestream process
+* of the backup-replicator) as a {@link SingletonService}.
+*
+* Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`
+* to its subprocesses. Instead, {@link runUntilKilled} stops it along with the
+* worker's other services, i.e. after they have drained, so that the
+* subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`
+* and `SIGABRT`), on the other hand, are forwarded immediately once the
+* service is running, and the subprocesses are killed if the worker exits
+* before they are stopped.
+*
+* `run()` settles when a subprocess exits before {@link stop()} is called,
+* which stops the worker: a clean exit resolves (and the parent in turn treats
+* the worker's clean exit outside of a drain as an intentional shutdown),
+* while any other exit rejects.
+*/
+var SupportingProcesses = class {
+	id = "supporting-processes";
+	#lc;
+	#parent;
+	#procs = /* @__PURE__ */ new Map();
+	#done = resolver();
+	#signaled;
+	constructor(lc, parent) {
+		this.#lc = lc.withContext("component", "supporting-processes");
+		this.#parent = parent;
+		this.#done.promise.catch(() => {});
+		parent.on("exit", (code) => this.#kill(code === 0 ? "SIGTERM" : code === 14 ? "SIGQUIT" : "SIGABRT"));
+	}
+	add(proc, name) {
+		this.#procs.set(proc, name);
+		proc.on("close", (code, sig) => this.#onClose(proc, name, code, sig));
+		proc.on("error", (err) => this.#lc[this.#signaled ? "warn" : "error"]?.(`error from ${name} ${proc.pid}`, err));
+		return proc;
+	}
+	run() {
+		for (const signal of FORCEFUL_SHUTDOWN) this.#parent.on(signal, () => this.#kill(signal));
+		return this.#done.promise;
+	}
+	async stop() {
+		if (!this.#signaled) this.#kill("SIGTERM");
+		if (this.#procs.size === 0) this.#done.resolve();
+		await this.#done.promise.catch(() => {});
+	}
+	#onClose(proc, name, code, sig) {
+		this.#procs.delete(proc);
+		const exit = sig ? `${name} (${proc.pid}) killed with (${sig})` : `${name} (${proc.pid}) exited with code (${code})`;
+		if (this.#signaled) {
+			this.#lc.info?.(exit);
+			if (this.#procs.size === 0) this.#done.resolve();
+		} else if (code === 0 || code === 14) {
+			this.#lc.info?.(exit);
+			this.#done.resolve();
+		} else this.#done.reject(new Error(exit));
+	}
+	#kill(signal) {
+		this.#signaled = signal;
+		for (const proc of this.#procs.keys()) try {
+			proc.kill(signal);
+		} catch (e) {
+			this.#lc.error?.(e);
+		}
+	}
+};
+/**
 * Coordinates a drain that is initiated by the server itself (e.g. by a
 * view-syncer that has been granted permission to restore a new replica),
 * as opposed to one initiated by the container runner with a `SIGTERM`.
@@ -195,14 +261,27 @@ var SelfDrain = class {
 	}
 };
 /**
+* The maximum time that {@link runUntilKilled} waits for its services to stop
+* (i.e. for their `run()` to settle) after stopping them.
+*/
+var STOP_TIMEOUT_MS = 6e4;
+/**
 * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
 * an optional {@link SingletonService.drain drain()}, or stopping them
-* without draining for `SIGQUIT`.
+* without draining for `SIGQUIT`. If a service stops (or throws) on its own,
+* the remaining services are stopped without draining.
 *
-* @returns a Promise that resolves/rejects when any of the services stops/throws.
+* @returns a Promise that settles once all services have stopped (or
+*          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting
+*          with the outcome of the first service to stop.
 */
 async function runUntilKilled(lc, parent, ...services) {
 	if (services.length === 0) return;
+	const stopped = /* @__PURE__ */ new Set();
+	const runs = services.map((svc) => svc.run().finally(() => stopped.add(svc)));
+	const allStopped = resolver();
+	let stopping;
+	const stopAll = (reason) => stopping ??= stopServices(lc, services, runs, stopped, reason).then(allStopped.resolve);
 	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, async () => {
 		if (GRACEFUL_SHUTDOWN.includes(signal)) {
 			const results = await Promise.allSettled(services.map(async (svc) => {
@@ -213,17 +292,26 @@ async function runUntilKilled(lc, parent, ...services) {
 			}));
 			for (const result of results) if (result.status === "rejected") lc.warn?.(`error while draining (${signal})`, result.reason);
 		}
-		services.forEach(async (svc) => {
-			lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);
-			await svc.stop();
-		});
+		await stopAll(signal);
 	});
 	try {
-		const svc = await Promise.race(services.map((svc) => svc.run().then(() => svc)));
-		lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
+		const svc = await Promise.race([...runs.map((run, i) => run.then(() => services[i])), allStopped.promise]);
+		if (svc) lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
 	} catch (e) {
 		lc.error?.(`exiting on error`, e);
 		throw e;
+	} finally {
+		await stopAll("stopped");
+	}
+}
+async function stopServices(lc, services, runs, stopped, reason) {
+	for (const svc of services) if (!stopped.has(svc)) {
+		lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);
+		svc.stop().catch((e) => lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e));
+	}
+	if (await orTimeout(Promise.allSettled(runs), 6e4) === "timed-out") {
+		const running = services.filter((svc) => !stopped.has(svc));
+		lc.warn?.(`services did not stop within ${STOP_TIMEOUT_MS} ms: ` + running.map((svc) => `${svc.constructor.name} ${svc.id}`).join(", "));
 	}
 }
 async function exitAfter(logContext, run) {
@@ -296,6 +384,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { HeartbeatMonitor, ProcessManager, SelfDrain, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { HeartbeatMonitor, ProcessManager, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#proc","#delayMs","#onRequested","#signaled","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, async () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      if (GRACEFUL_SIGNALS.includes(signal)) {\n        // Drain all services before stopping any of them. Stopping a service\n        // resolves its run() and thus ends runUntilKilled(), so a service\n        // without a drain() must not be stopped while others are draining.\n        const results = await Promise.allSettled(\n          services.map(async svc => {\n            if (svc.drain) {\n              lc.info?.(\n                `draining ${svc.constructor.name} ${svc.id} (${signal})`,\n              );\n              await svc.drain();\n            }\n          }),\n        );\n        for (const result of results) {\n          if (result.status === 'rejected') {\n            lc.warn?.(`error while draining (${signal})`, result.reason);\n          }\n        }\n      }\n      services.forEach(async svc => {\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;AAsCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKc,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKC,YAAY;GACjB,aAAa,KAAKC,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKF,WAC1B;EAEF,KAAKE,aAAa;EAClB,KAAKnB,IAAI,OACP,4CAA4C,KAAKe,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKE,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKD,WAAW;IACnB,KAAKjB,IAAI,OAAO,uBAAuB;IACvC,KAAKc,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,QAAQ,YAAY;EAG9B,IAAI,kBAAiB,SAAS,MAAM,GAAG;GAIrC,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OACD,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EACxD;KACA,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EACA,SAAS,QAAQ,OAAM,QAAO;GAC5B,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKf,MAAM;EACX,KAAKoB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKtB,IAAI,OACP,iCACE,KAAKoB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKpB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKsB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#parent","#procs","#done","#onClose","#signaled","#proc","#delayMs","#onRequested","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`. If a service stops (or throws) on its own,\n * the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, async () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      if (GRACEFUL_SIGNALS.includes(signal)) {\n        // Drain all services before stopping any of them. Stopping a service\n        // resolves its run() and thus ends runUntilKilled(), so a service\n        // without a drain() must not be stopped while others are draining.\n        const results = await Promise.allSettled(\n          services.map(async svc => {\n            if (svc.drain) {\n              lc.info?.(\n                `draining ${svc.constructor.name} ${svc.id} (${signal})`,\n              );\n              await svc.drain();\n            }\n          }),\n        );\n        for (const result of results) {\n          if (result.status === 'rejected') {\n            lc.warn?.(`error while draining (${signal})`, result.reason);\n          }\n        }\n      }\n      await stopAll(signal);\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKc,UAAU;EAEf,KAAKE,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAKT,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAKQ,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAKE,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAKjB,IAAI,KAAKkB,YAAY,SAAS,WACjC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAKJ,QAAQ,GAAG,cAAc,KAAKP,MAAM,MAAM,CAAC;EAElD,OAAO,KAAKS,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAKE,WACR,KAAKX,MAAM,SAAS;EAEtB,IAAI,KAAKQ,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAErB,MAAM,KAAKA,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAKD,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAKG,WAAW;GAClB,KAAKlB,IAAI,OAAO,IAAI;GACpB,IAAI,KAAKe,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAKhB,IAAI,OAAO,IAAI;GACpB,KAAKgB,MAAM,QAAQ;EACrB,OACE,KAAKA,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAKE,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAKH,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAKf,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKmB,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKH,YAAY;GACjB,aAAa,KAAKI,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKL,WAC1B;EAEF,KAAKK,aAAa;EAClB,KAAKvB,IAAI,OACP,4CAA4C,KAAKoB,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKC,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKJ,WAAW;IACnB,KAAKlB,IAAI,OAAO,uBAAuB;IACvC,KAAKmB,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;AAY/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,EAAE,KAC9D,WAAW,OACb;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,QAAQ,YAAY;EAG9B,IAAI,kBAAiB,SAAS,MAAM,GAAG;GAIrC,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OACD,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EACxD;KACA,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EACA,MAAM,QAAQ,MAAM;CACtB,CAAC;CAGH,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,EACL,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,EAAE,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKpB,MAAM;EACX,KAAKwB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAK1B,IAAI,OACP,iCACE,KAAKwB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKxB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK0B,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}

package/out/zero-client/src/client/version.js

@@ -2,7 +2,7 @@
 /**
 * The current version of Zero.
 */
-var version = "1.11.0-canary.39";
+var version = "1.11.0-canary.40";
 //#endregion
 export { version };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "1.11.0-canary.39",
+  "version": "1.11.0-canary.40",
   "description": "Zero is a web framework for serverless web development.",
   "homepage": "https://zero.rocicorp.dev",
   "bugs": {