@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.40

package/out/zero-cache/src/services/change-streamer/upstream-acker.js.map

@@ -1 +1 @@
-{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#maybeAck","#pgChangeLogWatermark","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  reset(upstream: Sink<ChangeSourceUpstream>) {\n    this.#upstream = upstream;\n    this.#lastTx = '';\n    this.#lastStatus = '';\n    this.#lastAck = '';\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        this.#lastTx = downstream[2].watermark;\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;CAEA,MAAM,UAAsC;EAC1C,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;CAClB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKD,cAAc,WAAW,GAAG;IAEnC,KAAKE,UAAU;IACf;GACF,KAAK;IACH,KAAKH,UAAU,WAAW,GAAG;IAC7B,KAAKG,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKC,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKD,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKE,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKF,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKL,eAC3B,KAAKM,wBACL,CAAC,KAAKP,oBACJ,KAAKQ,mBACL,IAAI,KAAKD,uBAAuB,KAAKC,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}
+{"version":3,"file":"upstream-acker.js","names":["#trackPgChangeLog","#trackBackup","#upstream","#lastTx","#lastStatus","#lastAck","#pgChangeLogWatermark","#maybeAck","#backupWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/upstream-acker.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\nimport {max, min} from '../../types/lexi-version.ts';\nimport type {Sink} from '../../types/streams.ts';\nimport type {\n  ChangeSourceUpstream,\n  ChangeStreamMessage,\n} from '../change-source/protocol/current.ts';\n\ntype Opts = {\n  trackPgChangeLog: boolean;\n  trackBackup: boolean;\n};\n\n/**\n * Tracks the progress (watermark) of multiple streams:\n * - downstream transactions and status messages (the latter\n *   indicating LSNs that are not relevant to the publication\n *   but nevertheless need to be ACKed)\n * - PG change-log commits\n * - backup watermarks\n *\n * and sends upstream ACKs accordingly. When both the PG change-log\n * and backup watermarks are being considered for upstream ACKs\n * (i.e. RMv1.5), only watermarks that have been reached by both\n * stores are acked.\n *\n * The UpstreamAcker also takes into account that an upstream\n * connection can be disconnected and {@link reset()}. In this case,\n * the progress of the persistent stores is retained and the acks\n * are resent on the new upstream connection as necessary.\n */\nexport class UpstreamAcker {\n  readonly #trackPgChangeLog: boolean;\n  readonly #trackBackup: boolean;\n\n  #pgChangeLogWatermark = '';\n  #backupWatermark = '';\n\n  #upstream: Sink<ChangeSourceUpstream> | undefined;\n  #lastTx = '';\n  #lastStatus = '';\n  #lastAck = '';\n\n  constructor({trackPgChangeLog, trackBackup}: Opts) {\n    assert(\n      trackPgChangeLog || trackBackup,\n      `At least one of trackPgChangeLog or trackBackup must be true`,\n    );\n    this.#trackPgChangeLog = trackPgChangeLog;\n    this.#trackBackup = trackBackup;\n  }\n\n  /**\n   * Starts tracking a new upstream connection, which resumes the change stream\n   * after `resumeWatermark`.\n   *\n   * Everything at or before `resumeWatermark` was committed on an earlier\n   * connection, so it counts as committed on this one: a status watermark past\n   * it waits for the tracked stores to reach it, as it would for a commit on\n   * this connection. Otherwise the first keepalive of a stream that resumes\n   * ahead of the stores -- a change log ahead of its backup, which is the\n   * normal state -- would move the replication slot past transactions that the\n   * backup has not persisted, and a task restored from the backup would then\n   * resume below the slot, which the upstream moves forward without a word.\n   *\n   * When the PG change log is tracked, the stream resumes from its head, so\n   * `resumeWatermark` has been persisted in it (by this or another\n   * change-streamer). Without this, status messages would not be acked on an\n   * idle upstream until a commit is stored.\n   */\n  reset(upstream: Sink<ChangeSourceUpstream>, resumeWatermark: string) {\n    this.#upstream = upstream;\n    this.#lastTx = resumeWatermark;\n    this.#lastStatus = '';\n    this.#lastAck = '';\n    if (this.#trackPgChangeLog) {\n      this.#pgChangeLogWatermark = max(\n        resumeWatermark,\n        this.#pgChangeLogWatermark,\n      );\n    }\n    // Re-ack what the stores have persisted on the new connection.\n    this.#maybeAck();\n  }\n\n  trackDownstream(downstream: ChangeStreamMessage) {\n    const [tag, msg] = downstream;\n    switch (tag) {\n      case 'status':\n        if (msg.ack) {\n          this.#lastStatus = downstream[2].watermark;\n        }\n        this.#maybeAck();\n        break;\n      case 'commit':\n        // A commit replayed from below the resume watermark does not lower it.\n        this.#lastTx = max(this.#lastTx, downstream[2].watermark);\n        this.#maybeAck();\n        break;\n    }\n  }\n\n  trackPgChangeLog(committedWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#pgChangeLogWatermark = max(\n      committedWatermark,\n      this.#pgChangeLogWatermark,\n    );\n    this.#maybeAck();\n  }\n\n  trackBackup(backedUpWatermark: string) {\n    // Watermarks should never move backwards, but use max() defensively.\n    this.#backupWatermark = max(backedUpWatermark, this.#backupWatermark);\n    this.#maybeAck();\n  }\n\n  #maybeAck() {\n    const currentWatermark = !this.#trackBackup\n      ? this.#pgChangeLogWatermark\n      : !this.#trackPgChangeLog\n        ? this.#backupWatermark\n        : min(this.#pgChangeLogWatermark, this.#backupWatermark);\n    if (currentWatermark > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {tag: 'commit'},\n        {watermark: currentWatermark},\n      ]);\n      this.#lastAck = currentWatermark;\n    }\n    // If all committed transactions have been acked, ack any outstanding\n    // status LSN's thereafter (i.e. non-publication upstream changes).\n    if (this.#lastAck >= this.#lastTx && this.#lastStatus > this.#lastAck) {\n      this.#upstream?.push([\n        'status',\n        {ack: true},\n        {watermark: this.#lastStatus},\n      ]);\n      this.#lastAck = this.#lastStatus;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+BA,IAAa,gBAAb,MAA2B;CACzB;CACA;CAEA,wBAAwB;CACxB,mBAAmB;CAEnB;CACA,UAAU;CACV,cAAc;CACd,WAAW;CAEX,YAAY,EAAC,kBAAkB,eAAoB;EACjD,OACE,oBAAoB,aACpB,8DACF;EACA,KAAKA,oBAAoB;EACzB,KAAKC,eAAe;CACtB;;;;;;;;;;;;;;;;;;;CAoBA,MAAM,UAAsC,iBAAyB;EACnE,KAAKC,YAAY;EACjB,KAAKC,UAAU;EACf,KAAKC,cAAc;EACnB,KAAKC,WAAW;EAChB,IAAI,KAAKL,mBACP,KAAKM,wBAAwB,IAC3B,iBACA,KAAKA,qBACP;EAGF,KAAKC,UAAU;CACjB;CAEA,gBAAgB,YAAiC;EAC/C,MAAM,CAAC,KAAK,OAAO;EACnB,QAAQ,KAAR;GACE,KAAK;IACH,IAAI,IAAI,KACN,KAAKH,cAAc,WAAW,GAAG;IAEnC,KAAKG,UAAU;IACf;GACF,KAAK;IAEH,KAAKJ,UAAU,IAAI,KAAKA,SAAS,WAAW,GAAG,SAAS;IACxD,KAAKI,UAAU;IACf;EACJ;CACF;CAEA,iBAAiB,oBAA4B;EAE3C,KAAKD,wBAAwB,IAC3B,oBACA,KAAKA,qBACP;EACA,KAAKC,UAAU;CACjB;CAEA,YAAY,mBAA2B;EAErC,KAAKC,mBAAmB,IAAI,mBAAmB,KAAKA,gBAAgB;EACpE,KAAKD,UAAU;CACjB;CAEA,YAAY;EACV,MAAM,mBAAmB,CAAC,KAAKN,eAC3B,KAAKK,wBACL,CAAC,KAAKN,oBACJ,KAAKQ,mBACL,IAAI,KAAKF,uBAAuB,KAAKE,gBAAgB;EAC3D,IAAI,mBAAmB,KAAKH,UAAU;GACpC,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,SAAQ;IACd,EAAC,WAAW,iBAAgB;GAC9B,CAAC;GACD,KAAKG,WAAW;EAClB;EAGA,IAAI,KAAKA,YAAY,KAAKF,WAAW,KAAKC,cAAc,KAAKC,UAAU;GACrE,KAAKH,WAAW,KAAK;IACnB;IACA,EAAC,KAAK,KAAI;IACV,EAAC,WAAW,KAAKE,YAAW;GAC9B,CAAC;GACD,KAAKC,WAAW,KAAKD;EACvB;CACF;AACF"}

package/out/zero-cache/src/services/http-service.d.ts

@@ -5,7 +5,7 @@ import type { Service } from './service.ts';
 export type Options = {
     port: number;
     keepaliveTimeoutMs: number | undefined;
-    readinessGate?: Promise<void> | undefined;
+    readinessGate?: Promise<unknown> | undefined;
 };
 /**
  * Common functionality for all HttpServices. These include:
@@ -22,6 +22,12 @@ export declare class HttpService implements Service {
     protected _onStop(): Promise<void>;
     start(): Promise<string>;
     run(): Promise<void>;
+    /**
+     * Responds to subsequent health checks (both "/" and "/keepalive") with a
+     * 503, e.g. to signal the container runner to stop routing to, and shut
+     * down, a server that is draining itself (see {@link SelfDrain}).
+     */
+    failHealthChecks(): void;
     stop(): Promise<void>;
 }
 //# sourceMappingURL=http-service.d.ts.map

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAMxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAoB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAOxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAGf;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}

package/out/zero-cache/src/services/http-service.js

@@ -17,6 +17,7 @@ var HttpService = class {
 	#heartbeatMonitor;
 	#init;
 	#ready = false;
+	#failHealthChecks = false;
 	constructor(id, lc, opts, init) {
 		const { port, keepaliveTimeoutMs, readinessGate = promiseVoid } = opts;
 		this.id = id;
@@ -34,11 +35,13 @@ var HttpService = class {
 	}
 	async start() {
 		this.#fastify.get("/", (_req, res) => {
-			if (this.#ready) res.send("OK");
+			if (this.#failHealthChecks) res.code(503).send("Draining");
+			else if (this.#ready) res.send("OK");
 		});
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
 			this.#heartbeatMonitor?.onHeartbeat(headers);
-			if (this.#ready) res.send("OK");
+			if (this.#failHealthChecks) res.code(503).send("Draining");
+			else if (this.#ready) res.send("OK");
 		});
 		await this.#init(this.#fastify);
 		const address = await this.#fastify.listen({
@@ -53,6 +56,15 @@ var HttpService = class {
 		await this.start();
 		await this._state.stopped();
 	}
+	/**
+	* Responds to subsequent health checks (both "/" and "/keepalive") with a
+	* 503, e.g. to signal the container runner to stop routing to, and shut
+	* down, a server that is draining itself (see {@link SelfDrain}).
+	*/
+	failHealthChecks() {
+		this._lc.info?.(`${this.id}: failing health checks to self drain`);
+		this.#failHealthChecks = true;
+	}
 	async stop() {
 		this._lc.info?.(`${this.id}: no longer accepting connections`);
 		this.#heartbeatMonitor?.stop();

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<void> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CAET,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKI,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKE,QACP,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKC,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init","#ready","#failHealthChecks"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the container runner to stop routing to, and shut\n   * down, a server that is draining itself (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    this._lc.info?.(`${this.id}: failing health checks to self drain`);\n    this.#failHealthChecks = true;\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKA,WAAW,QAAQ;EACxB,KAAKC,QAAQ;EACb,KAAKE,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAKD,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAKE,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAKJ,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAKK,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAKJ,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAKE,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAKG,mBACP,IAAI,KAAK,GAAG,EAAE,KAAK,UAAU;QACxB,IAAI,KAAKD,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAKD,MAAM,KAAKH,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAKA,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAKC;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;EACjE,KAAKI,oBAAoB;CAC3B;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAKH,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAKF,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -15,6 +15,9 @@ import type { SingletonService } from './service.ts';
  * are stopped without draining. `SIGQUIT` is used to represent an
  * intentional shutdown (for which draining is not beneficial), whereas
  * `SIGABRT` is used for unexpected process exits.
+ *
+ * A clean exit (i.e. code 0) of a worker of either type that is not the
+ * result of a drain is treated as an intentional shutdown of the server.
  */
 export type WorkerType = 'user-facing' | 'supporting';
 export declare const GRACEFUL_SHUTDOWN: readonly ['SIGTERM', 'SIGINT'];
@@ -36,12 +39,75 @@ export declare class ProcessManager {
     allWorkersReady(): Promise<void>;
     logErrorAndExit(err: unknown, name: string): void;
 }
+/**
+ * Runs the supporting subprocesses of a worker (e.g. the litestream process
+ * of the backup-replicator) as a {@link SingletonService}.
+ *
+ * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`
+ * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the
+ * worker's other services, i.e. after they have drained, so that the
+ * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`
+ * and `SIGABRT`), on the other hand, are forwarded immediately once the
+ * service is running, and the subprocesses are killed if the worker exits
+ * before they are stopped.
+ *
+ * `run()` settles when a subprocess exits before {@link stop()} is called,
+ * which stops the worker: a clean exit resolves (and the parent in turn treats
+ * the worker's clean exit outside of a drain as an intentional shutdown),
+ * while any other exit rejects.
+ */
+export declare class SupportingProcesses implements SingletonService {
+    #private;
+    readonly id = "supporting-processes";
+    constructor(lc: LogContext, parent: EventEmitter);
+    add<P extends Subprocess>(proc: P, name: string): P;
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
+export type SelfDrainOptions = {
+    /**
+     * The time after a self drain is requested at which the server sends itself
+     * a `SIGTERM`, unless one was received sooner (e.g. from the container
+     * runner).
+     */
+    delayMs: number;
+    /**
+     * Called when a self drain is requested. This should fail health checks
+     * (both readiness and liveness) so that the container runner stops routing
+     * to the server and initiates its normal shutdown process.
+     */
+    onRequested: () => void;
+};
+/**
+ * Coordinates a drain that is initiated by the server itself (e.g. by a
+ * view-syncer that has been granted permission to restore a new replica),
+ * as opposed to one initiated by the container runner with a `SIGTERM`.
+ *
+ * Health checks start failing immediately, upon which an orchestrator such
+ * as Kubernetes stops routing to the server and shuts it down as usual
+ * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,
+ * and for environments without liveness checks, the server sends itself a
+ * `SIGTERM` after the configured delay.
+ */
+export declare class SelfDrain {
+    #private;
+    constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions);
+    request(): void;
+}
+/**
+ * The maximum time that {@link runUntilKilled} waits for its services to stop
+ * (i.e. for their `run()` to settle) after stopping them.
+ */
+export declare const STOP_TIMEOUT_MS = 60000;
 /**
  * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
  * an optional {@link SingletonService.drain drain()}, or stopping them
- * without draining for `SIGQUIT`.
+ * without draining for `SIGQUIT`. If a service stops (or throws) on its own,
+ * the remaining services are stopped without draining.
  *
- * @returns a Promise that resolves/rejects when any of the services stops/throws.
+ * @returns a Promise that settles once all services have stopped (or
+ *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting
+ *          with the outcome of the first service to stop.
  */
 export declare function runUntilKilled(lc: LogContext, parent: EventEmitter, ...services: SingletonService[]): Promise<void>;
 export declare function exitAfter(logContext: LogContext | (() => LogContext), run: () => Promise<void>): Promise<void>;

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAQjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CA8EF;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBA6BzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA0Df;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -1,7 +1,9 @@
+import { AbortError } from "../../../shared/src/abort-error.js";
 import { singleProcessMode } from "../types/processes.js";
 import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../observability/metrics.js";
 import { ConfigurationError } from "../types/configuration-error.js";
 import { logLastChanceSQLiteCorruptionDiagnostics } from "../db/sqlite-corruption.js";
+import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
 import { resolver } from "@rocicorp/resolver";
 import { pid } from "node:process";
@@ -129,8 +131,8 @@ var ProcessManager = class {
 			this.#all.delete(worker);
 		}
 		const pid = worker?.pid ?? process.pid;
+		if (code === 0 && (this.#drainStart === 0 || type === "supporting" && this.#userFacing.size > 0)) code = 14;
 		if (type === "supporting") {
-			if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) code = 14;
 			const log = code === 0 || code === 14 ? "info" : "warn";
 			this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
 			return this.#exit(code);
@@ -139,9 +141,9 @@ var ProcessManager = class {
 		this.#lc[log]?.(sig ? `${name} (${pid}) killed with (${sig})` : `${name} (${pid}) exited with code (${code})`, err ?? "");
 		if (this.#userFacing.size === 0) {
 			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);
-			return this.#exit(this.#drainStart ? 0 : code || -1);
+			return this.#exit(this.#drainStart ? 0 : code);
 		}
-		if (this.#drainStart === 0) return this.#exit(code || -1);
+		if (this.#drainStart === 0) return this.#exit(code);
 	}
 	#kill(workers, signal) {
 		for (const worker of workers) try {
@@ -152,31 +154,164 @@ var ProcessManager = class {
 	}
 };
 /**
+* Runs the supporting subprocesses of a worker (e.g. the litestream process
+* of the backup-replicator) as a {@link SingletonService}.
+*
+* Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`
+* to its subprocesses. Instead, {@link runUntilKilled} stops it along with the
+* worker's other services, i.e. after they have drained, so that the
+* subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`
+* and `SIGABRT`), on the other hand, are forwarded immediately once the
+* service is running, and the subprocesses are killed if the worker exits
+* before they are stopped.
+*
+* `run()` settles when a subprocess exits before {@link stop()} is called,
+* which stops the worker: a clean exit resolves (and the parent in turn treats
+* the worker's clean exit outside of a drain as an intentional shutdown),
+* while any other exit rejects.
+*/
+var SupportingProcesses = class {
+	id = "supporting-processes";
+	#lc;
+	#parent;
+	#procs = /* @__PURE__ */ new Map();
+	#done = resolver();
+	#signaled;
+	constructor(lc, parent) {
+		this.#lc = lc.withContext("component", "supporting-processes");
+		this.#parent = parent;
+		this.#done.promise.catch(() => {});
+		parent.on("exit", (code) => this.#kill(code === 0 ? "SIGTERM" : code === 14 ? "SIGQUIT" : "SIGABRT"));
+	}
+	add(proc, name) {
+		this.#procs.set(proc, name);
+		proc.on("close", (code, sig) => this.#onClose(proc, name, code, sig));
+		proc.on("error", (err) => this.#lc[this.#signaled ? "warn" : "error"]?.(`error from ${name} ${proc.pid}`, err));
+		return proc;
+	}
+	run() {
+		for (const signal of FORCEFUL_SHUTDOWN) this.#parent.on(signal, () => this.#kill(signal));
+		return this.#done.promise;
+	}
+	async stop() {
+		if (!this.#signaled) this.#kill("SIGTERM");
+		if (this.#procs.size === 0) this.#done.resolve();
+		await this.#done.promise.catch(() => {});
+	}
+	#onClose(proc, name, code, sig) {
+		this.#procs.delete(proc);
+		const exit = sig ? `${name} (${proc.pid}) killed with (${sig})` : `${name} (${proc.pid}) exited with code (${code})`;
+		if (this.#signaled) {
+			this.#lc.info?.(exit);
+			if (this.#procs.size === 0) this.#done.resolve();
+		} else if (code === 0 || code === 14) {
+			this.#lc.info?.(exit);
+			this.#done.resolve();
+		} else this.#done.reject(new Error(exit));
+	}
+	#kill(signal) {
+		this.#signaled = signal;
+		for (const proc of this.#procs.keys()) try {
+			proc.kill(signal);
+		} catch (e) {
+			this.#lc.error?.(e);
+		}
+	}
+};
+/**
+* Coordinates a drain that is initiated by the server itself (e.g. by a
+* view-syncer that has been granted permission to restore a new replica),
+* as opposed to one initiated by the container runner with a `SIGTERM`.
+*
+* Health checks start failing immediately, upon which an orchestrator such
+* as Kubernetes stops routing to the server and shuts it down as usual
+* (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,
+* and for environments without liveness checks, the server sends itself a
+* `SIGTERM` after the configured delay.
+*/
+var SelfDrain = class {
+	#lc;
+	#proc;
+	#delayMs;
+	#onRequested;
+	#requested = false;
+	#signaled = false;
+	#timer;
+	constructor(lc, proc, opts) {
+		this.#lc = lc.withContext("component", "self-drain");
+		this.#proc = proc;
+		this.#delayMs = opts.delayMs;
+		this.#onRequested = opts.onRequested;
+		for (const signal of GRACEFUL_SHUTDOWN) proc.once(signal, () => {
+			this.#signaled = true;
+			clearTimeout(this.#timer);
+		});
+	}
+	request() {
+		if (this.#requested || this.#signaled) return;
+		this.#requested = true;
+		this.#lc.info?.(`self drain requested. sending SIGTERM in ${this.#delayMs} ms`);
+		this.#onRequested();
+		this.#timer = setTimeout(() => {
+			if (!this.#signaled) {
+				this.#lc.info?.(`initiating self drain`);
+				this.#proc.emit("SIGTERM", "SIGTERM");
+			}
+		}, this.#delayMs);
+	}
+};
+/**
+* The maximum time that {@link runUntilKilled} waits for its services to stop
+* (i.e. for their `run()` to settle) after stopping them.
+*/
+var STOP_TIMEOUT_MS = 6e4;
+/**
 * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
 * an optional {@link SingletonService.drain drain()}, or stopping them
-* without draining for `SIGQUIT`.
+* without draining for `SIGQUIT`. If a service stops (or throws) on its own,
+* the remaining services are stopped without draining.
 *
-* @returns a Promise that resolves/rejects when any of the services stops/throws.
+* @returns a Promise that settles once all services have stopped (or
+*          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting
+*          with the outcome of the first service to stop.
 */
 async function runUntilKilled(lc, parent, ...services) {
 	if (services.length === 0) return;
-	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, () => {
-		const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN;
-		services.forEach(async (svc) => {
-			if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {
-				lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
-				await svc.drain();
-			}
-			lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);
-			await svc.stop();
-		});
+	const stopped = /* @__PURE__ */ new Set();
+	const runs = services.map((svc) => svc.run().finally(() => stopped.add(svc)));
+	const allStopped = resolver();
+	let stopping;
+	const stopAll = (reason) => stopping ??= stopServices(lc, services, runs, stopped, reason).then(allStopped.resolve);
+	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, async () => {
+		if (GRACEFUL_SHUTDOWN.includes(signal)) {
+			const results = await Promise.allSettled(services.map(async (svc) => {
+				if (svc.drain) {
+					lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
+					await svc.drain();
+				}
+			}));
+			for (const result of results) if (result.status === "rejected") lc.warn?.(`error while draining (${signal})`, result.reason);
+		}
+		await stopAll(signal);
 	});
 	try {
-		const svc = await Promise.race(services.map((svc) => svc.run().then(() => svc)));
-		lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
+		const svc = await Promise.race([...runs.map((run, i) => run.then(() => services[i])), allStopped.promise]);
+		if (svc) lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
 	} catch (e) {
 		lc.error?.(`exiting on error`, e);
 		throw e;
+	} finally {
+		await stopAll("stopped");
+	}
+}
+async function stopServices(lc, services, runs, stopped, reason) {
+	for (const svc of services) if (!stopped.has(svc)) {
+		lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);
+		svc.stop().catch((e) => lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e));
+	}
+	if (await orTimeout(Promise.allSettled(runs), 6e4) === "timed-out") {
+		const running = services.filter((svc) => !stopped.has(svc));
+		lc.warn?.(`services did not stop within ${STOP_TIMEOUT_MS} ms: ` + running.map((svc) => `${svc.constructor.name} ${svc.id}`).join(", "));
 	}
 }
 async function exitAfter(logContext, run) {
@@ -191,7 +326,8 @@ async function exitAfter(logContext, run) {
 			lc.error?.(`exiting with configuration error: ${String(e)}`, e);
 			process.exit(0);
 		}
-		lc.error?.(`exiting with error: ${String(e)}`, e);
+		if (e instanceof AbortError) lc.warn?.(`exiting after abort: ${String(e)}`, e);
+		else lc.error?.(`exiting with error: ${String(e)}`, e);
 		try {
 			logLastChanceSQLiteCorruptionDiagnostics(lc, e);
 		} catch (diagnosticError) {
@@ -248,6 +384,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { HeartbeatMonitor, ProcessManager, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { HeartbeatMonitor, ProcessManager, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code || -1);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    lc.error?.(`exiting with error: ${String(e)}`, e);\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAkCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKO,gBAAgB,KAAK,KAAKR,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,QAAQ,EAAE;EACrD;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAChD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#parent","#procs","#done","#onClose","#signaled","#proc","#delayMs","#onRequested","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`. If a service stops (or throws) on its own,\n * the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, async () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      if (GRACEFUL_SIGNALS.includes(signal)) {\n        // Drain all services before stopping any of them. Stopping a service\n        // resolves its run() and thus ends runUntilKilled(), so a service\n        // without a drain() must not be stopped while others are draining.\n        const results = await Promise.allSettled(\n          services.map(async svc => {\n            if (svc.drain) {\n              lc.info?.(\n                `draining ${svc.constructor.name} ${svc.id} (${signal})`,\n              );\n              await svc.drain();\n            }\n          }),\n        );\n        for (const result of results) {\n          if (result.status === 'rejected') {\n            lc.warn?.(`error while draining (${signal})`, result.reason);\n          }\n        }\n      }\n      await stopAll(signal);\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKc,UAAU;EAEf,KAAKE,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAKT,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAKQ,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAKE,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAKjB,IAAI,KAAKkB,YAAY,SAAS,WACjC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAKJ,QAAQ,GAAG,cAAc,KAAKP,MAAM,MAAM,CAAC;EAElD,OAAO,KAAKS,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAKE,WACR,KAAKX,MAAM,SAAS;EAEtB,IAAI,KAAKQ,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAErB,MAAM,KAAKA,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAKD,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAKG,WAAW;GAClB,KAAKlB,IAAI,OAAO,IAAI;GACpB,IAAI,KAAKe,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAKhB,IAAI,OAAO,IAAI;GACpB,KAAKgB,MAAM,QAAQ;EACrB,OACE,KAAKA,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAKE,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAKH,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAKf,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKmB,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKH,YAAY;GACjB,aAAa,KAAKI,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKL,WAC1B;EAEF,KAAKK,aAAa;EAClB,KAAKvB,IAAI,OACP,4CAA4C,KAAKoB,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKC,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKJ,WAAW;IACnB,KAAKlB,IAAI,OAAO,uBAAuB;IACvC,KAAKmB,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;AAY/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,EAAE,KAC9D,WAAW,OACb;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,QAAQ,YAAY;EAG9B,IAAI,kBAAiB,SAAS,MAAM,GAAG;GAIrC,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OACD,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EACxD;KACA,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EACA,MAAM,QAAQ,MAAM;CACtB,CAAC;CAGH,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,EACL,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,EAAE,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKpB,MAAM;EACX,KAAKwB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAK1B,IAAI,OACP,iCACE,KAAKwB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKxB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK0B,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}