@rocicorp/zero 1.12.0-canary.10 → 1.12.0-canary.11

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -3,11 +3,11 @@ import { parse } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
 import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
+import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
-import { id } from "../../../types/sql.js";
 import { Queue } from "../../../../../shared/src/queue.js";
 import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -2,6 +2,7 @@ import { assert } from "../../../../../shared/src/asserts.js";
 import { must } from "../../../../../shared/src/must.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
 import { ALLOWED_APP_ID_CHARACTERS } from "../../../types/shards.js";
+import { id } from "../../../types/sql.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { connectPgClient } from "../../../types/pg.js";
 import { getOrCreateCounter, getOrCreateHistogram } from "../../../observability/metrics.js";
@@ -16,7 +17,6 @@ import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { runTx } from "../../../db/run-transaction.js";
 import { getPublicationInfo, warnForSkippedIndexes } from "./schema/published.js";
-import { id } from "../../../types/sql.js";
 import "./schema/replica-stage-enum.js";
 import { dropShard, getInternalShardConfig, getReplicaState, initInitialSyncReplica, validatePublications } from "./schema/shard.js";
 import { createLiteIndexStatement, createLiteTableStatement } from "../../../db/create.js";

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -1,10 +1,10 @@
 import { sleep } from "../../../../../../shared/src/sleep.js";
 import { childWorker } from "../../../../types/processes.js";
+import { id, lit } from "../../../../types/sql.js";
 import "../../../../types/pg.js";
 import { getOrCreateHistogram } from "../../../../observability/metrics.js";
 import { SLOT_KEEPER_URL } from "../../../../server/worker-urls.js";
 import { fromBigInt, toBigInt } from "../lsn.js";
-import { id, lit } from "../../../../types/sql.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { literalUnion, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { publishedSchema, publishedSchemaQuery } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { publishedSchema, publishedSchemaQuery } from "./published.js";
 import { literal } from "pg-format";
 //#region ../zero-cache/src/services/change-source/pg/schema/ddl.ts
 var PROTOCOL_VERSION = 1;

package/out/zero-cache/src/services/change-source/pg/schema/init.js

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { publishedSchema } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { publishedSchema } from "./published.js";
 import "./replica-stage-enum.js";
 import { getMutationsTableDefinition, legacyReplicationSlot, metadataPublicationName, setupTablesAndReplication, setupTriggers } from "./shard.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/schema/shard.js

@@ -2,8 +2,8 @@ import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { jsonObjectSchema, stringify } from "../../../../../../shared/src/bigint-json.js";
 import { appSchema, check, upstreamSchema } from "../../../../types/shards.js";
-import { getPublicationInfo, publishedSchema } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { getPublicationInfo, publishedSchema } from "./published.js";
 import { createEventFunctionStatements, createEventTriggerStatements } from "./ddl.js";
 import { replicaStageSchema } from "./replica-stage-enum.js";
 import { validate } from "./validation.js";

package/out/zero-cache/src/services/http-service.d.ts

@@ -28,6 +28,12 @@ export declare class HttpService implements Service {
      * is draining (see {@link SelfDrain}).
      */
     failHealthChecks(): void;
+    /**
+     * Responds to subsequent readiness checks ("/") with a 503, while liveness
+     * checks ("/keepalive") continue to succeed, e.g. to signal the load
+     * balancer to stop routing to a server that is about to drain.
+     */
+    failReadiness(): void;
     /**
      * Stops accepting new connections (e.g. when draining), while existing
      * connections, including websockets, continue to be served until

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAOxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAKf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAQxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAKf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}

package/out/zero-cache/src/services/http-service.js

@@ -17,6 +17,7 @@ var HttpService = class {
 	#heartbeatMonitor;
 	#init;
 	#ready = false;
+	#failReadiness = false;
 	#failHealthChecks = false;
 	constructor(id, lc, opts, init) {
 		const { port, keepaliveTimeoutMs, readinessGate = promiseVoid } = opts;
@@ -35,7 +36,7 @@ var HttpService = class {
 	}
 	async start() {
 		this.#fastify.get("/", (_req, res) => {
-			if (this.#failHealthChecks) res.code(503).send("Draining");
+			if (this.#failReadiness || this.#failHealthChecks) res.code(503).send("Draining");
 			else if (this.#ready) res.send("OK");
 		});
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
@@ -68,6 +69,17 @@ var HttpService = class {
 		}
 	}
 	/**
+	* Responds to subsequent readiness checks ("/") with a 503, while liveness
+	* checks ("/keepalive") continue to succeed, e.g. to signal the load
+	* balancer to stop routing to a server that is about to drain.
+	*/
+	failReadiness() {
+		if (!this.#failReadiness) {
+			this._lc.info?.(`${this.id}: failing readiness checks before draining`);
+			this.#failReadiness = true;
+		}
+	}
+	/**
 	* Stops accepting new connections (e.g. when draining), while existing
 	* connections, including websockets, continue to be served until
 	* {@link stop} is called.

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the load balancer to stop routing to a server that\n   * is draining (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    if (!this.#failHealthChecks) {\n      this._lc.info?.(`${this.id}: failing health checks while draining`);\n      this.#failHealthChecks = true;\n    }\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,IAAI,CAAC,KAAK,mBAAmB;GAC3B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,uCAAuC;GAClE,KAAK,oBAAoB;EAC3B;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failReadiness = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failReadiness || this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the load balancer to stop routing to a server that\n   * is draining (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    if (!this.#failHealthChecks) {\n      this._lc.info?.(`${this.id}: failing health checks while draining`);\n      this.#failHealthChecks = true;\n    }\n  }\n\n  /**\n   * Responds to subsequent readiness checks (\"/\") with a 503, while liveness\n   * checks (\"/keepalive\") continue to succeed, e.g. to signal the load\n   * balancer to stop routing to a server that is about to drain.\n   */\n  failReadiness() {\n    if (!this.#failReadiness) {\n      this._lc.info?.(`${this.id}: failing readiness checks before draining`);\n      this.#failReadiness = true;\n    }\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,iBAAiB;CACjB,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,kBAAkB,KAAK,mBAC9B,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,IAAI,CAAC,KAAK,mBAAmB;GAC3B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,uCAAuC;GAClE,KAAK,oBAAoB;EAC3B;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,CAAC,KAAK,gBAAgB;GACxB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,2CAA2C;GACtE,KAAK,iBAAiB;EACxB;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -1,3 +1,4 @@
+import { EventEmitter as NodeEventEmitter } from 'node:events';
 import type { IncomingHttpHeaders } from 'node:http';
 import type { EventEmitter } from 'stream';
 import type { LogContext } from '@rocicorp/logger';
@@ -94,6 +95,24 @@ export declare class SelfDrain {
     constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions);
     request(): void;
 }
+/**
+ * The event that a {@link PreDrain} emits immediately upon the first graceful
+ * shutdown signal, with that signal as its argument, before the delay.
+ */
+export declare const PRE_DRAIN = "preDrain";
+/**
+ * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+ * balancers can observe a failed readiness check before the server begins
+ * to drain. The first graceful signal is announced immediately with a
+ * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+ * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+ * delay has elapsed. Repeated graceful signals are dropped. All other events,
+ * including forceful signals, are forwarded immediately.
+ */
+export declare class PreDrain extends NodeEventEmitter {
+    #private;
+    constructor(lc: LogContext, proc: EventEmitter, delayMs: number);
+}
 /**
  * The maximum time that {@link runUntilKilled} waits for its services to stop
  * (i.e. for their `run()` to settle) after stopping them.

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,SAAS,aAAa,CAAC;AAEpC;;;;;;;;GAQG;AACH,qBAAa,QAAS,SAAQ,gBAAgB;;IAQ5C,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAW9D;CA4BF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -7,6 +7,7 @@ import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../o
 import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
 import { pid } from "node:process";
+import { EventEmitter } from "node:events";
 //#region ../zero-cache/src/services/life-cycle.ts
 var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
 var FORCEFUL_SHUTDOWN = ["SIGQUIT", "SIGABRT"];
@@ -261,6 +262,57 @@ var SelfDrain = class {
 	}
 };
 /**
+* The event that a {@link PreDrain} emits immediately upon the first graceful
+* shutdown signal, with that signal as its argument, before the delay.
+*/
+var PRE_DRAIN = "preDrain";
+/**
+* Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+* balancers can observe a failed readiness check before the server begins
+* to drain. The first graceful signal is announced immediately with a
+* {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+* as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+* delay has elapsed. Repeated graceful signals are dropped. All other events,
+* including forceful signals, are forwarded immediately.
+*/
+var PreDrain = class extends EventEmitter {
+	#lc;
+	#proc;
+	#delayMs;
+	#forwarded = /* @__PURE__ */ new Set(["newListener", PRE_DRAIN]);
+	#signaled = false;
+	#timer;
+	constructor(lc, proc, delayMs) {
+		super();
+		this.#lc = lc.withContext("component", "pre-drain");
+		this.#proc = proc;
+		this.#delayMs = delayMs;
+		for (const signal of GRACEFUL_SHUTDOWN) {
+			this.#forwarded.add(signal);
+			proc.on(signal, () => this.#onGracefulSignal(signal));
+		}
+		this.on("newListener", (event) => this.#forward(event));
+	}
+	#forward(event) {
+		if (this.#forwarded.has(event)) return;
+		this.#forwarded.add(event);
+		this.#proc.on(event, (...args) => {
+			if (event === "disconnect" || FORCEFUL_SHUTDOWN.includes(event)) clearTimeout(this.#timer);
+			this.emit(event, ...args);
+		});
+	}
+	#onGracefulSignal(signal) {
+		if (this.#signaled) {
+			this.#lc.info?.(`already shutting down. ignoring ${signal}`);
+			return;
+		}
+		this.#signaled = true;
+		this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);
+		this.emit(PRE_DRAIN, signal);
+		this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);
+	}
+};
+/**
 * The maximum time that {@link runUntilKilled} waits for its services to stop
 * (i.e. for their `run()` to settle) after stopping them.
 */
@@ -397,6 +449,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, PRE_DRAIN, PreDrain, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":[],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["NodeEventEmitter"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import {EventEmitter as NodeEventEmitter} from 'node:events';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The event that a {@link PreDrain} emits immediately upon the first graceful\n * shutdown signal, with that signal as its argument, before the delay.\n */\nexport const PRE_DRAIN = 'preDrain';\n\n/**\n * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load\n * balancers can observe a failed readiness check before the server begins\n * to drain. The first graceful signal is announced immediately with a\n * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such\n * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the\n * delay has elapsed. Repeated graceful signals are dropped. All other events,\n * including forceful signals, are forwarded immediately.\n */\nexport class PreDrain extends NodeEventEmitter {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #forwarded = new Set<string | symbol>(['newListener', PRE_DRAIN]);\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, delayMs: number) {\n    super();\n    this.#lc = lc.withContext('component', 'pre-drain');\n    this.#proc = proc;\n    this.#delayMs = delayMs;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      this.#forwarded.add(signal);\n      proc.on(signal, () => this.#onGracefulSignal(signal));\n    }\n    this.on('newListener', (event: string | symbol) => this.#forward(event));\n  }\n\n  #forward(event: string | symbol) {\n    if (this.#forwarded.has(event)) {\n      return;\n    }\n    this.#forwarded.add(event);\n    this.#proc.on(event, (...args: unknown[]) => {\n      if (\n        event === 'disconnect' ||\n        (FORCEFUL_SHUTDOWN as readonly (string | symbol)[]).includes(event)\n      ) {\n        clearTimeout(this.#timer); // Stopping without a drain.\n      }\n      this.emit(event, ...args);\n    });\n  }\n\n  #onGracefulSignal(signal: GracefulShutdownSignal) {\n    if (this.#signaled) {\n      this.#lc.info?.(`already shutting down. ignoring ${signal}`);\n      return;\n    }\n    this.#signaled = true;\n    this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);\n    this.emit(PRE_DRAIN, signal);\n    this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,YAAY;;;;;;;;;;AAWzB,IAAa,WAAb,cAA8BA,aAAiB;CAC7C;CACA;CACA;CACA,6BAAsB,IAAI,IAAqB,CAAC,eAAe,SAAS,CAAC;CACzE,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,SAAiB;EAC/D,MAAM;EACN,KAAK,MAAM,GAAG,YAAY,aAAa,WAAW;EAClD,KAAK,QAAQ;EACb,KAAK,WAAW;EAEhB,KAAK,MAAM,UAAU,mBAAmB;GACtC,KAAK,WAAW,IAAI,MAAM;GAC1B,KAAK,GAAG,cAAc,KAAK,kBAAkB,MAAM,CAAC;EACtD;EACA,KAAK,GAAG,gBAAgB,UAA2B,KAAK,SAAS,KAAK,CAAC;CACzE;CAEA,SAAS,OAAwB;EAC/B,IAAI,KAAK,WAAW,IAAI,KAAK,GAC3B;EAEF,KAAK,WAAW,IAAI,KAAK;EACzB,KAAK,MAAM,GAAG,QAAQ,GAAG,SAAoB;GAC3C,IACE,UAAU,gBACT,kBAAmD,SAAS,KAAK,GAElE,aAAa,KAAK,MAAM;GAE1B,KAAK,KAAK,OAAO,GAAG,IAAI;EAC1B,CAAC;CACH;CAEA,kBAAkB,QAAgC;EAChD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,mCAAmC,QAAQ;GAC3D;EACF;EACA,KAAK,YAAY;EACjB,KAAK,IAAI,OAAO,YAAY,OAAO,gBAAgB,KAAK,SAAS,IAAI;EACrE,KAAK,KAAK,WAAW,MAAM;EAC3B,KAAK,SAAS,iBAAiB,KAAK,KAAK,QAAQ,MAAM,GAAG,KAAK,QAAQ;CACzE;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -4,6 +4,7 @@ import { AbortError } from "../../../../shared/src/abort-error.js";
 import { must } from "../../../../shared/src/must.js";
 import { mapEntries } from "../../../../shared/src/objects.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
+import { id } from "../../types/sql.js";
 import { ZERO_VERSION_COLUMN_NAME } from "./schema/constants.js";
 import { liteRow } from "../../types/lite.js";
 import { liteTableName } from "../../types/names.js";
@@ -14,7 +15,6 @@ import { computeZqlSpecs, listTables } from "../../db/lite-tables.js";
 import { BackfillingTracker } from "./schema/backfilling.js";
 import { ChangeLog } from "./schema/change-log.js";
 import { updateReplicationWatermark } from "./schema/replication-state.js";
-import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
 import { acceptBackfill } from "../change-source/protocol/backfill-progress.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";