@rocicorp/zero 0.26.1-canary.9 → 0.26.2-canary.1

package/out/zero-cache/src/services/http-service.js

@@ -1,60 +1,62 @@
-import Fastify from "fastify";
 import { promiseVoid } from "../../../shared/src/resolved-promises.js";
-import { HeartbeatMonitor } from "./life-cycle.js";
 import { RunningState } from "./running-state.js";
-class HttpService {
-  id;
-  _lc;
-  #fastify;
-  #port;
-  _state;
-  #heartbeatMonitor;
-  #init;
-  constructor(id, lc, opts, init) {
-    this.id = id;
-    this._lc = lc.withContext("component", this.id);
-    this.#fastify = Fastify();
-    this.#port = opts.port;
-    this.#init = init;
-    this._state = new RunningState(id);
-    this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);
-  }
-  // Life-cycle hooks for subclass implementations
-  _onStart() {
-  }
-  _onStop() {
-    return promiseVoid;
-  }
-  // start() is used in unit tests.
-  // run() is the lifecycle method called by the ServiceRunner.
-  async start() {
-    this.#fastify.get("/", (_req, res) => res.send("OK"));
-    this.#fastify.get("/keepalive", ({ headers }, res) => {
-      this.#heartbeatMonitor.onHeartbeat(headers);
-      return res.send("OK");
-    });
-    await this.#init(this.#fastify);
-    const address = await this.#fastify.listen({
-      host: "::",
-      port: this.#port
-    });
-    this._lc.info?.(`${this.id} listening at ${address}`);
-    this._onStart();
-    return address;
-  }
-  async run() {
-    await this.start();
-    await this._state.stopped();
-  }
-  async stop() {
-    this._lc.info?.(`${this.id}: no longer accepting connections`);
-    this.#heartbeatMonitor.stop();
-    this._state.stop(this._lc);
-    await this.#fastify.close();
-    await this._onStop();
-  }
-}
-export {
-  HttpService
+import { HeartbeatMonitor } from "./life-cycle.js";
+import Fastify from "fastify";
+//#region ../zero-cache/src/services/http-service.ts
+/**
+* Common functionality for all HttpServices. These include:
+* * Responding to health checks at "/"
+* * Tracking optional heartbeats at "/keepalive" and draining when they stop.
+*/
+var HttpService = class {
+	id;
+	_lc;
+	#fastify;
+	#port;
+	_state;
+	#heartbeatMonitor;
+	#init;
+	constructor(id, lc, opts, init) {
+		this.id = id;
+		this._lc = lc.withContext("component", this.id);
+		this.#fastify = Fastify();
+		this.#port = opts.port;
+		this.#init = init;
+		this._state = new RunningState(id);
+		this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);
+	}
+	_onStart() {}
+	_onStop() {
+		return promiseVoid;
+	}
+	async start() {
+		this.#fastify.get("/", (_req, res) => res.send("OK"));
+		this.#fastify.get("/keepalive", ({ headers }, res) => {
+			this.#heartbeatMonitor.onHeartbeat(headers);
+			return res.send("OK");
+		});
+		await this.#init(this.#fastify);
+		const address = await this.#fastify.listen({
+			host: "::",
+			port: this.#port
+		});
+		this._lc.info?.(`${this.id} listening at ${address}`);
+		this._onStart();
+		return address;
+	}
+	async run() {
+		await this.start();
+		await this._state.stopped();
+	}
+	async stop() {
+		this._lc.info?.(`${this.id}: no longer accepting connections`);
+		this.#heartbeatMonitor.stop();
+		this._state.stop(this._lc);
+		await this.#fastify.close();
+		await this._onStop();
+	}
 };
-//# sourceMappingURL=http-service.js.map
+//#endregion
+export { HttpService };
+
+//# sourceMappingURL=http-service.js.map

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = opts.port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => res.send('OK'));\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor.onHeartbeat(headers);\n      return res.send('OK');\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"names":[],"mappings":";;;;AAgBO,MAAM,YAA+B;AAAA,EACjC;AAAA,EACU;AAAA,EACV;AAAA,EACA;AAAA,EACU;AAAA,EACV;AAAA,EACA;AAAA,EAET,YACE,IACA,IACA,MACA,MACA;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;AAC9C,SAAK,WAAW,QAAA;AAChB,SAAK,QAAQ,KAAK;AAClB,SAAK,QAAQ;AACb,SAAK,SAAS,IAAI,aAAa,EAAE;AACjC,SAAK,oBAAoB,IAAI,iBAAiB,KAAK,GAAG;AAAA,EACxD;AAAA;AAAA,EAGU,WAAW;AAAA,EAAC;AAAA,EACZ,UAAyB;AACjC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA,EAGA,MAAM,QAAyB;AAC7B,SAAK,SAAS,IAAI,KAAK,CAAC,MAAM,QAAQ,IAAI,KAAK,IAAI,CAAC;AACpD,SAAK,SAAS,IAAI,cAAc,CAAC,EAAC,QAAA,GAAU,QAAQ;AAClD,WAAK,kBAAkB,YAAY,OAAO;AAC1C,aAAO,IAAI,KAAK,IAAI;AAAA,IACtB,CAAC;AACD,UAAM,KAAK,MAAM,KAAK,QAAQ;AAC9B,UAAM,UAAU,MAAM,KAAK,SAAS,OAAO;AAAA,MACzC,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,IAAA,CACZ;AACD,SAAK,IAAI,OAAO,GAAG,KAAK,EAAE,iBAAiB,OAAO,EAAE;AACpD,SAAK,SAAA;AACL,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,KAAK,MAAA;AACX,UAAM,KAAK,OAAO,QAAA;AAAA,EACpB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,IAAI,OAAO,GAAG,KAAK,EAAE,mCAAmC;AAC7D,SAAK,kBAAkB,KAAA;AACvB,SAAK,OAAO,KAAK,KAAK,GAAG;AACzB,UAAM,KAAK,SAAS,MAAA;AACpB,UAAM,KAAK,QAAA;AAAA,EACb;AACF;"}
+{"version":3,"file":"http-service.js","names":["#fastify","#port","#heartbeatMonitor","#init"],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = opts.port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = new HeartbeatMonitor(this._lc);\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => res.send('OK'));\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor.onHeartbeat(headers);\n      return res.send('OK');\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,MACA,MACA;AACA,OAAK,KAAK;AACV,OAAK,MAAM,GAAG,YAAY,aAAa,KAAK,GAAG;AAC/C,QAAA,UAAgB,SAAS;AACzB,QAAA,OAAa,KAAK;AAClB,QAAA,OAAa;AACb,OAAK,SAAS,IAAI,aAAa,GAAG;AAClC,QAAA,mBAAyB,IAAI,iBAAiB,KAAK,IAAI;;CAIzD,WAAqB;CACrB,UAAmC;AACjC,SAAO;;CAIT,MAAM,QAAyB;AAC7B,QAAA,QAAc,IAAI,MAAM,MAAM,QAAQ,IAAI,KAAK,KAAK,CAAC;AACrD,QAAA,QAAc,IAAI,eAAe,EAAC,WAAU,QAAQ;AAClD,SAAA,iBAAuB,YAAY,QAAQ;AAC3C,UAAO,IAAI,KAAK,KAAK;IACrB;AACF,QAAM,MAAA,KAAW,MAAA,QAAc;EAC/B,MAAM,UAAU,MAAM,MAAA,QAAc,OAAO;GACzC,MAAM;GACN,MAAM,MAAA;GACP,CAAC;AACF,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,UAAU;AACrD,OAAK,UAAU;AACf,SAAO;;CAGT,MAAM,MAAqB;AACzB,QAAM,KAAK,OAAO;AAClB,QAAM,KAAK,OAAO,SAAS;;CAG7B,MAAM,OAAsB;AAC1B,OAAK,IAAI,OAAO,GAAG,KAAK,GAAG,mCAAmC;AAC9D,QAAA,iBAAuB,MAAM;AAC7B,OAAK,OAAO,KAAK,KAAK,IAAI;AAC1B,QAAM,MAAA,QAAc,OAAO;AAC3B,QAAM,KAAK,SAAS"}

package/out/zero-cache/src/services/life-cycle.js

@@ -1,221 +1,189 @@
-import { resolver } from "@rocicorp/resolver";
-import { pid } from "node:process";
 import { singleProcessMode } from "../types/processes.js";
 import { RunningState } from "./running-state.js";
-const GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
-const FORCEFUL_SHUTDOWN = ["SIGQUIT"];
-class ProcessManager {
-  #lc;
-  #userFacing = /* @__PURE__ */ new Set();
-  #all = /* @__PURE__ */ new Set();
-  #exitImpl;
-  #start = Date.now();
-  #ready = [];
-  #runningState = new RunningState("process-manager");
-  #drainStart = 0;
-  constructor(lc, proc) {
-    this.#lc = lc.withContext("component", "process-manager");
-    for (const signal of GRACEFUL_SHUTDOWN) {
-      proc.on(signal, () => this.#startDrain(signal));
-    }
-    proc.on(
-      "exit",
-      (code) => this.#kill(
-        this.#all,
-        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0]
-      )
-    );
-    for (const signal of FORCEFUL_SHUTDOWN) {
-      proc.on(signal, () => this.#exit(-1));
-    }
-    this.#exitImpl = (code) => {
-      if (singleProcessMode()) {
-        return proc.emit("exit", code);
-      }
-      process.exit(code);
-    };
-  }
-  done() {
-    return this.#runningState.stopped();
-  }
-  #exit(code) {
-    this.#lc.info?.("exiting with code", code);
-    this.#runningState.stop(this.#lc);
-    void this.#lc.flush().finally(() => this.#exitImpl(code));
-  }
-  #startDrain(signal = "SIGTERM") {
-    this.#lc.info?.(`initiating drain (${signal})`);
-    this.#drainStart = Date.now();
-    if (this.#userFacing.size) {
-      this.#kill(this.#userFacing, signal);
-    } else {
-      this.#kill(this.#all, signal);
-    }
-  }
-  addSubprocess(proc, type, name) {
-    if (type === "user-facing") {
-      this.#userFacing.add(proc);
-    }
-    this.#all.add(proc);
-    proc.on(
-      "error",
-      (err) => this.#lc.error?.(`error from ${name} ${proc.pid}`, err)
-    );
-    proc.on(
-      "close",
-      (code, signal) => this.#onExit(code, signal, null, type, name, proc)
-    );
-  }
-  #initializing = /* @__PURE__ */ new Map();
-  #nextID = 0;
-  addWorker(worker, type, name) {
-    this.addSubprocess(worker, type, name);
-    const id = ++this.#nextID;
-    this.#initializing.set(id, name);
-    const { promise, resolve } = resolver();
-    this.#ready.push(promise);
-    worker.onceMessageType("ready", () => {
-      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);
-      this.#initializing.delete(id);
-      resolve();
-    });
-    return worker;
-  }
-  initializing() {
-    return [...this.#initializing.values()];
-  }
-  async allWorkersReady() {
-    await Promise.all(this.#ready);
-  }
-  logErrorAndExit(err, name) {
-    this.#onExit(-1, null, err, "user-facing", name, void 0);
-  }
-  #onExit(code, sig, err, type, name, worker) {
-    if (worker) {
-      this.#userFacing.delete(worker);
-      this.#all.delete(worker);
-    }
-    const pid2 = worker?.pid ?? process.pid;
-    if (type === "supporting") {
-      const log2 = code === 0 && this.#userFacing.size === 0 ? "info" : "warn";
-      this.#lc[log2]?.(`${name} (${pid2}) exited with code (${code})`, err ?? "");
-      return this.#exit(log2 === "info" ? code : -1);
-    }
-    const log = this.#drainStart === 0 ? "error" : "warn";
-    if (sig) {
-      this.#lc[log]?.(`${name} (${pid2}) killed with (${sig})`, err ?? "");
-    } else if (code !== 0) {
-      this.#lc[log]?.(`${name} (${pid2}) exited with code (${code})`, err ?? "");
-    } else {
-      this.#lc.info?.(`${name} (${pid2}) exited with code (${code})`);
-    }
-    if (this.#userFacing.size === 0) {
-      this.#lc.info?.(
-        this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`
-      );
-      return this.#exit(0);
-    }
-    if (log === "error") {
-      return this.#exit(code || -1);
-    }
-    return void 0;
-  }
-  #kill(workers, signal) {
-    for (const worker of workers) {
-      try {
-        worker.kill(signal);
-      } catch (e) {
-        this.#lc.error?.(e);
-      }
-    }
-  }
-}
+import { resolver } from "@rocicorp/resolver";
+import { pid } from "node:process";
+//#region ../zero-cache/src/services/life-cycle.ts
+var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
+var FORCEFUL_SHUTDOWN = ["SIGQUIT"];
+/**
+* Handles readiness, termination signals, and coordination of graceful
+* shutdown.
+*/
+var ProcessManager = class {
+	#lc;
+	#userFacing = /* @__PURE__ */ new Set();
+	#all = /* @__PURE__ */ new Set();
+	#exitImpl;
+	#start = Date.now();
+	#ready = [];
+	#runningState = new RunningState("process-manager");
+	#drainStart = 0;
+	constructor(lc, proc) {
+		this.#lc = lc.withContext("component", "process-manager");
+		for (const signal of GRACEFUL_SHUTDOWN) proc.on(signal, () => this.#startDrain(signal));
+		proc.on("exit", (code) => this.#kill(this.#all, code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0]));
+		for (const signal of FORCEFUL_SHUTDOWN) proc.on(signal, () => this.#exit(-1));
+		this.#exitImpl = (code) => {
+			if (singleProcessMode()) return proc.emit("exit", code);
+			process.exit(code);
+		};
+	}
+	done() {
+		return this.#runningState.stopped();
+	}
+	#exit(code) {
+		this.#lc.info?.("exiting with code", code);
+		this.#runningState.stop(this.#lc);
+		this.#lc.flush().finally(() => this.#exitImpl(code));
+	}
+	#startDrain(signal = "SIGTERM") {
+		this.#lc.info?.(`initiating drain (${signal})`);
+		this.#drainStart = Date.now();
+		if (this.#userFacing.size) this.#kill(this.#userFacing, signal);
+		else this.#kill(this.#all, signal);
+	}
+	addSubprocess(proc, type, name) {
+		if (type === "user-facing") this.#userFacing.add(proc);
+		this.#all.add(proc);
+		proc.on("error", (err) => this.#lc.error?.(`error from ${name} ${proc.pid}`, err));
+		proc.on("close", (code, signal) => this.#onExit(code, signal, null, type, name, proc));
+	}
+	#initializing = /* @__PURE__ */ new Map();
+	#nextID = 0;
+	addWorker(worker, type, name) {
+		this.addSubprocess(worker, type, name);
+		const id = ++this.#nextID;
+		this.#initializing.set(id, name);
+		const { promise, resolve } = resolver();
+		this.#ready.push(promise);
+		worker.onceMessageType("ready", () => {
+			this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);
+			this.#initializing.delete(id);
+			resolve();
+		});
+		return worker;
+	}
+	initializing() {
+		return [...this.#initializing.values()];
+	}
+	async allWorkersReady() {
+		await Promise.all(this.#ready);
+	}
+	logErrorAndExit(err, name) {
+		this.#onExit(-1, null, err, "user-facing", name, void 0);
+	}
+	#onExit(code, sig, err, type, name, worker) {
+		if (worker) {
+			this.#userFacing.delete(worker);
+			this.#all.delete(worker);
+		}
+		const pid = worker?.pid ?? process.pid;
+		if (type === "supporting") {
+			const log = code === 0 && this.#userFacing.size === 0 ? "info" : "warn";
+			this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
+			return this.#exit(log === "info" ? code : -1);
+		}
+		const log = this.#drainStart === 0 ? "error" : "warn";
+		if (sig) this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? "");
+		else if (code !== 0) this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
+		else this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);
+		if (this.#userFacing.size === 0) {
+			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);
+			return this.#exit(0);
+		}
+		if (log === "error") return this.#exit(code || -1);
+	}
+	#kill(workers, signal) {
+		for (const worker of workers) try {
+			worker.kill(signal);
+		} catch (e) {
+			this.#lc.error?.(e);
+		}
+	}
+};
+/**
+* Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
+* an optional {@link SingletonService.drain drain()}, or stopping them
+* without draining for `SIGQUIT`.
+*
+* @returns a Promise that resolves/rejects when any of the services stops/throws.
+*/
 async function runUntilKilled(lc, parent, ...services) {
-  if (services.length === 0) {
-    return;
-  }
-  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {
-    parent.once(signal, () => {
-      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN;
-      services.forEach(async (svc) => {
-        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {
-          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
-          await svc.drain();
-        }
-        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);
-        await svc.stop();
-      });
-    });
-  }
-  try {
-    const svc = await Promise.race(
-      services.map((svc2) => svc2.run().then(() => svc2))
-    );
-    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
-  } catch (e) {
-    lc.error?.(`exiting on error`, e);
-    throw e;
-  }
+	if (services.length === 0) return;
+	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, () => {
+		const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN;
+		services.forEach(async (svc) => {
+			if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {
+				lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
+				await svc.drain();
+			}
+			lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);
+			await svc.stop();
+		});
+	});
+	try {
+		const svc = await Promise.race(services.map((svc) => svc.run().then(() => svc)));
+		lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);
+	} catch (e) {
+		lc.error?.(`exiting on error`, e);
+		throw e;
+	}
 }
 async function exitAfter(run) {
-  try {
-    await run();
-    console.info(`pid ${pid} exiting normally`);
-    process.exit(0);
-  } catch (e) {
-    console.error(`pid ${pid} exiting with error`, e);
-    process.exit(-1);
-  }
-}
-const DEFAULT_STOP_INTERVAL_MS = 2e4;
-class HeartbeatMonitor {
-  #stopInterval;
-  #lc;
-  #checkIntervalTimer;
-  #checkImmediateTimer;
-  #lastHeartbeat = 0;
-  constructor(lc, stopInterval = DEFAULT_STOP_INTERVAL_MS) {
-    this.#lc = lc;
-    this.#stopInterval = stopInterval;
-  }
-  onHeartbeat(reqHeaders) {
-    this.#lastHeartbeat = Date.now();
-    if (this.#checkIntervalTimer === void 0) {
-      this.#lc.info?.(
-        `starting heartbeat monitor at ${this.#stopInterval / 1e3} second interval`,
-        reqHeaders
-      );
-      this.#checkIntervalTimer = setInterval(
-        this.#checkStopInterval,
-        this.#stopInterval / 4
-      );
-    }
-  }
-  #checkStopInterval = () => {
-    this.#checkImmediateTimer = setImmediate(() => {
-      this.#checkImmediateTimer = void 0;
-      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;
-      if (timeSinceLastHeartbeat >= this.#stopInterval) {
-        this.#lc.info?.(
-          `last heartbeat received ${timeSinceLastHeartbeat / 1e3} seconds ago. draining.`
-        );
-        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);
-      }
-    });
-  };
-  stop() {
-    clearTimeout(this.#checkIntervalTimer);
-    if (this.#checkImmediateTimer) {
-      clearImmediate(this.#checkImmediateTimer);
-    }
-  }
+	try {
+		await run();
+		console.info(`pid ${pid} exiting normally`);
+		process.exit(0);
+	} catch (e) {
+		console.error(`pid ${pid} exiting with error`, e);
+		process.exit(-1);
+	}
 }
-export {
-  FORCEFUL_SHUTDOWN,
-  GRACEFUL_SHUTDOWN,
-  HeartbeatMonitor,
-  ProcessManager,
-  exitAfter,
-  runUntilKilled
+var DEFAULT_STOP_INTERVAL_MS = 2e4;
+/**
+* The HeartbeatMonitor monitors the cadence heartbeats (e.g. "/keepalive"
+* health checks made to HttpServices) that signal that the server
+* should continue processing requests. When a configurable `stopInterval`
+* elapses without receiving these heartbeats, the monitor initiates a
+* graceful shutdown of the server. This works with common load balancing
+* frameworks such as AWS Elastic Load Balancing.
+*
+* The HeartbeatMonitor is **opt-in** in that it only kicks in after it
+* starts receiving keepalives.
+*/
+var HeartbeatMonitor = class {
+	#stopInterval;
+	#lc;
+	#checkIntervalTimer;
+	#checkImmediateTimer;
+	#lastHeartbeat = 0;
+	constructor(lc, stopInterval = DEFAULT_STOP_INTERVAL_MS) {
+		this.#lc = lc;
+		this.#stopInterval = stopInterval;
+	}
+	onHeartbeat(reqHeaders) {
+		this.#lastHeartbeat = Date.now();
+		if (this.#checkIntervalTimer === void 0) {
+			this.#lc.info?.(`starting heartbeat monitor at ${this.#stopInterval / 1e3} second interval`, reqHeaders);
+			this.#checkIntervalTimer = setInterval(this.#checkStopInterval, this.#stopInterval / 4);
+		}
+	}
+	#checkStopInterval = () => {
+		this.#checkImmediateTimer = setImmediate(() => {
+			this.#checkImmediateTimer = void 0;
+			const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;
+			if (timeSinceLastHeartbeat >= this.#stopInterval) {
+				this.#lc.info?.(`last heartbeat received ${timeSinceLastHeartbeat / 1e3} seconds ago. draining.`);
+				process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);
+			}
+		});
+	};
+	stop() {
+		clearTimeout(this.#checkIntervalTimer);
+		if (this.#checkImmediateTimer) clearImmediate(this.#checkImmediateTimer);
+	}
 };
-//# sourceMappingURL=life-cycle.js.map
+//#endregion
+export { HeartbeatMonitor, ProcessManager, exitAfter, runUntilKilled };
+
+//# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    proc.on('error', err =>\n      this.#lc.error?.(`error from ${name} ${proc.pid}`, err),\n    );\n    proc.on('close', (code, signal) =>\n      this.#onExit(code, signal, null, type, name, proc),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"names":["pid","log","svc"],"mappings":";;;;AA2BO,MAAM,oBAAoB,CAAC,WAAW,QAAQ;AAC9C,MAAM,oBAAoB,CAAC,SAAS;AAMpC,MAAM,eAAe;AAAA,EACjB;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB,2BAAW,IAAA;AAAA,EACX;AAAA,EACA,SAAS,KAAK,IAAA;AAAA,EACd,SAA0B,CAAA;AAAA,EAEnC,gBAAgB,IAAI,aAAa,iBAAiB;AAAA,EAClD,cAAc;AAAA,EAEd,YAAY,IAAgB,MAAoB;AAC9C,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AAKxD,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,YAAY,MAAM,CAAC;AAAA,IAChD;AAGA,SAAK;AAAA,MAAG;AAAA,MAAQ,UACd,KAAK;AAAA,QACH,KAAK;AAAA,QACL,SAAS,IAAI,kBAAkB,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAAA;AAAA,IACzD;AAMF,eAAW,UAAU,mBAAmB;AACtC,WAAK,GAAG,QAAQ,MAAM,KAAK,MAAM,EAAE,CAAC;AAAA,IACtC;AAEA,SAAK,YAAY,CAAC,SAAiB;AACjC,UAAI,qBAAqB;AACvB,eAAO,KAAK,KAAK,QAAQ,IAAI;AAAA,MAC/B;AACA,cAAQ,KAAK,IAAI;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,OAAO;AACL,WAAO,KAAK,cAAc,QAAA;AAAA,EAC5B;AAAA,EAEA,MAAM,MAAc;AAClB,SAAK,IAAI,OAAO,qBAAqB,IAAI;AACzC,SAAK,cAAc,KAAK,KAAK,GAAG;AAChC,SAAK,KAAK,IAAI,MAAA,EAAQ,QAAQ,MAAM,KAAK,UAAU,IAAI,CAAC;AAAA,EAC1D;AAAA,EAEA,YAAY,SAA+B,WAAW;AACpD,SAAK,IAAI,OAAO,qBAAqB,MAAM,GAAG;AAC9C,SAAK,cAAc,KAAK,IAAA;AACxB,QAAI,KAAK,YAAY,MAAM;AACzB,WAAK,MAAM,KAAK,aAAa,MAAM;AAAA,IACrC,OAAO;AACL,WAAK,MAAM,KAAK,MAAM,MAAM;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,cAAc,MAAkB,MAAkB,MAAc;AAC9D,QAAI,SAAS,eAAe;AAC1B,WAAK,YAAY,IAAI,IAAI;AAAA,IAC3B;AACA,SAAK,KAAK,IAAI,IAAI;AAElB,SAAK;AAAA,MAAG;AAAA,MAAS,CAAA,QACf,KAAK,IAAI,QAAQ,cAAc,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG;AAAA,IAAA;AAExD,SAAK;AAAA,MAAG;AAAA,MAAS,CAAC,MAAM,WACtB,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;AAAA,IAAA;AAAA,EAErD;AAAA,EAES,oCAAoB,IAAA;AAAA,EAC7B,UAAU;AAAA,EAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,SAAK,cAAc,QAAQ,MAAM,IAAI;AAErC,UAAM,KAAK,EAAE,KAAK;AAClB,SAAK,cAAc,IAAI,IAAI,IAAI;AAC/B,UAAM,EAAC,SAAS,QAAA,IAAW,SAAA;AAC3B,SAAK,OAAO,KAAK,OAAO;AAExB,WAAO,gBAAgB,SAAS,MAAM;AACpC,WAAK,IAAI,QAAQ,GAAG,IAAI,WAAW,KAAK,IAAA,IAAQ,KAAK,MAAM,MAAM;AACjE,WAAK,cAAc,OAAO,EAAE;AAC5B,cAAA;AAAA,IACF,CAAC;AAED,WAAO;AAAA,EACT;AAAA,EAEA,eAAyB;AACvB,WAAO,CAAC,GAAG,KAAK,cAAc,QAAQ;AAAA,EACxC;AAAA,EAEA,MAAM,kBAAkB;AACtB,UAAM,QAAQ,IAAI,KAAK,MAAM;AAAA,EAC/B;AAAA,EAEA,gBAAgB,KAAc,MAAc;AAE1C,SAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,MAAS;AAAA,EAC5D;AAAA,EAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,QAAI,QAAQ;AACV,WAAK,YAAY,OAAO,MAAM;AAC9B,WAAK,KAAK,OAAO,MAAM;AAAA,IACzB;AAEA,UAAMA,OAAM,QAAQ,OAAO,QAAQ;AAEnC,QAAI,SAAS,cAAc;AAIzB,YAAMC,OAAM,SAAS,KAAK,KAAK,YAAY,SAAS,IAAI,SAAS;AACjE,WAAK,IAAIA,IAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AACxE,aAAO,KAAK,MAAMC,SAAQ,SAAS,OAAO,EAAE;AAAA,IAC9C;AAEA,UAAM,MAAM,KAAK,gBAAgB,IAAI,UAAU;AAC/C,QAAI,KAAK;AACP,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKD,IAAG,kBAAkB,GAAG,KAAK,OAAO,EAAE;AAAA,IACpE,WAAW,SAAS,GAAG;AACrB,WAAK,IAAI,GAAG,IAAI,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,KAAK,OAAO,EAAE;AAAA,IAC1E,OAAO;AACL,WAAK,IAAI,OAAO,GAAG,IAAI,KAAKA,IAAG,uBAAuB,IAAI,GAAG;AAAA,IAC/D;AAGA,QAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,WAAK,IAAI;AAAA,QACP,KAAK,cACD,oCACE,KAAK,QAAQ,KAAK,WACpB,SACA;AAAA,MAAA;AAEN,aAAO,KAAK,MAAM,CAAC;AAAA,IACrB;AAIA,QAAI,QAAQ,SAAS;AACnB,aAAO,KAAK,MAAM,QAAQ,EAAE;AAAA,IAC9B;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,SAA+B,QAAwB;AAC3D,eAAW,UAAU,SAAS;AAC5B,UAAI;AACF,eAAO,KAAK,MAAM;AAAA,MACpB,SAAS,GAAG;AACV,aAAK,IAAI,QAAQ,CAAC;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;AAUA,eAAsB,eACpB,IACA,WACG,UACY;AACf,MAAI,SAAS,WAAW,GAAG;AACzB;AAAA,EACF;AACA,aAAW,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAAG;AACjE,WAAO,KAAK,QAAQ,MAAM;AACxB,YAAM,mBAAmB;AAEzB,eAAS,QAAQ,OAAM,QAAO;AAC5B,YAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;AAClD,aAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,gBAAM,IAAI,MAAA;AAAA,QACZ;AACA,WAAG,OAAO,YAAY,IAAI,YAAY,IAAI,IAAI,IAAI,EAAE,KAAK,MAAM,GAAG;AAClE,cAAM,IAAI,KAAA;AAAA,MACZ,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAEA,MAAI;AAEF,UAAM,MAAM,MAAM,QAAQ;AAAA,MACxB,SAAS,IAAI,CAAAE,SAAOA,KAAI,MAAM,KAAK,MAAMA,IAAG,CAAC;AAAA,IAAA;AAE/C,OAAG,OAAO,GAAG,IAAI,YAAY,IAAI,KAAK,IAAI,EAAE,WAAW;AAAA,EACzD,SAAS,GAAG;AACV,OAAG,QAAQ,oBAAoB,CAAC;AAChC,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,UAAU,KAA0B;AACxD,MAAI;AACF,UAAM,IAAA;AAEN,YAAQ,KAAK,OAAO,GAAG,mBAAmB;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB,SAAS,GAAG;AAEV,YAAQ,MAAM,OAAO,GAAG,uBAAuB,CAAC;AAChD,YAAQ,KAAK,EAAE;AAAA,EACjB;AACF;AAEA,MAAM,2BAA2B;AAa1B,MAAM,iBAAiB;AAAA,EACnB;AAAA,EAET;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,SAAK,MAAM;AACX,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,YAAY,YAAiC;AAC3C,SAAK,iBAAiB,KAAK,IAAA;AAC3B,QAAI,KAAK,wBAAwB,QAAW;AAC1C,WAAK,IAAI;AAAA,QACP,iCACE,KAAK,gBAAgB,GACvB;AAAA,QACA;AAAA,MAAA;AAIF,WAAK,sBAAsB;AAAA,QACzB,KAAK;AAAA,QACL,KAAK,gBAAgB;AAAA,MAAA;AAAA,IAEzB;AAAA,EACF;AAAA,EAEA,qBAAqB,MAAM;AAsBzB,SAAK,uBAAuB,aAAa,MAAM;AAC7C,WAAK,uBAAuB;AAC5B,YAAM,yBAAyB,KAAK,IAAA,IAAQ,KAAK;AACjD,UAAI,0BAA0B,KAAK,eAAe;AAChD,aAAK,IAAI;AAAA,UACP,2BACE,yBAAyB,GAC3B;AAAA,QAAA;AAEF,gBAAQ,KAAK,QAAQ,KAAK,kBAAkB,CAAC,CAAC;AAAA,MAChD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,OAAO;AACL,iBAAa,KAAK,mBAAmB;AACrC,QAAI,KAAK,sBAAsB;AAC7B,qBAAe,KAAK,oBAAoB;AAAA,IAC1C;AAAA,EACF;AACF;"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` to all workers. For this signal, workers are\n    // stopped immediately without draining. See `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#exit(-1));\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: 'SIGTERM' | 'SIGINT' = 'SIGTERM') {\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    proc.on('error', err =>\n      this.#lc.error?.(`error from ${name} ${proc.pid}`, err),\n    );\n    proc.on('close', (code, signal) =>\n      this.#onExit(code, signal, null, type, name, proc),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown | null,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // The replication-manager has no user-facing workers.\n      // In this case, code === 0 shutdowns are not errors.\n      // Non-zero exits are warnings (not errors) since they're often transient issues.\n      const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(log === 'info' ? code : -1);\n    }\n\n    const log = this.#drainStart === 0 ? 'error' : 'warn';\n    if (sig) {\n      this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n    } else if (code !== 0) {\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n    } else {\n      this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n    }\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    // Exit only if not draining. If a user-facing worker exits unexpectedly\n    // during a drain, log a warning but let other user-facing workers drain.\n    if (log === 'error') {\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n  try {\n    await run();\n    // oxlint-disable-next-line no-console\n    console.info(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    // oxlint-disable-next-line no-console\n    console.error(`pid ${pid} exiting with error`, e);\n    process.exit(-1);\n  }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;AA2BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,UAAU;;;;;AAM5C,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IAAI,kBAAkB,KAAK,kBAAkB,GACvD,CACF;AAKD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,KAAW,GAAG,CAAC;AAGvC,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,SAA+B,WAAW;AACpD,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;AAEnB,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,QAAQ,cAAc,KAAK,GAAG,KAAK,OAAO,IAAI,CACxD;AACD,OAAK,GAAG,UAAU,MAAM,WACtB,MAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK,CACnD;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;GAIzB,MAAM,MAAM,SAAS,KAAK,MAAA,WAAiB,SAAS,IAAI,SAAS;AACjE,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,QAAQ,SAAS,OAAO,GAAG;;EAG/C,MAAM,MAAM,MAAA,eAAqB,IAAI,UAAU;AAC/C,MAAI,IACF,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,IAAI,OAAO,GAAG;WAC1D,SAAS,EAClB,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;MAEzE,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,GAAG;AAIhE,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAKtB,MAAI,QAAQ,QACV,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}