npm - @rocicorp/zero - Versions diffs - 1.2.0 → 1.3.0-canary.0 - Mend

@rocicorp/zero 1.2.0 → 1.3.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (303) hide show

package/out/zero-cache/src/services/life-cycle.js CHANGED Viewed

@@ -4,7 +4,8 @@ import { resolver } from "@rocicorp/resolver";
 import { pid } from "node:process";
 //#region ../zero-cache/src/services/life-cycle.ts
 var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
-var FORCEFUL_SHUTDOWN = ["SIGQUIT"];
+var FORCEFUL_SHUTDOWN = ["SIGQUIT", "SIGABRT"];
+var INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;
 /**
 * Handles readiness, termination signals, and coordination of graceful
 * shutdown.
@@ -21,8 +22,8 @@ var ProcessManager = class {
 	constructor(lc, proc) {
 		this.#lc = lc.withContext("component", "process-manager");
 		for (const signal of GRACEFUL_SHUTDOWN) proc.on(signal, () => this.#startDrain(signal));
-		proc.on("exit", (code) => this.#kill(this.#all, code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0]));
-		for (const signal of FORCEFUL_SHUTDOWN) proc.on(signal, () => this.#exit(-1));
+		proc.on("exit", (code) => this.#kill(this.#all, code === 0 ? "SIGTERM" : code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "SIGQUIT" : "SIGABRT"));
+		for (const signal of FORCEFUL_SHUTDOWN) proc.on(signal, () => this.#exit(signal === "SIGQUIT" ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1));
 		this.#exitImpl = (code) => {
 			if (singleProcessMode()) return proc.emit("exit", code);
 			process.exit(code);
@@ -36,7 +37,7 @@ var ProcessManager = class {
 		this.#runningState.stop(this.#lc);
 		this.#lc.flush().finally(() => this.#exitImpl(code));
 	}
-	#startDrain(signal = "SIGTERM") {
+	#startDrain(signal) {
 		this.#lc.info?.(`initiating drain (${signal})`);
 		this.#drainStart = Date.now();
 		if (this.#userFacing.size) this.#kill(this.#userFacing, signal);
@@ -83,19 +84,18 @@ var ProcessManager = class {
 		}
 		const pid = worker?.pid ?? process.pid;
 		if (type === "supporting") {
-			const log = code === 0 && this.#userFacing.size === 0 ? "info" : "warn";
+			if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) code = INTENTIONAL_SHUTDOWN_ERROR_CODE;
+			const log = code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "info" : "warn";
 			this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
-			return this.#exit(log === "info" || code !== 0 ? code : -1);
+			return this.#exit(code);
 		}
-		const log = this.#drainStart > 0 || code === 13 ? "warn" : "error";
-		if (sig) this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? "");
-		else if (code !== 0) this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
-		else this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);
+		const log = code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "info" : this.#drainStart > 0 || code === 13 ? "warn" : "error";
+		this.#lc[log]?.(sig ? `${name} (${pid}) killed with (${sig})` : `${name} (${pid}) exited with code (${code})`, err ?? "");
 		if (this.#userFacing.size === 0) {
 			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);
 			return this.#exit(0);
 		}
-		if (log === "error") return this.#exit(code || -1);
+		if (this.#drainStart === 0) return this.#exit(code || -1);
 	}
 	#kill(workers, signal) {
 		for (const worker of workers) try {

package/out/zero-cache/src/services/life-cycle.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/*\n * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n \n * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n \n For other kill signals, such as `SIGQUIT`, all workers\n * are stopped without draining. Additionally, if any worker exits\n * unexpectedly, all workers sent an immediate `SIGQUIT` signal.\n /\nexport type WorkerType = 'user-facing' \| 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT'] as const;\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n/\n Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n /\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0 ? GRACEFUL_SHUTDOWN[0] : FORCEFUL_SHUTDOWN[0],\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` to all workers. For this signal, workers are\n // stopped immediately without draining. See `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#exit(-1));\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n done() {\n return this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: 'SIGTERM' \| 'SIGINT' = 'SIGTERM') {\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[isOpen ? 'error' : 'warn']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals \| null,\n err: unknown \| null,\n type: WorkerType,\n name: string,\n worker: Subprocess \| undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // The replication-manager has no user-facing workers.\n // In this case, code === 0 shutdowns are not errors.\n // Non-zero exits are warnings (not errors) since they're often transient issues.\n const log = code === 0 && this.#userFacing.size === 0 ? 'info' : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(log === 'info' \|\| code !== 0 ? code : -1);\n }\n\n const log =\n this.#drainStart > 0 \|\| code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n if (sig) {\n this.#lc[log]?.(`${name} (${pid}) killed with (${sig})`, err ?? '');\n } else if (code !== 0) {\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n } else {\n this.#lc.info?.(`${name} (${pid}) exited with code (${code})`);\n }\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(0);\n }\n\n // Exit only if not draining. If a user-facing worker exits unexpectedly\n // during a drain, log a warning but let other user-facing workers drain.\n if (log === 'error') {\n return this.#exit(code \|\| -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/\n Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n \n @returns a Promise that resolves/rejects when any of the services stops/throws.\n /\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n try {\n await run();\n // oxlint-disable-next-line no-console\n console.info(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n // oxlint-disable-next-line no-console\n console.error(`pid ${pid} exiting with error`, e);\n process.exit(-1);\n }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/\n The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n \n The HeartbeatMonitor is opt-in in that it only kicks in after it\n * starts receiving keepalives.\n /\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout \| undefined;\n #checkImmediateTimer: NodeJS.Immediate \| undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run before the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop after\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;AA2BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,UAAU;;;;;AAW5C,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IAAI,kBAAkB,KAAK,kBAAkB,GACvD,CACF;AAKD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,KAAW,GAAG,CAAC;AAGvC,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,SAA+B,WAAW;AACpD,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;GAIzB,MAAM,MAAM,SAAS,KAAK,MAAA,WAAiB,SAAS,IAAI,SAAS;AACjE,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,QAAQ,UAAU,SAAS,IAAI,OAAO,GAAG;;EAG7D,MAAM,MACJ,MAAA,aAAmB,KAAK,SAAA,KACpB,SACA;AACN,MAAI,IACF,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,IAAI,OAAO,GAAG;WAC1D,SAAS,EAClB,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;MAEzE,OAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,GAAG;AAIhE,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAKtB,MAAI,QAAQ,QACV,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
1	+ {"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/*\n * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n \n * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n \n For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n /\nexport type WorkerType = 'user-facing' \| 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nconst INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/\n Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n /\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0\n ? 'SIGTERM' // graceful, drained shutdown\n : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'SIGQUIT' // intentional abort without drain\n : 'SIGABRT', // unintentional shutdown, alertable error\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n // shutdown) to all workers. For these signals, workers are stopped\n // immediately without draining, since there is no merit to slowly draining\n // when supporting workers have stopped.\n //\n // The logic for handling these signals is in `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () =>\n this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n );\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n done() {\n return this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: GracefulShutdownSignal) {\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[isOpen ? 'error' : 'warn']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals \| null,\n err: unknown,\n type: WorkerType,\n name: string,\n worker: Subprocess \| undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // Supporting workers like the replication-manager shut down without a\n // drain signal when receiving protocol-specific instructions (like auto\n // reset). In this case, a special error code is used to signal that the\n // server should be shut down without draining, but it is otherwise not\n // considered an unexpected/alertable error.\n if (code === 0 && (this.#drainStart === 0 \|\| this.#userFacing.size > 0)) {\n code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n }\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(code);\n }\n\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : this.#drainStart > 0 \|\| code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n this.#lc[log]?.(\n sig\n ? `${name} (${pid}) killed with (${sig})`\n : `${name} (${pid}) exited with code (${code})`,\n err ?? '',\n );\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(0);\n }\n\n if (this.#drainStart === 0) {\n // If a user-facing worker exits without receiving a drain signal,\n // shutdown the server.\n return this.#exit(code \|\| -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/\n Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n \n @returns a Promise that resolves/rejects when any of the services stops/throws.\n /\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n try {\n await run();\n // oxlint-disable-next-line no-console\n console.info(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n // oxlint-disable-next-line no-console\n console.error(`pid ${pid} exiting with error`, e);\n process.exit(-1);\n }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/\n The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n \n The HeartbeatMonitor is opt-in in that it only kicks in after it\n * starts receiving keepalives.\n /\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout \| undefined;\n #checkImmediateTimer: NodeJS.Immediate \| undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run before the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop after\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;AAYvD,IAAM,kCAAkC;;;;;AAMxC,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAS,kCACP,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAY,kCAAkC,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAO;GAET,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/litestream/commands.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;~~AAiKD~~,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}
1	+ {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA2KD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}

package/out/zero-cache/src/services/litestream/commands.js CHANGED Viewed

@@ -33,11 +33,11 @@ async function restoreReplica(lc, config) {
 	}
 	throw new Error(`max attempts exceeded restoring replica`);
 }
-function getLitestream(config, logLevelOverride, backupURLOverride) {
-	const { executable, backupURL, logLevel, configPath, endpoint, port = config.port + 2, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config.litestream;
+function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
+	const { executable, executableV5, restoreUsingV5, backupURL, logLevel, configPath, endpoint, port = config.port + 2, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config.litestream;
 	const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;
 	return {
-		litestream: must(executable, `Missing --litestream-executable`),
+		litestream: (mode === "restore" && restoreUsingV5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`),
 		env: {
 			...process.env,
 			["ZERO_REPLICA_FILE"]: config.replica.file,
@@ -65,7 +65,7 @@ async function tryRestore(lc, config) {
 		snapshotStatus = await firstMessage;
 		lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);
 	} else firstMessage.catch((e) => lc.debug?.(e));
-	const { litestream, env } = getLitestream(config, "debug", snapshotStatus?.backupURL);
+	const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
 	const { restoreParallelism: parallelism } = config.litestream;
 	const proc = spawn(litestream, [
 		"restore",
@@ -117,7 +117,7 @@ function replicaIsValid(lc, replica, snapshot) {
 	}
 }
 function startReplicaBackupProcess(lc, config) {
-	const { litestream, env } = getLitestream(config);
+	const { litestream, env } = getLitestream("replicate", config);
 	lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);
 	return spawn(litestream, ["replicate"], {
 		env,

package/out/zero-cache/src/services/litestream/commands.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/*\n @returns The time at which the last restore started\n * (i.e. not counting failed attempts).\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const {changeStreamer} = config;\n\n for (let i = 0; i < MAX_RETRIES; i++) {\n if (i > 0) {\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n const start = new Date();\n const restored = await tryRestore(lc, config);\n if (restored) {\n return start;\n }\n if (\n changeStreamer.mode === 'dedicated' &&\n changeStreamer.uri === undefined\n ) {\n lc.info?.('no litestream backup found');\n return start;\n }\n }\n throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n // Set the snapshot interval to something smaller than x hours so that\n // the hourly check triggers on the hour, rather than the hour after.\n const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n return {\n litestream: must(executable, `Missing --litestream-executable`),\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n snapshotBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n },\n };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n const {changeStreamer} = config;\n\n const isViewSyncer =\n changeStreamer.mode === 'discover' \|\| changeStreamer.uri !== undefined;\n\n // Fire off a snapshot reservation to the current replication-manager\n // (if there is one).\n const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n let snapshotStatus: SnapshotStatus \| undefined;\n if (isViewSyncer) {\n // The return value is required by view-syncers ...\n snapshotStatus = await firstMessage;\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n } else {\n // but it is also useful to pause change-log cleanup when a new\n // replication-manager is starting up. In this case, the request is\n // best-effort. In particular, there may not be a previous\n // replication-manager running at all.\n void firstMessage.catch(e => lc.debug?.(e));\n }\n\n const {litestream, env} = getLitestream(\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {restoreParallelism: parallelism} = config.litestream;\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: 'inherit', windowsHide: true},\n );\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n await promise;\n if (!existsSync(config.replica.file)) {\n return false;\n }\n if (\n snapshotStatus &&\n !replicaIsValid(lc, config.replica.file, snapshotStatus)\n ) {\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return false;\n }\n return true;\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n snapshot: SnapshotStatus,\n) {\n const db = new Database(lc, replica);\n try {\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== snapshot.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n snapshot,\n );\n return false;\n }\n if (watermark < snapshot.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n snapshot,\n );\n return true;\n } catch (e) {\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n return spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown \| string = '';\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // (i.e. BackupMonitor) returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n if (!isViewSyncer) {\n return reject(err);\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;;;;;AAM1B,eAAsB,eACpB,IACA,QACe;CACf,MAAM,EAAC,mBAAkB;AAEzB,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,MAAI,IAAI,GAAG;AACT,MAAG,OACD,kCAAkC,oBAAoB,IAAK,UAC5D;AACD,SAAM,MAAM,kBAAkB;;EAEhC,MAAM,wBAAQ,IAAI,MAAM;AAExB,MADiB,MAAM,WAAW,IAAI,OAAO,CAE3C,QAAO;AAET,MACE,eAAe,SAAS,eACxB,eAAe,QAAQ,KAAA,GACvB;AACA,MAAG,OAAO,6BAA6B;AACvC,UAAO;;;AAGX,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,SAAS,cACP,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,WACA,UACA,YACA,UACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;AAEzE,QAAO;EACL,YAAY,KAAK,YAAY,kCAAkC;EAC/D,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,UAAU;IACnE,8CAA8C,OAC7C,uBACD;IACA,8CAA8C,OAC7C,uBACD;IACA,wDAAwD,OACvD,iCACD;IACA,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,8BACD;IACA,0CAA0C,OAAO,qBAAqB;IACtE,mCAAmC,OAAO,cAAc;IACxD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,KAAK;GACjC,GAAI,WAAW,GAAE,6BAA6B,UAAS,GAAG,EAAE;GAC7D;EACF;;AAGH,eAAe,WAAW,IAAgB,QAAoB;CAC5D,MAAM,EAAC,mBAAkB;CAEzB,MAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ,KAAA;CAI/D,MAAM,eAAe,4BAA4B,IAAI,QAAQ,aAAa;CAC1E,IAAI;AACJ,KAAI,cAAc;AAEhB,mBAAiB,MAAM;AACvB,KAAG,OAAO,yBAAyB,eAAe,YAAY;OAMzD,cAAa,OAAM,MAAK,GAAG,QAAQ,EAAE,CAAC;CAG7C,MAAM,EAAC,YAAY,QAAO,cACxB,QACA,SACA,gBAAgB,UACjB;CACD,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,YAAY;EACnB,OAAO,QAAQ;EAChB,EACD;EAAC;EAAK,OAAO;EAAW,aAAa;EAAK,CAC3C;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,UAAU;AAC7C,MAAK,GAAG,SAAS,OAAO;AACxB,MAAK,GAAG,UAAU,MAAM,WAAW;AACjC,MAAI,OACF,QAAO,0BAA0B,SAAS;WACjC,SAAS,EAClB,QAAO,+BAA+B,OAAO;MAE7C,UAAS;GAEX;AACF,OAAM;AACN,KAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAClC,QAAO;AAET,KACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,eAAe,EACxD;AACA,KAAG,OAAO,8CAA8C;AACxD,eAAa,OAAO,QAAQ,KAAK;AACjC,SAAO;;AAET,QAAO;;AAGT,SAAS,eACP,IACA,SACA,UACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,QAAQ;AACpC,KAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,GAAG,CACxB;AACD,MAAI,mBAAmB,SAAS,gBAAgB;AAC9C,MAAG,OACD,yBAAyB,eAAe,iDAAiD,SAAS,kBAClG,SACD;AACD,UAAO;;AAET,MAAI,YAAY,SAAS,cAAc;AACrC,MAAG,OACD,2BAA2B,UAAU,gDAAgD,SAAS,eAC/F;AACD,UAAO;;AAET,KAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,sCACtE,SACD;AACD,SAAO;UACA,GAAG;AACV,KAAG,QAAQ,2CAA2C,EAAE;AACxD,SAAO;WACC;AACR,KAAG,OAAO;;;AAId,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,OAAO;AAC/C,IAAG,OAAO,iCAAiC,OAAO,WAAW,YAAY;AACzE,QAAO,MAAM,YAAY,CAAC,YAAY,EAAE;EACtC;EACA,OAAO;EACP,aAAa;EACd,CAAC;;AAGJ,SAAS,4BACP,IACA,QACA,cACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,UAA0B;AAErE,EAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,iBAAiB;AACnC,UAAQ,GAAG,gBAAgB,MAAM,OAAO,CAAC;AACzC,UAAQ,GAAG,iBAAiB,MAAM,OAAO,CAAC;AAE1C,OAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,MAAwB;AAC5B,OAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,OAAO;AAChD,eAAW,MAAM,OAAO,QAAQ;AAI9B,aAAQ,IAAI,GAAG;AACf,gBAAW;;AAIb,QAAI,SACF;YAEK,GAAG;AACV,UAAM;;AAER,OAAI,CAAC,aACH,QAAO,OAAO,IAAI;AAUpB,MAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,IAAI,CACZ;AACD,OAAI;AACF,UAAM,MAAM,KAAM,MAAM,OAAO;YACxB,GAAG;AACV,WAAO,OAAO,EAAE;;;KAGlB;AAEJ,QAAO;;AAGT,SAAS,gBACP,IACA,QACkC;AAClC,kBAAiB,OAAO;CACxB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;AAUzC,QAP6B,IAAI,yBAC/B,IAHc,eAAe,OAAO,EAKpC,OAAO,IACP,eAAe,IAChB,CAE2B,gBAAgB,OAAO"}
1	+ {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/*\n @returns The time at which the last restore started\n * (i.e. not counting failed attempts).\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const {changeStreamer} = config;\n\n for (let i = 0; i < MAX_RETRIES; i++) {\n if (i > 0) {\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n const start = new Date();\n const restored = await tryRestore(lc, config);\n if (restored) {\n return start;\n }\n if (\n changeStreamer.mode === 'dedicated' &&\n changeStreamer.uri === undefined\n ) {\n lc.info?.('no litestream backup found');\n return start;\n }\n }\n throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n // Set the snapshot interval to something smaller than x hours so that\n // the hourly check triggers on the hour, rather than the hour after.\n const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n const litestream =\n // The v0.5.8+ litestream executable can restore from either the new LTX\n // format or the legacy WAL format, allowing forwards-compatibility /\n // rollback safety with zero-cache versions that backup to LTX.\n (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n snapshotBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n },\n };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n const {changeStreamer} = config;\n\n const isViewSyncer =\n changeStreamer.mode === 'discover' \|\| changeStreamer.uri !== undefined;\n\n // Fire off a snapshot reservation to the current replication-manager\n // (if there is one).\n const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n let snapshotStatus: SnapshotStatus \| undefined;\n if (isViewSyncer) {\n // The return value is required by view-syncers ...\n snapshotStatus = await firstMessage;\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n } else {\n // but it is also useful to pause change-log cleanup when a new\n // replication-manager is starting up. In this case, the request is\n // best-effort. In particular, there may not be a previous\n // replication-manager running at all.\n void firstMessage.catch(e => lc.debug?.(e));\n }\n\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {restoreParallelism: parallelism} = config.litestream;\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: 'inherit', windowsHide: true},\n );\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n await promise;\n if (!existsSync(config.replica.file)) {\n return false;\n }\n if (\n snapshotStatus &&\n !replicaIsValid(lc, config.replica.file, snapshotStatus)\n ) {\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return false;\n }\n return true;\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n snapshot: SnapshotStatus,\n) {\n const db = new Database(lc, replica);\n try {\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== snapshot.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n snapshot,\n );\n return false;\n }\n if (watermark < snapshot.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n snapshot,\n );\n return true;\n } catch (e) {\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n return spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown \| string = '';\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // (i.e. BackupMonitor) returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n if (!isViewSyncer) {\n return reject(err);\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;;;;;AAM1B,eAAsB,eACpB,IACA,QACe;CACf,MAAM,EAAC,mBAAkB;AAEzB,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,MAAI,IAAI,GAAG;AACT,MAAG,OACD,kCAAkC,oBAAoB,IAAK,UAC5D;AACD,SAAM,MAAM,kBAAkB;;EAEhC,MAAM,wBAAQ,IAAI,MAAM;AAExB,MADiB,MAAM,WAAW,IAAI,OAAO,CAE3C,QAAO;AAET,MACE,eAAe,SAAS,eACxB,eAAe,QAAQ,KAAA,GACvB;AACA,MAAG,OAAO,6BAA6B;AACvC,UAAO;;;AAGX,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;AAQzE,QAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,kCAAkC;EAGnD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,UAAU;IACnE,8CAA8C,OAC7C,uBACD;IACA,8CAA8C,OAC7C,uBACD;IACA,wDAAwD,OACvD,iCACD;IACA,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,8BACD;IACA,0CAA0C,OAAO,qBAAqB;IACtE,mCAAmC,OAAO,cAAc;IACxD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,KAAK;GACjC,GAAI,WAAW,GAAE,6BAA6B,UAAS,GAAG,EAAE;GAC7D;EACF;;AAGH,eAAe,WAAW,IAAgB,QAAoB;CAC5D,MAAM,EAAC,mBAAkB;CAEzB,MAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ,KAAA;CAI/D,MAAM,eAAe,4BAA4B,IAAI,QAAQ,aAAa;CAC1E,IAAI;AACJ,KAAI,cAAc;AAEhB,mBAAiB,MAAM;AACvB,KAAG,OAAO,yBAAyB,eAAe,YAAY;OAMzD,cAAa,OAAM,MAAK,GAAG,QAAQ,EAAE,CAAC;CAG7C,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,UACjB;CACD,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,YAAY;EACnB,OAAO,QAAQ;EAChB,EACD;EAAC;EAAK,OAAO;EAAW,aAAa;EAAK,CAC3C;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,UAAU;AAC7C,MAAK,GAAG,SAAS,OAAO;AACxB,MAAK,GAAG,UAAU,MAAM,WAAW;AACjC,MAAI,OACF,QAAO,0BAA0B,SAAS;WACjC,SAAS,EAClB,QAAO,+BAA+B,OAAO;MAE7C,UAAS;GAEX;AACF,OAAM;AACN,KAAI,CAAC,WAAW,OAAO,QAAQ,KAAK,CAClC,QAAO;AAET,KACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,eAAe,EACxD;AACA,KAAG,OAAO,8CAA8C;AACxD,eAAa,OAAO,QAAQ,KAAK;AACjC,SAAO;;AAET,QAAO;;AAGT,SAAS,eACP,IACA,SACA,UACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,QAAQ;AACpC,KAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,GAAG,CACxB;AACD,MAAI,mBAAmB,SAAS,gBAAgB;AAC9C,MAAG,OACD,yBAAyB,eAAe,iDAAiD,SAAS,kBAClG,SACD;AACD,UAAO;;AAET,MAAI,YAAY,SAAS,cAAc;AACrC,MAAG,OACD,2BAA2B,UAAU,gDAAgD,SAAS,eAC/F;AACD,UAAO;;AAET,KAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,sCACtE,SACD;AACD,SAAO;UACA,GAAG;AACV,KAAG,QAAQ,2CAA2C,EAAE;AACxD,SAAO;WACC;AACR,KAAG,OAAO;;;AAId,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,OAAO;AAC5D,IAAG,OAAO,iCAAiC,OAAO,WAAW,YAAY;AACzE,QAAO,MAAM,YAAY,CAAC,YAAY,EAAE;EACtC;EACA,OAAO;EACP,aAAa;EACd,CAAC;;AAGJ,SAAS,4BACP,IACA,QACA,cACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,UAA0B;AAErE,EAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,iBAAiB;AACnC,UAAQ,GAAG,gBAAgB,MAAM,OAAO,CAAC;AACzC,UAAQ,GAAG,iBAAiB,MAAM,OAAO,CAAC;AAE1C,OAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI,MAAwB;AAC5B,OAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,OAAO;AAChD,eAAW,MAAM,OAAO,QAAQ;AAI9B,aAAQ,IAAI,GAAG;AACf,gBAAW;;AAIb,QAAI,SACF;YAEK,GAAG;AACV,UAAM;;AAER,OAAI,CAAC,aACH,QAAO,OAAO,IAAI;AAUpB,MAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,IAAI,CACZ;AACD,OAAI;AACF,UAAM,MAAM,KAAM,MAAM,OAAO;YACxB,GAAG;AACV,WAAO,OAAO,EAAE;;;KAGlB;AAEJ,QAAO;;AAGT,SAAS,gBACP,IACA,QACkC;AAClC,kBAAiB,OAAO;CACxB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;AAUzC,QAP6B,IAAI,yBAC/B,IAHc,eAAe,OAAO,EAKpC,OAAO,IACP,eAAe,IAChB,CAE2B,gBAAgB,OAAO"}

package/out/zero-cache/src/services/mutagen/pusher.d.ts CHANGED Viewed

@@ -6,12 +6,12 @@ import type { Source } from '../../types/streams.ts';
 import { Subscription } from '../../types/subscription.ts';
 import type { HandlerResult, StreamResult } from '../../workers/connection.ts';
 import type { RefCountedService, Service } from '../service.ts';
+import type { ConnectionContext, ConnectionContextManager, ConnectionSelector } from '../view-syncer/connection-context-manager.ts';
 export interface Pusher extends RefCountedService {
-    readonly pushURL: string | undefined;
-    initConnection(clientID: string, wsID: string, userPushURL: string | undefined, userPushHeaders: Record<string, string> | undefined, onAuthFailure?: () => void): Source<Downstream>;
-    enqueuePush(clientID: string, push: PushBody, auth: string | undefined, httpCookie: string | undefined, origin: string | undefined): HandlerResult;
-    ackMutationResponses(upToID: MutationID): Promise<void>;
-    deleteClientMutations(clientIDs: string[]): Promise<void>;
+    initConnection(selector: ConnectionSelector): Source<Downstream>;
+    enqueuePush(selector: ConnectionSelector, push: PushBody): HandlerResult;
+    ackMutationResponses(requester: ConnectionSelector, upToID: MutationID): Promise<void>;
+    deleteClientMutations(requester: ConnectionSelector, clientIDs: string[]): Promise<void>;
 }
 type Config = Pick<ZeroConfig, 'app' | 'shard'>;
 /**
@@ -30,11 +30,8 @@ type Config = Pick<ZeroConfig, 'app' | 'shard'>;
 export declare class PusherService implements Service, Pusher {
     #private;
     readonly id: string;
-    constructor(appConfig: Config, pushConfig: ZeroConfig['push'] & {
-        url: string[];
-    }, lc: LogContext, clientGroupID: string);
-    get pushURL(): string | undefined;
-    initConnection(clientID: string, wsID: string, userPushURL: string | undefined, userPushHeaders: Record<string, string> | undefined, onAuthFailure?: () => void): Subscription<["deleteClients", {
+    constructor(appConfig: Config, lc: LogContext, clientGroupID: string, contextManager: ConnectionContextManager);
+    initConnection(selector: ConnectionSelector): Subscription<["deleteClients", {
         readonly clientIDs?: readonly string[] | undefined;
         readonly clientGroupIDs?: readonly string[] | undefined;
     }] | ["connected", {
@@ -633,9 +630,15 @@ export declare class PusherService implements Service, Pusher {
         requestID: string;
         lastMutationIDChanges: Record<string, number>;
     }]>;
-    enqueuePush(clientID: string, push: PushBody, auth: string | undefined, httpCookie: string | undefined, origin: string | undefined): Exclude<HandlerResult, StreamResult>;
-    ackMutationResponses(upToID: MutationID): Promise<void>;
-    deleteClientMutations(clientIDs: string[]): Promise<void>;
+    enqueuePush(selector: ConnectionSelector, push: PushBody): Exclude<HandlerResult, StreamResult>;
+    ackMutationResponses(requester: ConnectionSelector, upToID: MutationID): Promise<void>;
+    /**
+     * Bulk cleanup is routed through the requester's push context.
+     *
+     * This assumes the client group shares a compatible push endpoint/auth
+     * context.
+     */
+    deleteClientMutations(requester: ConnectionSelector, clientIDs: string[]): Promise<void>;
     ref(): void;
     unref(): void;
     hasRefs(): boolean;
@@ -644,17 +647,14 @@ export declare class PusherService implements Service, Pusher {
 }
 type PusherEntry = {
     push: PushBody;
-    auth: string | undefined;
-    httpCookie: string | undefined;
-    origin: string | undefined;
-    clientID: string;
+    context: ConnectionContext;
 };
 type PusherEntryOrStop = PusherEntry | 'stop';
 /**
- * Pushes for different clientIDs could theoretically be interleaved.
+ * Pushes for different clients, sockets, or auth revisions could be interleaved.
  *
- * In order to do efficient batching to the user's API server,
- * we collect all pushes for the same clientID into a single push.
+ * In order to batch safely, we only combine pushes from the same
+ * clientID/wsID/revision snapshot.
  */
 export declare function combinePushes(entries: readonly (PusherEntryOrStop | undefined)[]): [PusherEntry[], boolean];
 export {};

package/out/zero-cache/src/services/mutagen/pusher.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"~~AAAA~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;~~AAC~~/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;~~AAE9D~~,~~MAAM~~,~~WAAW~~,~~MAAO~~,~~SAAQ,~~iBAAiB~~;IAC/C~~,~~QAAQ~~,~~CAAC~~,~~OAAO~~,~~EAAE~~,MAAM,~~GAAG~~,~~SAAS,~~CAAC;~~IAErC~~,~~cAAc,CACZ,QAAQ,EAAE,~~MAAM,~~EAChB,IAAI,EAAE,MAAM,EACZ,~~WAAW,~~EAAE~~,~~MAAM~~,~~GAAG,SAAS,EAC~~/B,~~eAAe~~,~~EAAE,MAAM,~~CAAC,~~MAAM~~,EAAE,~~MAAM~~,~~CAAC,~~GAAG,~~SAAS,EACnD,aAAa,CAAC,EAAE,~~MAAM,~~IAAI,GACzB,MAAM,~~CAAC,UAAU,CAAC,CAAC;~~IACtB~~,WAAW,~~CACT~~,QAAQ,EAAE,~~MAAM~~,~~EAChB~~,IAAI,EAAE,QAAQ,~~EACd,IAAI,EAAE,MAAM,~~GAAG,~~SAAS~~,~~EACxB~~,~~UAAU~~,~~EAAE~~,~~MAAM,GAAG,~~SAAS,~~EAC9B,MAAM,~~EAAE,~~MAAM~~,~~GAAG~~,~~SAAS,GACzB,aAAa,CAAC;IACjB,oBAAoB,CAAC,~~MAAM,EAAE,UAAU,~~GAAG~~,OAAO,CAAC,IAAI,CAAC,CAAC;~~IACxD~~,qBAAqB,~~CAAC~~,SAAS,EAAE,MAAM,EAAE,~~GAAG~~,OAAO,CAAC,IAAI,CAAC,CAAC;~~CAC3D~~;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;~~gBAYlB~~,SAAS,EAAE,MAAM,EACjB,~~UAAU,~~EAAE,~~UAAU,CAAC,MAAM,CAAC,GAAG;QAAC,GAAG,~~EAAE,~~MAAM,EAAE,CAAA;KAAC,EAChD,EAAE,EAAE,~~UAAU,EACd,aAAa,EAAE,MAAM~~;IAkBvB~~,~~IAAI~~,~~OAAO~~,~~IAAI~~,~~MAAM,GAAG,SAAS,CAEhC~~;~~IAED~~,cAAc,~~CACZ~~,QAAQ,EAAE,~~MAAM~~,~~EAChB~~,~~IAAI~~,EAAE,~~MAAM~~,~~EACZ~~,~~WAAW~~,EAAE,~~MAAM~~,~~GAAG~~,~~SAAS~~,~~EAC/B~~,~~eAAe~~,EAAE,~~MAAM~~,CAAC,~~MAAM~~,~~EAAE~~,~~MAAM,CAAC,GAAG,~~SAAS,~~EACnD~~,~~aAAa~~,~~CAAC~~,~~EAAE,~~MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAW5B,WAAW,CACT,QAAQ,EAAE,~~MAAM~~,~~EAChB~~,~~IAAI~~,~~EAAE~~,~~QAAQ,EACd,~~IAAI,~~EAAE~~,~~MAAM~~,~~GAAG~~,SAAS,~~EACxB,UAAU,~~EAAE,~~MAAM~~,~~GAAG~~,SAAS,~~EAC9B~~,MAAM,EAAE,~~MAAM~~,~~GAAG,SAAS,GACzB,~~OAAO,CAAC,~~aAAa~~,~~EAAE,YAAY,~~CAAC;~~IAWjC~~,~~oBAAoB,CAAC,MAAM,EAAE,UAAU;IAiDvC,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE;IAmD/C,~~GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,~~IAAI~~,EAAE,~~MAAM~~,~~GAAG,SAAS,~~CAAC;~~IACzB~~,~~UAAU,EAAE,MAAM,GAAG,SAAS,~~CAAC;~~IAC/B,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;~~AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;~~AAuV9C~~;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}
1	+ {"version":3,"file":"pusher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/pusher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AAStE,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EAEd,MAAM,uCAAuC,CAAC;AAE/C,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AACzD,OAAO,KAAK,EAAC,aAAa,EAAE,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EACV,iBAAiB,EACjB,wBAAwB,EACxB,kBAAkB,EACnB,MAAM,8CAA8C,CAAC;AAEtD,MAAM,WAAW,MAAO,SAAQ,iBAAiB;IAC/C,cAAc,CAAC,QAAQ,EAAE,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACjE,WAAW,CAAC,QAAQ,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IACzE,oBAAoB,CAClB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,qBAAqB,CACnB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC,CAAC;CAClB;AAED,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;AAEhD;;;;;;;;;;;;GAYG;AACH,qBAAa,aAAc,YAAW,OAAO,EAAE,MAAM;;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAWlB,SAAS,EAAE,MAAM,EACjB,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,wBAAwB;IAe1C,cAAc,CAAC,QAAQ,EAAE,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI3C,WAAW,CACT,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC;IAWjC,oBAAoB,CACxB,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IA+ChB;;;;;OAKG;IACG,qBAAqB,CACzB,SAAS,EAAE,kBAAkB,EAC7B,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IAkDhB,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAKpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtB;AAED,KAAK,WAAW,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAAC;AACF,KAAK,iBAAiB,GAAG,WAAW,GAAG,MAAM,CAAC;AAsU9C;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,SAAS,CAAC,iBAAiB,GAAG,SAAS,CAAC,EAAE,GAClD,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,CAqC1B"}