@rocicorp/zero 1.11.0-canary.36 → 1.11.0-canary.37

package/out/zero-cache/src/services/life-cycle.js

@@ -130,8 +130,8 @@ var ProcessManager = class {
 			this.#all.delete(worker);
 		}
 		const pid = worker?.pid ?? process.pid;
+		if (code === 0 && (this.#drainStart === 0 || type === "supporting" && this.#userFacing.size > 0)) code = 14;
 		if (type === "supporting") {
-			if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) code = 14;
 			const log = code === 0 || code === 14 ? "info" : "warn";
 			this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
 			return this.#exit(code);
@@ -140,9 +140,9 @@ var ProcessManager = class {
 		this.#lc[log]?.(sig ? `${name} (${pid}) killed with (${sig})` : `${name} (${pid}) exited with code (${code})`, err ?? "");
 		if (this.#userFacing.size === 0) {
 			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);
-			return this.#exit(this.#drainStart ? 0 : code || -1);
+			return this.#exit(this.#drainStart ? 0 : code);
 		}
-		if (this.#drainStart === 0) return this.#exit(code || -1);
+		if (this.#drainStart === 0) return this.#exit(code);
 	}
 	#kill(workers, signal) {
 		for (const worker of workers) try {
@@ -153,6 +153,48 @@ var ProcessManager = class {
 	}
 };
 /**
+* Coordinates a drain that is initiated by the server itself (e.g. by a
+* view-syncer that has been granted permission to restore a new replica),
+* as opposed to one initiated by the container runner with a `SIGTERM`.
+*
+* Health checks start failing immediately, upon which an orchestrator such
+* as Kubernetes stops routing to the server and shuts it down as usual
+* (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,
+* and for environments without liveness checks, the server sends itself a
+* `SIGTERM` after the configured delay.
+*/
+var SelfDrain = class {
+	#lc;
+	#proc;
+	#delayMs;
+	#onRequested;
+	#requested = false;
+	#signaled = false;
+	#timer;
+	constructor(lc, proc, opts) {
+		this.#lc = lc.withContext("component", "self-drain");
+		this.#proc = proc;
+		this.#delayMs = opts.delayMs;
+		this.#onRequested = opts.onRequested;
+		for (const signal of GRACEFUL_SHUTDOWN) proc.once(signal, () => {
+			this.#signaled = true;
+			clearTimeout(this.#timer);
+		});
+	}
+	request() {
+		if (this.#requested || this.#signaled) return;
+		this.#requested = true;
+		this.#lc.info?.(`self drain requested. sending SIGTERM in ${this.#delayMs} ms`);
+		this.#onRequested();
+		this.#timer = setTimeout(() => {
+			if (!this.#signaled) {
+				this.#lc.info?.(`initiating self drain`);
+				this.#proc.emit("SIGTERM", "SIGTERM");
+			}
+		}, this.#delayMs);
+	}
+};
+/**
 * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with
 * an optional {@link SingletonService.drain drain()}, or stopping them
 * without draining for `SIGQUIT`.
@@ -161,13 +203,17 @@ var ProcessManager = class {
 */
 async function runUntilKilled(lc, parent, ...services) {
 	if (services.length === 0) return;
-	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, () => {
-		const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN;
+	for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) parent.once(signal, async () => {
+		if (GRACEFUL_SHUTDOWN.includes(signal)) {
+			const results = await Promise.allSettled(services.map(async (svc) => {
+				if (svc.drain) {
+					lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
+					await svc.drain();
+				}
+			}));
+			for (const result of results) if (result.status === "rejected") lc.warn?.(`error while draining (${signal})`, result.reason);
+		}
 		services.forEach(async (svc) => {
-			if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {
-				lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);
-				await svc.drain();
-			}
 			lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);
 			await svc.stop();
 		});
@@ -250,6 +296,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { HeartbeatMonitor, ProcessManager, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { HeartbeatMonitor, ProcessManager, SelfDrain, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code || -1);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;AAmCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKO,gBAAgB,KAAK,KAAKR,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,QAAQ,EAAE;EACrD;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#proc","#delayMs","#onRequested","#signaled","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, async () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      if (GRACEFUL_SIGNALS.includes(signal)) {\n        // Drain all services before stopping any of them. Stopping a service\n        // resolves its run() and thus ends runUntilKilled(), so a service\n        // without a drain() must not be stopped while others are draining.\n        const results = await Promise.allSettled(\n          services.map(async svc => {\n            if (svc.drain) {\n              lc.info?.(\n                `draining ${svc.constructor.name} ${svc.id} (${signal})`,\n              );\n              await svc.drain();\n            }\n          }),\n        );\n        for (const result of results) {\n          if (result.status === 'rejected') {\n            lc.warn?.(`error while draining (${signal})`, result.reason);\n          }\n        }\n      }\n      services.forEach(async svc => {\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;AAsCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKc,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKC,YAAY;GACjB,aAAa,KAAKC,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKF,WAC1B;EAEF,KAAKE,aAAa;EAClB,KAAKnB,IAAI,OACP,4CAA4C,KAAKe,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKE,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKD,WAAW;IACnB,KAAKjB,IAAI,OAAO,uBAAuB;IACvC,KAAKc,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,QAAQ,YAAY;EAG9B,IAAI,kBAAiB,SAAS,MAAM,GAAG;GAIrC,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OACD,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EACxD;KACA,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EACA,SAAS,QAAQ,OAAM,QAAO;GAC5B,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKf,MAAM;EACX,KAAKoB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKtB,IAAI,OACP,iCACE,KAAKoB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKpB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKsB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts

@@ -42,6 +42,14 @@ export declare function tryRestore(lc: LogContext, config: LitestreamConfig, rep
  * directory, `.<file>-litestream`).
  */
 export declare function deleteLitestreamMetaDir(replicaFile: string): void;
+/**
+ * Returns the replica version and watermark of the local replica file, or
+ * `undefined` if it does not exist or cannot be read.
+ */
+export declare function readLocalReplica(lc: LogContext, replica: string): {
+    replicaVersion: string;
+    watermark: string;
+} | undefined;
 export declare function startReplicaBackupProcess(lc: LogContext, config: LitestreamConfig, replicaFile: string): ChildProcess;
 /**
  * Returns the time of the most recent object (snapshot or WAL segment)

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,mBAAmB,CAAC,GAChE,MAAM,GAAG,SAAS,CAEpB;AA6FD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAsND;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AA2GD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAK3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;IACtB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC9B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,+EAA+E;IAC/E,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,mBAAmB,CAAC,GAChE,MAAM,GAAG,SAAS,CAEpB;AA6FD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,GAClC,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAsND;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,QAK1D;AAwDD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,MAAM,GACd;IAAC,cAAc,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAC,GAAG,SAAS,CAiBzD;AAqDD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -278,6 +278,27 @@ function logRestoreDirectoryContents(lc, replicaFile, phase) {
 	}
 }
 /**
+* Returns the replica version and watermark of the local replica file, or
+* `undefined` if it does not exist or cannot be read.
+*/
+function readLocalReplica(lc, replica) {
+	if (!existsSync(replica)) return;
+	let db;
+	try {
+		db = new Database(lc, replica, { readonly: true });
+		const { replicaVersion, watermark } = getSubscriptionState(new StatementRunner(db));
+		return {
+			replicaVersion,
+			watermark
+		};
+	} catch (e) {
+		lc.warn?.(`unable to read the subscription state of ${replica}`, e);
+		return;
+	} finally {
+		db?.close();
+	}
+}
+/**
 * Opens and validates the restored replica. Returns the open connection if
 * the replica is valid, which the caller must close; otherwise closes it and
 * returns `undefined`.
@@ -443,6 +464,6 @@ function parseBackupCreatedTimes(lc, command, output) {
 	return times;
 }
 //#endregion
-export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, startReplicaBackupProcess, tryRestore };
+export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, readLocalReplica, startReplicaBackupProcess, tryRestore };
 
 //# sourceMappingURL=commands.js.map

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Returns the replica version and watermark of the local replica file, or\n * `undefined` if it does not exist or cannot be read.\n */\nexport function readLocalReplica(\n  lc: LogContext,\n  replica: string,\n): {replicaVersion: string; watermark: string} | undefined {\n  if (!existsSync(replica)) {\n    return undefined;\n  }\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica, {readonly: true});\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    return {replicaVersion, watermark};\n  } catch (e) {\n    lc.warn?.(`unable to read the subscription state of ${replica}`, e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;AAMA,SAAgB,iBACd,IACA,SACyD;CACzD,IAAI,CAAC,WAAW,OAAO,GACrB;CAEF,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,SAAS,EAAC,UAAU,KAAI,CAAC;EAC/C,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,OAAO;GAAC;GAAgB;EAAS;CACnC,SAAS,GAAG;EACV,GAAG,OAAO,4CAA4C,WAAW,CAAC;EAClE;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/replicator/drain-handler.d.ts

@@ -0,0 +1,74 @@
+import type { LogContext } from '@rocicorp/logger';
+import { type Sized, type Source } from '../../types/streams.ts';
+import type { ChangeStreamer, SubscriberContext } from '../change-streamer/change-streamer.ts';
+import type { SubscribeDownstream } from '../change-streamer/subscribe.ts';
+import type { RunningState } from '../running-state.ts';
+export type SwitchoverOptions = {
+    /**
+     * Called when a replication-manager with a newer replica version grants
+     * permission to restore its replica. This should initiate a graceful
+     * restart of the server (during which the current subscription continues
+     * to be served).
+     */
+    onRestoreGranted: () => void;
+    /**
+     * The replica file, from which the state of the last committed
+     * transaction is read for switchover subscriptions.
+     */
+    replicaFile: string;
+    /** For testing. */
+    timeoutMs?: number | undefined;
+};
+export type Downstream = Source<Sized<SubscribeDownstream>>;
+/**
+ * Handles the switchover of a serving replica's subscription from a draining
+ * replication-manager to another one, without interrupting the current
+ * subscription.
+ *
+ * Upon {@link onChangeStreamerDrain}, the handler repeatedly (with backoff)
+ * opens a new `switchover` subscription, at the watermark of the replica's
+ * last committed transaction, to whichever replication-manager the
+ * connection is routed to. The first message on the new subscription
+ * determines the outcome:
+ * * `status`: The subscription was accepted (i.e. the replica is compatible).
+ *   The current subscription is cancelled (aborting any transaction in
+ *   progress), and the new one is made available via
+ *   {@link getNextConnection}. Note that the new subscription may start with
+ *   transactions that were committed from the current subscription in the
+ *   meantime, which the consumer must skip.
+ * * `status` with `restore: queued`: The replication-manager has a newer
+ *   replica version, and is holding the subscription until it grants
+ *   permission to restore. The held subscription is monitored for the grant.
+ * * `status` with `restore: granted`: Permission to restore is granted, and
+ *   {@link SwitchoverOptions.onRestoreGranted} is called. The granted
+ *   subscription is kept open until the process exits, so that the
+ *   replication-manager can track the grant until the restore begins.
+ * * Anything else (e.g. an error, or a `draining` replication-manager):
+ *   The switchover is retried later.
+ *
+ * The current subscription continues to be consumed throughout. A
+ * switchover is scoped to the subscription from which it switches: If that
+ * subscription ends first (e.g. the replication-manager shuts down), the
+ * switchover is abandoned, including any held (but not granted)
+ * subscription, as the consumer reconnects on its own.
+ */
+export declare class DrainHandler {
+    #private;
+    /**
+     * @param committedContext Returns the subscriber context as of the
+     *        replica's last committed transaction.
+     */
+    constructor(lc: LogContext, changeStreamer: ChangeStreamer, state: RunningState, options: SwitchoverOptions, committedContext: () => SubscriberContext);
+    /**
+     * Called when the change-streamer of the `current` subscription is
+     * draining.
+     */
+    onChangeStreamerDrain(current: Downstream): void;
+    get hasNextConnection(): boolean;
+    /**
+     * Returns (and clears) the subscription that replaces the (now cancelled)
+     * current one, if a switchover succeeded.
+     */
+    getNextConnection(): Downstream | undefined;
+}
+//# sourceMappingURL=drain-handler.d.ts.map

package/out/zero-cache/src/services/replicator/drain-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drain-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/drain-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAW,KAAK,KAAK,EAAE,KAAK,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzE,OAAO,KAAK,EACV,cAAc,EAEd,iBAAiB,EAClB,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,iCAAiC,CAAC;AACzE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAQtD,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,IAAI,CAAC;IAE7B;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB,mBAAmB;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAI5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,YAAY;;IAYvB;;;OAGG;IACH,YACE,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,cAAc,EAC9B,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,iBAAiB,EAC1B,gBAAgB,EAAE,MAAM,iBAAiB,EAO1C;IAED;;;OAGG;IACH,qBAAqB,CAAC,OAAO,EAAE,UAAU,QAaxC;IAED,IAAI,iBAAiB,IAAI,OAAO,CAE/B;IAED;;;OAGG;IACH,iBAAiB,IAAI,UAAU,GAAG,SAAS,CAI1C;CAyIF"}