@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.11

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#parent","#procs","#done","#onClose","#signaled","#proc","#delayMs","#onRequested","#timer","#requested","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAKO,gBAAgB,KACnB,SAAS,gBAAgB,KAAKR,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAKA,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAKc,UAAU;EAEf,KAAKE,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAKT,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAKQ,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAKE,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAKjB,IAAI,KAAKkB,YAAY,SAAS,WACjC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAKJ,QAAQ,GAAG,cAAc,KAAKP,MAAM,MAAM,CAAC;EAElD,OAAO,KAAKS,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAKE,WACR,KAAKX,MAAM,SAAS;EAEtB,IAAI,KAAKQ,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAErB,MAAM,KAAKA,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAKD,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAKG,WAAW;GAClB,KAAKlB,IAAI,OAAO,IAAI;GACpB,IAAI,KAAKe,OAAO,SAAS,GACvB,KAAKC,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAKhB,IAAI,OAAO,IAAI;GACpB,KAAKgB,MAAM,QAAQ;EACrB,OACE,KAAKA,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAKE,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAKH,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAKf,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAKA,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAKmB,QAAQ;EACb,KAAKC,WAAW,KAAK;EACrB,KAAKC,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAKH,YAAY;GACjB,aAAa,KAAKI,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAKC,cAAc,KAAKL,WAC1B;EAEF,KAAKK,aAAa;EAClB,KAAKvB,IAAI,OACP,4CAA4C,KAAKoB,SAAS,IAC5D;EACA,KAAKC,aAAa;EAClB,KAAKC,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAKJ,WAAW;IACnB,KAAKlB,IAAI,OAAO,uBAAuB;IACvC,KAAKmB,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAKC,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,EAAE,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,EACL,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,EAAE,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKpB,MAAM;EACX,KAAKwB,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAK1B,IAAI,OACP,iCACE,KAAKwB,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKxB,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK0B,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["NodeEventEmitter"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import {EventEmitter as NodeEventEmitter} from 'node:events';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The event that a {@link PreDrain} emits immediately upon the first graceful\n * shutdown signal, with that signal as its argument, before the delay.\n */\nexport const PRE_DRAIN = 'preDrain';\n\n/**\n * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load\n * balancers can observe a failed readiness check before the server begins\n * to drain. The first graceful signal is announced immediately with a\n * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such\n * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the\n * delay has elapsed. Repeated graceful signals are dropped. All other events,\n * including forceful signals, are forwarded immediately.\n */\nexport class PreDrain extends NodeEventEmitter {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #forwarded = new Set<string | symbol>(['newListener', PRE_DRAIN]);\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, delayMs: number) {\n    super();\n    this.#lc = lc.withContext('component', 'pre-drain');\n    this.#proc = proc;\n    this.#delayMs = delayMs;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      this.#forwarded.add(signal);\n      proc.on(signal, () => this.#onGracefulSignal(signal));\n    }\n    this.on('newListener', (event: string | symbol) => this.#forward(event));\n  }\n\n  #forward(event: string | symbol) {\n    if (this.#forwarded.has(event)) {\n      return;\n    }\n    this.#forwarded.add(event);\n    this.#proc.on(event, (...args: unknown[]) => {\n      if (\n        event === 'disconnect' ||\n        (FORCEFUL_SHUTDOWN as readonly (string | symbol)[]).includes(event)\n      ) {\n        clearTimeout(this.#timer); // Stopping without a drain.\n      }\n      this.emit(event, ...args);\n    });\n  }\n\n  #onGracefulSignal(signal: GracefulShutdownSignal) {\n    if (this.#signaled) {\n      this.#lc.info?.(`already shutting down. ignoring ${signal}`);\n      return;\n    }\n    this.#signaled = true;\n    this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);\n    this.emit(PRE_DRAIN, signal);\n    this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,YAAY;;;;;;;;;;AAWzB,IAAa,WAAb,cAA8BA,aAAiB;CAC7C;CACA;CACA;CACA,6BAAsB,IAAI,IAAqB,CAAC,eAAe,SAAS,CAAC;CACzE,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,SAAiB;EAC/D,MAAM;EACN,KAAK,MAAM,GAAG,YAAY,aAAa,WAAW;EAClD,KAAK,QAAQ;EACb,KAAK,WAAW;EAEhB,KAAK,MAAM,UAAU,mBAAmB;GACtC,KAAK,WAAW,IAAI,MAAM;GAC1B,KAAK,GAAG,cAAc,KAAK,kBAAkB,MAAM,CAAC;EACtD;EACA,KAAK,GAAG,gBAAgB,UAA2B,KAAK,SAAS,KAAK,CAAC;CACzE;CAEA,SAAS,OAAwB;EAC/B,IAAI,KAAK,WAAW,IAAI,KAAK,GAC3B;EAEF,KAAK,WAAW,IAAI,KAAK;EACzB,KAAK,MAAM,GAAG,QAAQ,GAAG,SAAoB;GAC3C,IACE,UAAU,gBACT,kBAAmD,SAAS,KAAK,GAElE,aAAa,KAAK,MAAM;GAE1B,KAAK,KAAK,OAAO,GAAG,IAAI;EAC1B,CAAC;CACH;CAEA,kBAAkB,QAAgC;EAChD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,mCAAmC,QAAQ;GAC3D;EACF;EACA,KAAK,YAAY;EACjB,KAAK,IAAI,OAAO,YAAY,OAAO,gBAAgB,KAAK,SAAS,IAAI;EACrE,KAAK,KAAK,WAAW,MAAM;EAC3B,KAAK,SAAS,iBAAiB,KAAK,KAAK,QAAQ,MAAM,GAAG,KAAK,QAAQ;CACzE;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js

@@ -58,8 +58,10 @@ var SlidingWindowLimiter = class {
 		if (now - this.#windowSizeMs > this.#nextWindow.start + this.#windowSizeMs) [this.#priorWindow, this.#nextWindow] = this.#newWindows();
 		if (now - this.#windowSizeMs >= this.#nextWindow.start) this.#rotateWindows();
 		const canDo = this.totalCallsForTime(now) < this.#maxMutations;
-		if (canDo) if (now < this.#nextWindow.start) this.#priorWindow.count++;
-		else this.#nextWindow.count++;
+		if (canDo) {
+			if (now < this.#nextWindow.start) this.#priorWindow.count++;
+			else this.#nextWindow.count++;
+		}
 		return canDo;
 	}
 	totalCallsForTime(now) {

package/out/zero-cache/src/services/limiter/sliding-window-limiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"sliding-window-limiter.js","names":["#windowSizeMs","#maxMutations","#priorWindow","#nextWindow","#newWindows","#rotateWindows"],"sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\ntype Window = {\n  start: number;\n  count: number;\n};\n\n/**\n * The limiter is a sliding window rate limiter that allows a maximum\n * number of mutations per window.\n *\n * The window is divided into two parts: the prev window and the next window.\n * The prev window is the part of the window that has already passed, and the\n * next window is the part of the window that is still in the future.\n *\n * The limiter keeps track of the number of mutations in each window. When the\n * current time moves completely past the next window, the limiter creates new\n * windows (this only happens if the limiter is not called for a long time).\n * When the current time moves completely into the next window, the\n * limiter rotates the windows.\n *\n * Case 1: need new windows\n * |----|----|\n *             |----| (sliding window)\n *\n * Case 2: rotate windows\n * |----|----|\n *      |----| (sliding window)\n *\n * or\n *\n * |----|----|\n *         |----| (sliding window)\n *\n * The leading edge of the sliding window is current time.\n *\n * The limiter increments the count for the prev window if the current time is\n * in the prior window. Otherwise, it increments the count for the next\n * window.\n *\n * The limiter computes the total mutations by: taking the number of mutation in the next window and\n * adding the number of mutation from the prev window, weighted by\n * the fraction of the window that has passed. The total mutations must be less\n * than the maximum number of mutations allowed per window.\n *\n * |----|----|\n *    |----| (sliding window)\n * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.\n * The entirety of the next window's count counts.\n */\nexport class SlidingWindowLimiter {\n  readonly #windowSizeMs: number;\n  readonly #maxMutations: number;\n\n  #priorWindow: Window;\n  #nextWindow: Window;\n\n  constructor(windowSizeMs: number, maxMutations: number) {\n    this.#windowSizeMs = windowSizeMs;\n    this.#maxMutations = maxMutations;\n\n    [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n  }\n\n  canDo(): boolean {\n    const now = Date.now();\n\n    // If the current sliding window is completely past the next window, we need new windows.\n    if (\n      now - this.#windowSizeMs >\n      this.#nextWindow.start + this.#windowSizeMs\n    ) {\n      [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n    }\n\n    // Has the current sliding window moved completely into the next window?\n    // Then rotate the windows.\n    if (now - this.#windowSizeMs >= this.#nextWindow.start) {\n      this.#rotateWindows();\n    }\n\n    // Now compute the total mutations in the current window.\n    // Weighted by the fraction of the window that has passed.\n    const totalCalls = this.totalCallsForTime(now);\n    const canDo = totalCalls < this.#maxMutations;\n\n    // We don't increment counts if we throttled the user. This is so the user can\n    // eventually do the action they are trying to do. If we incremented on every\n    // attempt they can end up in a case where their excessive retries lock them out\n    // continuously.\n    if (canDo) {\n      // If \"now\" in the prior or next window? Increment the count for the correct window.\n      if (now < this.#nextWindow.start) {\n        this.#priorWindow.count++;\n      } else {\n        this.#nextWindow.count++;\n      }\n    }\n\n    return canDo;\n  }\n\n  totalCallsForTime(now: number): number {\n    let fraction: number;\n    if (now < this.#priorWindow.start + this.#windowSizeMs) {\n      fraction = 1;\n    } else {\n      fraction =\n        (this.#priorWindow.start +\n          (this.#windowSizeMs - 1) -\n          (now - this.#windowSizeMs)) /\n        this.#windowSizeMs;\n    }\n    if (fraction < 0) {\n      fraction = 0;\n    }\n    assert(\n      fraction <= 1,\n      'The past cannot contribute more than the a full window.',\n    );\n    const totalCalls =\n      this.#priorWindow.count * fraction + this.#nextWindow.count;\n    return totalCalls;\n  }\n\n  get priorWindow(): Window {\n    return this.#priorWindow;\n  }\n\n  get nextWindow(): Window {\n    return this.#nextWindow;\n  }\n\n  /**\n   * If there were no mutations in over `windowSizeMs` we need new windows.\n   */\n  #newWindows() {\n    const now = Date.now();\n    const start = now - (now % this.#windowSizeMs);\n    return [\n      {\n        // clamp to a `windowSizeMs` boundary to start the prior window.\n        start,\n        count: 0,\n      },\n      {\n        start: start + this.#windowSizeMs,\n        count: 0,\n      },\n    ];\n  }\n\n  #rotateWindows() {\n    this.#priorWindow = this.#nextWindow;\n    this.#nextWindow = {\n      start: this.#priorWindow.start + this.#windowSizeMs,\n      count: 0,\n    };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAa,uBAAb,MAAkC;CAChC;CACA;CAEA;CACA;CAEA,YAAY,cAAsB,cAAsB;EACtD,KAAKA,gBAAgB;EACrB,KAAKC,gBAAgB;EAErB,CAAC,KAAKC,cAAc,KAAKC,eAAe,KAAKC,YAAY;CAC3D;CAEA,QAAiB;EACf,MAAM,MAAM,KAAK,IAAI;EAGrB,IACE,MAAM,KAAKJ,gBACX,KAAKG,YAAY,QAAQ,KAAKH,eAE9B,CAAC,KAAKE,cAAc,KAAKC,eAAe,KAAKC,YAAY;EAK3D,IAAI,MAAM,KAAKJ,iBAAiB,KAAKG,YAAY,OAC/C,KAAKE,eAAe;EAMtB,MAAM,QADa,KAAK,kBAAkB,GAC5B,IAAa,KAAKJ;EAMhC,IAAI,OAEF,IAAI,MAAM,KAAKE,YAAY,OACzB,KAAKD,aAAa;OAElB,KAAKC,YAAY;EAIrB,OAAO;CACT;CAEA,kBAAkB,KAAqB;EACrC,IAAI;EACJ,IAAI,MAAM,KAAKD,aAAa,QAAQ,KAAKF,eACvC,WAAW;OAEX,YACG,KAAKE,aAAa,SAChB,KAAKF,gBAAgB,MACrB,MAAM,KAAKA,kBACd,KAAKA;EAET,IAAI,WAAW,GACb,WAAW;EAEb,OACE,YAAY,GACZ,yDACF;EAGA,OADE,KAAKE,aAAa,QAAQ,WAAW,KAAKC,YAAY;CAE1D;CAEA,IAAI,cAAsB;EACxB,OAAO,KAAKD;CACd;CAEA,IAAI,aAAqB;EACvB,OAAO,KAAKC;CACd;;;;CAKA,cAAc;EACZ,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,QAAQ,MAAO,MAAM,KAAKH;EAChC,OAAO,CACL;GAEE;GACA,OAAO;EACT,GACA;GACE,OAAO,QAAQ,KAAKA;GACpB,OAAO;EACT,CACF;CACF;CAEA,iBAAiB;EACf,KAAKE,eAAe,KAAKC;EACzB,KAAKA,cAAc;GACjB,OAAO,KAAKD,aAAa,QAAQ,KAAKF;GACtC,OAAO;EACT;CACF;AACF"}
+{"version":3,"file":"sliding-window-limiter.js","names":[],"sources":["../../../../../../zero-cache/src/services/limiter/sliding-window-limiter.ts"],"sourcesContent":["import {assert} from '../../../../shared/src/asserts.ts';\n\ntype Window = {\n  start: number;\n  count: number;\n};\n\n/**\n * The limiter is a sliding window rate limiter that allows a maximum\n * number of mutations per window.\n *\n * The window is divided into two parts: the prev window and the next window.\n * The prev window is the part of the window that has already passed, and the\n * next window is the part of the window that is still in the future.\n *\n * The limiter keeps track of the number of mutations in each window. When the\n * current time moves completely past the next window, the limiter creates new\n * windows (this only happens if the limiter is not called for a long time).\n * When the current time moves completely into the next window, the\n * limiter rotates the windows.\n *\n * Case 1: need new windows\n * |----|----|\n *             |----| (sliding window)\n *\n * Case 2: rotate windows\n * |----|----|\n *      |----| (sliding window)\n *\n * or\n *\n * |----|----|\n *         |----| (sliding window)\n *\n * The leading edge of the sliding window is current time.\n *\n * The limiter increments the count for the prev window if the current time is\n * in the prior window. Otherwise, it increments the count for the next\n * window.\n *\n * The limiter computes the total mutations by: taking the number of mutation in the next window and\n * adding the number of mutation from the prev window, weighted by\n * the fraction of the window that has passed. The total mutations must be less\n * than the maximum number of mutations allowed per window.\n *\n * |----|----|\n *    |----| (sliding window)\n * ^-- the sliding window covers 25% of the prior window. Only 25% of that window's count counts.\n * The entirety of the next window's count counts.\n */\nexport class SlidingWindowLimiter {\n  readonly #windowSizeMs: number;\n  readonly #maxMutations: number;\n\n  #priorWindow: Window;\n  #nextWindow: Window;\n\n  constructor(windowSizeMs: number, maxMutations: number) {\n    this.#windowSizeMs = windowSizeMs;\n    this.#maxMutations = maxMutations;\n\n    [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n  }\n\n  canDo(): boolean {\n    const now = Date.now();\n\n    // If the current sliding window is completely past the next window, we need new windows.\n    if (\n      now - this.#windowSizeMs >\n      this.#nextWindow.start + this.#windowSizeMs\n    ) {\n      [this.#priorWindow, this.#nextWindow] = this.#newWindows();\n    }\n\n    // Has the current sliding window moved completely into the next window?\n    // Then rotate the windows.\n    if (now - this.#windowSizeMs >= this.#nextWindow.start) {\n      this.#rotateWindows();\n    }\n\n    // Now compute the total mutations in the current window.\n    // Weighted by the fraction of the window that has passed.\n    const totalCalls = this.totalCallsForTime(now);\n    const canDo = totalCalls < this.#maxMutations;\n\n    // We don't increment counts if we throttled the user. This is so the user can\n    // eventually do the action they are trying to do. If we incremented on every\n    // attempt they can end up in a case where their excessive retries lock them out\n    // continuously.\n    if (canDo) {\n      // If \"now\" in the prior or next window? Increment the count for the correct window.\n      if (now < this.#nextWindow.start) {\n        this.#priorWindow.count++;\n      } else {\n        this.#nextWindow.count++;\n      }\n    }\n\n    return canDo;\n  }\n\n  totalCallsForTime(now: number): number {\n    let fraction: number;\n    if (now < this.#priorWindow.start + this.#windowSizeMs) {\n      fraction = 1;\n    } else {\n      fraction =\n        (this.#priorWindow.start +\n          (this.#windowSizeMs - 1) -\n          (now - this.#windowSizeMs)) /\n        this.#windowSizeMs;\n    }\n    if (fraction < 0) {\n      fraction = 0;\n    }\n    assert(\n      fraction <= 1,\n      'The past cannot contribute more than the a full window.',\n    );\n    const totalCalls =\n      this.#priorWindow.count * fraction + this.#nextWindow.count;\n    return totalCalls;\n  }\n\n  get priorWindow(): Window {\n    return this.#priorWindow;\n  }\n\n  get nextWindow(): Window {\n    return this.#nextWindow;\n  }\n\n  /**\n   * If there were no mutations in over `windowSizeMs` we need new windows.\n   */\n  #newWindows() {\n    const now = Date.now();\n    const start = now - (now % this.#windowSizeMs);\n    return [\n      {\n        // clamp to a `windowSizeMs` boundary to start the prior window.\n        start,\n        count: 0,\n      },\n      {\n        start: start + this.#windowSizeMs,\n        count: 0,\n      },\n    ];\n  }\n\n  #rotateWindows() {\n    this.#priorWindow = this.#nextWindow;\n    this.#nextWindow = {\n      start: this.#priorWindow.start + this.#windowSizeMs,\n      count: 0,\n    };\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAa,uBAAb,MAAkC;CAChC;CACA;CAEA;CACA;CAEA,YAAY,cAAsB,cAAsB;EACtD,KAAK,gBAAgB;EACrB,KAAK,gBAAgB;EAErB,CAAC,KAAK,cAAc,KAAK,eAAe,KAAK,YAAY;CAC3D;CAEA,QAAiB;EACf,MAAM,MAAM,KAAK,IAAI;EAGrB,IACE,MAAM,KAAK,gBACX,KAAK,YAAY,QAAQ,KAAK,eAE9B,CAAC,KAAK,cAAc,KAAK,eAAe,KAAK,YAAY;EAK3D,IAAI,MAAM,KAAK,iBAAiB,KAAK,YAAY,OAC/C,KAAK,eAAe;EAMtB,MAAM,QADa,KAAK,kBAAkB,GAC5B,IAAa,KAAK;EAMhC,IAAI,OAAO;GAET,IAAI,MAAM,KAAK,YAAY,OACzB,KAAK,aAAa;QAElB,KAAK,YAAY;EAErB;EAEA,OAAO;CACT;CAEA,kBAAkB,KAAqB;EACrC,IAAI;EACJ,IAAI,MAAM,KAAK,aAAa,QAAQ,KAAK,eACvC,WAAW;OAEX,YACG,KAAK,aAAa,SAChB,KAAK,gBAAgB,MACrB,MAAM,KAAK,kBACd,KAAK;EAET,IAAI,WAAW,GACb,WAAW;EAEb,OACE,YAAY,GACZ,yDACF;EAGA,OADE,KAAK,aAAa,QAAQ,WAAW,KAAK,YAAY;CAE1D;CAEA,IAAI,cAAsB;EACxB,OAAO,KAAK;CACd;CAEA,IAAI,aAAqB;EACvB,OAAO,KAAK;CACd;;;;CAKA,cAAc;EACZ,MAAM,MAAM,KAAK,IAAI;EACrB,MAAM,QAAQ,MAAO,MAAM,KAAK;EAChC,OAAO,CACL;GAEE;GACA,OAAO;EACT,GACA;GACE,OAAO,QAAQ,KAAK;GACpB,OAAO;EACT,CACF;CACF;CAEA,iBAAiB;EACf,KAAK,eAAe,KAAK;EACzB,KAAK,cAAc;GACjB,OAAO,KAAK,aAAa,QAAQ,KAAK;GACtC,OAAO;EACT;CACF;AACF"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -1,15 +1,15 @@
+import { resolver } from "../../../../shared/src/resolver.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
 import { must } from "../../../../shared/src/must.js";
 import "../../config/normalize.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import { getSubscriptionState } from "../replicator/schema/replication-state.js";
-import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
+import { StatementRunner } from "../../db/statements.js";
+import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { deleteChangeLogDB } from "../replicator/change-log-db.js";
 import { litestreamSocketPath } from "./litestream-controller.js";
 import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreValidationDuration, litestreamRestoredDbBytes } from "./metrics.js";
-import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
 import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
 import { basename, dirname, join } from "node:path";
@@ -464,6 +464,6 @@ function parseBackupCreatedTimes(lc, command, output) {
 	return times;
 }
 //#endregion
-export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, readLocalReplica, startReplicaBackupProcess, tryRestore };
+export { backupDestinationURL, deleteLitestreamMetaDir, getLastBackupTime, parseBackupCreatedTimes, readLocalReplica, startReplicaBackupProcess, tryRestore };
 
 //# sourceMappingURL=commands.js.map

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Returns the replica version and watermark of the local replica file, or\n * `undefined` if it does not exist or cannot be read.\n */\nexport function readLocalReplica(\n  lc: LogContext,\n  replica: string,\n): {replicaVersion: string; watermark: string} | undefined {\n  if (!existsSync(replica)) {\n    return undefined;\n  }\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica, {readonly: true});\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    return {replicaVersion, watermark};\n  } catch (e) {\n    lc.warn?.(`unable to read the subscription state of ${replica}`, e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;AAMA,SAAgB,iBACd,IACA,SACyD;CACzD,IAAI,CAAC,WAAW,OAAO,GACrB;CAEF,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,SAAS,EAAC,UAAU,KAAI,CAAC;EAC/C,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,OAAO;GAAC;GAAgB;EAAS;CACnC,SAAS,GAAG;EACV,GAAG,OAAO,4CAA4C,WAAW,CAAC;EAClE;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
+{"version":3,"file":"commands.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Returns the replica version and watermark of the local replica file, or\n * `undefined` if it does not exist or cannot be read.\n */\nexport function readLocalReplica(\n  lc: LogContext,\n  replica: string,\n): {replicaVersion: string; watermark: string} | undefined {\n  if (!existsSync(replica)) {\n    return undefined;\n  }\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica, {readonly: true});\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    return {replicaVersion, watermark};\n  } catch (e) {\n    lc.warn?.(`unable to read the subscription state of ${replica}`, e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAOjC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,CAAC,CAAC,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,CAAC,CAAC,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,CAAC,CAC5B,KAAI,UAAS,MAAM,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,CAAC,CAAC,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,CAAC,CAAC,IAAI,SAAS,WAAW,CAAC,CAAC,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,CAAC,CAAC,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0BA,eAAE,OAAO,EACvC,MAAMA,eACH,OAAO;CACN,QAAQA,eAAE,QAAQ;CAClB,MAAMA,eAAE,OAAO,CAAC,CAAC,SAAS;AAC5B,CAAC,CAAC,CACD,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAASC,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,CAAC,CAAC,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,CAAC,CAC9C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;AAMA,SAAgB,iBACd,IACA,SACyD;CACzD,IAAI,CAAC,WAAW,OAAO,GACrB;CAEF,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,SAAS,EAAC,UAAU,KAAI,CAAC;EAC/C,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,OAAO;GAAC;GAAgB;EAAS;CACnC,SAAS,GAAG;EACV,GAAG,OAAO,4CAA4C,WAAW,CAAC;EAClE;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,CAAC,CAAC,IAAI,GAAG,MAAM;EAC3C,gCAAgC,CAAC,CAAC,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js

@@ -2,7 +2,7 @@ import { AbortError } from "../../../../shared/src/abort-error.js";
 import { RunningState } from "../running-state.js";
 //#region ../zero-cache/src/services/litestream/litestream-checkpointer.ts
 var SOFT_CHECKPOINT_WAIT_MS = 1e4;
-var LITESTREAM_SYNC_TIMEOUT_MS = 5 * 6e4;
+var LITESTREAM_SYNC_TIMEOUT_MS = 3e5;
 var PAUSE_POLL_INTERVAL_MS = 1e3;
 var PAUSE_LOG_INTERVAL_POLLS = 30;
 /**
@@ -126,6 +126,6 @@ var LitestreamCheckpointer = class {
 	}
 };
 //#endregion
-export { LitestreamCheckpointer };
+export { LitestreamCheckpointer, SOFT_CHECKPOINT_WAIT_MS };
 
 //# sourceMappingURL=litestream-checkpointer.js.map

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-checkpointer.js","names":["#lc","#db","#litestream","#attemptChunk","#maxWalPages","#state","#nextWalThreshold","#getNumWalPages","#pendingSync","#sync","#waitFor","#pauseUntilWalDrained"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\n// How long the soft path blocks the write path waiting for an in-flight sync\n// before letting the writer proceed. This is a backpressure knob, *not* a\n// request timeout: letting it elapse leaves the sync running (see #pendingSync)\n// rather than abandoning it. In normal operation litestream seals + runs its\n// PASSIVE checkpoint well within this budget; a sync that regularly exceeds it\n// indicates a litestream-side problem to fix (e.g. unintended snapshot), not a\n// value to raise.\nexport const SOFT_CHECKPOINT_WAIT_MS = 10_000;\n// Liveness bound on the /sync request itself, well above the soft wait budget.\n// It exists only to recover from a genuinely wedged litestream (so #pendingSync\n// can't hang forever).\nconst LITESTREAM_SYNC_TIMEOUT_MS = 5 * 60_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint) and block the write path until it\n *    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the\n *    backpressure: it throttles upstream reads to litestream's WAL-seal rate\n *    and, in the common case, gives litestream a window to checkpoint without\n *    writer interference. A single sync is kept in flight at a time\n *    (`#pendingSync`): if one is still running when the budget elapses, the\n *    writer proceeds and the next commit waits for that sync rather than\n *    issuing (and orphaning) another. Backs off by a full threshold's worth of\n *    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream\n *    is mid-snapshot and its checkpoints are being skipped), so repeated\n *    failures don't reattempt on every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n  #pendingSync: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    let walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    // Single-flight: reuse the outstanding sync if one is already running.\n    this.#pendingSync ??= this.#sync(walPages);\n\n    // Soft backpressure: block the write path until the sync completes, but no\n    // longer than the wait budget. Letting the budget elapse lets the writer\n    // proceed *without* cancelling the sync (it keeps running, tracked by\n    // #pendingSync); unbounded WAL growth is caught by the hard cap below.\n    await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);\n\n    walPages = this.#getNumWalPages();\n    if (this.#maxWalPages !== undefined && walPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = walPages + this.#attemptChunk;\n  }\n\n  async #waitFor(p: Promise<void>, timeoutMs: number): Promise<void> {\n    const result = await Promise.race([\n      p,\n      this.#state.sleep(timeoutMs).then(() => 'timed-out' as const),\n    ]);\n    if (result === 'timed-out') {\n      this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);\n    }\n  }\n\n  /**\n   * Runs a single litestream sync, logs its outcome, and clears\n   * {@link #pendingSync} when it settles. Never rejects: sync failures and\n   * post-sync bookkeeping errors (e.g. the db being closed during shutdown)\n   * are caught and logged, so the shared promise can be awaited from multiple\n   * places without risking an unhandled rejection.\n   */\n  async #sync(walPagesBefore: number): Promise<void> {\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      try {\n        result = await this.#litestream.sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        );\n      } catch (e) {\n        err = e;\n      }\n\n      const elapsed = performance.now() - start;\n      const newWalPages = this.#getNumWalPages();\n      if (newWalPages < this.#nextWalThreshold) {\n        this.#lc.info?.(\n          `checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else if (result) {\n        this.#lc.warn?.(\n          `checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else {\n        this.#lc.warn?.(\n          `checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n          err,\n        );\n      }\n    } catch (e) {\n      this.#lc.warn?.('litestream-checkpointer sync post-processing failed', e);\n    } finally {\n      this.#pendingSync = undefined;\n    }\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      // Single-flight: reuse the outstanding sync if one is already running.\n      this.#pendingSync ??= this.#sync(this.#getNumWalPages());\n      // await the full liveness timeout configured in #sync()\n      await this.#pendingSync;\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAiBA,IAAa,0BAA0B;AAIvC,IAAM,6BAA6B,IAAI;AACvC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CACA;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKC,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,gBAAgB,OAAO;EAC5B,KAAKG,oBAAoB,OAAO;EAChC,KAAKF,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAKH,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,IAAI,WAAW,KAAKM,gBAAgB;EACpC,IAAI,WAAW,KAAKD,mBAClB;EAIF,KAAKE,iBAAiB,KAAKC,MAAM,QAAQ;EAMzC,MAAM,KAAKC,SAAS,KAAKF,cAAc,uBAAuB;EAE9D,WAAW,KAAKD,gBAAgB;EAChC,IAAI,KAAKH,iBAAiB,KAAA,KAAa,WAAW,KAAKA,cAOrD,WAAW,MAAM,KAAKO,sBAAsB,KAAKP,YAAY;EAO/D,KAAKE,oBAAoB,WAAW,KAAKH;CAC3C;CAEA,MAAMO,SAAS,GAAkB,WAAkC;EAKjE,IAAI,MAJiB,QAAQ,KAAK,CAChC,GACA,KAAKL,OAAO,MAAM,SAAS,EAAE,WAAW,WAAoB,CAC9D,CAAC,MACc,aACb,KAAKL,IAAI,OAAO,uCAAuC,UAAU,IAAI;CAEzE;;;;;;;;CASA,MAAMS,MAAM,gBAAuC;EACjD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IACF,SAAS,MAAM,KAAKP,YAAY,KAC9B;KAAC,MAAM;KAAO,WAAW;IAA0B,GACnD,KAAKG,OAAO,MACd;GACF,SAAS,GAAG;IACV,MAAM;GACR;GAEA,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,cAAc,KAAKE,gBAAgB;GACzC,IAAI,cAAc,KAAKD,mBACrB,KAAKN,IAAI,OACP,gBAAgB,eAAe,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAClF,MACF;QACK,IAAI,QACT,KAAKA,IAAI,OACP,sBAAsB,eAAe,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OAC7G,MACF;QAEA,KAAKA,IAAI,OACP,qBAAqB,eAAe,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC7E,GACF;EAEJ,SAAS,GAAG;GACV,KAAKA,IAAI,OAAO,uDAAuD,CAAC;EAC1E,UAAU;GACR,KAAKQ,eAAe,KAAA;EACtB;CACF;CAEA,MAAMG,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAKN,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAKE,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAKP,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAKA,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAKK,OAAO,MAAM,sBAAsB;GAE9C,KAAKG,iBAAiB,KAAKC,MAAM,KAAKF,gBAAgB,CAAC;GAEvD,MAAM,KAAKC;EACb;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAKN,YAAY,MAAM;EACvB,KAAKG,OAAO,KAAK,KAAKL,GAAG;CAC3B;AACF"}
+{"version":3,"file":"litestream-checkpointer.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\n// How long the soft path blocks the write path waiting for an in-flight sync\n// before letting the writer proceed. This is a backpressure knob, *not* a\n// request timeout: letting it elapse leaves the sync running (see #pendingSync)\n// rather than abandoning it. In normal operation litestream seals + runs its\n// PASSIVE checkpoint well within this budget; a sync that regularly exceeds it\n// indicates a litestream-side problem to fix (e.g. unintended snapshot), not a\n// value to raise.\nexport const SOFT_CHECKPOINT_WAIT_MS = 10_000;\n// Liveness bound on the /sync request itself, well above the soft wait budget.\n// It exists only to recover from a genuinely wedged litestream (so #pendingSync\n// can't hang forever).\nconst LITESTREAM_SYNC_TIMEOUT_MS = 5 * 60_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint) and block the write path until it\n *    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the\n *    backpressure: it throttles upstream reads to litestream's WAL-seal rate\n *    and, in the common case, gives litestream a window to checkpoint without\n *    writer interference. A single sync is kept in flight at a time\n *    (`#pendingSync`): if one is still running when the budget elapses, the\n *    writer proceeds and the next commit waits for that sync rather than\n *    issuing (and orphaning) another. Backs off by a full threshold's worth of\n *    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream\n *    is mid-snapshot and its checkpoints are being skipped), so repeated\n *    failures don't reattempt on every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n  #pendingSync: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    let walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    // Single-flight: reuse the outstanding sync if one is already running.\n    this.#pendingSync ??= this.#sync(walPages);\n\n    // Soft backpressure: block the write path until the sync completes, but no\n    // longer than the wait budget. Letting the budget elapse lets the writer\n    // proceed *without* cancelling the sync (it keeps running, tracked by\n    // #pendingSync); unbounded WAL growth is caught by the hard cap below.\n    await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);\n\n    walPages = this.#getNumWalPages();\n    if (this.#maxWalPages !== undefined && walPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = walPages + this.#attemptChunk;\n  }\n\n  async #waitFor(p: Promise<void>, timeoutMs: number): Promise<void> {\n    const result = await Promise.race([\n      p,\n      this.#state.sleep(timeoutMs).then(() => 'timed-out' as const),\n    ]);\n    if (result === 'timed-out') {\n      this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);\n    }\n  }\n\n  /**\n   * Runs a single litestream sync, logs its outcome, and clears\n   * {@link #pendingSync} when it settles. Never rejects: sync failures and\n   * post-sync bookkeeping errors (e.g. the db being closed during shutdown)\n   * are caught and logged, so the shared promise can be awaited from multiple\n   * places without risking an unhandled rejection.\n   */\n  async #sync(walPagesBefore: number): Promise<void> {\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      try {\n        result = await this.#litestream.sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        );\n      } catch (e) {\n        err = e;\n      }\n\n      const elapsed = performance.now() - start;\n      const newWalPages = this.#getNumWalPages();\n      if (newWalPages < this.#nextWalThreshold) {\n        this.#lc.info?.(\n          `checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else if (result) {\n        this.#lc.warn?.(\n          `checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else {\n        this.#lc.warn?.(\n          `checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n          err,\n        );\n      }\n    } catch (e) {\n      this.#lc.warn?.('litestream-checkpointer sync post-processing failed', e);\n    } finally {\n      this.#pendingSync = undefined;\n    }\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      // Single-flight: reuse the outstanding sync if one is already running.\n      this.#pendingSync ??= this.#sync(this.#getNumWalPages());\n      // await the full liveness timeout configured in #sync()\n      await this.#pendingSync;\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAiBA,IAAa,0BAA0B;AAIvC,IAAM,6BAA6B;AACnC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CACA;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,gBAAgB,OAAO;EAC5B,KAAK,oBAAoB,OAAO;EAChC,KAAK,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAK,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,IAAI,WAAW,KAAK,gBAAgB;EACpC,IAAI,WAAW,KAAK,mBAClB;EAIF,KAAK,iBAAiB,KAAK,MAAM,QAAQ;EAMzC,MAAM,KAAK,SAAS,KAAK,cAAc,uBAAuB;EAE9D,WAAW,KAAK,gBAAgB;EAChC,IAAI,KAAK,iBAAiB,KAAA,KAAa,WAAW,KAAK,cAOrD,WAAW,MAAM,KAAK,sBAAsB,KAAK,YAAY;EAO/D,KAAK,oBAAoB,WAAW,KAAK;CAC3C;CAEA,MAAM,SAAS,GAAkB,WAAkC;EAKjE,IAAI,MAJiB,QAAQ,KAAK,CAChC,GACA,KAAK,OAAO,MAAM,SAAS,CAAC,CAAC,WAAW,WAAoB,CAC9D,CAAC,MACc,aACb,KAAK,IAAI,OAAO,uCAAuC,UAAU,IAAI;CAEzE;;;;;;;;CASA,MAAM,MAAM,gBAAuC;EACjD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IACF,SAAS,MAAM,KAAK,YAAY,KAC9B;KAAC,MAAM;KAAO,WAAW;IAA0B,GACnD,KAAK,OAAO,MACd;GACF,SAAS,GAAG;IACV,MAAM;GACR;GAEA,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,cAAc,KAAK,gBAAgB;GACzC,IAAI,cAAc,KAAK,mBACrB,KAAK,IAAI,OACP,gBAAgB,eAAe,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAClF,MACF;QACK,IAAI,QACT,KAAK,IAAI,OACP,sBAAsB,eAAe,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OAC7G,MACF;QAEA,KAAK,IAAI,OACP,qBAAqB,eAAe,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC7E,GACF;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,uDAAuD,CAAC;EAC1E,UAAU;GACR,KAAK,eAAe,KAAA;EACtB;CACF;CAEA,MAAM,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAK,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAK,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAK,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAK,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAK,OAAO,MAAM,sBAAsB;GAE9C,KAAK,iBAAiB,KAAK,MAAM,KAAK,gBAAgB,CAAC;GAEvD,MAAM,KAAK;EACb;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAK,YAAY,MAAM;EACvB,KAAK,OAAO,KAAK,KAAK,GAAG;CAC3B;AACF"}

package/out/zero-cache/src/services/litestream/litestream-controller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-controller.js","names":["#lc","#replicaFile","#socketPath","#agent","#post"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean | undefined;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number | undefined;\n\n  /**\n   * Server-side deadline in seconds, sent as litestream's `timeout`. When set,\n   * litestream ends the request itself (with an error response) once it\n   * elapses. Set it below {@link timeoutMs} so that litestream, rather than\n   * the client, ends a slow request. (litestream defaults to 30s when `wait`\n   * is set, and to no deadline otherwise.)\n   */\n  serverTimeoutSeconds?: number | undefined;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {\n      wait = false,\n      timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,\n      serverTimeoutSeconds,\n    } = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {\n        path: this.#replicaFile,\n        wait,\n        ...(serverTimeoutSeconds ? {timeout: serverTimeoutSeconds} : {}),\n      },\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAgDnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAKC,eAAe;EACpB,KAAKC,cAAc;EAGnB,KAAKC,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EACJ,OAAO,OACP,YAAY,4BACZ,yBACE;EACJ,MAAM,SAAS,MAAM,KAAKC,MACxB,SACA;GACE,MAAM,KAAKH;GACX;GACA,GAAI,uBAAuB,EAAC,SAAS,qBAAoB,IAAI,CAAC;EAChE,GACA,WACA,MACF;EACA,KAAKD,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAKG,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAKD;IACjB,QAAQ;IACR;IACA,OAAO,KAAKC;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,EAAE,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}
+{"version":3,"file":"litestream-controller.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean | undefined;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number | undefined;\n\n  /**\n   * Server-side deadline in seconds, sent as litestream's `timeout`. When set,\n   * litestream ends the request itself (with an error response) once it\n   * elapses. Set it below {@link timeoutMs} so that litestream, rather than\n   * the client, ends a slow request. (litestream defaults to 30s when `wait`\n   * is set, and to no deadline otherwise.)\n   */\n  serverTimeoutSeconds?: number | undefined;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {\n      wait = false,\n      timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,\n      serverTimeoutSeconds,\n    } = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {\n        path: this.#replicaFile,\n        wait,\n        ...(serverTimeoutSeconds ? {timeout: serverTimeoutSeconds} : {}),\n      },\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAgDnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAK,eAAe;EACpB,KAAK,cAAc;EAGnB,KAAK,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EACJ,OAAO,OACP,YAAY,4BACZ,yBACE;EACJ,MAAM,SAAS,MAAM,KAAK,MACxB,SACA;GACE,MAAM,KAAK;GACX;GACA,GAAI,uBAAuB,EAAC,SAAS,qBAAoB,IAAI,CAAC;EAChE,GACA,WACA,MACF;EACA,KAAK,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAK,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAK;IACjB,QAAQ;IACR;IACA,OAAO,KAAK;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,CAAC,CAAC,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}