@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  done() {\n    return this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(0);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    lc.error?.(`exiting with error: ${String(e)}`, e);\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAkCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,OAAO;EACL,OAAO,KAAKM,cAAc,QAAQ;CACpC;CAEA,MAAM,MAAc;EAClB,KAAKT,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKS,cAAc,KAAK,KAAKT,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKU,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKT,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKU,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKG;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKQ,gBAAgB,KAAK,KAAKT,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKE,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKV,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKU,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKF,MAAM,CAAC;EACrB;EAEA,IAAI,KAAKE,gBAAgB,GAGvB,OAAO,KAAKF,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAChD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#drainStart","#runningState","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    if (type === 'supporting') {\n      // Supporting workers like the replication-manager shut down without a\n      // drain signal when receiving protocol-specific instructions (like auto\n      // reset). In this case, a special error code is used to signal that the\n      // server should be shut down without draining, but it is otherwise not\n      // considered an unexpected/alertable error.\n      if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) {\n        code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n      }\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code || -1);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code || -1);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n *\n * @returns a Promise that resolves/rejects when any of the services stops/throws.\n */\n\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.once(signal, () => {\n      const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n      services.forEach(async svc => {\n        if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n          lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n          await svc.drain();\n        }\n        lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n        await svc.stop();\n      });\n    });\n  }\n\n  try {\n    // Run all services and resolve when any of them stops.\n    const svc = await Promise.race(\n      services.map(svc => svc.run().then(() => svc)),\n    );\n    lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    lc.error?.(`exiting with error: ${String(e)}`, e);\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAkCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,EAAE,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAKM,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAKC,MACH,KAAKL,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAKM,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAKL,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAKM,gBAAgB,GACvB,KAAKH,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAKI,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAKV,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAKU,cAAc,KAAK,KAAKV,GAAG;EAChC,KAAUA,IAAI,MAAM,EAAE,cAAc,KAAKG,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAKD,KAAK,SAAS,GAAG;GAExB,KAAKF,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAKQ,MAAM,CAAC;EACd;EACA,KAAKR,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAKS,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAKR,YAAY,MACnB,KAAKM,MAAM,KAAKN,aAAa,MAAM;OAEnC,KAAKM,MAAM,KAAKL,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAKD,YAAY,IAAI,IAAI;EAE3B,KAAKC,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAKS,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAKX,IAAI,CAAC,UAAU,KAAKS,cAAc,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAKI;EAClB,KAAKD,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAKP,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,EAAE,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAKL,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAKI,OAAO,KAAK;GACjE,KAAKQ,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAKA,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAKP,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAKM,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAKV,YAAY,OAAO,MAAM;GAC9B,KAAKC,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EAEnC,IAAI,SAAS,cAAc;GAMzB,IAAI,SAAS,MAAM,KAAKO,gBAAgB,KAAK,KAAKR,YAAY,OAAO,IACnE,OAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAKD,IAAI,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAKQ,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAKC,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAKT,IAAI,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAKC,YAAY,SAAS,GAAG;GAC/B,KAAKD,IAAI,OACP,KAAKS,cACD,oCACE,KAAK,IAAI,IAAI,KAAKA,YACnB,QACD,gCACN;GACA,OAAO,KAAKD,MAAM,KAAKC,cAAc,IAAI,QAAQ,EAAE;EACrD;EAEA,IAAI,KAAKA,gBAAgB,GAGvB,OAAO,KAAKD,MAAM,QAAQ,EAAE;CAIhC;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAKR,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;AAUA,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;EAEzB,SAAS,QAAQ,OAAM,QAAO;GAC5B,IAAI,iBAAiB,SAAS,MAAM,KAAK,IAAI,OAAO;IAClD,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;IAClE,MAAM,IAAI,MAAM;GAClB;GACA,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;GAClE,MAAM,IAAI,KAAK;EACjB,CAAC;CACH,CAAC;CAGH,IAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,IAAI,EAAE,WAAW,GAAG,CAAC,CAC/C;EACA,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CACzD,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,EAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAChD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAKA,MAAM;EACX,KAAKc,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAKC,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAKC,wBAAwB,KAAA,GAAW;GAC1C,KAAKhB,IAAI,OACP,iCACE,KAAKc,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAKE,sBAAsB,YACzB,KAAKC,oBACL,KAAKH,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAKI,uBAAuB,mBAAmB;GAC7C,KAAKA,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAKH;GACjD,IAAI,0BAA0B,KAAKD,eAAe;IAChD,KAAKd,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAKgB,mBAAmB;EACrC,IAAI,KAAKE,sBACP,eAAe,KAAKA,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/litestream/commands.d.ts

@@ -1,22 +1,23 @@
 import type { ChildProcess } from 'node:child_process';
 import type { LogContext } from '@rocicorp/logger';
-import type { ZeroConfig } from '../../config/zero-config.ts';
-type ReplicaConstraints = {
+import { type LitestreamConfig } from '../../config/normalize.ts';
+import { type LitestreamRole } from './metrics.ts';
+export type ReplicaConstraints = {
     replicaVersion: string;
     minWatermark: string;
 };
+export type RestoreResult = 'success' | 'no_backup' | 'invalid_replica' | 'error';
+type RestoreAttempt = {
+    restored: boolean;
+    backupURL: string | undefined;
+    result: RestoreResult;
+};
 export declare class BackupNotFoundException extends Error {
     static readonly name = "BackupNotFoundException";
     constructor(backupURL: string | undefined);
 }
-/**
- * @param replicaConstraints The constraints of the restored backup when
- *        restoring for the change-streamer (replication-manager). For the
- *        view-syncer, this should be unspecified so that the constraints are
- *        retrieved from the replication-manager via the snapshot protocol.
- */
-export declare function restoreReplica(lc: LogContext, config: ZeroConfig, replicaConstraints: ReplicaConstraints | null): Promise<void>;
-export declare function startReplicaBackupProcess(lc: LogContext, config: ZeroConfig): ChildProcess;
+export declare function tryRestore(lc: LogContext, config: LitestreamConfig, replicaFile: string, replicaConstraints: ReplicaConstraints | undefined, role: LitestreamRole): Promise<RestoreAttempt>;
+export declare function startReplicaBackupProcess(lc: LogContext, config: LitestreamConfig, replicaFile: string): ChildProcess;
 /**
  * Returns the time of the most recent object (snapshot or WAL segment)
  * actually uploaded to the backup replica destination, as listed by the
@@ -31,7 +32,7 @@ export declare function startReplicaBackupProcess(lc: LogContext, config: ZeroCo
  * Rejects if the backup state cannot be determined (spawn error, non-zero
  * exit, timeout, or empty/unparseable listing).
  */
-export declare function getLastBackupTime(lc: LogContext, config: ZeroConfig): Promise<Date>;
+export declare function getLastBackupTime(lc: LogContext, config: LitestreamConfig, replicaFile: string): Promise<Date>;
 /**
  * Parses the `created` column (the last, RFC3339-formatted column) from the
  * tab-formatted output of `litestream snapshots` / `litestream wal`. The

package/out/zero-cache/src/services/litestream/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAkC5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAUF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBAiF9C;AAmQD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA6DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAI3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAShE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,KAAK,cAAc,GAAG;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;CACvB,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AA6ED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,cAAc,CAAC,CAwIzB;AAoGD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}

package/out/zero-cache/src/services/litestream/commands.js

@@ -1,114 +1,57 @@
-import { sleep } from "../../../../shared/src/sleep.js";
 import { must } from "../../../../shared/src/must.js";
-import { getShardConfig } from "../../types/shards.js";
-import { assertNormalized } from "../../config/normalize.js";
+import "../../config/normalize.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../db/statements.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../db/sqlite-corruption.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
-import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-http.js";
-import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoreWaitDuration, litestreamRestoredDbBytes } from "./metrics.js";
+import { deleteChangeLogDB } from "../replicator/change-log-db.js";
+import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreValidationDuration, litestreamRestoredDbBytes } from "./metrics.js";
 import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
-import { existsSync, statSync } from "node:fs";
+import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
 //#region ../zero-cache/src/services/litestream/commands.ts
-var RETRY_INTERVAL_MS = 3e3;
-var BackupNotFoundException = class extends Error {
-	static name = "BackupNotFoundException";
-	constructor(backupURL) {
-		super(`backup not found at ${backupURL}`);
-	}
-};
-/**
-* @param replicaConstraints The constraints of the restored backup when
-*        restoring for the change-streamer (replication-manager). For the
-*        view-syncer, this should be unspecified so that the constraints are
-*        retrieved from the replication-manager via the snapshot protocol.
-*/
-async function restoreReplica(lc, config, replicaConstraints) {
-	const start = performance.now();
-	const role = replicaConstraints ? "replication_manager" : "view_syncer";
-	let backupURL = config.litestream.backupURL;
-	let result;
-	let consecutiveErrors = 0;
-	try {
-		for (;;) {
-			try {
-				const attempt = await tryRestore(lc, config, replicaConstraints, role, consecutiveErrors >= 1);
-				backupURL = attempt.backupURL;
-				if (attempt.restored) {
-					result = attempt.result;
-					return;
-				}
-				consecutiveErrors = 0;
-				if (replicaConstraints) {
-					result = attempt.result;
-					throw new BackupNotFoundException(config.litestream.backupURL);
-				}
-			} catch (e) {
-				if (e instanceof BackupNotFoundException) throw e;
-				if (++consecutiveErrors <= 1) {
-					lc.warn?.(`restore attempt failed. retrying once`, e);
-					continue;
-				}
-				throw e;
-			}
-			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
-			await sleep(RETRY_INTERVAL_MS);
-		}
-	} catch (e) {
-		if (e instanceof BackupNotFoundException && result === void 0) result = "no_backup";
-		throw e;
-	} finally {
-		const labels = {
-			...litestreamRestoreMetricAttrs(config, role, backupURL),
-			result: result ?? "error"
-		};
-		litestreamRestoreRuns().add(1, labels);
-		litestreamRestoreDuration().recordMs(performance.now() - start, labels);
-	}
-}
-function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
-	const { executable, executableV5, restoreUsingV5, backupURL, logLevel, configPath, endpoint, region, port = config.port + 2, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config.litestream;
-	const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;
+var MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;
+var LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;
+function getLitestream(mode, config, replicaFile, logLevelOverride) {
+	const { executable, executableV5, restoreUsingV5, backupUsingV5, backupURL, logLevel, configPath, configPathV5, endpoint, region, port, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, incrementalBackupIntervalSeconds, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config;
+	const v5 = mode === "restore" && restoreUsingV5 || mode === "replicate" && backupUsingV5;
+	const litestream = (v5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`);
+	const litestreamConfig = v5 ? configPathV5 : configPath;
 	return {
-		litestream: (mode === "restore" && restoreUsingV5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`),
+		litestream,
 		env: {
 			...process.env,
-			["ZERO_REPLICA_FILE"]: config.replica.file,
-			["ZERO_LITESTREAM_BACKUP_URL"]: must(backupURLOverride ?? backupURL),
+			["ZERO_REPLICA_FILE"]: replicaFile,
+			["ZERO_LITESTREAM_BACKUP_URL"]: must(backupURL),
 			["ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT"]: String(minCheckpointPageCount),
 			["ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT"]: String(maxCheckpointPageCount),
 			["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES"]: String(incrementalBackupIntervalMinutes),
+			["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS"]: String(incrementalBackupIntervalSeconds),
 			["ZERO_LITESTREAM_LOG_LEVEL"]: logLevelOverride ?? logLevel,
-			["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES"]: String(snapshotBackupIntervalMinutes),
+			["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS"]: String(snapshotBackupIntervalHours),
+			["ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS"]: String(snapshotBackupIntervalHours + 6),
 			["ZERO_LITESTREAM_MULTIPART_CONCURRENCY"]: String(multipartConcurrency),
 			["ZERO_LITESTREAM_MULTIPART_SIZE"]: String(multipartSize),
-			["ZERO_LOG_FORMAT"]: config.log.format,
-			["LITESTREAM_CONFIG"]: configPath,
+			["ZERO_LOG_FORMAT"]: "json",
+			["LITESTREAM_CONFIG"]: litestreamConfig,
 			["LITESTREAM_PORT"]: String(port),
 			...endpoint ? { ["ZERO_LITESTREAM_ENDPOINT"]: endpoint } : {},
 			...region ? { ["ZERO_LITESTREAM_REGION"]: region } : {}
 		}
 	};
 }
-async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt) {
-	let snapshotStatus;
-	if (!replicaConstraints) {
-		const waitStart = performance.now();
-		snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);
-		litestreamRestoreWaitDuration().recordMs(performance.now() - waitStart, litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL));
-		lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);
-		replicaConstraints = snapshotStatus;
-	}
-	const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;
+async function tryRestore(lc, config, replicaFile, replicaConstraints, role) {
+	const { backupURL } = config;
 	const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);
 	let result = "error";
 	try {
-		const replicaExistedBeforeRestore = existsSync(config.replica.file);
-		const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
-		const { restoreParallelism: parallelism, multipartConcurrency, multipartSize } = config.litestream;
+		logRestoreDirectoryContents(lc, replicaFile, "before-temp-cleanup");
+		deleteRestoreTempFiles(replicaFile);
+		const replicaExistedBeforeRestore = existsSync(replicaFile);
+		const { litestream, env } = getLitestream("restore", config, replicaFile, "debug");
+		const { restoreParallelism: parallelism, multipartConcurrency, multipartSize } = config;
 		lc.info?.(`starting litestream restore`, {
 			restoreParallelism: parallelism,
 			multipartConcurrency,
@@ -120,7 +63,7 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 			"-if-replica-exists",
 			"-parallelism",
 			String(parallelism),
-			config.replica.file
+			replicaFile
 		], {
 			env,
 			stdio: [
@@ -134,8 +77,14 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 		let stderr = "";
 		proc.stdout.setEncoding("utf-8");
 		proc.stderr.setEncoding("utf-8");
-		proc.stdout.on("data", (chunk) => stdout += chunk);
-		proc.stderr.on("data", (chunk) => stderr += chunk);
+		proc.stdout.on("data", (chunk) => {
+			process.stdout.write(chunk);
+			stdout += chunk;
+		});
+		proc.stderr.on("data", (chunk) => {
+			process.stderr.write(chunk);
+			stderr += chunk;
+		});
 		const { promise, resolve, reject } = resolver();
 		proc.on("error", reject);
 		proc.on("close", (code, signal) => {
@@ -150,19 +99,15 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 				...attrs,
 				result: "success"
 			});
-			if (stdout) process.stdout.write(stdout);
-			if (stderr) process.stderr.write(stderr);
 		} catch (e) {
 			litestreamRestoreProcessDuration().recordMs(performance.now() - processStart, {
 				...attrs,
 				result: "error"
 			});
-			const output = (stdout + stderr).trim();
-			if (isFinalAttempt) lc.error?.(`litestream restore failed after retry`, output || e);
-			else lc.error?.(`litestream restore attempt failed, will retry`, output || e);
-			throw e;
+			const output = [stdout, stderr].map((value) => value.trim()).filter(Boolean).join("\n");
+			throw new Error(output ? `${String(e)}\n${output}` : String(e), { cause: e });
 		}
-		if (!existsSync(config.replica.file)) {
+		if (!existsSync(replicaFile)) {
 			result = "no_backup";
 			return {
 				restored: false,
@@ -171,7 +116,7 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 			};
 		}
 		const validationStart = performance.now();
-		const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);
+		const valid = replicaIsValid(lc, replicaFile, replicaConstraints);
 		litestreamRestoreValidationDuration().recordMs(performance.now() - validationStart, {
 			...attrs,
 			result: valid ? "success" : "invalid_replica"
@@ -179,7 +124,8 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 		if (!valid) {
 			result = "invalid_replica";
 			lc.info?.(`Deleting local replica and retrying restore`);
-			deleteLiteDB(config.replica.file);
+			deleteLiteDB(replicaFile);
+			deleteChangeLogDB(replicaFile);
 			return {
 				restored: false,
 				backupURL,
@@ -187,36 +133,87 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 			};
 		}
 		result = "success";
-		if (!replicaExistedBeforeRestore) litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {
-			...attrs,
-			result: "success"
-		});
+		if (!replicaExistedBeforeRestore) {
+			deleteChangeLogDB(replicaFile);
+			litestreamRestoredDbBytes().add(statSync(replicaFile).size, {
+				...attrs,
+				result: "success"
+			});
+		}
 		return {
 			restored: true,
 			backupURL,
 			result
 		};
 	} finally {
+		try {
+			deleteRestoreTempFiles(replicaFile);
+		} catch (e) {
+			lc.warn?.("Unable to clean up Litestream restore temporary files", {
+				replicaFile,
+				error: String(e)
+			});
+		}
+		logRestoreDirectoryContents(lc, replicaFile, "after-temp-cleanup");
 		litestreamRestoreAttempts().add(1, {
 			...attrs,
 			result
 		});
 	}
 }
+function deleteRestoreTempFiles(replicaFile) {
+	const temporaryReplicaFile = `${replicaFile}.tmp`;
+	deleteLiteDB(temporaryReplicaFile);
+	const directory = dirname(temporaryReplicaFile);
+	if (!existsSync(directory)) return;
+	const prefix = `${basename(temporaryReplicaFile)}-`;
+	for (const name of readdirSync(directory)) {
+		if (!name.startsWith(prefix) || !name.endsWith("-wal")) continue;
+		const index = name.slice(prefix.length, -4);
+		if (LEGACY_RESTORE_WAL_INDEX.test(index)) rmSync(join(directory, name), { force: true });
+	}
+}
+function logRestoreDirectoryContents(lc, replicaFile, phase) {
+	const directory = dirname(replicaFile);
+	try {
+		const names = readdirSync(directory).sort();
+		const entries = names.slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES).map((name) => ({
+			name,
+			sizeBytes: statSync(join(directory, name)).size
+		}));
+		lc.info?.("Litestream restore directory contents", {
+			phase,
+			directory,
+			replicaFile,
+			entryCount: names.length,
+			omittedEntryCount: Math.max(0, names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES),
+			entries
+		});
+	} catch (e) {
+		lc.warn?.("Unable to inspect Litestream restore directory", {
+			phase,
+			directory,
+			replicaFile,
+			error: String(e)
+		});
+	}
+}
 function replicaIsValid(lc, replica, constraints) {
 	let db;
 	try {
 		db = new Database(lc, replica);
 		const { replicaVersion, watermark } = getSubscriptionState(new StatementRunner(db));
-		if (replicaVersion !== constraints.replicaVersion) {
-			lc.warn?.(`Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`, constraints);
-			return false;
-		}
-		if (watermark < constraints.minWatermark) {
-			lc.warn?.(`Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`);
-			return false;
+		if (constraints) {
+			if (replicaVersion !== constraints.replicaVersion) {
+				lc.warn?.(`Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`, constraints);
+				return false;
+			}
+			if (watermark < constraints.minWatermark) {
+				lc.warn?.(`Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`);
+				return false;
+			}
+			lc.info?.(`Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`, constraints);
 		}
-		lc.info?.(`Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`, constraints);
 		return true;
 	} catch (e) {
 		if (isSQLiteCorruption(e)) logSQLiteCorruptionDiagnostics(lc, "restored replica", replica, e);
@@ -226,10 +223,10 @@ function replicaIsValid(lc, replica, constraints) {
 		db?.close();
 	}
 }
-function startReplicaBackupProcess(lc, config) {
-	const { litestream, env } = getLitestream("replicate", config);
+function startReplicaBackupProcess(lc, config, replicaFile) {
+	const { litestream, env } = getLitestream("replicate", config, replicaFile);
 	const attrs = litestreamBackupProcessMetricAttrs(config);
-	lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);
+	lc.info?.(`starting litestream backup to ${config.backupURL}`);
 	const start = performance.now();
 	const proc = spawn(litestream, ["replicate"], {
 		env,
@@ -279,10 +276,10 @@ var wsRe = /\s+/;
 * Rejects if the backup state cannot be determined (spawn error, non-zero
 * exit, timeout, or empty/unparseable listing).
 */
-async function getLastBackupTime(lc, config) {
-	const [snapshots, wal] = await Promise.all([listBackupCreatedTimes(lc, config, "snapshots"), listBackupCreatedTimes(lc, config, "wal")]);
+async function getLastBackupTime(lc, config, replicaFile) {
+	const [snapshots, wal] = await Promise.all([listBackupCreatedTimes(lc, config, replicaFile, "snapshots"), listBackupCreatedTimes(lc, config, replicaFile, "wal")]);
 	const times = [...snapshots, ...wal];
-	if (times.length === 0) throw new Error(`no snapshots or WAL segments listed at ${config.litestream.backupURL}`);
+	if (times.length === 0) throw new Error(`no snapshots or WAL segments listed at ${config.backupURL}`);
 	return new Date(Math.max(...times.map((time) => time.getTime())));
 }
 /**
@@ -296,11 +293,11 @@ async function getLastBackupTime(lc, config) {
 * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z
 * ```
 */
-async function listBackupCreatedTimes(lc, config, command) {
+async function listBackupCreatedTimes(lc, config, replicaFile, command) {
 	const start = performance.now();
 	let result = "error";
-	const { litestream, env } = getLitestream("replicate", config);
-	const proc = spawn(litestream, [command, config.replica.file], {
+	const { litestream, env } = getLitestream("replicate", config, replicaFile);
+	const proc = spawn(litestream, [command, replicaFile], {
 		env,
 		stdio: [
 			"ignore",
@@ -359,41 +356,7 @@ function parseBackupCreatedTimes(lc, command, output) {
 	}
 	return times;
 }
-function reserveAndGetSnapshotStatus(lc, config) {
-	const { promise: status, resolve, reject } = resolver();
-	(async function() {
-		const abort = new AbortController();
-		process.on("SIGINT", () => abort.abort());
-		process.on("SIGTERM", () => abort.abort());
-		for (let i = 0;; i++) {
-			let err;
-			try {
-				let resolved = false;
-				const stream = await reserveSnapshot(lc, config);
-				for await (const msg of stream) {
-					resolve(msg[1]);
-					resolved = true;
-				}
-				if (resolved) break;
-			} catch (e) {
-				err = e;
-			}
-			lc.warn?.(`Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`, String(err));
-			try {
-				await sleep(5e3, abort.signal);
-			} catch (e) {
-				return reject(e);
-			}
-		}
-	})();
-	return status;
-}
-function reserveSnapshot(lc, config) {
-	assertNormalized(config);
-	const { taskID, change, changeStreamer } = config;
-	return new ChangeStreamerHttpClient(lc, getShardConfig(config), change.db, changeStreamer.uri).reserveSnapshot(taskID);
-}
 //#endregion
-export { BackupNotFoundException, getLastBackupTime, restoreReplica, startReplicaBackupProcess };
+export { getLastBackupTime, startReplicaBackupProcess, tryRestore };
 
 //# sourceMappingURL=commands.js.map