@rocicorp/zero 1.12.0-canary.10 → 1.12.0-canary.12

package/out/zero-cache/src/services/life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.js","names":[],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AAuCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}
+{"version":3,"file":"life-cycle.js","names":["NodeEventEmitter"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import {EventEmitter as NodeEventEmitter} from 'node:events';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../observability/metrics.ts';\nimport {ConfigurationError} from '../types/configuration-error.ts';\nimport {\n  singleProcessMode,\n  type Subprocess,\n  type Worker,\n} from '../types/processes.ts';\nimport {orTimeout} from '../types/timeout.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/**\n * * `user-facing` workers serve external requests and are the first to\n *   receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n *\n * * `supporting` workers support `user-facing` workers and are sent\n *   the `SIGTERM` signal only after all `user-facing` workers have\n *   exited.\n *\n * For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n *\n * A clean exit (i.e. code 0) of a worker of either type that is not the\n * result of a drain is treated as an intentional shutdown of the server.\n */\nexport type WorkerType = 'user-facing' | 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\nfunction workerStartupMetricName(name: string) {\n  if (name === 'zero-cache') {\n    return undefined;\n  }\n  if (name.startsWith('change-streamer')) {\n    return 'change_streamer';\n  }\n  if (name.startsWith('replicator')) {\n    if (name.includes('(backup)')) {\n      return 'backup_replicator';\n    }\n    if (name.includes('(serving-copy)')) {\n      return 'serving_copy_replicator';\n    }\n    if (name.includes('(serving)')) {\n      return 'serving_replicator';\n    }\n    return 'other';\n  }\n  if (name.startsWith('syncer')) {\n    return 'syncer';\n  }\n  if (name.startsWith('reaper')) {\n    return 'reaper';\n  }\n  if (name.startsWith('shadow-syncer')) {\n    return 'shadow_syncer';\n  }\n  if (name.startsWith('mutator')) {\n    return 'mutator';\n  }\n  return 'other';\n}\n\nfunction startupDuration() {\n  return getOrCreateHistogram('server', 'startup_duration', {\n    description: 'Duration from starting zero-cache to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\nexport function recordStartupDurationMs(durationMs: number) {\n  startupDuration().recordMs(durationMs, {component: 'dispatcher'});\n}\n\nfunction workerStartupDuration() {\n  return getOrCreateHistogram('server', 'worker_startup_duration', {\n    description: 'Duration from starting a worker to its ready signal.',\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a worker exiting cleanly outside of a drain,\n// such as for an auto reset), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/**\n * Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n */\nexport class ProcessManager {\n  readonly #lc: LogContext;\n  readonly #userFacing = new Set<Subprocess>();\n  readonly #all = new Set<Subprocess>();\n  readonly #exitImpl: (code: number) => never;\n  readonly #start = Date.now();\n  readonly #ready: Promise<void>[] = [];\n\n  #runningState = new RunningState('process-manager');\n  #drainStart = 0;\n\n  constructor(lc: LogContext, proc: EventEmitter) {\n    this.#lc = lc.withContext('component', 'process-manager');\n\n    // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n    // initiating a graceful shutdown. The parent process will\n    // exit once all user-facing workers have exited ...\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.on(signal, () => this.#startDrain(signal));\n    }\n\n    // ... which will result in sending `SIGTERM` to the remaining workers.\n    proc.on('exit', code =>\n      this.#kill(\n        this.#all,\n        code === 0\n          ? 'SIGTERM' // graceful, drained shutdown\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT' // intentional abort without drain\n            : 'SIGABRT', // unintentional shutdown, alertable error\n      ),\n    );\n\n    // For other (catchable) kill signals, exit with a non-zero error code\n    // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n    // shutdown) to all workers. For these signals, workers are stopped\n    // immediately without draining, since there is no merit to slowly draining\n    // when supporting workers have stopped.\n    //\n    // The logic for handling these signals is in `runUntilKilled()`.\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      proc.on(signal, () =>\n        this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n      );\n    }\n\n    this.#exitImpl = (code: number) => {\n      if (singleProcessMode()) {\n        return proc.emit('exit', code) as never; // For unit / integration tests.\n      }\n      process.exit(code);\n    };\n  }\n\n  async shutdown() {\n    if (this.#drainStart === 0) {\n      this.#startDrain(GRACEFUL_SHUTDOWN[0]);\n    }\n    await this.#runningState.stopped();\n  }\n\n  #exit(code: number) {\n    this.#lc.info?.('exiting with code', code);\n    this.#runningState.stop(this.#lc);\n    void this.#lc.flush().finally(() => this.#exitImpl(code));\n  }\n\n  #startDrain(signal: GracefulShutdownSignal) {\n    if (this.#all.size === 0) {\n      // Shutdown if a signal is received before any subprocesses are added.\n      this.#lc.info?.(`exiting on ${signal}`);\n      this.#exit(0);\n    }\n    this.#lc.info?.(`initiating drain (${signal})`);\n    this.#drainStart = Date.now();\n    if (this.#userFacing.size) {\n      this.#kill(this.#userFacing, signal);\n    } else {\n      this.#kill(this.#all, signal);\n    }\n  }\n\n  addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n    if (type === 'user-facing') {\n      this.#userFacing.add(proc);\n    }\n    this.#all.add(proc);\n\n    let isOpen = true;\n    proc.on('close', (code, signal) => {\n      isOpen = false;\n      this.#onExit(code, signal, null, type, name, proc);\n    });\n\n    // As per https://nodejs.org/api/child_process.html#event-error\n    // 'error' events can happen when sending a message to a child process\n    // fails. This is not really an error when the server is shutting down,\n    // so log any post-close errors at 'warn'.\n    proc.on('error', err =>\n      this.#lc[!isOpen || this.#drainStart > 0 ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n  }\n\n  readonly #initializing = new Map<number, string>();\n  #nextID = 0;\n\n  addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n    this.addSubprocess(worker, type, name);\n\n    const start = performance.now();\n    const id = ++this.#nextID;\n    this.#initializing.set(id, name);\n    const {promise, resolve} = resolver();\n    this.#ready.push(promise);\n\n    worker.onceMessageType('ready', () => {\n      const elapsed = performance.now() - start;\n      const workerMetricName = workerStartupMetricName(name);\n      if (workerMetricName) {\n        workerStartupDuration().recordMs(elapsed, {\n          worker: workerMetricName,\n          type,\n        });\n      }\n      this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n      this.#initializing.delete(id);\n      resolve();\n    });\n\n    return worker;\n  }\n\n  initializing(): string[] {\n    return [...this.#initializing.values()];\n  }\n\n  async allWorkersReady() {\n    await Promise.all(this.#ready);\n  }\n\n  logErrorAndExit(err: unknown, name: string) {\n    // only accessible by the main (i.e. user-facing) process.\n    this.#onExit(-1, null, err, 'user-facing', name, undefined);\n  }\n\n  #onExit(\n    code: number,\n    sig: NodeJS.Signals | null,\n    err: unknown,\n    type: WorkerType,\n    name: string,\n    worker: Subprocess | undefined,\n  ) {\n    // Remove the worker from maps to avoid attempting to send more signals to it.\n    if (worker) {\n      this.#userFacing.delete(worker);\n      this.#all.delete(worker);\n    }\n\n    const pid = worker?.pid ?? process.pid;\n\n    // A clean exit outside of a drain is an intentional shutdown, e.g. the\n    // replication-manager shutting down for protocol-specific instructions\n    // (like auto reset), or a configuration error. In this case, a special\n    // error code is used to signal that the server should be shut down\n    // without draining, but it is otherwise not considered an\n    // unexpected/alertable error. The same applies to a supporting worker\n    // that exits while user-facing workers are still draining.\n    if (\n      code === 0 &&\n      (this.#drainStart === 0 ||\n        (type === 'supporting' && this.#userFacing.size > 0))\n    ) {\n      code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n    }\n\n    if (type === 'supporting') {\n      const log =\n        code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n          ? 'info'\n          : 'warn';\n      this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n      return this.#exit(code);\n    }\n\n    const log =\n      code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n        ? 'info'\n        : this.#drainStart > 0 || code === UNHANDLED_EXCEPTION_ERROR_CODE\n          ? 'warn'\n          : 'error';\n    this.#lc[log]?.(\n      sig\n        ? `${name} (${pid}) killed with (${sig})`\n        : `${name} (${pid}) exited with code (${code})`,\n      err ?? '',\n    );\n\n    // user-facing workers exited or finished draining.\n    if (this.#userFacing.size === 0) {\n      this.#lc.info?.(\n        this.#drainStart\n          ? `all user-facing workers drained (${\n              Date.now() - this.#drainStart\n            } ms)`\n          : `all user-facing workers exited`,\n      );\n      return this.#exit(this.#drainStart ? 0 : code);\n    }\n\n    if (this.#drainStart === 0) {\n      // If a user-facing worker exits without receiving a drain signal,\n      // shutdown the server.\n      return this.#exit(code);\n    }\n\n    return undefined;\n  }\n\n  #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n    for (const worker of workers) {\n      try {\n        worker.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\n/**\n * Runs the supporting subprocesses of a worker (e.g. the litestream process\n * of the backup-replicator) as a {@link SingletonService}.\n *\n * Unlike a {@link ProcessManager}, it does not forward `SIGTERM` or `SIGINT`\n * to its subprocesses. Instead, {@link runUntilKilled} stops it along with the\n * worker's other services, i.e. after they have drained, so that the\n * subprocesses keep supporting them until then. Forceful signals (`SIGQUIT`\n * and `SIGABRT`), on the other hand, are forwarded immediately once the\n * service is running, and the subprocesses are killed if the worker exits\n * before they are stopped.\n *\n * `run()` settles when a subprocess exits before {@link stop()} is called,\n * which stops the worker: a clean exit resolves (and the parent in turn treats\n * the worker's clean exit outside of a drain as an intentional shutdown),\n * while any other exit rejects.\n */\nexport class SupportingProcesses implements SingletonService {\n  readonly id = 'supporting-processes';\n  readonly #lc: LogContext;\n  readonly #parent: EventEmitter;\n  readonly #procs = new Map<Subprocess, string>();\n  readonly #done = resolver<void>();\n  #signaled: NodeJS.Signals | undefined;\n\n  constructor(lc: LogContext, parent: EventEmitter) {\n    this.#lc = lc.withContext('component', 'supporting-processes');\n    this.#parent = parent;\n    // run() may never be called if the worker fails before running services.\n    this.#done.promise.catch(() => {});\n\n    parent.on('exit', code =>\n      this.#kill(\n        code === 0\n          ? 'SIGTERM'\n          : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n            ? 'SIGQUIT'\n            : 'SIGABRT',\n      ),\n    );\n  }\n\n  add<P extends Subprocess>(proc: P, name: string): P {\n    this.#procs.set(proc, name);\n    proc.on('close', (code: number | null, sig: NodeJS.Signals | null) =>\n      this.#onClose(proc, name, code, sig),\n    );\n    // As with the ProcessManager, 'error' events after a signal (e.g. failing\n    // to send a message to an exiting process) are logged at 'warn'.\n    proc.on('error', err =>\n      this.#lc[this.#signaled ? 'warn' : 'error']?.(\n        `error from ${name} ${proc.pid}`,\n        err,\n      ),\n    );\n    return proc;\n  }\n\n  run(): Promise<void> {\n    // Signal listeners are only installed when running, i.e. alongside those\n    // of runUntilKilled(). Installing them any earlier would consume signals\n    // received during worker initialization, which would otherwise terminate\n    // the worker (or be handled by the initialization logic).\n    for (const signal of FORCEFUL_SHUTDOWN) {\n      this.#parent.on(signal, () => this.#kill(signal));\n    }\n    return this.#done.promise;\n  }\n\n  async stop(): Promise<void> {\n    if (!this.#signaled) {\n      this.#kill('SIGTERM');\n    }\n    if (this.#procs.size === 0) {\n      this.#done.resolve();\n    }\n    await this.#done.promise.catch(() => {});\n  }\n\n  #onClose(\n    proc: Subprocess,\n    name: string,\n    code: number | null,\n    sig: NodeJS.Signals | null,\n  ) {\n    this.#procs.delete(proc);\n    const exit = sig\n      ? `${name} (${proc.pid}) killed with (${sig})`\n      : `${name} (${proc.pid}) exited with code (${code})`;\n\n    if (this.#signaled) {\n      this.#lc.info?.(exit);\n      if (this.#procs.size === 0) {\n        this.#done.resolve();\n      }\n    } else if (code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE) {\n      this.#lc.info?.(exit);\n      this.#done.resolve();\n    } else {\n      this.#done.reject(new Error(exit));\n    }\n  }\n\n  #kill(signal: NodeJS.Signals) {\n    this.#signaled = signal;\n    for (const proc of this.#procs.keys()) {\n      try {\n        proc.kill(signal);\n      } catch (e) {\n        this.#lc.error?.(e);\n      }\n    }\n  }\n}\n\nexport type SelfDrainOptions = {\n  /**\n   * The time after a self drain is requested at which the server sends itself\n   * a `SIGTERM`, unless one was received sooner (e.g. from the container\n   * runner).\n   */\n  delayMs: number;\n\n  /**\n   * Called when a self drain is requested. This should fail health checks\n   * (both readiness and liveness) so that the container runner stops routing\n   * to the server and initiates its normal shutdown process.\n   */\n  onRequested: () => void;\n};\n\n/**\n * Coordinates a drain that is initiated by the server itself (e.g. by a\n * view-syncer that has been granted permission to restore a new replica),\n * as opposed to one initiated by the container runner with a `SIGTERM`.\n *\n * Health checks start failing immediately, upon which an orchestrator such\n * as Kubernetes stops routing to the server and shuts it down as usual\n * (i.e. running any pre-stop hooks before sending `SIGTERM`). As a backstop,\n * and for environments without liveness checks, the server sends itself a\n * `SIGTERM` after the configured delay.\n */\nexport class SelfDrain {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #onRequested: () => void;\n  #requested = false;\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions) {\n    this.#lc = lc.withContext('component', 'self-drain');\n    this.#proc = proc;\n    this.#delayMs = opts.delayMs;\n    this.#onRequested = opts.onRequested;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      proc.once(signal, () => {\n        this.#signaled = true;\n        clearTimeout(this.#timer);\n      });\n    }\n  }\n\n  request() {\n    if (this.#requested || this.#signaled) {\n      return; // Already draining.\n    }\n    this.#requested = true;\n    this.#lc.info?.(\n      `self drain requested. sending SIGTERM in ${this.#delayMs} ms`,\n    );\n    this.#onRequested();\n    this.#timer = setTimeout(() => {\n      if (!this.#signaled) {\n        this.#lc.info?.(`initiating self drain`);\n        this.#proc.emit('SIGTERM', 'SIGTERM');\n      }\n    }, this.#delayMs);\n  }\n}\n\n/**\n * The event that a {@link PreDrain} emits immediately upon the first graceful\n * shutdown signal, with that signal as its argument, before the delay.\n */\nexport const PRE_DRAIN = 'preDrain';\n\n/**\n * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load\n * balancers can observe a failed readiness check before the server begins\n * to drain. The first graceful signal is announced immediately with a\n * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such\n * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the\n * delay has elapsed. Repeated graceful signals are dropped. All other events,\n * including forceful signals, are forwarded immediately.\n */\nexport class PreDrain extends NodeEventEmitter {\n  readonly #lc: LogContext;\n  readonly #proc: EventEmitter;\n  readonly #delayMs: number;\n  readonly #forwarded = new Set<string | symbol>(['newListener', PRE_DRAIN]);\n  #signaled = false;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n\n  constructor(lc: LogContext, proc: EventEmitter, delayMs: number) {\n    super();\n    this.#lc = lc.withContext('component', 'pre-drain');\n    this.#proc = proc;\n    this.#delayMs = delayMs;\n\n    for (const signal of GRACEFUL_SHUTDOWN) {\n      this.#forwarded.add(signal);\n      proc.on(signal, () => this.#onGracefulSignal(signal));\n    }\n    this.on('newListener', (event: string | symbol) => this.#forward(event));\n  }\n\n  #forward(event: string | symbol) {\n    if (this.#forwarded.has(event)) {\n      return;\n    }\n    this.#forwarded.add(event);\n    this.#proc.on(event, (...args: unknown[]) => {\n      if (\n        event === 'disconnect' ||\n        (FORCEFUL_SHUTDOWN as readonly (string | symbol)[]).includes(event)\n      ) {\n        clearTimeout(this.#timer); // Stopping without a drain.\n      }\n      this.emit(event, ...args);\n    });\n  }\n\n  #onGracefulSignal(signal: GracefulShutdownSignal) {\n    if (this.#signaled) {\n      this.#lc.info?.(`already shutting down. ignoring ${signal}`);\n      return;\n    }\n    this.#signaled = true;\n    this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);\n    this.emit(PRE_DRAIN, signal);\n    this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);\n  }\n}\n\n/**\n * The maximum time that {@link runUntilKilled} waits for its services to stop\n * (i.e. for their `run()` to settle) after stopping them.\n */\nexport const STOP_TIMEOUT_MS = 60_000;\n\n/**\n * Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`, or when the parent process disconnects.\n * Repeated graceful signals are ignored. If a service stops (or throws) on its\n * own, the remaining services are stopped without draining.\n *\n * @returns a Promise that settles once all services have stopped (or\n *          {@link STOP_TIMEOUT_MS} has elapsed), resolving or rejecting\n *          with the outcome of the first service to stop.\n */\nexport async function runUntilKilled(\n  lc: LogContext,\n  parent: EventEmitter,\n  ...services: SingletonService[]\n): Promise<void> {\n  if (services.length === 0) {\n    return;\n  }\n  const stopped = new Set<SingletonService>();\n  const runs = services.map(svc => svc.run().finally(() => stopped.add(svc)));\n\n  const allStopped = resolver<void>();\n  let stopping: Promise<void> | undefined;\n  const stopAll = (reason: string) =>\n    (stopping ??= stopServices(lc, services, runs, stopped, reason).then(\n      allStopped.resolve,\n    ));\n\n  const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n  let draining = false;\n  const onSignal = async (signal: NodeJS.Signals) => {\n    if (GRACEFUL_SIGNALS.includes(signal)) {\n      if (draining || stopping) {\n        // e.g. A SIGINT from both the terminal and a wrapping process.\n        lc.info?.(`already shutting down. ignoring ${signal}`);\n        return;\n      }\n      draining = true;\n      // Drain all services before stopping any of them. Stopping a service\n      // resolves its run() and thus ends runUntilKilled(), so a service\n      // without a drain() must not be stopped while others are draining.\n      const results = await Promise.allSettled(\n        services.map(async svc => {\n          if (svc.drain) {\n            lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n            await svc.drain();\n          }\n        }),\n      );\n      for (const result of results) {\n        if (result.status === 'rejected') {\n          lc.warn?.(`error while draining (${signal})`, result.reason);\n        }\n      }\n    }\n    // A forceful signal stops the services without waiting for (or after)\n    // a drain.\n    await stopAll(signal);\n  };\n  // The listeners remain installed for the lifetime of the process, as a\n  // signal without a listener would terminate the process (e.g. a repeated\n  // SIGINT during a drain).\n  for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n    parent.on(signal, () => void onSignal(signal));\n  }\n  // A forked worker whose parent has exited (e.g. was killed by a signal,\n  // in which case it cannot signal its workers) stops without draining, so\n  // that it (and in turn its own subprocesses) is not orphaned.\n  parent.once('disconnect', () => {\n    lc.warn?.(`parent process disconnected. stopping without draining`);\n    void stopAll('disconnect');\n  });\n\n  try {\n    // Run all services and resolve when any of them stops, or when stopping\n    // them (after a signal) has timed out.\n    const svc = await Promise.race([\n      ...runs.map((run, i) => run.then(() => services[i])),\n      allStopped.promise,\n    ]);\n    if (svc) {\n      lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n    }\n  } catch (e) {\n    lc.error?.(`exiting on error`, e);\n    throw e;\n  } finally {\n    await stopAll('stopped');\n  }\n}\n\nasync function stopServices(\n  lc: LogContext,\n  services: SingletonService[],\n  runs: Promise<void>[],\n  stopped: Set<SingletonService>,\n  reason: string,\n) {\n  for (const svc of services) {\n    if (!stopped.has(svc)) {\n      lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${reason})`);\n      void svc\n        .stop()\n        .catch(e =>\n          lc.warn?.(`error stopping ${svc.constructor.name} ${svc.id}`, e),\n        );\n    }\n  }\n  if (\n    (await orTimeout(Promise.allSettled(runs), STOP_TIMEOUT_MS)) === 'timed-out'\n  ) {\n    const running = services.filter(svc => !stopped.has(svc));\n    lc.warn?.(\n      `services did not stop within ${STOP_TIMEOUT_MS} ms: ` +\n        running.map(svc => `${svc.constructor.name} ${svc.id}`).join(', '),\n    );\n  }\n}\n\nexport async function exitAfter(\n  logContext: LogContext | (() => LogContext),\n  run: () => Promise<void>,\n) {\n  const getLogContext =\n    typeof logContext === 'function' ? logContext : () => logContext;\n  try {\n    await run();\n    const lc = getLogContext();\n    lc.info?.(`pid ${pid} exiting normally`);\n    process.exit(0);\n  } catch (e) {\n    const lc = getLogContext();\n    if (e instanceof ConfigurationError) {\n      lc.error?.(`exiting with configuration error: ${String(e)}`, e);\n      process.exit(0);\n    }\n    if (e instanceof AbortError) {\n      // AbortErrors signal an intentional shutdown or restart (e.g. SIGTERM\n      // during a rollout, AutoResetSignal) and should not raise alerts. The\n      // non-zero exit code is kept so that restart semantics are unchanged.\n      lc.warn?.(`exiting after abort: ${String(e)}`, e);\n    } else {\n      lc.error?.(`exiting with error: ${String(e)}`, e);\n    }\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, e);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n    try {\n      await lc.flush();\n    } finally {\n      process.exit(-1);\n    }\n  }\n}\n\n/**\n * The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n *\n * The HeartbeatMonitor is **opt-in** in that it only kicks in after it\n * starts receiving keepalives.\n */\nexport class HeartbeatMonitor {\n  readonly #stopInterval: number;\n\n  #lc: LogContext;\n  #checkIntervalTimer: NodeJS.Timeout | undefined;\n  #checkImmediateTimer: NodeJS.Immediate | undefined;\n  #lastHeartbeat = 0;\n\n  constructor(lc: LogContext, stopInterval: number) {\n    this.#lc = lc;\n    this.#stopInterval = stopInterval;\n  }\n\n  onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n    this.#lastHeartbeat = Date.now();\n    if (this.#checkIntervalTimer === undefined) {\n      this.#lc.info?.(\n        `starting heartbeat monitor at ${\n          this.#stopInterval / 1000\n        } second interval`,\n        reqHeaders,\n      );\n      // e.g. check every 5 seconds to see if it's been over 20 seconds\n      //      since the last heartbeat.\n      this.#checkIntervalTimer = setInterval(\n        this.#checkStopInterval,\n        this.#stopInterval / 4,\n      );\n    }\n  }\n\n  #checkStopInterval = () => {\n    // In the Node.js event loop, timers like setInterval and setTimeout\n    // run *before* I/O events coming from network sockets or file reads/writes.\n    // When this process gets starved of CPU resources for long periods of time,\n    // for example when other processes are monopolizing all available cores,\n    // pathological behavior can emerge:\n    // - keepalive network request comes in, but is queued in Node internals waiting\n    //   for time on the event loop\n    // - CPU is starved/monopolized by other processes for longer than the time\n    //   configured via this.#stopInterval\n    // - When CPU becomes available and the event loop wakes up, this stop interval\n    //   check is run *before* the keepalive request is processed. The value of\n    //   this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n    //   even though keepalive requests were about to be processed and update\n    //   this.#lastHeartbeat. Downtime ensues.\n    //\n    // To avoid this, we push the check out to a phase of the event loop *after*\n    // I/O events are processed, using setImmediate():\n    // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n    //\n    // This ensures we see a value for this.#lastHeartbeat that reflects\n    // any keepalive requests that came in during the current event loop turn.\n    this.#checkImmediateTimer = setImmediate(() => {\n      this.#checkImmediateTimer = undefined;\n      const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n      if (timeSinceLastHeartbeat >= this.#stopInterval) {\n        this.#lc.info?.(\n          `last heartbeat received ${\n            timeSinceLastHeartbeat / 1000\n          } seconds ago. draining.`,\n        );\n        process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n      }\n    });\n  };\n\n  stop() {\n    clearTimeout(this.#checkIntervalTimer);\n    if (this.#checkImmediateTimer) {\n      clearImmediate(this.#checkImmediateTimer);\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,oBAAoB,CAAC,WAAW,QAAQ;AACrD,IAAa,oBAAoB,CAAC,WAAW,SAAS;AAItD,SAAS,wBAAwB,MAAc;CAC7C,IAAI,SAAS,cACX;CAEF,IAAI,KAAK,WAAW,iBAAiB,GACnC,OAAO;CAET,IAAI,KAAK,WAAW,YAAY,GAAG;EACjC,IAAI,KAAK,SAAS,UAAU,GAC1B,OAAO;EAET,IAAI,KAAK,SAAS,gBAAgB,GAChC,OAAO;EAET,IAAI,KAAK,SAAS,WAAW,GAC3B,OAAO;EAET,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,eAAe,GACjC,OAAO;CAET,IAAI,KAAK,WAAW,SAAS,GAC3B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,kBAAkB;CACzB,OAAO,qBAAqB,UAAU,oBAAoB;EACxD,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;AAEA,SAAgB,wBAAwB,YAAoB;CAC1D,gBAAgB,CAAC,CAAC,SAAS,YAAY,EAAC,WAAW,aAAY,CAAC;AAClE;AAEA,SAAS,wBAAwB;CAC/B,OAAO,qBAAqB,UAAU,2BAA2B;EAC/D,aAAa;EACb,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;AAgBA,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,IAAgB;CAC3C,uBAAgB,IAAI,IAAgB;CACpC;CACA,SAAkB,KAAK,IAAI;CAC3B,SAAmC,CAAC;CAEpC,gBAAgB,IAAI,aAAa,iBAAiB;CAClD,cAAc;CAEd,YAAY,IAAgB,MAAoB;EAC9C,KAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;EAKxD,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cAAc,KAAK,YAAY,MAAM,CAAC;EAIhD,KAAK,GAAG,SAAQ,SACd,KAAK,MACH,KAAK,MACL,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;EASA,KAAK,MAAM,UAAU,mBACnB,KAAK,GAAG,cACN,KAAK,MAAM,WAAW,YAAA,KAA8C,EAAE,CACxE;EAGF,KAAK,aAAa,SAAiB;GACjC,IAAI,kBAAkB,GACpB,OAAO,KAAK,KAAK,QAAQ,IAAI;GAE/B,QAAQ,KAAK,IAAI;EACnB;CACF;CAEA,MAAM,WAAW;EACf,IAAI,KAAK,gBAAgB,GACvB,KAAK,YAAY,kBAAkB,EAAE;EAEvC,MAAM,KAAK,cAAc,QAAQ;CACnC;CAEA,MAAM,MAAc;EAClB,KAAK,IAAI,OAAO,qBAAqB,IAAI;EACzC,KAAK,cAAc,KAAK,KAAK,GAAG;EAChC,KAAU,IAAI,MAAM,CAAC,CAAC,cAAc,KAAK,UAAU,IAAI,CAAC;CAC1D;CAEA,YAAY,QAAgC;EAC1C,IAAI,KAAK,KAAK,SAAS,GAAG;GAExB,KAAK,IAAI,OAAO,cAAc,QAAQ;GACtC,KAAK,MAAM,CAAC;EACd;EACA,KAAK,IAAI,OAAO,qBAAqB,OAAO,EAAE;EAC9C,KAAK,cAAc,KAAK,IAAI;EAC5B,IAAI,KAAK,YAAY,MACnB,KAAK,MAAM,KAAK,aAAa,MAAM;OAEnC,KAAK,MAAM,KAAK,MAAM,MAAM;CAEhC;CAEA,cAAc,MAAkB,MAAkB,MAAc;EAC9D,IAAI,SAAS,eACX,KAAK,YAAY,IAAI,IAAI;EAE3B,KAAK,KAAK,IAAI,IAAI;EAElB,IAAI,SAAS;EACb,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,SAAS;GACT,KAAK,QAAQ,MAAM,QAAQ,MAAM,MAAM,MAAM,IAAI;EACnD,CAAC;EAMD,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,CAAC,UAAU,KAAK,cAAc,IAAI,SAAS,QAAQ,GAC1D,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;CACF;CAEA,gCAAyB,IAAI,IAAoB;CACjD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;EAChE,KAAK,cAAc,QAAQ,MAAM,IAAI;EAErC,MAAM,QAAQ,YAAY,IAAI;EAC9B,MAAM,KAAK,EAAE,KAAK;EAClB,KAAK,cAAc,IAAI,IAAI,IAAI;EAC/B,MAAM,EAAC,SAAS,YAAW,SAAS;EACpC,KAAK,OAAO,KAAK,OAAO;EAExB,OAAO,gBAAgB,eAAe;GACpC,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,mBAAmB,wBAAwB,IAAI;GACrD,IAAI,kBACF,sBAAsB,CAAC,CAAC,SAAS,SAAS;IACxC,QAAQ;IACR;GACF,CAAC;GAEH,KAAK,IAAI,QAAQ,GAAG,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,KAAK;GACjE,KAAK,cAAc,OAAO,EAAE;GAC5B,QAAQ;EACV,CAAC;EAED,OAAO;CACT;CAEA,eAAyB;EACvB,OAAO,CAAC,GAAG,KAAK,cAAc,OAAO,CAAC;CACxC;CAEA,MAAM,kBAAkB;EACtB,MAAM,QAAQ,IAAI,KAAK,MAAM;CAC/B;CAEA,gBAAgB,KAAc,MAAc;EAE1C,KAAK,QAAQ,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,CAAS;CAC5D;CAEA,QACE,MACA,KACA,KACA,MACA,MACA,QACA;EAEA,IAAI,QAAQ;GACV,KAAK,YAAY,OAAO,MAAM;GAC9B,KAAK,KAAK,OAAO,MAAM;EACzB;EAEA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EASnC,IACE,SAAS,MACR,KAAK,gBAAgB,KACnB,SAAS,gBAAgB,KAAK,YAAY,OAAO,IAEpD,OAAA;EAGF,IAAI,SAAS,cAAc;GACzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;GACN,KAAK,IAAI,IAAI,GAAG,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,EAAE;GACxE,OAAO,KAAK,MAAM,IAAI;EACxB;EAEA,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,KAAK,cAAc,KAAK,SAAA,KACtB,SACA;EACR,KAAK,IAAI,IAAI,GACX,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,EACT;EAGA,IAAI,KAAK,YAAY,SAAS,GAAG;GAC/B,KAAK,IAAI,OACP,KAAK,cACD,oCACE,KAAK,IAAI,IAAI,KAAK,YACnB,QACD,gCACN;GACA,OAAO,KAAK,MAAM,KAAK,cAAc,IAAI,IAAI;EAC/C;EAEA,IAAI,KAAK,gBAAgB,GAGvB,OAAO,KAAK,MAAM,IAAI;CAI1B;CAEA,MAAM,SAA+B,QAAwB;EAC3D,KAAK,MAAM,UAAU,SACnB,IAAI;GACF,OAAO,KAAK,MAAM;EACpB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,MAA6D;CAC3D,KAAc;CACd;CACA;CACA,yBAAkB,IAAI,IAAwB;CAC9C,QAAiB,SAAe;CAChC;CAEA,YAAY,IAAgB,QAAsB;EAChD,KAAK,MAAM,GAAG,YAAY,aAAa,sBAAsB;EAC7D,KAAK,UAAU;EAEf,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;EAEjC,OAAO,GAAG,SAAQ,SAChB,KAAK,MACH,SAAS,IACL,YACA,SAAA,KACE,YACA,SACR,CACF;CACF;CAEA,IAA0B,MAAS,MAAiB;EAClD,KAAK,OAAO,IAAI,MAAM,IAAI;EAC1B,KAAK,GAAG,UAAU,MAAqB,QACrC,KAAK,SAAS,MAAM,MAAM,MAAM,GAAG,CACrC;EAGA,KAAK,GAAG,UAAS,QACf,KAAK,IAAI,KAAK,YAAY,SAAS,QAAQ,GACzC,cAAc,KAAK,GAAG,KAAK,OAC3B,GACF,CACF;EACA,OAAO;CACT;CAEA,MAAqB;EAKnB,KAAK,MAAM,UAAU,mBACnB,KAAK,QAAQ,GAAG,cAAc,KAAK,MAAM,MAAM,CAAC;EAElD,OAAO,KAAK,MAAM;CACpB;CAEA,MAAM,OAAsB;EAC1B,IAAI,CAAC,KAAK,WACR,KAAK,MAAM,SAAS;EAEtB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAErB,MAAM,KAAK,MAAM,QAAQ,YAAY,CAAC,CAAC;CACzC;CAEA,SACE,MACA,MACA,MACA,KACA;EACA,KAAK,OAAO,OAAO,IAAI;EACvB,MAAM,OAAO,MACT,GAAG,KAAK,IAAI,KAAK,IAAI,iBAAiB,IAAI,KAC1C,GAAG,KAAK,IAAI,KAAK,IAAI,sBAAsB,KAAK;EAEpD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,IAAI;GACpB,IAAI,KAAK,OAAO,SAAS,GACvB,KAAK,MAAM,QAAQ;EAEvB,OAAO,IAAI,SAAS,KAAK,SAAA,IAA0C;GACjE,KAAK,IAAI,OAAO,IAAI;GACpB,KAAK,MAAM,QAAQ;EACrB,OACE,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC;CAErC;CAEA,MAAM,QAAwB;EAC5B,KAAK,YAAY;EACjB,KAAK,MAAM,QAAQ,KAAK,OAAO,KAAK,GAClC,IAAI;GACF,KAAK,KAAK,MAAM;EAClB,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,CAAC;EACpB;CAEJ;AACF;;;;;;;;;;;;AA6BA,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CACA,aAAa;CACb,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,MAAwB;EACtE,KAAK,MAAM,GAAG,YAAY,aAAa,YAAY;EACnD,KAAK,QAAQ;EACb,KAAK,WAAW,KAAK;EACrB,KAAK,eAAe,KAAK;EAEzB,KAAK,MAAM,UAAU,mBACnB,KAAK,KAAK,cAAc;GACtB,KAAK,YAAY;GACjB,aAAa,KAAK,MAAM;EAC1B,CAAC;CAEL;CAEA,UAAU;EACR,IAAI,KAAK,cAAc,KAAK,WAC1B;EAEF,KAAK,aAAa;EAClB,KAAK,IAAI,OACP,4CAA4C,KAAK,SAAS,IAC5D;EACA,KAAK,aAAa;EAClB,KAAK,SAAS,iBAAiB;GAC7B,IAAI,CAAC,KAAK,WAAW;IACnB,KAAK,IAAI,OAAO,uBAAuB;IACvC,KAAK,MAAM,KAAK,WAAW,SAAS;GACtC;EACF,GAAG,KAAK,QAAQ;CAClB;AACF;;;;;AAMA,IAAa,YAAY;;;;;;;;;;AAWzB,IAAa,WAAb,cAA8BA,aAAiB;CAC7C;CACA;CACA;CACA,6BAAsB,IAAI,IAAqB,CAAC,eAAe,SAAS,CAAC;CACzE,YAAY;CACZ;CAEA,YAAY,IAAgB,MAAoB,SAAiB;EAC/D,MAAM;EACN,KAAK,MAAM,GAAG,YAAY,aAAa,WAAW;EAClD,KAAK,QAAQ;EACb,KAAK,WAAW;EAEhB,KAAK,MAAM,UAAU,mBAAmB;GACtC,KAAK,WAAW,IAAI,MAAM;GAC1B,KAAK,GAAG,cAAc,KAAK,kBAAkB,MAAM,CAAC;EACtD;EACA,KAAK,GAAG,gBAAgB,UAA2B,KAAK,SAAS,KAAK,CAAC;CACzE;CAEA,SAAS,OAAwB;EAC/B,IAAI,KAAK,WAAW,IAAI,KAAK,GAC3B;EAEF,KAAK,WAAW,IAAI,KAAK;EACzB,KAAK,MAAM,GAAG,QAAQ,GAAG,SAAoB;GAC3C,IACE,UAAU,gBACT,kBAAmD,SAAS,KAAK,GAElE,aAAa,KAAK,MAAM;GAE1B,KAAK,KAAK,OAAO,GAAG,IAAI;EAC1B,CAAC;CACH;CAEA,kBAAkB,QAAgC;EAChD,IAAI,KAAK,WAAW;GAClB,KAAK,IAAI,OAAO,mCAAmC,QAAQ;GAC3D;EACF;EACA,KAAK,YAAY;EACjB,KAAK,IAAI,OAAO,YAAY,OAAO,gBAAgB,KAAK,SAAS,IAAI;EACrE,KAAK,KAAK,WAAW,MAAM;EAC3B,KAAK,SAAS,iBAAiB,KAAK,KAAK,QAAQ,MAAM,GAAG,KAAK,QAAQ;CACzE;AACF;;;;;AAMA,IAAa,kBAAkB;;;;;;;;;;;;AAa/B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;CACf,IAAI,SAAS,WAAW,GACtB;CAEF,MAAM,0BAAU,IAAI,IAAsB;CAC1C,MAAM,OAAO,SAAS,KAAI,QAAO,IAAI,IAAI,CAAC,CAAC,cAAc,QAAQ,IAAI,GAAG,CAAC,CAAC;CAE1E,MAAM,aAAa,SAAe;CAClC,IAAI;CACJ,MAAM,WAAW,WACd,aAAa,aAAa,IAAI,UAAU,MAAM,SAAS,MAAM,CAAC,CAAC,KAC9D,WAAW,OACb;CAEF,MAAM,mBAAmB;CACzB,IAAI,WAAW;CACf,MAAM,WAAW,OAAO,WAA2B;EACjD,IAAI,iBAAiB,SAAS,MAAM,GAAG;GACrC,IAAI,YAAY,UAAU;IAExB,GAAG,OAAO,mCAAmC,QAAQ;IACrD;GACF;GACA,WAAW;GAIX,MAAM,UAAU,MAAM,QAAQ,WAC5B,SAAS,IAAI,OAAM,QAAO;IACxB,IAAI,IAAI,OAAO;KACb,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;KAClE,MAAM,IAAI,MAAM;IAClB;GACF,CAAC,CACH;GACA,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,WAAW,YACpB,GAAG,OAAO,yBAAyB,OAAO,IAAI,OAAO,MAAM;EAGjE;EAGA,MAAM,QAAQ,MAAM;CACtB;CAIA,KAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,iBAAiB,GAC9D,OAAO,GAAG,cAAc,KAAK,SAAS,MAAM,CAAC;CAK/C,OAAO,KAAK,oBAAoB;EAC9B,GAAG,OAAO,wDAAwD;EAClE,QAAa,YAAY;CAC3B,CAAC;CAED,IAAI;EAGF,MAAM,MAAM,MAAM,QAAQ,KAAK,CAC7B,GAAG,KAAK,KAAK,KAAK,MAAM,IAAI,WAAW,SAAS,EAAE,CAAC,GACnD,WAAW,OACb,CAAC;EACD,IAAI,KACF,GAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,UAAU;CAE3D,SAAS,GAAG;EACV,GAAG,QAAQ,oBAAoB,CAAC;EAChC,MAAM;CACR,UAAU;EACR,MAAM,QAAQ,SAAS;CACzB;AACF;AAEA,eAAe,aACb,IACA,UACA,MACA,SACA,QACA;CACA,KAAK,MAAM,OAAO,UAChB,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG;EACrB,GAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,EAAE;EAClE,IACG,KAAK,CAAC,CACN,OAAM,MACL,GAAG,OAAO,kBAAkB,IAAI,YAAY,KAAK,GAAG,IAAI,MAAM,CAAC,CACjE;CACJ;CAEF,IACG,MAAM,UAAU,QAAQ,WAAW,IAAI,GAAA,GAAkB,MAAO,aACjE;EACA,MAAM,UAAU,SAAS,QAAO,QAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;EACxD,GAAG,OACD,gCAAgC,gBAAgB,SAC9C,QAAQ,KAAI,QAAO,GAAG,IAAI,YAAY,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,CACrE;CACF;AACF;AAEA,eAAsB,UACpB,YACA,KACA;CACA,MAAM,gBACJ,OAAO,eAAe,aAAa,mBAAmB;CACxD,IAAI;EACF,MAAM,IAAI;EAEV,cAAA,CAAA,CAAG,OAAO,OAAO,IAAI,kBAAkB;EACvC,QAAQ,KAAK,CAAC;CAChB,SAAS,GAAG;EACV,MAAM,KAAK,cAAc;EACzB,IAAI,aAAa,oBAAoB;GACnC,GAAG,QAAQ,qCAAqC,OAAO,CAAC,KAAK,CAAC;GAC9D,QAAQ,KAAK,CAAC;EAChB;EACA,IAAI,aAAa,YAIf,GAAG,OAAO,wBAAwB,OAAO,CAAC,KAAK,CAAC;OAEhD,GAAG,QAAQ,uBAAuB,OAAO,CAAC,KAAK,CAAC;EAElD,IAAI;GACF,yCAAyC,IAAI,CAAC;EAChD,SAAS,iBAAiB;GACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;EACH;EACA,IAAI;GACF,MAAM,GAAG,MAAM;EACjB,UAAU;GACR,QAAQ,KAAK,EAAE;EACjB;CACF;AACF;;;;;;;;;;;;AAaA,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,cAAsB;EAChD,KAAK,MAAM;EACX,KAAK,gBAAgB;CACvB;CAEA,YAAY,YAAiC;EAC3C,KAAK,iBAAiB,KAAK,IAAI;EAC/B,IAAI,KAAK,wBAAwB,KAAA,GAAW;GAC1C,KAAK,IAAI,OACP,iCACE,KAAK,gBAAgB,IACtB,mBACD,UACF;GAGA,KAAK,sBAAsB,YACzB,KAAK,oBACL,KAAK,gBAAgB,CACvB;EACF;CACF;CAEA,2BAA2B;EAsBzB,KAAK,uBAAuB,mBAAmB;GAC7C,KAAK,uBAAuB,KAAA;GAC5B,MAAM,yBAAyB,KAAK,IAAI,IAAI,KAAK;GACjD,IAAI,0BAA0B,KAAK,eAAe;IAChD,KAAK,IAAI,OACP,2BACE,yBAAyB,IAC1B,wBACH;IACA,QAAQ,KAAK,QAAQ,KAAK,kBAAkB,EAAE;GAChD;EACF,CAAC;CACH;CAEA,OAAO;EACL,aAAa,KAAK,mBAAmB;EACrC,IAAI,KAAK,sBACP,eAAe,KAAK,oBAAoB;CAE5C;AACF"}

package/out/zero-cache/src/services/replicator/change-processor.js

@@ -4,6 +4,7 @@ import { AbortError } from "../../../../shared/src/abort-error.js";
 import { must } from "../../../../shared/src/must.js";
 import { mapEntries } from "../../../../shared/src/objects.js";
 import { stringify } from "../../../../shared/src/bigint-json.js";
+import { id } from "../../types/sql.js";
 import { ZERO_VERSION_COLUMN_NAME } from "./schema/constants.js";
 import { liteRow } from "../../types/lite.js";
 import { liteTableName } from "../../types/names.js";
@@ -14,7 +15,6 @@ import { computeZqlSpecs, listTables } from "../../db/lite-tables.js";
 import { BackfillingTracker } from "./schema/backfilling.js";
 import { ChangeLog } from "./schema/change-log.js";
 import { updateReplicationWatermark } from "./schema/replication-state.js";
-import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
 import { acceptBackfill } from "../change-source/protocol/backfill-progress.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";

package/out/zero-cache/src/services/replicator/serving-checkpointer.js

@@ -0,0 +1,85 @@
+import { Database } from "../../../../zqlite/src/db.js";
+import { RunningState } from "../running-state.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
+//#region ../zero-cache/src/services/replicator/serving-checkpointer.ts
+var CHECKPOINT_INTERVAL_MS = 100;
+var SLOW_CHECKPOINT_MS = 1e3;
+var WAL_WARNING_BYTES = 67108864;
+var LOG_WINDOW_MS = 6e4;
+var MB = 1048576;
+/**
+* Checkpoints the serving (wal2) replica from its own connection, so that the
+* replicator's writer, which runs with `wal_autocheckpoint = 0`, never waits
+* on a checkpoint's disk syncs.
+*
+* In wal2 mode the writer switches to the other wal file once the current
+* one reaches its size limit, provided the other one has been checkpointed.
+* Until then, it keeps appending to the current file rather than waiting. A
+* `PASSIVE` checkpoint copies the whole non-current file, or does nothing
+* (without any I/O) if that file is already checkpointed, empty, or still
+* being read (e.g. by an old snapshot). Polling it on a short interval thus
+* keeps the wal files near their size limit without any coordination with
+* the writer.
+*
+* A failed checkpoint is logged and retried rather than stopping the
+* replicator. This matches the writer's automatic checkpoints that this
+* replaces, whose errors SQLite ignores: a persistent failure (e.g. a full or
+* failing disk) surfaces as the writer's own failure once the wal can no
+* longer grow.
+*/
+var ServingCheckpointer = class {
+	id = "serving-checkpointer";
+	#lc;
+	#db;
+	#pageSize;
+	#intervalMs;
+	#walWarningBytes;
+	#state = new RunningState(this.id);
+	#logThrottle = new LogThrottle({ windowMs: LOG_WINDOW_MS });
+	constructor(lc, dbPath, opts = {}) {
+		this.#lc = lc.withContext("component", this.id);
+		this.#db = new Database(this.#lc, dbPath, { fileMustExist: true }, SLOW_CHECKPOINT_MS);
+		[{page_size: this.#pageSize}] = this.#db.pragma("page_size");
+		this.#intervalMs = opts.intervalMs ?? CHECKPOINT_INTERVAL_MS;
+		this.#walWarningBytes = opts.walWarningBytes ?? WAL_WARNING_BYTES;
+	}
+	async run() {
+		try {
+			while (this.#state.shouldRun()) {
+				this.#checkpoint();
+				await this.#state.sleep(this.#intervalMs);
+			}
+		} finally {
+			this.#db.close();
+		}
+	}
+	#checkpoint() {
+		let result;
+		try {
+			[result] = this.#db.pragma("wal_checkpoint(PASSIVE)");
+		} catch (e) {
+			const suppressed = this.#logThrottle.admit("error");
+			if (suppressed !== void 0) this.#lc.error?.(`error checkpointing the replica` + (suppressed ? ` (${suppressed} more since last logged)` : ""), e);
+			return;
+		}
+		const { busy, log } = result;
+		const walBytes = log * this.#pageSize;
+		if (walBytes >= this.#walWarningBytes) {
+			const reason = busy ? "a reader (e.g. a long-running snapshot) is still reading the older wal file" : "checkpoints are not keeping up with writes";
+			const suppressed = this.#logThrottle.admit(busy ? "busy" : "behind");
+			if (suppressed !== void 0) this.#lc.warn?.(`replica wal is ${(walBytes / MB).toFixed(1)} MB: ${reason}`, {
+				walBytes,
+				busy: !!busy,
+				suppressed
+			});
+		}
+	}
+	stop() {
+		this.#state.stop(this.#lc);
+		return Promise.resolve();
+	}
+};
+//#endregion
+export { ServingCheckpointer };
+
+//# sourceMappingURL=serving-checkpointer.js.map

package/out/zero-cache/src/services/replicator/serving-checkpointer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serving-checkpointer.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/serving-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {LogThrottle} from '../../observability/log-throttle.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst CHECKPOINT_INTERVAL_MS = 100;\n\n// A checkpoint copies at most one wal file (~4 MB with the default 1000 page\n// limit and 4 KB pages), so only a disk that is struggling makes one slow.\nconst SLOW_CHECKPOINT_MS = 1000;\n\n// The wal files normally hold about two files' worth of frames (~8 MB). A wal\n// several times that is not being checkpointed as fast as it is written.\nconst WAL_WARNING_BYTES = 64 * 1024 * 1024;\n\nconst LOG_WINDOW_MS = 60 * 1000;\n\nconst MB = 1024 * 1024;\n\ntype Options = {\n  intervalMs?: number | undefined;\n  walWarningBytes?: number | undefined;\n};\n\n/**\n * Checkpoints the serving (wal2) replica from its own connection, so that the\n * replicator's writer, which runs with `wal_autocheckpoint = 0`, never waits\n * on a checkpoint's disk syncs.\n *\n * In wal2 mode the writer switches to the other wal file once the current\n * one reaches its size limit, provided the other one has been checkpointed.\n * Until then, it keeps appending to the current file rather than waiting. A\n * `PASSIVE` checkpoint copies the whole non-current file, or does nothing\n * (without any I/O) if that file is already checkpointed, empty, or still\n * being read (e.g. by an old snapshot). Polling it on a short interval thus\n * keeps the wal files near their size limit without any coordination with\n * the writer.\n *\n * A failed checkpoint is logged and retried rather than stopping the\n * replicator. This matches the writer's automatic checkpoints that this\n * replaces, whose errors SQLite ignores: a persistent failure (e.g. a full or\n * failing disk) surfaces as the writer's own failure once the wal can no\n * longer grow.\n */\nexport class ServingCheckpointer implements Service {\n  readonly id = 'serving-checkpointer';\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #pageSize: number;\n  readonly #intervalMs: number;\n  readonly #walWarningBytes: number;\n  readonly #state = new RunningState(this.id);\n  readonly #logThrottle = new LogThrottle({windowMs: LOG_WINDOW_MS});\n\n  constructor(lc: LogContext, dbPath: string, opts: Options = {}) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#db = new Database(\n      this.#lc,\n      dbPath,\n      {fileMustExist: true},\n      SLOW_CHECKPOINT_MS,\n    );\n    [{page_size: this.#pageSize}] = this.#db.pragma<{page_size: number}>(\n      'page_size',\n    );\n    this.#intervalMs = opts.intervalMs ?? CHECKPOINT_INTERVAL_MS;\n    this.#walWarningBytes = opts.walWarningBytes ?? WAL_WARNING_BYTES;\n  }\n\n  async run(): Promise<void> {\n    try {\n      while (this.#state.shouldRun()) {\n        this.#checkpoint();\n        await this.#state.sleep(this.#intervalMs);\n      }\n    } finally {\n      this.#db.close();\n    }\n  }\n\n  #checkpoint() {\n    let result;\n    try {\n      [result] = this.#db.pragma<{busy: number; log: number}>(\n        'wal_checkpoint(PASSIVE)',\n      );\n    } catch (e) {\n      const suppressed = this.#logThrottle.admit('error');\n      if (suppressed !== undefined) {\n        this.#lc.error?.(\n          `error checkpointing the replica` +\n            (suppressed ? ` (${suppressed} more since last logged)` : ''),\n          e,\n        );\n      }\n      return;\n    }\n\n    // In wal2 mode `log` counts the frames in both wal files, and `busy` is\n    // set if a reader prevented the checkpoint.\n    const {busy, log} = result;\n    const walBytes = log * this.#pageSize;\n    if (walBytes >= this.#walWarningBytes) {\n      const reason = busy\n        ? 'a reader (e.g. a long-running snapshot) is still reading the older wal file'\n        : 'checkpoints are not keeping up with writes';\n      const suppressed = this.#logThrottle.admit(busy ? 'busy' : 'behind');\n      if (suppressed !== undefined) {\n        this.#lc.warn?.(\n          `replica wal is ${(walBytes / MB).toFixed(1)} MB: ${reason}`,\n          {walBytes, busy: !!busy, suppressed},\n        );\n      }\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#state.stop(this.#lc);\n    return Promise.resolve();\n  }\n}\n"],"mappings":";;;;AAMA,IAAM,yBAAyB;AAI/B,IAAM,qBAAqB;AAI3B,IAAM,oBAAoB;AAE1B,IAAM,gBAAgB;AAEtB,IAAM,KAAK;;;;;;;;;;;;;;;;;;;;;AA2BX,IAAa,sBAAb,MAAoD;CAClD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C,eAAwB,IAAI,YAAY,EAAC,UAAU,cAAa,CAAC;CAEjE,YAAY,IAAgB,QAAgB,OAAgB,CAAC,GAAG;EAC9D,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,MAAM,IAAI,SACb,KAAK,KACL,QACA,EAAC,eAAe,KAAI,GACpB,kBACF;EACA,CAAC,CAAC,WAAW,KAAK,cAAc,KAAK,IAAI,OACvC,WACF;EACA,KAAK,cAAc,KAAK,cAAc;EACtC,KAAK,mBAAmB,KAAK,mBAAmB;CAClD;CAEA,MAAM,MAAqB;EACzB,IAAI;GACF,OAAO,KAAK,OAAO,UAAU,GAAG;IAC9B,KAAK,YAAY;IACjB,MAAM,KAAK,OAAO,MAAM,KAAK,WAAW;GAC1C;EACF,UAAU;GACR,KAAK,IAAI,MAAM;EACjB;CACF;CAEA,cAAc;EACZ,IAAI;EACJ,IAAI;GACF,CAAC,UAAU,KAAK,IAAI,OAClB,yBACF;EACF,SAAS,GAAG;GACV,MAAM,aAAa,KAAK,aAAa,MAAM,OAAO;GAClD,IAAI,eAAe,KAAA,GACjB,KAAK,IAAI,QACP,qCACG,aAAa,KAAK,WAAW,4BAA4B,KAC5D,CACF;GAEF;EACF;EAIA,MAAM,EAAC,MAAM,QAAO;EACpB,MAAM,WAAW,MAAM,KAAK;EAC5B,IAAI,YAAY,KAAK,kBAAkB;GACrC,MAAM,SAAS,OACX,gFACA;GACJ,MAAM,aAAa,KAAK,aAAa,MAAM,OAAO,SAAS,QAAQ;GACnE,IAAI,eAAe,KAAA,GACjB,KAAK,IAAI,OACP,mBAAmB,WAAW,GAAA,CAAI,QAAQ,CAAC,EAAE,OAAO,UACpD;IAAC;IAAU,MAAM,CAAC,CAAC;IAAM;GAAU,CACrC;EAEJ;CACF;CAEA,OAAsB;EACpB,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,OAAO,QAAQ,QAAQ;CACzB;AACF"}

package/out/zero-cache/src/services/view-syncer/pipeline-driver.js

@@ -2,24 +2,27 @@ import { assert, unreachable } from "../../../../shared/src/asserts.js";
 import { deepEqual } from "../../../../shared/src/json.js";
 import { getOrInsertComputed } from "../../../../shared/src/map.js";
 import { must } from "../../../../shared/src/must.js";
+import { h32 } from "../../../../shared/src/hash.js";
 import { assignProperty } from "../../../../shared/src/objects.js";
 import { skipYields } from "../../../../zql/src/ivm/skip-yields.js";
 import { throwOutput } from "../../../../zql/src/ivm/operator.js";
 import { MeasurePushOperator } from "../../../../zql/src/query/measure-push-operator.js";
 import { makeSourceChangeAdd, makeSourceChangeEdit, makeSourceChangeRemove } from "../../../../zql/src/ivm/source.js";
-import { buildPipeline } from "../../../../zql/src/builder/builder.js";
+import { buildPipeline, isAggregateTableName, topLevelAggregateTableName } from "../../../../zql/src/builder/builder.js";
+import { stringify } from "../../../../shared/src/bigint-json.js";
 import { Debug, runtimeDebugFlags } from "../../../../zql/src/builder/debug-delegate.js";
 import { TableSource } from "../../../../zqlite/src/table-source.js";
 import { randInt } from "../../../../shared/src/rand.js";
 import { getOrCreateCounter, getOrCreateLatencyHistogram } from "../../observability/metrics.js";
 import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
 import { computeZqlSpecs, mustGetTableSpec } from "../../db/lite-tables.js";
-import "../../types/row-key.js";
+import { rowKeyString } from "../../types/row-key.js";
 import { getSubscriptionState } from "../replicator/schema/replication-state.js";
+import { versionToLexi } from "../../types/lexi-version.js";
+import { LogThrottle } from "../../observability/log-throttle.js";
 import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
 import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-scalar-subqueries.js";
 import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
-import { LogThrottle } from "../../observability/log-throttle.js";
 import { checkClientSchema } from "./client-schema.js";
 import { planWarningMessage } from "./plan-warnings.js";
 import { PushCutoff } from "./push-cutoff.js";
@@ -82,6 +85,13 @@ function formatAdvancementBreakdown(advanceContext, elapsed) {
 */
 var PipelineDriver = class {
 	#tables = /* @__PURE__ */ new Map();
+	/**
+	* The latest synthetic aggregate row streamed for each key, by synthetic
+	* table. Aggregate tables have no {@link TableSource}, so this is what
+	* {@link getRow} reads for them when the view-syncer rebuilds a put patch on
+	* client catch-up. Entries live as long as the query that produced them.
+	*/
+	#aggregateRows = /* @__PURE__ */ new Map();
 	#pipelines = /* @__PURE__ */ new Map();
 	/**
 	* XOR signature of the set of rows currently attached to each active
@@ -196,6 +206,7 @@ var PipelineDriver = class {
 		this.#abandoned.clear();
 		this.#costBasedAdvanceIncomplete = false;
 		this.#tables.clear();
+		this.#aggregateRows.clear();
 		this.#allTableNames.clear();
 		this.#rowSetSignatures.clear();
 		this.#changedRowSetSignatures.clear();
@@ -211,7 +222,7 @@ var PipelineDriver = class {
 		for (const table of fullTables.keys()) this.#allTableNames.add(table);
 		const primaryKeys = buildRowKeys(clientSchema, this.#tableSpecs, this.#primaryKeys ?? void 0);
 		this.#primaryKeys = primaryKeys;
-		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs);
+		this.#streamer ??= new Streamer(primaryKeys, this.#tableSpecs, () => this.currentVersion(), this.#aggregateRows);
 		const { replicaVersion } = getSubscriptionState(db);
 		this.#replicaVersion = replicaVersion;
 	}
@@ -282,6 +293,7 @@ var PipelineDriver = class {
 			this.#destroyPipeline(queryID, pipeline, "destroy");
 		}
 		this.#tables.clear();
+		this.#aggregateRows.clear();
 		this.#rowSetSignatures.clear();
 		this.#changedRowSetSignatures.clear();
 		this.#storage.destroy();
@@ -672,6 +684,12 @@ var PipelineDriver = class {
 		}
 		this.#rowSetSignatures.delete(queryID);
 		this.#changedRowSetSignatures.delete(queryID);
+		const topLevel = topLevelAggregateTableName(queryID);
+		const prefix = topLevel + ":";
+		for (const [table, byKey] of this.#aggregateRows) if (table === topLevel || table.startsWith(prefix)) {
+			if (stopReason === "remove-query") this.#aggregateRows.delete(table);
+			else for (const entry of byKey.values()) entry.refs = 0;
+		}
 		if (stopReason !== "replace-query") this.#abandoned.delete(queryID);
 		if (stopReason === "remove-query") this.#pushCosts.forget(queryID);
 	}
@@ -748,6 +766,7 @@ var PipelineDriver = class {
 	*/
 	getRow(table, pk) {
 		assert(this.initialized(), "Not yet initialized");
+		if (isAggregateTableName(table)) return this.#aggregateRows.get(table)?.get(rowKeyString(pk))?.row;
 		return must(this.#tables.get(table)).getRow(pk);
 	}
 	/**
@@ -1386,9 +1405,15 @@ var PipelineDriver = class {
 var Streamer = class {
 	#primaryKeys;
 	#tableSpecs;
-	constructor(primaryKeys, tableSpecs) {
+	#version;
+	#aggregateRows;
+	#stampVersion;
+	#stampCount = 0;
+	constructor(primaryKeys, tableSpecs, version, aggregateRows) {
 		this.#primaryKeys = primaryKeys;
 		this.#tableSpecs = tableSpecs;
+		this.#version = version;
+		this.#aggregateRows = aggregateRows;
 	}
 	/** The row changes for a change pushed out of the pipeline of `queryID`. */
 	*streamChange(queryID, schema, change) {
@@ -1417,8 +1442,9 @@ var Streamer = class {
 	*streamNodes(queryID, schema, op, nodes) {
 		const { tableName: table, system } = schema;
 		if (system === "permissions") return;
-		const primaryKey = must(this.#primaryKeys.get(table));
-		const spec = must(this.#tableSpecs.get(table)).tableSpec;
+		const isAggregate = schema.isAggregate ?? false;
+		const primaryKey = isAggregate ? schema.primaryKey : must(this.#primaryKeys.get(table));
+		const minRowVersion = isAggregate ? void 0 : must(this.#tableSpecs.get(table)).tableSpec.minRowVersion;
 		for (const node of nodes) {
 			if (node === "yield") {
 				yield node;
@@ -1428,12 +1454,16 @@ var Streamer = class {
 			let { row } = node;
 			const rowKey = getRowKey(primaryKey, row);
 			if (op !== 1) {
-				const rowVersion = row[ZERO_VERSION_COLUMN_NAME];
-				if (typeof rowVersion === "string" && rowVersion < (spec.minRowVersion ?? "00")) row = {
-					...row,
-					[ZERO_VERSION_COLUMN_NAME]: spec.minRowVersion
-				};
+				if (isAggregate) row = this.#stampAggregateRow(table, rowKey, row, op);
+				else {
+					const rowVersion = row[ZERO_VERSION_COLUMN_NAME];
+					if (typeof rowVersion === "string" && rowVersion < (minRowVersion ?? "00")) row = {
+						...row,
+						[ZERO_VERSION_COLUMN_NAME]: minRowVersion
+					};
+				}
 			}
+			if (op === 1 && isAggregate) this.#releaseAggregateRow(table, rowKeyString(rowKey));
 			yield {
 				type: op,
 				queryID,
@@ -1447,6 +1477,64 @@ var Streamer = class {
 			}
 		}
 	}
+	/**
+	* Stamps a synthetic aggregate row with the replica state version and
+	* remembers it for {@link PipelineDriver.getRow}.
+	*
+	* The CVR only sends a put whose version is strictly greater than the last
+	* one it sent for that row, and the rows of one advance reach it in pages.
+	* An aggregate row is emitted once per changed child, so a stamp is the
+	* replica version plus a count of the stamps made at it, which keeps a row's
+	* stamps strictly increasing however often it is removed and re-added. The
+	* count is a LexiVersion, and a LexiVersion is never a proper prefix of a
+	* later one, so the stamp stays below the next replica version.
+	*
+	* The CVR also takes an equal version to mean equal contents, but the count
+	* starts over in a new process, so the stamp ends in a hash of the contents.
+	*/
+	#stampAggregateRow(table, rowKey, row, op) {
+		const key = rowKeyString(rowKey);
+		const version = this.#version();
+		if (version !== this.#stampVersion) {
+			this.#stampVersion = version;
+			this.#stampCount = 0;
+		}
+		const count = versionToLexi(this.#stampCount++);
+		const hash = h32(stringify(row)).toString(36);
+		const stamped = {
+			...row,
+			[ZERO_VERSION_COLUMN_NAME]: `${version}.${count}.${hash}`
+		};
+		this.#rememberAggregateRow(table, key, stamped, op === 0);
+		return stamped;
+	}
+	#rememberAggregateRow(table, key, row, add) {
+		const rows = this.#aggregateRows;
+		if (!rows) return;
+		let byKey = rows.get(table);
+		if (!byKey) {
+			byKey = /* @__PURE__ */ new Map();
+			rows.set(table, byKey);
+		}
+		const entry = byKey.get(key);
+		if (entry) {
+			entry.row = row;
+			entry.refs += add ? 1 : 0;
+		} else byKey.set(key, {
+			row,
+			refs: add ? 1 : 0
+		});
+	}
+	/**
+	* Drops one reference to an aggregate row. Parents that share a correlation
+	* value share the row, so it is forgotten only when the last one is removed,
+	* as the CVR counts it.
+	*/
+	#releaseAggregateRow(table, key) {
+		const byKey = this.#aggregateRows?.get(table);
+		const entry = byKey?.get(key);
+		if (entry && --entry.refs <= 0) byKey?.delete(key);
+	}
 };
 var QueryFailureLoggingOperator = class {
 	#lc;
@@ -1530,7 +1618,7 @@ function isKeyedBy(rowKey, primaryKey) {
 * of a table outside `clientSchema` are keyed by its key in the replica.
 */
 function hydrate(input, hash, clientSchema, tableSpecs) {
-	return new Streamer(buildRowKeys(clientSchema, tableSpecs), tableSpecs).streamNodes(hash, input.getSchema(), 0, input.fetch({}));
+	return new Streamer(buildRowKeys(clientSchema, tableSpecs), tableSpecs, () => "00").streamNodes(hash, input.getSchema(), 0, input.fetch({}));
 }
 /**
 * Returns the value of `childField` in the row of `input`, the pipeline of a