@rocicorp/zero 1.12.0-canary.10 → 1.12.0-canary.12

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process. It is\n  // started after the write worker has prepared the db file in order to\n  // avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {SERVING_CHECKPOINTER_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig();\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process, which\n  // checkpoints the backup replica. A serving replicator instead manages a\n  // checkpointer for its replica. Either is started after the write worker\n  // has prepared the db file in order to avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  } else {\n    processes.add(\n      childWorker(SERVING_CHECKPOINTER_URL, env, dbPath),\n      'serving-checkpointer',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB;CAChC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAMrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;MAEA,UAAU,IACR,YAAY,0BAA0B,KAAK,MAAM,GACjD,sBACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/runner/run-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAa1C,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,0BAA0B,CAAC;AAsBlC;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAuEf"}
+{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAe1C,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,0BAA0B,CAAC;AAsBlC;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAuFf"}

package/out/zero-cache/src/server/runner/run-worker.js

@@ -5,7 +5,7 @@ import { colorConsole } from "../../../../shared/src/logging.js";
 import { childWorker } from "../../types/processes.js";
 import { normalizeZeroConfig } from "../../config/normalize.js";
 import { getServerVersion, getZeroConfig } from "../../config/zero-config.js";
-import { ProcessManager, SelfDrain, runUntilKilled } from "../../services/life-cycle.js";
+import { PRE_DRAIN, PreDrain, ProcessManager, SelfDrain, runUntilKilled } from "../../services/life-cycle.js";
 import { createLogContext } from "../logging.js";
 import { MAIN_URL } from "../worker-urls.js";
 import { getTaskID } from "./runtime.js";
@@ -33,20 +33,23 @@ async function runWorker(parent, env) {
 	const lc = createLogContext(cfg, "runner");
 	const defaultTaskID = await getTaskID(lc);
 	const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);
-	const processes = new ProcessManager(lc, parent ?? process);
-	const { port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs } = config;
+	const { port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs, preDrainDelayMs } = config;
 	const serverVersion = getServerVersion(config);
 	lc.info?.(`starting server${!serverVersion ? "" : `@${serverVersion}`} `, {
-		protocolVersion: 53,
+		protocolVersion: 57,
 		taskID: config.taskID,
 		app: config.app,
 		shard: config.shard,
 		port: config.port
 	});
+	const preDrain = preDrainDelayMs > 0 ? new PreDrain(lc, parent ?? process, preDrainDelayMs) : void 0;
+	const signals = preDrain ?? parent ?? process;
+	const processes = new ProcessManager(lc, signals);
 	const dispatcher = new ZeroDispatcher(config, lc, {
 		port,
 		keepaliveTimeoutMs
 	}, startZeroCache, () => printStartupMessage(env), () => processes.shutdown());
+	preDrain?.on(PRE_DRAIN, () => dispatcher.failReadiness());
 	const selfDrain = new SelfDrain(lc, parent ?? process, {
 		delayMs: selfDrainDelayMs,
 		onRequested: () => dispatcher.failHealthChecks()
@@ -70,7 +73,7 @@ async function runWorker(parent, env) {
 	await processes.allWorkersReady();
 	parent?.send(["ready", { ready: true }]);
 	try {
-		await runUntilKilled(lc, parent ?? process, dispatcher);
+		await runUntilKilled(lc, signals, dispatcher);
 	} catch (err) {
 		processes.logErrorAndExit(err, "main");
 	} finally {

package/out/zero-cache/src/server/runner/run-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n    // Keep the listener open until the zero-cache has drained and exited.\n    () => processes.shutdown(),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAEzC,MAAM,gBAAgB,MAAM,UAAU,EAAE;CACxC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,aAAa;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,SAEvB,UAAU,SAAS,CAC3B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,CAAC,CACjC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,CAAC,CACD,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}
+{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {resolver, type Resolver} from '../../../../shared/src/resolver.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  PRE_DRAIN,\n  PreDrain,\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n\n  const {\n    port,\n    keepaliveTimeoutMs,\n    lazyStartup,\n    selfDrainDelayMs,\n    preDrainDelayMs,\n  } = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  // With a pre-drain delay, graceful shutdown signals reach the\n  // ProcessManager and the dispatcher's drain only after the delay.\n  const preDrain =\n    preDrainDelayMs > 0\n      ? new PreDrain(lc, parent ?? process, preDrainDelayMs)\n      : undefined;\n  const signals = preDrain ?? parent ?? process;\n  const processes = new ProcessManager(lc, signals);\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n    // Keep the listener open until the zero-cache has drained and exited.\n    () => processes.shutdown(),\n  );\n  // Fail readiness checks during the delay, while continuing to serve.\n  preDrain?.on(PRE_DRAIN, () => dispatcher.failReadiness());\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, signals, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAyBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAEzC,MAAM,gBAAgB,MAAM,UAAU,EAAE;CACxC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,aAAa;CAE9D,MAAM,EACJ,MACA,oBACA,aACA,kBACA,oBACE;CACJ,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAID,MAAM,WACJ,kBAAkB,IACd,IAAI,SAAS,IAAI,UAAU,SAAS,eAAe,IACnD,KAAA;CACN,MAAM,UAAU,YAAY,UAAU;CACtC,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,SAEvB,UAAU,SAAS,CAC3B;CAEA,UAAU,GAAG,iBAAiB,WAAW,cAAc,CAAC;CACxD,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,CAAC,CACjC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,CAAC,CACD,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,SAAS,UAAU;CAC9C,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}

package/out/zero-cache/src/server/serving-checkpointer.js

@@ -0,0 +1,41 @@
+import { assert } from "../../../shared/src/asserts.js";
+import { parentWorker, singleProcessMode } from "../types/processes.js";
+import { getLogConfig } from "../config/zero-config.js";
+import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
+import { createLogContext } from "./logging.js";
+import { startOtelAuto } from "./otel-start.js";
+import { ServingCheckpointer } from "../services/replicator/serving-checkpointer.js";
+import { LogContext, consoleLogSink } from "@rocicorp/logger";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+//#region ../zero-cache/src/server/serving-checkpointer.ts
+var lc = new LogContext("info", {}, consoleLogSink);
+/**
+* Checkpoints a serving replica on behalf of its replicator, which starts
+* this worker as one of its supporting processes once the replica is set up,
+* passing the path of its replica file as the first argument.
+*
+* Only the logging config is parsed from the remaining arguments, which
+* ignores everything else. In particular, `childWorker()` forwards the
+* replicator's own arguments, which start with its replica file mode and
+* would fail a full config parse.
+*/
+function runWorker(parent, env, ...args) {
+	const [dbPath, ...argv] = args;
+	assert(dbPath, `replica file not specified`);
+	const logConfig = getLogConfig({
+		env,
+		argv
+	});
+	const workerName = "serving-checkpointer";
+	startOtelAuto(createLogContext(logConfig, workerName, 0, false), workerName, 0);
+	lc = createLogContext(logConfig, workerName);
+	return runUntilKilled(lc, parent, new ServingCheckpointer(lc, dbPath));
+}
+var isMain = typeof process.argv[1] === "string" && fileURLToPath(import.meta.url) === resolve(process.argv[1]);
+var parent = parentWorker;
+if (isMain && parent !== null && !singleProcessMode()) exitAfter(() => lc, () => runWorker(parent, process.env, ...process.argv.slice(2)));
+//#endregion
+export { runWorker as default };
+
+//# sourceMappingURL=serving-checkpointer.js.map

package/out/zero-cache/src/server/serving-checkpointer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serving-checkpointer.js","names":[],"sources":["../../../../../zero-cache/src/server/serving-checkpointer.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ServingCheckpointer} from '../services/replicator/serving-checkpointer.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n/**\n * Checkpoints a serving replica on behalf of its replicator, which starts\n * this worker as one of its supporting processes once the replica is set up,\n * passing the path of its replica file as the first argument.\n *\n * Only the logging config is parsed from the remaining arguments, which\n * ignores everything else. In particular, `childWorker()` forwards the\n * replicator's own arguments, which start with its replica file mode and\n * would fail a full config parse.\n */\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const [dbPath, ...argv] = args;\n  assert(dbPath, `replica file not specified`);\n  const logConfig = getLogConfig({env, argv});\n  const workerName = 'serving-checkpointer';\n  startOtelAuto(\n    createLogContext(logConfig, workerName, 0, false),\n    workerName,\n    0,\n  );\n  lc = createLogContext(logConfig, workerName);\n\n  return runUntilKilled(lc, parent, new ServingCheckpointer(lc, dbPath));\n}\n\n// fork()\nconst isMain =\n  typeof process.argv[1] === 'string' &&\n  fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(parent, process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;;;;;;;;;;;AAYlD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,CAAC,QAAQ,GAAG,QAAQ;CAC1B,OAAO,QAAQ,4BAA4B;CAC3C,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,aAAa;CACnB,cACE,iBAAiB,WAAW,YAAY,GAAG,KAAK,GAChD,YACA,CACF;CACA,KAAK,iBAAiB,WAAW,UAAU;CAE3C,OAAO,eAAe,IAAI,QAAQ,IAAI,oBAAoB,IAAI,MAAM,CAAC;AACvE;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,YAAY,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}

package/out/zero-cache/src/server/worker-urls.d.ts

@@ -3,6 +3,7 @@ export declare const MAIN_URL: URL;
 export declare const MUTATOR_URL: URL;
 export declare const REAPER_URL: URL;
 export declare const REPLICATOR_URL: URL;
+export declare const SERVING_CHECKPOINTER_URL: URL;
 export declare const SHADOW_SYNCER_URL: URL;
 export declare const SLOT_KEEPER_URL: URL;
 export declare const SYNCER_URL: URL;

package/out/zero-cache/src/server/worker-urls.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,eAAe,KAA8B,CAAC;AAC3D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
+{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,wBAAwB,KAAuC,CAAC;AAC7E,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,eAAe,KAA8B,CAAC;AAC3D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}

package/out/zero-cache/src/server/worker-urls.js

@@ -10,11 +10,12 @@ var MAIN_URL = resolve("./main.ts");
 resolve("./mutator.ts");
 var REAPER_URL = resolve("./reaper.ts");
 var REPLICATOR_URL = resolve("./replicator.ts");
+var SERVING_CHECKPOINTER_URL = resolve("./serving-checkpointer.ts");
 var SHADOW_SYNCER_URL = resolve("./shadow-syncer.ts");
 var SLOT_KEEPER_URL = resolve("./slot-keeper.ts");
 var SYNCER_URL = resolve("./syncer.ts");
 var WRITE_WORKER_URL = resolve("./write-worker.ts");
 //#endregion
-export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SLOT_KEEPER_URL, SYNCER_URL, WRITE_WORKER_URL };
+export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SERVING_CHECKPOINTER_URL, SHADOW_SYNCER_URL, SLOT_KEEPER_URL, SYNCER_URL, WRITE_WORKER_URL };
 
 //# sourceMappingURL=worker-urls.js.map

package/out/zero-cache/src/server/worker-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO;CACd,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
+{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SERVING_CHECKPOINTER_URL = resolve('./serving-checkpointer.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO;CACd,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,2BAA2B,QAAQ,2BAA2B;AAC3E,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js

@@ -3,11 +3,11 @@ import { parse } from "../../../../../shared/src/valita.js";
 import { must } from "../../../../../shared/src/must.js";
 import { randomCharacters } from "../../../../../shared/src/random-values.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
+import { id } from "../../../types/sql.js";
 import { connectPgClient, pgClient } from "../../../types/pg.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { getPublicationInfo } from "./schema/published.js";
-import { id } from "../../../types/sql.js";
 import { Queue } from "../../../../../shared/src/queue.js";
 import { resumePoint } from "../protocol/backfill-progress.js";
 import { SchemaIncompatibilityError } from "../common/backfill-manager.js";

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -2,6 +2,7 @@ import { assert } from "../../../../../shared/src/asserts.js";
 import { must } from "../../../../../shared/src/must.js";
 import { equals } from "../../../../../shared/src/set-utils.js";
 import { ALLOWED_APP_ID_CHARACTERS } from "../../../types/shards.js";
+import { id } from "../../../types/sql.js";
 import { Database } from "../../../../../zqlite/src/db.js";
 import { connectPgClient } from "../../../types/pg.js";
 import { getOrCreateCounter, getOrCreateHistogram } from "../../../observability/metrics.js";
@@ -16,7 +17,6 @@ import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { runTx } from "../../../db/run-transaction.js";
 import { getPublicationInfo, warnForSkippedIndexes } from "./schema/published.js";
-import { id } from "../../../types/sql.js";
 import "./schema/replica-stage-enum.js";
 import { dropShard, getInternalShardConfig, getReplicaState, initInitialSyncReplica, validatePublications } from "./schema/shard.js";
 import { createLiteIndexStatement, createLiteTableStatement } from "../../../db/create.js";

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js

@@ -1,10 +1,10 @@
 import { sleep } from "../../../../../../shared/src/sleep.js";
 import { childWorker } from "../../../../types/processes.js";
+import { id, lit } from "../../../../types/sql.js";
 import "../../../../types/pg.js";
 import { getOrCreateHistogram } from "../../../../observability/metrics.js";
 import { SLOT_KEEPER_URL } from "../../../../server/worker-urls.js";
 import { fromBigInt, toBigInt } from "../lsn.js";
-import { id, lit } from "../../../../types/sql.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { literalUnion, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { publishedSchema, publishedSchemaQuery } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { publishedSchema, publishedSchemaQuery } from "./published.js";
 import { literal } from "pg-format";
 //#region ../zero-cache/src/services/change-source/pg/schema/ddl.ts
 var PROTOCOL_VERSION = 1;

package/out/zero-cache/src/services/change-source/pg/schema/init.js

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { publishedSchema } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { publishedSchema } from "./published.js";
 import "./replica-stage-enum.js";
 import { getMutationsTableDefinition, legacyReplicationSlot, metadataPublicationName, setupTablesAndReplication, setupTriggers } from "./shard.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/schema/shard.js

@@ -2,8 +2,8 @@ import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { jsonObjectSchema, stringify } from "../../../../../../shared/src/bigint-json.js";
 import { appSchema, check, upstreamSchema } from "../../../../types/shards.js";
-import { getPublicationInfo, publishedSchema } from "./published.js";
 import { id } from "../../../../types/sql.js";
+import { getPublicationInfo, publishedSchema } from "./published.js";
 import { createEventFunctionStatements, createEventTriggerStatements } from "./ddl.js";
 import { replicaStageSchema } from "./replica-stage-enum.js";
 import { validate } from "./validation.js";

package/out/zero-cache/src/services/http-service.d.ts

@@ -28,6 +28,12 @@ export declare class HttpService implements Service {
      * is draining (see {@link SelfDrain}).
      */
     failHealthChecks(): void;
+    /**
+     * Responds to subsequent readiness checks ("/") with a 503, while liveness
+     * checks ("/keepalive") continue to succeed, e.g. to signal the load
+     * balancer to stop routing to a server that is about to drain.
+     */
+    failReadiness(): void;
     /**
      * Stops accepting new connections (e.g. when draining), while existing
      * connections, including websockets, continue to be served until

package/out/zero-cache/src/services/http-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAOxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAKf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}
+{"version":3,"file":"http-service.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/http-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAgB,EAAC,KAAK,eAAe,EAAC,MAAM,SAAS,CAAC;AAGtD,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;IAGvC,aAAa,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,qBAAa,WAAY,YAAW,OAAO;;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IAGnC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAQxC,YACE,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAczD;IAGD,SAAS,CAAC,QAAQ,SAAK;IACvB,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAGK,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAwB7B;IAEK,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;;;OAIG;IACH,gBAAgB,SAKf;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAED;;;;OAIG;IACH,aAAa,SAKZ;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAM1B;CACF"}

package/out/zero-cache/src/services/http-service.js

@@ -17,6 +17,7 @@ var HttpService = class {
 	#heartbeatMonitor;
 	#init;
 	#ready = false;
+	#failReadiness = false;
 	#failHealthChecks = false;
 	constructor(id, lc, opts, init) {
 		const { port, keepaliveTimeoutMs, readinessGate = promiseVoid } = opts;
@@ -35,7 +36,7 @@ var HttpService = class {
 	}
 	async start() {
 		this.#fastify.get("/", (_req, res) => {
-			if (this.#failHealthChecks) res.code(503).send("Draining");
+			if (this.#failReadiness || this.#failHealthChecks) res.code(503).send("Draining");
 			else if (this.#ready) res.send("OK");
 		});
 		this.#fastify.get("/keepalive", ({ headers }, res) => {
@@ -68,6 +69,17 @@ var HttpService = class {
 		}
 	}
 	/**
+	* Responds to subsequent readiness checks ("/") with a 503, while liveness
+	* checks ("/keepalive") continue to succeed, e.g. to signal the load
+	* balancer to stop routing to a server that is about to drain.
+	*/
+	failReadiness() {
+		if (!this.#failReadiness) {
+			this._lc.info?.(`${this.id}: failing readiness checks before draining`);
+			this.#failReadiness = true;
+		}
+	}
+	/**
 	* Stops accepting new connections (e.g. when draining), while existing
 	* connections, including websockets, continue to be served until
 	* {@link stop} is called.

package/out/zero-cache/src/services/http-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the load balancer to stop routing to a server that\n   * is draining (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    if (!this.#failHealthChecks) {\n      this._lc.info?.(`${this.id}: failing health checks while draining`);\n      this.#failHealthChecks = true;\n    }\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,IAAI,CAAC,KAAK,mBAAmB;GAC3B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,uCAAuC;GAClE,KAAK,oBAAoB;EAC3B;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}
+{"version":3,"file":"http-service.js","names":[],"sources":["../../../../../zero-cache/src/services/http-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport Fastify, {type FastifyInstance} from 'fastify';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {HeartbeatMonitor} from './life-cycle.ts';\nimport {RunningState} from './running-state.ts';\nimport type {Service} from './service.ts';\n\nexport type Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n\n  // Wait for the readinessGate to resolve before responding to health checks.\n  readinessGate?: Promise<unknown> | undefined;\n};\n\n/**\n * Common functionality for all HttpServices. These include:\n * * Responding to health checks at \"/\"\n * * Tracking optional heartbeats at \"/keepalive\" and draining when they stop.\n */\nexport class HttpService implements Service {\n  readonly id: string;\n  protected readonly _lc: LogContext;\n  readonly #fastify: FastifyInstance;\n  readonly #port: number;\n  protected readonly _state: RunningState;\n  readonly #heartbeatMonitor: HeartbeatMonitor | undefined;\n  readonly #init: (fastify: FastifyInstance) => void | Promise<void>;\n\n  #ready = false;\n  #failReadiness = false;\n  #failHealthChecks = false;\n\n  constructor(\n    id: string,\n    lc: LogContext,\n    opts: Options,\n    init: (fastify: FastifyInstance) => void | Promise<void>,\n  ) {\n    const {port, keepaliveTimeoutMs, readinessGate = promiseVoid} = opts;\n    this.id = id;\n    this._lc = lc.withContext('component', this.id);\n    this.#fastify = Fastify();\n    this.#port = port;\n    this.#init = init;\n    this._state = new RunningState(id);\n    this.#heartbeatMonitor = keepaliveTimeoutMs\n      ? new HeartbeatMonitor(this._lc, keepaliveTimeoutMs)\n      : undefined;\n\n    void readinessGate.then(() => (this.#ready = true));\n  }\n\n  // Life-cycle hooks for subclass implementations\n  protected _onStart() {}\n  protected _onStop(): Promise<void> {\n    return promiseVoid;\n  }\n  // start() is used in unit tests.\n  // run() is the lifecycle method called by the ServiceRunner.\n  async start(): Promise<string> {\n    this.#fastify.get('/', (_req, res) => {\n      if (this.#failReadiness || this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    this.#fastify.get('/keepalive', ({headers}, res) => {\n      this.#heartbeatMonitor?.onHeartbeat(headers);\n      if (this.#failHealthChecks) {\n        res.code(503).send('Draining');\n      } else if (this.#ready) {\n        res.send('OK');\n      }\n    });\n    await this.#init(this.#fastify);\n    const address = await this.#fastify.listen({\n      host: '::',\n      port: this.#port,\n    });\n    this._lc.info?.(`${this.id} listening at ${address}`);\n    this._onStart();\n    return address;\n  }\n\n  async run(): Promise<void> {\n    await this.start();\n    await this._state.stopped();\n  }\n\n  /**\n   * Responds to subsequent health checks (both \"/\" and \"/keepalive\") with a\n   * 503, e.g. to signal the load balancer to stop routing to a server that\n   * is draining (see {@link SelfDrain}).\n   */\n  failHealthChecks() {\n    if (!this.#failHealthChecks) {\n      this._lc.info?.(`${this.id}: failing health checks while draining`);\n      this.#failHealthChecks = true;\n    }\n  }\n\n  /**\n   * Responds to subsequent readiness checks (\"/\") with a 503, while liveness\n   * checks (\"/keepalive\") continue to succeed, e.g. to signal the load\n   * balancer to stop routing to a server that is about to drain.\n   */\n  failReadiness() {\n    if (!this.#failReadiness) {\n      this._lc.info?.(`${this.id}: failing readiness checks before draining`);\n      this.#failReadiness = true;\n    }\n  }\n\n  /**\n   * Stops accepting new connections (e.g. when draining), while existing\n   * connections, including websockets, continue to be served until\n   * {@link stop} is called.\n   */\n  stopAccepting() {\n    if (this.#fastify.server.listening) {\n      this._lc.info?.(`${this.id}: no longer accepting new connections`);\n      this.#fastify.server.close();\n    }\n  }\n\n  async stop(): Promise<void> {\n    this._lc.info?.(`${this.id}: no longer accepting connections`);\n    this.#heartbeatMonitor?.stop();\n    this._state.stop(this._lc);\n    await this.#fastify.close();\n    await this._onStop();\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAS;CACT,iBAAiB;CACjB,oBAAoB;CAEpB,YACE,IACA,IACA,MACA,MACA;EACA,MAAM,EAAC,MAAM,oBAAoB,gBAAgB,gBAAe;EAChE,KAAK,KAAK;EACV,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,WAAW,QAAQ;EACxB,KAAK,QAAQ;EACb,KAAK,QAAQ;EACb,KAAK,SAAS,IAAI,aAAa,EAAE;EACjC,KAAK,oBAAoB,qBACrB,IAAI,iBAAiB,KAAK,KAAK,kBAAkB,IACjD,KAAA;EAEJ,cAAmB,WAAY,KAAK,SAAS,IAAK;CACpD;CAGA,WAAqB,CAAC;CACtB,UAAmC;EACjC,OAAO;CACT;CAGA,MAAM,QAAyB;EAC7B,KAAK,SAAS,IAAI,MAAM,MAAM,QAAQ;GACpC,IAAI,KAAK,kBAAkB,KAAK,mBAC9B,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,KAAK,SAAS,IAAI,eAAe,EAAC,WAAU,QAAQ;GAClD,KAAK,mBAAmB,YAAY,OAAO;GAC3C,IAAI,KAAK,mBACP,IAAI,KAAK,GAAG,CAAC,CAAC,KAAK,UAAU;QACxB,IAAI,KAAK,QACd,IAAI,KAAK,IAAI;EAEjB,CAAC;EACD,MAAM,KAAK,MAAM,KAAK,QAAQ;EAC9B,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO;GACzC,MAAM;GACN,MAAM,KAAK;EACb,CAAC;EACD,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,gBAAgB,SAAS;EACpD,KAAK,SAAS;EACd,OAAO;CACT;CAEA,MAAM,MAAqB;EACzB,MAAM,KAAK,MAAM;EACjB,MAAM,KAAK,OAAO,QAAQ;CAC5B;;;;;;CAOA,mBAAmB;EACjB,IAAI,CAAC,KAAK,mBAAmB;GAC3B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,uCAAuC;GAClE,KAAK,oBAAoB;EAC3B;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,CAAC,KAAK,gBAAgB;GACxB,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,2CAA2C;GACtE,KAAK,iBAAiB;EACxB;CACF;;;;;;CAOA,gBAAgB;EACd,IAAI,KAAK,SAAS,OAAO,WAAW;GAClC,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,sCAAsC;GACjE,KAAK,SAAS,OAAO,MAAM;EAC7B;CACF;CAEA,MAAM,OAAsB;EAC1B,KAAK,IAAI,OAAO,GAAG,KAAK,GAAG,kCAAkC;EAC7D,KAAK,mBAAmB,KAAK;EAC7B,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,MAAM,KAAK,SAAS,MAAM;EAC1B,MAAM,KAAK,QAAQ;CACrB;AACF"}

package/out/zero-cache/src/services/life-cycle.d.ts

@@ -1,3 +1,4 @@
+import { EventEmitter as NodeEventEmitter } from 'node:events';
 import type { IncomingHttpHeaders } from 'node:http';
 import type { EventEmitter } from 'stream';
 import type { LogContext } from '@rocicorp/logger';
@@ -94,6 +95,24 @@ export declare class SelfDrain {
     constructor(lc: LogContext, proc: EventEmitter, opts: SelfDrainOptions);
     request(): void;
 }
+/**
+ * The event that a {@link PreDrain} emits immediately upon the first graceful
+ * shutdown signal, with that signal as its argument, before the delay.
+ */
+export declare const PRE_DRAIN = "preDrain";
+/**
+ * Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+ * balancers can observe a failed readiness check before the server begins
+ * to drain. The first graceful signal is announced immediately with a
+ * {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+ * as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+ * delay has elapsed. Repeated graceful signals are dropped. All other events,
+ * including forceful signals, are forwarded immediately.
+ */
+export declare class PreDrain extends NodeEventEmitter {
+    #private;
+    constructor(lc: LogContext, proc: EventEmitter, delayMs: number);
+}
 /**
  * The maximum time that {@link runUntilKilled} waits for its services to stop
  * (i.e. for their `run()` to settle) after stopping them.

package/out/zero-cache/src/services/life-cycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}
+{"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,QAAQ,CAAU,CAAC;AAChE,eAAO,MAAM,iBAAiB,YAAI,SAAS,EAAE,SAAS,CAAU,CAAC;AA8CjE,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,QAEzD;AAaD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;IAWzB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAyC7C;IAEK,QAAQ,kBAKb;IAuBD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,QAsB7D;IAKD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAwBhE;IAED,YAAY,IAAI,MAAM,EAAE,CAEvB;IAEK,eAAe,kBAEpB;IAED,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,QAGzC;CAqFF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,mBAAoB,YAAW,gBAAgB;;IAC1D,QAAQ,CAAC,EAAE,0BAA0B;IAOrC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAe/C;IAED,GAAG,CAAC,CAAC,SAAS,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAclD;IAED,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CASnB;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ1B;CAoCF;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,WAAW,EAAE,MAAM,IAAI,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;;IASpB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAYrE;IAED,OAAO,SAeN;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,SAAS,aAAa,CAAC;AAEpC;;;;;;;;GAQG;AACH,qBAAa,QAAS,SAAQ,gBAAgB;;IAQ5C,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAW9D;CA4BF;AAED;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAEtC;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA2Ef;AA8BD,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,GAAG,CAAC,MAAM,UAAU,CAAC,EAC3C,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAoCzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;IAQ3B,YAAY,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAG/C;IAED,WAAW,CAAC,UAAU,EAAE,mBAAmB,QAgB1C;IAsCD,IAAI,SAKH;CACF"}

package/out/zero-cache/src/services/life-cycle.js

@@ -7,6 +7,7 @@ import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateHistogram } from "../o
 import { orTimeout } from "../types/timeout.js";
 import { RunningState } from "./running-state.js";
 import { pid } from "node:process";
+import { EventEmitter } from "node:events";
 //#region ../zero-cache/src/services/life-cycle.ts
 var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
 var FORCEFUL_SHUTDOWN = ["SIGQUIT", "SIGABRT"];
@@ -261,6 +262,57 @@ var SelfDrain = class {
 	}
 };
 /**
+* The event that a {@link PreDrain} emits immediately upon the first graceful
+* shutdown signal, with that signal as its argument, before the delay.
+*/
+var PRE_DRAIN = "preDrain";
+/**
+* Delays graceful shutdown signals (`SIGTERM`, `SIGINT`), e.g. so that load
+* balancers can observe a failed readiness check before the server begins
+* to drain. The first graceful signal is announced immediately with a
+* {@link PRE_DRAIN} event, and is re-emitted to listeners of the signal, such
+* as those of the {@link ProcessManager} and {@link runUntilKilled}, once the
+* delay has elapsed. Repeated graceful signals are dropped. All other events,
+* including forceful signals, are forwarded immediately.
+*/
+var PreDrain = class extends EventEmitter {
+	#lc;
+	#proc;
+	#delayMs;
+	#forwarded = /* @__PURE__ */ new Set(["newListener", PRE_DRAIN]);
+	#signaled = false;
+	#timer;
+	constructor(lc, proc, delayMs) {
+		super();
+		this.#lc = lc.withContext("component", "pre-drain");
+		this.#proc = proc;
+		this.#delayMs = delayMs;
+		for (const signal of GRACEFUL_SHUTDOWN) {
+			this.#forwarded.add(signal);
+			proc.on(signal, () => this.#onGracefulSignal(signal));
+		}
+		this.on("newListener", (event) => this.#forward(event));
+	}
+	#forward(event) {
+		if (this.#forwarded.has(event)) return;
+		this.#forwarded.add(event);
+		this.#proc.on(event, (...args) => {
+			if (event === "disconnect" || FORCEFUL_SHUTDOWN.includes(event)) clearTimeout(this.#timer);
+			this.emit(event, ...args);
+		});
+	}
+	#onGracefulSignal(signal) {
+		if (this.#signaled) {
+			this.#lc.info?.(`already shutting down. ignoring ${signal}`);
+			return;
+		}
+		this.#signaled = true;
+		this.#lc.info?.(`received ${signal}. draining in ${this.#delayMs} ms`);
+		this.emit(PRE_DRAIN, signal);
+		this.#timer = setTimeout(() => this.emit(signal, signal), this.#delayMs);
+	}
+};
+/**
 * The maximum time that {@link runUntilKilled} waits for its services to stop
 * (i.e. for their `run()` to settle) after stopping them.
 */
@@ -397,6 +449,6 @@ var HeartbeatMonitor = class {
 	}
 };
 //#endregion
-export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
+export { FORCEFUL_SHUTDOWN, GRACEFUL_SHUTDOWN, HeartbeatMonitor, PRE_DRAIN, PreDrain, ProcessManager, STOP_TIMEOUT_MS, SelfDrain, SupportingProcesses, exitAfter, recordStartupDurationMs, runUntilKilled };
 
 //# sourceMappingURL=life-cycle.js.map