@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.41

package/out/zero-cache/src/server/main.js

@@ -35,10 +35,14 @@ async function runWorker(parent, env) {
 		"--cvr-max-conns-per-worker",
 		String(Math.floor(config.cvr.maxConns / numSyncers))
 	];
+	const allSubWorkers = [];
 	function loadWorker(moduleUrl, type, id, ...args) {
 		const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);
 		const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : "");
-		return processes.addWorker(worker, type, name);
+		const w = processes.addWorker(worker, type, name);
+		allSubWorkers.push(w);
+		w.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
+		return w;
 	}
 	const { taskID, litestream } = config;
 	const runChangeStreamer = runsChangeStreamer(config);
@@ -46,7 +50,7 @@ async function runWorker(parent, env) {
 	let changeStreamer;
 	if (runChangeStreamer) {
 		const { promise: changeStreamerReady, resolve: changeStreamerStarted } = resolver();
-		changeStreamer = loadWorker(CHANGE_STREAMER_URL, "supporting").once("message", changeStreamerStarted);
+		changeStreamer = loadWorker(CHANGE_STREAMER_URL, numSyncers > 0 ? "supporting" : "user-facing").once("message", changeStreamerStarted);
 		await changeStreamerReady;
 	}
 	if (numSyncers > 0) {
@@ -67,6 +71,10 @@ async function runWorker(parent, env) {
 			subscribeTo(lc, replicator);
 			resolve();
 		});
+		replicator.onMessageType("drainRequested", (msg) => {
+			lc.info?.(`drain requested by the replicator`);
+			parent.send(["drainRequested", msg]);
+		});
 		await replicaReady;
 		const notifier = createNotifierFrom(lc, replicator);
 		for (let i = 0; i < numSyncers; i++) syncers.push(loadWorker(SYNCER_URL, "user-facing", i, mode, String(i)));
@@ -81,6 +89,9 @@ async function runWorker(parent, env) {
 	lc.info?.(`all workers ready (${startupDurationMs} ms)`);
 	recordStartupDurationMs(startupDurationMs);
 	parent.send(["ready", { ready: true }]);
+	parent.onMessageType("profile", (req) => {
+		for (const w of allSubWorkers) w.send(["profile", req]);
+	});
 	try {
 		await runUntilKilled(lc, parent, new WorkerDispatcher(lc, taskID, parent, syncers, mutator, changeStreamer));
 	} catch (err) {

package/out/zero-cache/src/server/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  type ReplicaFileMode,\n  subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    return processes.addWorker(worker, type, name);\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n      'message',\n      changeStreamerStarted,\n    );\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA0CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,OAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;CAC/C;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EACX,iBAAiB,WAAW,qBAAqB,YAAY,EAAE,KAC7D,WACA,qBACF;EAGA,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}
+{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  subscribeTo,\n  type ReplicaFileMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    // In a dedicated replication-manager (i.e. no syncers), the\n    // change-streamer is the user-facing worker: it receives the SIGTERM\n    // first so that it can drain subscribers before other supporting\n    // workers are stopped. When syncers run in the same task, the\n    // change-streamer supports them and thus must outlive them.\n    changeStreamer = loadWorker(\n      CHANGE_STREAMER_URL,\n      numSyncers > 0 ? 'supporting' : 'user-facing',\n    ).once('message', changeStreamerStarted);\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    // Relay drain requests to the runner, which coordinates the SelfDrain.\n    replicator.onMessageType<DrainRequestedMessage>('drainRequested', msg => {\n      lc.info?.(`drain requested by the replicator`);\n      parent.send<DrainRequestedMessage>(['drainRequested', msg]);\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EAMX,iBAAiB,WACf,qBACA,aAAa,IAAI,eAAe,aAClC,EAAE,KAAK,WAAW,qBAAqB;EAGvC,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EAED,WAAW,cAAqC,mBAAkB,QAAO;GACvE,GAAG,OAAO,mCAAmC;GAC7C,OAAO,KAA4B,CAAC,kBAAkB,GAAG,CAAC;EAC5D,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}

package/out/zero-cache/src/server/mutator.js

@@ -3,6 +3,7 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { Mutator } from "../workers/mutator.js";
@@ -10,6 +11,7 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/mutator.ts
 var lc = new LogContext("info", {}, consoleLogSink);
 function runWorker(parent, env, ...args) {
+	installProfileHandler(parent, "mutator");
 	const config = getNormalizedZeroConfig({
 		env,
 		argv: args.slice(1)

package/out/zero-cache/src/server/mutator.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n  lc = createLogContext(config, 'mutator');\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;AAeA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,cAAc,iBAAiB,QAAQ,WAAW,GAAG,KAAK,GAAG,WAAW,CAAC;CACzE,KAAK,iBAAiB,QAAQ,SAAS;CACvC,cAAc,IAAI,MAAM;CAGxB,OAAO,eAAe,IAAI,QAAQ,IAAI,QAAQ,CAAC;AACjD;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'mutator');\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n  lc = createLogContext(config, 'mutator');\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,SAAS;CACvC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,cAAc,iBAAiB,QAAQ,WAAW,GAAG,KAAK,GAAG,WAAW,CAAC;CACzE,KAAK,iBAAiB,QAAQ,SAAS;CACvC,cAAc,IAAI,MAAM;CAGxB,OAAO,eAAe,IAAI,QAAQ,IAAI,QAAQ,CAAC;AACjD;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/reaper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"AASA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA+Bf"}
+{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"AASA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAY/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAgCf"}

package/out/zero-cache/src/server/reaper.js

@@ -5,6 +5,7 @@ import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { connectPgClient } from "../types/pg.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { startAnonymousTelemetry } from "./anonymous-otel-start.js";
@@ -16,6 +17,7 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
 var MS_PER_HOUR = 1e3 * 60 * 60;
 var lc = new LogContext("info", {}, consoleLogSink);
 async function runWorker(parent, env, ...argv) {
+	installProfileHandler(parent, "reaper");
 	const config = getNormalizedZeroConfig({
 		env,
 		argv

package/out/zero-cache/src/server/reaper.js.map

@@ -1 +1 @@
-{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAmBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,IAC9B,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'reaper');\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAoBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,QAAQ;CACtC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,IAC9B,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/replicator.d.ts

@@ -1,3 +1,10 @@
+import { LogContext } from '@rocicorp/logger';
+import type { NormalizedZeroConfig } from '../config/normalize.ts';
+import { type ReservationFollowup, type SnapshotReserver } from '../services/change-streamer/change-streamer-http.ts';
 import { type Worker } from '../types/processes.ts';
+import { type WalMode } from '../workers/replicator.ts';
 export default function runWorker(parent: Worker, env: NodeJS.ProcessEnv, ...args: string[]): Promise<void>;
+export declare function setupMetrics(lc: LogContext, file: string, walMode: WalMode, pageSize: number): void;
+/** Exported for testing. */
+export declare function restoreReplica(lc: LogContext, config: NormalizedZeroConfig, changeStreamer: SnapshotReserver): Promise<ReservationFollowup>;
 //# sourceMappingURL=replicator.d.ts.map

package/out/zero-cache/src/server/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAsCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoHf"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAGA,OAAO,EAAiB,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAM5D,OAAO,KAAK,EAEV,oBAAoB,EACrB,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,EACtB,MAAM,qDAAqD,CAAC;AA8B7D,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAML,KAAK,OAAO,EACb,MAAM,0BAA0B,CAAC;AAOlC,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAkJf;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,QAwDjB;AAyFD,4BAA4B;AAC5B,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,cAAc,EAAE,gBAAgB,GAC/B,OAAO,CAAC,mBAAmB,CAAC,CAuF9B"}

package/out/zero-cache/src/server/replicator.js

@@ -5,20 +5,23 @@ import { must } from "../../../shared/src/must.js";
 import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
+import { Database } from "../../../zqlite/src/db.js";
 import { getOrCreateGauge } from "../observability/metrics.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
-import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
+import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
 import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../services/litestream/metrics.js";
-import { tryRestore } from "../services/litestream/commands.js";
-import { reserveAndGetSnapshotStatus } from "../services/change-streamer/snapshot.js";
+import { readLocalReplica, startReplicaBackupProcess, tryRestore } from "../services/litestream/commands.js";
 import { ChangeStreamerHttpClient } from "../services/change-streamer/change-streamer-http.js";
 import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observability.js";
 import { startOtelAuto } from "./otel-start.js";
 import { ThreadWriteWorkerClient } from "../services/replicator/write-worker-client.js";
 import { deleteStaleChangeLog, getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
+import { reserveAndGetSnapshotStatus } from "../services/change-streamer/snapshot.js";
+import { RestoreProgressReporter } from "../services/litestream/restore-progress.js";
 import { ReplicatorService } from "../services/replicator/replicator.js";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 import { pid } from "node:process";
@@ -35,6 +38,7 @@ async function runWorker(parent, env, ...args) {
 	const mode = fileMode === "backup" ? "backup" : "serving";
 	const runningLocalChangeStreamer = config.changeStreamer.mode === "dedicated" && !config.changeStreamer.uri;
 	const workerName = `${mode}-replicator`;
+	installProfileHandler(parent, workerName);
 	startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);
 	lc = createLogContext(config, workerName);
 	const unregisterInitialCorruptionDiagnosticTarget = registerSQLiteCorruptionDiagnosticTarget({
@@ -42,7 +46,11 @@ async function runWorker(parent, env, ...args) {
 		dbPath: config.replica.file
 	}, config.sqliteCorruptionChecks);
 	initEventSink(lc, config);
-	if (fileMode === "serving" && !runningLocalChangeStreamer && (config.litestream.executable || config.litestream.executableV5)) await restoreReplica(lc, config);
+	const shard = getShardConfig(config);
+	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 }, replicationLag: { initialReadinessThresholdMs, maxReadinessDelayMs } } = config;
+	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
+	let initialConnection;
+	if (fileMode === "serving" && !runningLocalChangeStreamer && (config.litestream.executable || config.litestream.executableV5)) initialConnection = await restoreReplica(lc, config, changeStreamer);
 	const { file: dbPath, walMode, pageSize } = await setupReplica(lc, fileMode, config.replica);
 	unregisterInitialCorruptionDiagnosticTarget();
 	registerSQLiteCorruptionDiagnosticTarget({
@@ -50,7 +58,7 @@ async function runWorker(parent, env, ...args) {
 		dbPath
 	}, config.sqliteCorruptionChecks);
 	deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);
-	setupMetrics(lc, dbPath, walMode);
+	setupMetrics(lc, dbPath, walMode, pageSize);
 	const pragmas = getPragmaConfig(fileMode);
 	const checkpoint = mode === "backup" && config.litestream.backupURL && config.litestream.backupUsingV5 && config.litestream.forceCheckpointThresholdMB ? {
 		checkpointThresholdPages: config.litestream.forceCheckpointThresholdMB * 1024 ** 2 / pageSize,
@@ -58,19 +66,31 @@ async function runWorker(parent, env, ...args) {
 	} : null;
 	const workerClient = new ThreadWriteWorkerClient();
 	await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);
-	const shard = getShardConfig(config);
-	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
-	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
-	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
+	const processes = new SupportingProcesses(lc, parent);
+	if (mode === "backup") processes.add(startReplicaBackupProcess(lc, config.litestream, dbPath), "litestream");
+	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null, {
+		readinessThresholdMs: initialReadinessThresholdMs,
+		maxReadinessDelayMs
+	}, initialConnection, runningLocalChangeStreamer ? void 0 : {
+		onRestoreGranted: () => parent.send(["drainRequested", {}]),
+		replicaFile: dbPath
+	});
 	setUpMessageHandlers(lc, replicator, parent);
-	const running = runUntilKilled(lc, parent, replicator);
-	for await (const _ of replicator.subscribe()) {
+	replicator.ready().then(() => {
 		parent.send(["ready", { ready: true }]);
-		break;
-	}
-	return running;
+	});
+	return runUntilKilled(lc, parent, replicator, processes);
 }
-function setupMetrics(lc, file, walMode) {
+function setupMetrics(lc, file, walMode, pageSize) {
+	let db;
+	const getDb = () => {
+		if (!db) try {
+			db = new Database(lc, file, { readonly: true });
+		} catch (e) {
+			lc.warn?.(`unable to open ${file} for wal metrics`, e);
+		}
+		return db;
+	};
 	getOrCreateGauge("replica", "db_size", {
 		description: "The size of the replica's main db file, which does not include the wal file(s)",
 		unit: "bytes"
@@ -83,6 +103,14 @@ function setupMetrics(lc, file, walMode) {
 		description: `The size of the replica's wal2 file`,
 		unit: "bytes"
 	}).addCallback(observeFileSize(lc, `${file}-wal2`));
+	getOrCreateGauge("replica", "wal_active_bytes", {
+		description: "The actual size of active WAL frames currently in use (excluding pre-allocated or reclaimed space).",
+		unit: "bytes"
+	}).addCallback(observeWalActiveBytes(lc, getDb, pageSize));
+	getOrCreateGauge("replica", "wal_uncheckpointed_bytes", {
+		description: "The size of un-checkpointed WAL frames that have not yet been backfilled into the main database.",
+		unit: "bytes"
+	}).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));
 	getOrCreateGauge("replica", "file_bytes", {
 		description: "The replica's total on-disk footprint: its main db file plus its wal sidecars.",
 		unit: "By"
@@ -98,30 +126,83 @@ function observeFileSize(lc, file) {
 		}
 	};
 }
+function observeWalActiveBytes(lc, getDb, pageSize) {
+	return (o) => {
+		try {
+			const db = getDb();
+			if (!db) return;
+			const [{ log }] = db.pragma("wal_checkpoint(NOOP)");
+			if (typeof log === "number") o.observe(Math.max(0, log) * pageSize);
+		} catch (e) {
+			lc.warn?.("unable to inspect wal_checkpoint for active bytes metric", e);
+		}
+	};
+}
+function observeWalUncheckpointedBytes(lc, getDb, pageSize) {
+	return (o) => {
+		try {
+			const db = getDb();
+			if (!db) return;
+			const [{ log, checkpointed }] = db.pragma("wal_checkpoint(NOOP)");
+			if (typeof log === "number" && typeof checkpointed === "number") o.observe(Math.max(0, log - checkpointed) * pageSize);
+		} catch (e) {
+			lc.warn?.("unable to inspect wal_checkpoint for uncheckpointed bytes metric", e);
+		}
+	};
+}
 function observeSQLiteFileBytes(lc, file) {
 	return async (o) => o.observe(await sqliteFileBytes(lc, file));
 }
 var RETRY_INTERVAL_MS = 3e3;
-async function restoreReplica(lc, config) {
+var MAX_OLDER_REPLICA_VERSION_RETRIES = 20;
+/** Exported for testing. */
+async function restoreReplica(lc, config, changeStreamer) {
 	const start = performance.now();
 	let backupURL;
 	let result;
+	const progress = new RestoreProgressReporter(lc, config.replica.file);
+	let olderReplicaVersionRetries = 0;
+	const localReplica = readLocalReplica(lc, config.replica.file);
+	const localReplicaVersion = localReplica?.replicaVersion;
 	try {
 		for (;;) {
-			const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);
-			({backupURL} = snapshotStatus);
-			const attempt = await tryRestore(lc, {
+			const { reserved, followup } = await reserveAndGetSnapshotStatus(lc, config.taskID, changeStreamer, localReplica);
+			if (localReplicaVersion !== void 0 && reserved.replicaVersion < localReplicaVersion && olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES) {
+				followup.cancel();
+				lc.info?.(`replication-manager is at an older replica version (${reserved.replicaVersion}) than the local replica (${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
+				await sleep(RETRY_INTERVAL_MS);
+				continue;
+			}
+			({backupURL} = reserved);
+			const litestream = {
 				...config.litestream,
 				backupURL
-			}, config.replica.file, snapshotStatus, "view_syncer");
+			};
+			progress.start(reserved.replicaSize);
+			let attempt;
+			try {
+				attempt = await tryRestore(lc, litestream, config.replica.file, reserved, "view_syncer", followup.signal);
+			} catch (e) {
+				if (followup.signal.aborted) {
+					lc.info?.(`reservation disconnected during restore. retrying`, String(e));
+					continue;
+				}
+				followup.cancel();
+				throw e;
+			} finally {
+				progress.stop();
+			}
 			if (attempt.restored) {
 				result = attempt.result;
-				return;
+				progress.done();
+				return followup;
 			}
+			followup.cancel();
 			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
 			await sleep(RETRY_INTERVAL_MS);
 		}
 	} finally {
+		progress.stop();
 		const labels = {
 			...litestreamRestoreMetricAttrs(config.litestream, "view_syncer", backupURL),
 			result: result ?? "error"
@@ -132,6 +213,6 @@ async function restoreReplica(lc, config) {
 }
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
 //#endregion
-export { runWorker as default };
+export { runWorker as default, restoreReplica, setupMetrics };
 
 //# sourceMappingURL=replicator.js.map

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAwDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAKhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n  ChangeStreamerHttpClient,\n  type ReservationFollowup,\n  type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n  readLocalReplica,\n  startReplicaBackupProcess,\n  tryRestore,\n  type RestoreAttempt,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n    replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  //\n  // The reservation's connection (`initialConnection`) is threaded through to\n  // IncrementalSyncer's first subscribe(), so the replication-manager that\n  // reserved the change log for this restore is the one that serves the\n  // subscription (single-connection handoff).\n  let initialConnection: ReservationFollowup | undefined;\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    initialConnection = await restoreReplica(lc, config, changeStreamer);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode, pageSize);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  // The backup-replicator manages the litestream backup process. It is\n  // started after the write worker has prepared the db file in order to\n  // avoid contending on the sqlite lock.\n  const processes = new SupportingProcesses(lc, parent);\n  if (mode === 'backup') {\n    processes.add(\n      startReplicaBackupProcess(lc, config.litestream, dbPath),\n      'litestream',\n    );\n  }\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n    {\n      readinessThresholdMs: initialReadinessThresholdMs,\n      maxReadinessDelayMs,\n    },\n    initialConnection,\n    // View-syncers (i.e. with a remote change-streamer) switch over to\n    // another replication-manager when theirs drains.\n    runningLocalChangeStreamer\n      ? undefined\n      : {\n          onRestoreGranted: () =>\n            parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n          replicaFile: dbPath,\n        },\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  void replicator.ready().then(() => {\n    parent.send(['ready', {ready: true}]);\n  });\n\n  return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n  lc: LogContext,\n  file: string,\n  walMode: WalMode,\n  pageSize: number,\n) {\n  let db: Database | undefined;\n  const getDb = () => {\n    if (!db) {\n      try {\n        db = new Database(lc, file, {readonly: true});\n      } catch (e) {\n        lc.warn?.(`unable to open ${file} for wal metrics`, e);\n      }\n    }\n    return db;\n  };\n\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  getOrCreateGauge('replica', 'wal_active_bytes', {\n    description:\n      `The actual size of active WAL frames currently in use ` +\n      `(excluding pre-allocated or reclaimed space).`,\n    unit: 'bytes',\n  }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n  getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n    description:\n      `The size of un-checkpointed WAL frames that have not yet ` +\n      `been backfilled into the main database.`,\n    unit: 'bytes',\n  }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeWalActiveBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number') {\n        o.observe(Math.max(0, log) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n    }\n  };\n}\n\nfunction observeWalUncheckpointedBytes(\n  lc: LogContext,\n  getDb: () => Database | undefined,\n  pageSize: number,\n): ObservableCallback {\n  return o => {\n    try {\n      const db = getDb();\n      if (!db) {\n        return;\n      }\n      const [{log, checkpointed}] = db.pragma<{\n        log: number;\n        checkpointed: number;\n      }>('wal_checkpoint(NOOP)');\n      if (typeof log === 'number' && typeof checkpointed === 'number') {\n        o.observe(Math.max(0, log - checkpointed) * pageSize);\n      }\n    } catch (e) {\n      lc.warn?.(\n        'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n        e,\n      );\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  const progress = new RestoreProgressReporter(lc, config.replica.file);\n  let olderReplicaVersionRetries = 0;\n  // Describing the local replica (if any) allows the replication-manager to\n  // account for replicas that are being replaced (see RestoreAdmission). It\n  // is read once, before the first attempt, as a failed attempt may have\n  // already deleted it; it continues to describe the replica being replaced\n  // (and the version not to downgrade from) on every retry.\n  const localReplica = readLocalReplica(lc, config.replica.file);\n  const localReplicaVersion = localReplica?.replicaVersion;\n  try {\n    for (;;) {\n      const {reserved, followup} = await reserveAndGetSnapshotStatus(\n        lc,\n        config.taskID,\n        changeStreamer,\n        localReplica,\n      );\n      if (\n        localReplicaVersion !== undefined &&\n        reserved.replicaVersion < localReplicaVersion &&\n        olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n      ) {\n        // Don't replace a newer local replica with an older one, e.g. from\n        // a replication-manager that has yet to be replaced by a resync.\n        followup.cancel();\n        lc.info?.(\n          `replication-manager is at an older replica version ` +\n            `(${reserved.replicaVersion}) than the local replica ` +\n            `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n        );\n        await sleep(RETRY_INTERVAL_MS);\n        continue;\n      }\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = reserved);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      progress.start(reserved.replicaSize);\n      let attempt: RestoreAttempt;\n      try {\n        attempt = await tryRestore(\n          lc,\n          litestream,\n          config.replica.file,\n          reserved,\n          'view_syncer',\n          followup.signal, // abort the restore if the reservation disconnects\n        );\n      } catch (e) {\n        if (followup.signal.aborted) {\n          lc.info?.(\n            `reservation disconnected during restore. retrying`,\n            String(e),\n          );\n          continue;\n        }\n        followup.cancel();\n        throw e;\n      } finally {\n        progress.stop();\n      }\n      if (attempt.restored) {\n        result = attempt.result;\n        progress.done();\n        return followup; // connection to be converted to a subscription\n      }\n      // This attempt's reservation is stale; a fresh one is made on retry.\n      followup.cancel();\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    progress.stop();\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,EAAE,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/runner/run-worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAS1C,OAAO,EAAc,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAsBlE;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAgEf"}
+{"version":3,"file":"run-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"names":[],"mappings":"AAAA,OAAO,kCAAkC,CAAC;AAa1C,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,0BAA0B,CAAC;AAsBlC;;;;;;GAMG;AACH,wBAAsB,SAAS,CAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAqEf"}

package/out/zero-cache/src/server/runner/run-worker.js

@@ -4,7 +4,7 @@ import { colorConsole } from "../../../../shared/src/logging.js";
 import { childWorker } from "../../types/processes.js";
 import { normalizeZeroConfig } from "../../config/normalize.js";
 import { getServerVersion, getZeroConfig } from "../../config/zero-config.js";
-import { ProcessManager, runUntilKilled } from "../../services/life-cycle.js";
+import { ProcessManager, SelfDrain, runUntilKilled } from "../../services/life-cycle.js";
 import { createLogContext } from "../logging.js";
 import { MAIN_URL } from "../worker-urls.js";
 import { getTaskID } from "./runtime.js";
@@ -33,7 +33,7 @@ async function runWorker(parent, env) {
 	const lc = createLogContext(cfg, "runner");
 	const config = normalizeZeroConfig(lc, cfg, env, await getTaskID(lc));
 	const processes = new ProcessManager(lc, parent ?? process);
-	const { port, keepaliveTimeoutMs, lazyStartup } = config;
+	const { port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs } = config;
 	const serverVersion = getServerVersion(config);
 	lc.info?.(`starting server${!serverVersion ? "" : `@${serverVersion}`} `, {
 		protocolVersion: 53,
@@ -42,6 +42,14 @@ async function runWorker(parent, env) {
 		shard: config.shard,
 		port: config.port
 	});
+	const dispatcher = new ZeroDispatcher(config, lc, {
+		port,
+		keepaliveTimeoutMs
+	}, startZeroCache, () => printStartupMessage(env));
+	const selfDrain = new SelfDrain(lc, parent ?? process, {
+		delayMs: selfDrainDelayMs,
+		onRequested: () => dispatcher.failHealthChecks()
+	});
 	let zeroCache;
 	function startZeroCache() {
 		if (zeroCache === void 0) {
@@ -52,6 +60,7 @@ async function runWorker(parent, env) {
 				r.resolve(w);
 				lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);
 			}).once("error", r.reject);
+			w.onMessageType("drainRequested", () => selfDrain.request());
 			processes.addWorker(w, "user-facing", "zero-cache");
 		}
 		return zeroCache.promise;
@@ -60,10 +69,7 @@ async function runWorker(parent, env) {
 	await processes.allWorkersReady();
 	parent?.send(["ready", { ready: true }]);
 	try {
-		await runUntilKilled(lc, parent ?? process, new ZeroDispatcher(config, lc, {
-			port,
-			keepaliveTimeoutMs
-		}, startZeroCache, () => printStartupMessage(env)));
+		await runUntilKilled(lc, parent ?? process, dispatcher);
 	} catch (err) {
 		processes.logErrorAndExit(err, "main");
 	} finally {

package/out/zero-cache/src/server/runner/run-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {ProcessManager, runUntilKilled} from '../../services/life-cycle.ts';\nimport {childWorker, type Worker} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent ?? process,\n      new ZeroDispatcher(\n        config,\n        lc,\n        {port, keepaliveTimeoutMs},\n        startZeroCache,\n        () => printStartupMessage(env),\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAGzC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,MADrB,UAAU,EAAE,CACsB;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,gBAAe;CAChD,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,EAChC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,EACA,KAAK,SAAS,EAAE,MAAM;GAEzB,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eACJ,IACA,UAAU,SACV,IAAI,eACF,QACA,IACA;GAAC;GAAM;EAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}
+{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {\n  ProcessManager,\n  runUntilKilled,\n  SelfDrain,\n} from '../../services/life-cycle.ts';\nimport {\n  childWorker,\n  type DrainRequestedMessage,\n  type Worker,\n} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup, selfDrainDelayMs} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  const dispatcher = new ZeroDispatcher(\n    config,\n    lc,\n    {port, keepaliveTimeoutMs},\n    startZeroCache,\n    () => printStartupMessage(env),\n  );\n  const selfDrain = new SelfDrain(lc, parent ?? process, {\n    delayMs: selfDrainDelayMs,\n    onRequested: () => dispatcher.failHealthChecks(),\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n      w.onMessageType<DrainRequestedMessage>('drainRequested', () =>\n        selfDrain.request(),\n      );\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(lc, parent ?? process, dispatcher);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAuBA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAGzC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,MADrB,UAAU,EAAE,CACsB;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,aAAa,qBAAoB;CAClE,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,MAAM,aAAa,IAAI,eACrB,QACA,IACA;EAAC;EAAM;CAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B;CACA,MAAM,YAAY,IAAI,UAAU,IAAI,UAAU,SAAS;EACrD,SAAS;EACT,mBAAmB,WAAW,iBAAiB;CACjD,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,EAChC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,EACA,KAAK,SAAS,EAAE,MAAM;GACzB,EAAE,cAAqC,wBACrC,UAAU,QAAQ,CACpB;GAEA,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eAAe,IAAI,UAAU,SAAS,UAAU;CACxD,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}

package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAGzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAarB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}
+{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAQzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAsBrB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}

package/out/zero-cache/src/server/runner/zero-dispatcher.js

@@ -1,5 +1,7 @@
 import { handleHeapzRequest } from "../../services/heapz.js";
 import { HttpService } from "../../services/http-service.js";
+import { handlePlannerzRequest } from "../../services/plannerz.js";
+import { handleProfrmzRequest, handleProfzRequest } from "../../services/profz.js";
 import { handleStatzRequest } from "../../services/statz.js";
 import { installWebSocketHandoff } from "../../types/websocket-handoff.js";
 //#region ../zero-cache/src/server/runner/zero-dispatcher.ts
@@ -10,7 +12,10 @@ var ZeroDispatcher = class extends HttpService {
 	constructor(config, lc, opts, getWorker, onStart) {
 		super(`zero-dispatcher`, lc, opts, (fastify) => {
 			fastify.get("/statz", (req, res) => handleStatzRequest(lc, config, req, res));
+			fastify.get("/plannerz", (req, res) => handlePlannerzRequest(lc, config, req, res));
 			fastify.get("/heapz", (req, res) => handleHeapzRequest(lc, config, req, res));
+			fastify.get("/profz", (req, res) => handleProfzRequest(lc, config, req, res, this.#getWorker));
+			fastify.get("/profrmz", (req, res) => handleProfrmzRequest(lc, config, req, res, this.#getWorker));
 			installWebSocketHandoff(lc, this.#handoff, fastify.server);
 		});
 		this.#getWorker = getWorker;