@rocicorp/zero 1.11.0-canary.4 → 1.11.0-canary.6

package/out/zero-cache/src/server/change-streamer.js

@@ -1,7 +1,7 @@
 import { assert } from "../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../shared/src/resolved-promises.js";
 import { must } from "../../../shared/src/must.js";
-import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
+import { broadcastWorker, childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { DatabaseInitError } from "../../../zqlite/src/db.js";
@@ -11,6 +11,7 @@ import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruptio
 import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { REPLICATOR_URL } from "./worker-urls.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { getServerContext } from "../config/server-context.js";
 import { deleteLiteDB } from "../db/delete-lite-db.js";
 import { warmupConnections } from "../db/warmup.js";
@@ -33,11 +34,12 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/change-streamer.ts
 var lc = new LogContext("info", {}, consoleLogSink);
 async function runWorker(parent, env, ...argv) {
+	installProfileHandler(parent, "change-streamer");
 	const config = getNormalizedZeroConfig({
 		env,
 		argv
 	});
-	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
+	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, pgChangeLogEnabled, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
 	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
 	lc = createLogContext(config, "change-streamer");
 	registerSQLiteCorruptionDiagnosticTarget({
@@ -49,7 +51,7 @@ async function runWorker(parent, env, ...argv) {
 	warmupConnections(lc, changeDB, "change").catch(() => {});
 	const shard = getShardConfig(config);
 	await initChangeStreamerSchema(lc, changeDB, shard);
-	let purgeLock = litestream.backupURL && litestream.executable ? await new PurgeLocker(lc, shard, changeDB).acquire() : null;
+	let purgeLock = pgChangeLogEnabled && litestream.backupURL && litestream.executable ? await new PurgeLocker(lc, shard, changeDB).acquire() : null;
 	const restoreOptions = {
 		litestream,
 		constraints: purgeLock ?? void 0
@@ -82,6 +84,7 @@ async function runWorker(parent, env, ...argv) {
 			backupURL: destinationBackupURL,
 			litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy"
 		} : null, purgeLock, autoReset ?? false, {
+			pgChangeLogEnabled,
 			backPressureLimitHeapProportion,
 			flowControlConsensusTimeoutProportion,
 			flowControlSlowSubscriberGracePeriodMs: flowControlSlowSubscriberGracePeriodSeconds > 0 ? flowControlSlowSubscriberGracePeriodSeconds * 1e3 : void 0,
@@ -104,7 +107,7 @@ async function runWorker(parent, env, ...argv) {
 				retentionMs: sqliteChangeLogRetentionMs,
 				batchRows: sqliteChangeLogPurgeBatchRows
 			} : void 0,
-			sqliteChangeLogCompare: sqliteChangeLogComparing ? {
+			sqliteChangeLogCompare: pgChangeLogEnabled && sqliteChangeLogComparing ? {
 				replicaFile: replica.file,
 				comparePercent: sqliteChangeLogComparePercent,
 				retentionMs: sqliteChangeLogRetentionMs,
@@ -133,14 +136,17 @@ async function runWorker(parent, env, ...argv) {
 	}
 	assert(changeStreamer, `resetting replica did not advance replicaVersion`);
 	const processes = new ProcessManager(lc, parent);
+	const profileSubWorkers = [];
 	if (backupURL) {
 		lc.info?.("setting up backup to", backupURL);
 		litestream.backupURL = backupURL;
 		const { promise: backupStarted, resolve } = resolver();
-		processes.addWorker(childWorker(REPLICATOR_URL, env, "backup"), "supporting", "backup-replicator").once("message", () => {
+		const backupReplicator = processes.addWorker(childWorker(REPLICATOR_URL, env, "backup"), "supporting", "backup-replicator").once("message", () => {
 			processes.addSubprocess(startReplicaBackupProcess(lc, litestream, replica.file), "supporting", "litestream");
 			resolve();
 		});
+		profileSubWorkers.push(backupReplicator);
+		backupReplicator.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
 		await backupStarted;
 	}
 	const backupMonitor = createBackupCleanupMonitor({
@@ -158,11 +164,14 @@ async function runWorker(parent, env, ...argv) {
 			lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
 		});
 	}
+	const getProfileWorker = profileSubWorkers.length > 0 ? () => Promise.resolve(broadcastWorker(profileSubWorkers)) : void 0;
 	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
 		port,
 		keepaliveTimeoutMs,
 		startupDelayMs,
-		readinessGate
+		readinessGate,
+		config,
+		getProfileWorker
 	}, parent, changeStreamer);
 	readinessGate.then(() => parent.send(["ready", { ready: true }]));
 	try {

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n                installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare: sqliteChangeLogComparing\n            ? {\n                replicaFile: replica.file,\n                comparePercent: sqliteChangeLogComparePercent,\n                retentionMs: sqliteChangeLogRetentionMs,\n                readBatchRows: sqliteChangeLogReadBatchRows,\n              }\n            : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  // When restoring from litestream, acquire a lock to prevent change-log\n  // purges. This ensures that (this) change-streamer will be able to resume\n  // from the backup.\n  let purgeLock =\n    pgChangeLogEnabled && litestream.backupURL && litestream.executable\n      ? await new PurgeLocker(lc, shard, changeDB).acquire()\n      : null;\n  const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  for (const first of [true, false]) {\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n      } =\n        upstream.type === 'pg'\n          ? await initializePostgresChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              {\n                ...initialSync,\n                replicationSlotFailover: upstream.pgReplicationSlotFailover,\n                installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n              },\n              context,\n              replicationLag.reportIntervalMs,\n              restoreOptions,\n              {backupV5: litestream.backupUsingV5},\n              purgeLock,\n              upstream.pgStreamInboundTimeoutMs,\n            )\n          : await initializeCustomChangeSource(\n              lc,\n              upstream.db,\n              shard,\n              replica.file,\n              context,\n              restoreOptions,\n            );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      break;\n    } catch (e) {\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        // Release the purge lock before retrying. This is safe because the\n        // purge lock exists to preserve change-log entries so the new\n        // change-streamer can resume from the backup replica's watermark.\n        // An AutoResetSignal means we cant resume from the backup replica\n        // (e.g. its replication slot is gone), so the change-log entries the lock\n        // was protecting are no longer needed. The retry performs a fresh\n        // initial sync with a new replication slot, independent of the old\n        // change-log. Releasing is also necessary to avoid a\n        // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n        // CREATE_REPLICATION_SLOT waits for all older transactions to\n        // finish, including this lock's open transaction.\n        await purgeLock?.release();\n        purgeLock = null;\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  const processes = new ProcessManager(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  if (backupURL) {\n    lc.info?.('setting up backup to', backupURL);\n    litestream.backupURL = backupURL;\n    const {promise: backupStarted, resolve} = resolver();\n\n    // Start a backup replicator and corresponding litestream backup process.\n    const backupReplicator = processes\n      .addWorker(\n        childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n        'supporting',\n        'backup-replicator',\n      )\n      // Wait for the replicator's first message (i.e. \"ready\") before starting\n      // litestream backup in order to avoid contending on the sqlite lock\n      // when the replicator first prepares the db file.\n      .once('message', () => {\n        processes.addSubprocess(\n          startReplicaBackupProcess(lc, litestream, replica.file),\n          'supporting',\n          'litestream',\n        );\n        resolve();\n      });\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n    await backupStarted;\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let readinessGate = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    readinessGate = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  const getProfileWorker =\n    profileSubWorkers.length > 0\n      ? () => Promise.resolve(broadcastWorker(profileSubWorkers))\n      : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      startupDelayMs,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  try {\n    await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n  } catch (err) {\n    processes.logErrorAndExit(err, 'change-streamer');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        await publishCriticalEvent(\n          lc,\n          replicationStatusError(lc, 'Initializing', e),\n        );\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAKxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,sBAAsB,WAAW,aAAa,WAAW,aACrD,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GAClC,6BAA6B,SAAS;EACxC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBACE,sBAAsB,2BAClB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIN,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,MAAM,oBAA8B,CAAC;CACrC,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,MAAM,mBAAmB,UACtB,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;EACA,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,mBACJ,kBAAkB,SAAS,UACjB,QAAQ,QAAQ,gBAAgB,iBAAiB,CAAC,IACxD,KAAA;CAEN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EACA;EACA;EACA;EACA;CACF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAwB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAoKf"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAML,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAwB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CA+Kf"}

package/out/zero-cache/src/server/main.js

@@ -35,10 +35,14 @@ async function runWorker(parent, env) {
 		"--cvr-max-conns-per-worker",
 		String(Math.floor(config.cvr.maxConns / numSyncers))
 	];
+	const allSubWorkers = [];
 	function loadWorker(moduleUrl, type, id, ...args) {
 		const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);
 		const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : "");
-		return processes.addWorker(worker, type, name);
+		const w = processes.addWorker(worker, type, name);
+		allSubWorkers.push(w);
+		w.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
+		return w;
 	}
 	const { taskID, litestream } = config;
 	const runChangeStreamer = runsChangeStreamer(config);
@@ -81,6 +85,9 @@ async function runWorker(parent, env) {
 	lc.info?.(`all workers ready (${startupDurationMs} ms)`);
 	recordStartupDurationMs(startupDurationMs);
 	parent.send(["ready", { ready: true }]);
+	parent.onMessageType("profile", (req) => {
+		for (const w of allSubWorkers) w.send(["profile", req]);
+	});
 	try {
 		await runUntilKilled(lc, parent, new WorkerDispatcher(lc, taskID, parent, syncers, mutator, changeStreamer));
 	} catch (err) {

package/out/zero-cache/src/server/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  type ReplicaFileMode,\n  subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    return processes.addWorker(worker, type, name);\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n      'message',\n      changeStreamerStarted,\n    );\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA0CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,OAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;CAC/C;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EACX,iBAAiB,WAAW,qBAAqB,YAAY,EAAE,KAC7D,WACA,qBACF;EAGA,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}
+{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  type ReplicaFileMode,\n  subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n      'message',\n      changeStreamerStarted,\n    );\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA4CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EACX,iBAAiB,WAAW,qBAAqB,YAAY,EAAE,KAC7D,WACA,qBACF;EAGA,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}

package/out/zero-cache/src/server/mutator.js

@@ -3,6 +3,7 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { Mutator } from "../workers/mutator.js";
@@ -10,6 +11,7 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/mutator.ts
 var lc = new LogContext("info", {}, consoleLogSink);
 function runWorker(parent, env, ...args) {
+	installProfileHandler(parent, "mutator");
 	const config = getNormalizedZeroConfig({
 		env,
 		argv: args.slice(1)

package/out/zero-cache/src/server/mutator.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n  lc = createLogContext(config, 'mutator');\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;AAeA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,cAAc,iBAAiB,QAAQ,WAAW,GAAG,KAAK,GAAG,WAAW,CAAC;CACzE,KAAK,iBAAiB,QAAQ,SAAS;CACvC,cAAc,IAAI,MAAM;CAGxB,OAAO,eAAe,IAAI,QAAQ,IAAI,QAAQ,CAAC;AACjD;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nfunction runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'mutator');\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n  lc = createLogContext(config, 'mutator');\n  initEventSink(lc, config);\n\n  // TODO: create `PusherFactory`\n  return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,SAAS;CACvC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,cAAc,iBAAiB,QAAQ,WAAW,GAAG,KAAK,GAAG,WAAW,CAAC;CACzE,KAAK,iBAAiB,QAAQ,SAAS;CACvC,cAAc,IAAI,MAAM;CAGxB,OAAO,eAAe,IAAI,QAAQ,IAAI,QAAQ,CAAC;AACjD;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/reaper.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"AASA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA+Bf"}
+{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"AASA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAY/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAgCf"}

package/out/zero-cache/src/server/reaper.js

@@ -5,6 +5,7 @@ import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { connectPgClient } from "../types/pg.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { startAnonymousTelemetry } from "./anonymous-otel-start.js";
@@ -16,6 +17,7 @@ import { LogContext, consoleLogSink } from "@rocicorp/logger";
 var MS_PER_HOUR = 1e3 * 60 * 60;
 var lc = new LogContext("info", {}, consoleLogSink);
 async function runWorker(parent, env, ...argv) {
+	installProfileHandler(parent, "reaper");
 	const config = getNormalizedZeroConfig({
 		env,
 		argv

package/out/zero-cache/src/server/reaper.js.map

@@ -1 +1 @@
-{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAmBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,IAC9B,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'reaper');\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAoBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,QAAQ;CACtC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,IAC9B,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/replicator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAsCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoHf"}
+{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAsCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAiB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAqHf"}

package/out/zero-cache/src/server/replicator.js

@@ -9,6 +9,7 @@ import { getOrCreateGauge } from "../observability/metrics.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { initEventSink } from "../observability/events.js";
 import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
 import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../services/litestream/metrics.js";
@@ -35,6 +36,7 @@ async function runWorker(parent, env, ...args) {
 	const mode = fileMode === "backup" ? "backup" : "serving";
 	const runningLocalChangeStreamer = config.changeStreamer.mode === "dedicated" && !config.changeStreamer.uri;
 	const workerName = `${mode}-replicator`;
+	installProfileHandler(parent, workerName);
 	startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);
 	lc = createLogContext(config, workerName);
 	const unregisterInitialCorruptionDiagnosticTarget = registerSQLiteCorruptionDiagnosticTarget({

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAwDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAKhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n  ThreadWriteWorkerClient,\n  type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  installProfileHandler(parent, workerName);\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  // Remote view-syncers restore from the replication-manager, which supplies\n  // the backup URL. A local change-streamer owns the canonical replica and\n  // must not infer backup configuration from bundled executable paths.\n  if (\n    fileMode === 'serving' &&\n    !runningLocalChangeStreamer &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {\n    file: dbPath,\n    walMode,\n    pageSize,\n  } = await setupReplica(lc, fileMode, config.replica);\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes. Enable write-path\n  // checkpoint backpressure only for the backup replicator on litestream v5,\n  // which owns the replica litestream backs up.\n  const pragmas = getPragmaConfig(fileMode);\n  const checkpoint: ForceCheckpointConfig | null =\n    mode === 'backup' &&\n    config.litestream.backupURL &&\n    config.litestream.backupUsingV5 &&\n    config.litestream.forceCheckpointThresholdMB\n      ? {\n          checkpointThresholdPages:\n            (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n            pageSize,\n          maxWalPages: !config.litestream.maxWalSizeMB\n            ? undefined\n            : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n        }\n      : null;\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAKhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CACN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAErE,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/runner/zero-dispatcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAGzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAarB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}
+{"version":3,"file":"zero-dispatcher.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAEpE,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,EAAC,MAAM,gCAAgC,CAAC;AAOzE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,qBAAa,cAAe,SAAQ,WAAW;;IAC7C,QAAQ,CAAC,EAAE,qBAAqB;IAIhC,YACE,MAAM,EAAE,oBAAoB,EAC5B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,IAAI,EAmBrB;IAED,UAAmB,QAAQ,SAE1B;CAYF"}

package/out/zero-cache/src/server/runner/zero-dispatcher.js

@@ -1,5 +1,6 @@
 import { handleHeapzRequest } from "../../services/heapz.js";
 import { HttpService } from "../../services/http-service.js";
+import { handleProfrmzRequest, handleProfzRequest } from "../../services/profz.js";
 import { handleStatzRequest } from "../../services/statz.js";
 import { installWebSocketHandoff } from "../../types/websocket-handoff.js";
 //#region ../zero-cache/src/server/runner/zero-dispatcher.ts
@@ -11,6 +12,8 @@ var ZeroDispatcher = class extends HttpService {
 		super(`zero-dispatcher`, lc, opts, (fastify) => {
 			fastify.get("/statz", (req, res) => handleStatzRequest(lc, config, req, res));
 			fastify.get("/heapz", (req, res) => handleHeapzRequest(lc, config, req, res));
+			fastify.get("/profz", (req, res) => handleProfzRequest(lc, config, req, res, this.#getWorker));
+			fastify.get("/profrmz", (req, res) => handleProfrmzRequest(lc, config, req, res, this.#getWorker));
 			installWebSocketHandoff(lc, this.#handoff, fastify.server);
 		});
 		this.#getWorker = getWorker;

package/out/zero-cache/src/server/runner/zero-dispatcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"zero-dispatcher.js","names":["#getWorker","#onStart","#handoff"],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n} from '../../types/websocket-handoff.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff = (\n    _req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n  ) => {\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n}\n"],"mappings":";;;;;AAYA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CAEA,YACE,QACA,IACA,MACA,WACA,SACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,wBAAwB,IAAI,KAAKE,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAKF,aAAa;EAClB,KAAKC,WAAW;CAClB;CAEA,WAA8B;EAC5B,KAAKA,WAAW;CAClB;CAEA,YACE,MACA,UACA,YACG;EACH,KAAUD,WAAW,EAAE,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;AACF"}
+{"version":3,"file":"zero-dispatcher.js","names":["#getWorker","#onStart","#handoff"],"sources":["../../../../../../zero-cache/src/server/runner/zero-dispatcher.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {handleHeapzRequest} from '../../services/heapz.ts';\nimport {HttpService, type Options} from '../../services/http-service.ts';\nimport {\n  handleProfrmzRequest,\n  handleProfzRequest,\n} from '../../services/profz.ts';\nimport {handleStatzRequest} from '../../services/statz.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport type {Worker} from '../../types/processes.ts';\nimport {\n  installWebSocketHandoff,\n  type HandoffSpec,\n} from '../../types/websocket-handoff.ts';\n\nexport class ZeroDispatcher extends HttpService {\n  readonly id = 'zero-dispatcher';\n  readonly #getWorker: () => Promise<Worker>;\n  readonly #onStart: (() => void) | undefined;\n\n  constructor(\n    config: NormalizedZeroConfig,\n    lc: LogContext,\n    opts: Options,\n    getWorker: () => Promise<Worker>,\n    onStart?: () => void,\n  ) {\n    super(`zero-dispatcher`, lc, opts, fastify => {\n      fastify.get('/statz', (req, res) =>\n        handleStatzRequest(lc, config, req, res),\n      );\n      fastify.get('/heapz', (req, res) =>\n        handleHeapzRequest(lc, config, req, res),\n      );\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(lc, config, req, res, this.#getWorker),\n      );\n      fastify.get('/profrmz', (req, res) =>\n        handleProfrmzRequest(lc, config, req, res, this.#getWorker),\n      );\n      installWebSocketHandoff(lc, this.#handoff, fastify.server);\n    });\n    this.#getWorker = getWorker;\n    this.#onStart = onStart;\n  }\n\n  protected override _onStart() {\n    this.#onStart?.();\n  }\n\n  readonly #handoff = (\n    _req: IncomingMessageSubset,\n    dispatch: (h: HandoffSpec<string>) => void,\n    onError: (error: unknown) => void,\n  ) => {\n    void this.#getWorker().then(\n      sender => dispatch({payload: 'unused', sender}),\n      onError,\n    );\n  };\n}\n"],"mappings":";;;;;;AAgBA,IAAa,iBAAb,cAAoC,YAAY;CAC9C,KAAc;CACd;CACA;CAEA,YACE,QACA,IACA,MACA,WACA,SACA;EACA,MAAM,mBAAmB,IAAI,OAAM,YAAW;GAC5C,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,GAAG,CACzC;GACA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBAAmB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC1D;GACA,QAAQ,IAAI,aAAa,KAAK,QAC5B,qBAAqB,IAAI,QAAQ,KAAK,KAAK,KAAKA,UAAU,CAC5D;GACA,wBAAwB,IAAI,KAAKE,UAAU,QAAQ,MAAM;EAC3D,CAAC;EACD,KAAKF,aAAa;EAClB,KAAKC,WAAW;CAClB;CAEA,WAA8B;EAC5B,KAAKA,WAAW;CAClB;CAEA,YACE,MACA,UACA,YACG;EACH,KAAUD,WAAW,EAAE,MACrB,WAAU,SAAS;GAAC,SAAS;GAAU;EAAM,CAAC,GAC9C,OACF;CACF;AACF"}

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAiCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAiNf"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAiCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAqC/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAkNf"}

package/out/zero-cache/src/server/syncer.js

@@ -12,6 +12,7 @@ import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
+import { installProfileHandler } from "../types/profiler.js";
 import { warmupConnections } from "../db/warmup.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
@@ -53,6 +54,7 @@ async function runWorker(parent, env, ...args) {
 	assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);
 	const fileMode = parse(args[0], replicaFileModeSchema);
 	const workerIndex = Number(args[1]);
+	installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);
 	const config = getNormalizedZeroConfig({
 		env,
 		argv: args.slice(2)

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAGF,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,KAAK,GAC1C,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n  type ConnectionContextManager,\n  ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    apiKey: queryConfig.apiKey,\n    allowedClientHeaders: queryConfig.allowedClientHeaders,\n    allowedRequestHeaders: queryConfig.allowedRequestHeaders,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n  const workerIndex = Number(args[1]);\n  installProfileHandler(parent, `syncer-${workerIndex}`, workerIndex);\n  const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n  startOtelAuto(\n    createLogContext(config, 'syncer', workerIndex, false),\n    'syncer',\n    workerIndex,\n  );\n  lc = createLogContext(config, 'syncer', workerIndex);\n  initEventSink(lc, config);\n\n  const {cvr, upstream, enableCrudMutations} = config;\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'syncer replica',\n      dbPath: replicaFile,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n    max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n  });\n\n  const upstreamDB =\n    enableCrudMutations && upstream.type === 'pg'\n      ? await connectPgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n          max: must(\n            upstream.maxConnsPerWorker,\n            'upstream.maxConnsPerWorker must be set',\n          ),\n        })\n      : undefined;\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n  const customQueryConfig = getCustomQueryConfig(config);\n  const pushConfig =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : {\n          ...config.push,\n          ...config.mutate,\n          url: must(\n            config.push.url ?? config.mutate.url,\n            'No push or mutate URL configured',\n          ),\n        };\n\n  /** @deprecated used in JWT validation */\n  let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n  const tokenOptions = tokenConfigOptions(config.auth ?? {});\n  if (tokenOptions.length === 1) {\n    validateLegacyJWT = async (token, {userID}) => {\n      if (!userID) {\n        throw new ProtocolErrorWithLevel(\n          {\n            kind: 'Unauthorized',\n            message: 'UserID is required for JWT validation.',\n            origin: 'zeroCache',\n          },\n          'warn',\n        );\n      }\n\n      const decoded = await verifyToken(config.auth, token, {\n        subject: userID,\n        ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n        ...(config.auth?.audience && {\n          audience: config.auth.audience,\n        }),\n      });\n      return {\n        type: 'jwt',\n        raw: token,\n        decoded,\n      };\n    };\n  }\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('taskID', config.taskID)\n      .withContext('component', 'view-syncer')\n      .withContext('appID', shard.appID)\n      .withContext('shardNum', shard.shardNum)\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n\n    const customQueryTransformer =\n      customQueryConfig && new CustomQueryTransformer(logger, shard);\n    const connContextManager = new ConnectionContextManagerImpl(\n      logger,\n      config.auth.revalidateIntervalSeconds,\n      config.auth.retransformIntervalSeconds,\n      customQueryConfig,\n      pushConfig,\n      validateLegacyJWT,\n    );\n\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n        config,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      connContextManager,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = upstreamDB\n    ? (id: string) =>\n        new MutagenService(\n          lc\n            .withContext('component', 'mutagen')\n            .withContext('clientGroupID', id),\n          shard,\n          id,\n          upstreamDB,\n          config,\n          writeAuthzStorage,\n        )\n    : undefined;\n\n  const pusherFactory =\n    pushConfig === undefined\n      ? undefined\n      : (id: string, connContextManager: ConnectionContextManager) =>\n          new PusherService(\n            config,\n            lc.withContext('clientGroupID', id),\n            id,\n            connContextManager,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n    validateLegacyJWT,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,SAAS,WAAW;CAClB,OAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;CAE9D,IAAI,CAAC,aAAa,KAChB;CAGF,OAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,uBAAuB,YAAY;EACnC,gBAAgB,YAAY,kBAAkB;CAChD;AACF;AAGA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,UAAU,GAAG,4CAA4C;CACrE,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CACvD,MAAM,cAAc,OAAO,KAAK,EAAE;CAClC,sBAAsB,QAAQ,UAAU,eAAe,WAAW;CAClE,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CAEjE,cACE,iBAAiB,QAAQ,UAAU,aAAa,KAAK,GACrD,UACA,WACF;CACA,KAAK,iBAAiB,QAAQ,UAAU,WAAW;CACnD,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;CACjE,yCACE;EACE,WAAW;EACX,QAAQ;CACV,GACA,OAAO,sBACT;CACA,GAAG,QAAQ,0BAA0B,aAAa;CAElD,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EACxE,KAAK,KAAK,IAAI,mBAAmB,mCAAmC,EACtE,CAAC;CAED,MAAM,aACJ,uBAAuB,SAAS,SAAS,OACrC,MAAM,gBAAgB,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACpE,KAAK,KACH,SAAS,mBACT,wCACF,EACF,CAAC,IACD,KAAA;CAEN,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,KAAK,GAClC,aAAa,kBAAkB,IAAI,YAAY,UAAU,IAAI,WAC/D,CAAC;CAED,MAAM,SAAS,OAAO,mBAAmB,OAAO;CAChD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,WAAW,GAAG,CACjD;CACA,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,WAAW,GAAG,CAC7C;CAEA,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,oBAAoB,qBAAqB,MAAM;CACrD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,kCACF;CACF;;CAGN,IAAI,oBAAmD,KAAA;CAGvD,IADqB,mBAAmB,OAAO,QAAQ,CAAC,CACpD,EAAa,WAAW,GAC1B,oBAAoB,OAAO,OAAO,EAAC,aAAY;EAC7C,IAAI,CAAC,QACH,MAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;EACV,GACA,MACF;EAUF,OAAO;GACL,MAAM;GACN,KAAK;GACL,SAAA,MAVoB,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,OAAM;IACtD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,SACxB;GACF,CAAC;EAKD;CACF;CAGF,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,UAAU,OAAO,MAAM,EACnC,YAAY,aAAa,aAAa,EACtC,YAAY,SAAS,MAAM,KAAK,EAChC,YAAY,YAAY,MAAM,QAAQ,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,SAAS,CAAC;EAErC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,KAAK;EAC/D,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,iBACF;EAEA,GAAG,QACD,gDAAgD,OAAO,oBACzD;EAEA,MAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;EAEtE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,CACF;EACA,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;EAElE,OAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,KAAK,GAC1C,OACA,gBAAgB,yBAAyB,EAAE,GAC3C,IACA,yBAEE,oBAAoB,IAChB,oCACA,wBACN,OAAO,oBACP,MACF,GACA,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,aACF;CACF;CA2BA,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,SAAS,EAClC,YAAY,iBAAiB,EAAE,GAClC,OACA,IACA,YACA,QACA,iBACF,IACF,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,EAAE,GAClC,IACA,kBACF,GAQN,QACA,iBACF;CAEA,wBAAwB,IAAI,MAAM;CAElC,SAAc,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CAE9D,OAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/change-source/pg/initial-sync.js

@@ -12,6 +12,7 @@ import { getOrCreateCounter, getOrCreateHistogram } from "../../../observability
 import { initReplicationState } from "../../replicator/schema/replication-state.js";
 import { id } from "../../../types/sql.js";
 import { connectPgClient } from "../../../types/pg.js";
+import { CpuProfiler } from "../../../types/profiler.js";
 import { toStateVersionString } from "./lsn.js";
 import { READONLY } from "../../../db/mode-enum.js";
 import { runTx } from "../../../db/run-transaction.js";
@@ -24,7 +25,6 @@ import { BinaryCopyParser, hasBinaryDecoder, makeBinaryDecoder, textCastDecoder
 import { TsvParser } from "../../../db/pg-copy.js";
 import { getTypeParsers } from "../../../db/pg-type-parser.js";
 import { TransactionPool, importSnapshot } from "../../../db/transaction-pool.js";
-import { CpuProfiler } from "../../../types/profiler.js";
 import { createReplicaAndSlot } from "./replication-slots.js";
 import { ensureShardSchema } from "./schema/init.js";
 import { platform, tmpdir } from "node:os";