@rocicorp/zero 1.12.0-canary.0 → 1.12.0-canary.10

package/out/zero-cache/src/server/change-streamer.js

@@ -1,13 +1,14 @@
 import { assert } from "../../../shared/src/asserts.js";
 import { promiseVoid } from "../../../shared/src/resolved-promises.js";
+import { resolver } from "../../../shared/src/resolver.js";
 import { must } from "../../../shared/src/must.js";
 import { broadcastWorker, childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { DatabaseInitError } from "../../../zqlite/src/db.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
 import { connectPgClient } from "../types/pg.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { REPLICATOR_URL } from "./worker-urls.js";
@@ -31,8 +32,8 @@ import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observ
 import { PurgeLocker } from "../services/change-streamer/storer.js";
 import { initializeStreamer } from "../services/change-streamer/change-streamer-service.js";
 import { initChangeStreamerSchema } from "../services/change-streamer/schema/init.js";
+import { ServerStartMetrics } from "../services/change-streamer/server-start-metrics.js";
 import { startOtelAuto } from "./otel-start.js";
-import { resolver } from "@rocicorp/resolver";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/change-streamer.ts
 var lc = new LogContext("info", {}, consoleLogSink);
@@ -52,12 +53,11 @@ async function runWorker(parent, env, ...argv) {
 	}, sqliteCorruptionChecks);
 	initEventSink(lc, config);
 	if (upstream.type === "pg") warnIfPostgresUnpatched(lc);
-	const changeDB = await connectPgClient(lc, change.db, "change-streamer-init", { max: 5 }, { sendStringAsJson: true });
-	warmupConnections(lc, changeDB, "change").catch(() => {});
 	const shard = getShardConfig(config);
-	await initChangeStreamerSchema(lc, changeDB, shard);
+	const changeDB = pgChangeLogEnabled ? await connectChangeDB(lc, change.db, shard) : null;
 	let purgeLock = null;
 	let changeStreamer;
+	let backupMonitor;
 	let backupURL;
 	const context = getServerContext(config);
 	const sqliteChangeLogEnabled = sqliteChangeLogMode !== "off";
@@ -73,13 +73,14 @@ async function runWorker(parent, env, ...argv) {
 			unit: "By"
 		}).addCallback(async (o) => o.observe(await sqliteFileBytes(lc, changeLogFile)));
 	}
+	const startMetrics = new ServerStartMetrics();
 	let waitForFirstBackupBeforeServing = false;
 	let newBackupLineage = false;
 	for (const first of [true, false]) {
 		const cleanup = new InitCleanup(lc);
 		const restoreOptions = {
 			litestream,
-			acquirePurgeLock: pgChangeLogEnabled && litestream.backupURL && litestream.executable ? async (slotWatermark) => {
+			acquirePurgeLock: changeDB && litestream.backupURL && litestream.executable ? async (slotWatermark) => {
 				const lock = await new PurgeLocker(lc, shard, changeDB).acquire(slotWatermark);
 				purgeLock = lock === "behind-slot" ? null : lock;
 				if (purgeLock) {
@@ -95,7 +96,7 @@ async function runWorker(parent, env, ...argv) {
 		};
 		try {
 			const { pgReplicationEpoch: epoch, pgReplicationSlotPerReplica: slotPerReplica, pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs } = upstream;
-			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, pgChangeLogBehindSlot } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
+			const { changeSource, subscriptionState, destinationBackupURL, replicaID, waitForBackupBeforeServing, newBackupLineage: initNewBackupLineage, pgChangeLogBehindSlot, startTimings } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, {
 				...initialSync,
 				replicationSlotFailover: upstream.pgReplicationSlotFailover,
 				installPartialIndexTriggers: upstream.pgPartialIndexTriggers
@@ -107,12 +108,22 @@ async function runWorker(parent, env, ...argv) {
 				pgChangeLog: pgChangeLogEnabled
 			}, upstream.pgStreamInboundTimeoutMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context, restoreOptions);
 			const replicationStatusPublisher = ReplicationStatusPublisher.forReplicaFile(replica.file);
+			backupMonitor = createBackupCleanupMonitor({
+				lc,
+				config: destinationBackupURL ? {
+					...config,
+					litestream: {
+						...config.litestream,
+						backupURLOverride: destinationBackupURL
+					}
+				} : config,
+				replicaFile: replica.file
+			});
 			changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, replicationStatusPublisher, subscriptionState, destinationBackupURL ? {
 				backupURL: destinationBackupURL,
 				litestreamVersion: litestream.backupUsingV5 ? "v5" : "legacy",
 				replicaFile: replica.file
 			} : null, purgeLock, autoReset ?? false, {
-				pgChangeLogEnabled,
 				pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,
 				backPressureLimitHeapProportion,
 				flowControlConsensusTimeoutProportion,
@@ -148,14 +159,21 @@ async function runWorker(parent, env, ...argv) {
 					readPercent: sqliteChangeLogReadPercent,
 					coldReadPercent: sqliteChangeLogColdReadPercent,
 					retentionMs: sqliteChangeLogRetentionMs
-				} : void 0
+				} : void 0,
+				backupMonitor
 			}, setTimeout);
 			backupURL = destinationBackupURL;
 			waitForFirstBackupBeforeServing = waitForBackupBeforeServing;
+			startMetrics.setKind(startTimings?.kind);
+			startMetrics.record("slot_create", startTimings?.slotCreateMs);
+			startMetrics.record("fork_wait", startTimings?.forkWaitMs);
+			startMetrics.record("restore", startTimings?.restoreMs);
 			newBackupLineage = initNewBackupLineage;
 			break;
 		} catch (e) {
 			await cleanup.release();
+			backupMonitor?.stop();
+			backupMonitor = void 0;
 			if (first && e instanceof AutoResetSignal) {
 				lc.warn?.(`resetting replica ${replica.file}`, e);
 				deleteLiteDB(replica.file);
@@ -167,9 +185,14 @@ async function runWorker(parent, env, ...argv) {
 		}
 	}
 	assert(changeStreamer, `resetting replica did not advance replicaVersion`);
+	assert(backupMonitor, `backup monitor is created with the change-streamer`);
 	const processes = new SupportingProcesses(lc, parent);
 	const profileSubWorkers = [];
 	const { promise: replicatorReady, resolve } = resolver();
+	if (backupURL) {
+		const catchUpStart = performance.now();
+		replicatorReady.then(() => startMetrics.record("catch_up", performance.now() - catchUpStart));
+	}
 	if (!backupURL) resolve();
 	else {
 		lc.info?.("setting up backup to", backupURL);
@@ -182,12 +205,6 @@ async function runWorker(parent, env, ...argv) {
 		profileSubWorkers.push(backupReplicator);
 		backupReplicator.onMessageType("profileResponse", (res) => parent.send(["profileResponse", res]));
 	}
-	const backupMonitor = createBackupCleanupMonitor({
-		lc,
-		config,
-		replicaFile: replica.file,
-		changeStreamer
-	});
 	let backupReady = promiseVoid;
 	if (waitForFirstBackupBeforeServing) {
 		const start = performance.now();
@@ -195,6 +212,7 @@ async function runWorker(parent, env, ...argv) {
 		backupReady = backupMonitor.firstBackupReceived().then(() => {
 			const elapsed = performance.now() - start;
 			lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
+			startMetrics.record("backup_confirm", elapsed);
 		});
 	}
 	const readinessGate = pgChangeLogEnabled ? backupReady : Promise.all([backupReady, replicatorReady]);
@@ -207,10 +225,25 @@ async function runWorker(parent, env, ...argv) {
 		config,
 		getProfileWorker: profileWorker ? () => Promise.resolve(profileWorker) : void 0
 	}, parent, changeStreamer);
-	readinessGate.then(() => parent.send(["ready", { ready: true }]));
+	readinessGate.then(() => {
+		startMetrics.ready();
+		parent.send(["ready", { ready: true }]);
+	});
 	running = true;
 	await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor, processes);
 }
+/**
+* Connects the startup-time client used for change-streamer initialization
+* and handoff / takeover, and ensures that the change DB schema is
+* initialized / up-to-date. Steady-state clients are managed by the
+* change-streamer (Storer) implementation.
+*/
+async function connectChangeDB(lc, connectionURI, shard) {
+	const changeDB = await connectPgClient(lc, connectionURI, "change-streamer-init", { max: 5 }, { sendStringAsJson: true });
+	warmupConnections(lc, changeDB, "change").catch(() => {});
+	await initChangeStreamerSchema(lc, changeDB, shard);
+	return changeDB;
+}
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)).catch(async (e) => {
 	if (!running) await publishCriticalEvent(lc, replicationStatusError(lc, "Initializing", e));
 	throw e;

package/out/zero-cache/src/server/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  // Startup-time client used for for change-streamer initialization\n  // and handoff / takeover. Steady-state clients are managed by the\n  // change-streamer (Storer) implementation.\n  const changeDB = await connectPgClient(\n    lc,\n    change.db,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n  const shard = getShardConfig(config);\n\n  // Ensure the change DB schema is initialized/up-to-date.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      pgChangeLogEnabled && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogEnabled,\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  const backupMonitor = createBackupCleanupMonitor({\n    lc,\n    config,\n    replicaFile: replica.file,\n    changeStreamer,\n  });\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAM5B,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAElD,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,sBAAsB,WAAW,aAAa,WAAW,aACrD,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,0BACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE;IACA,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;GACR,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GACtB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CAIzE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,EACC,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,EAAE,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {InitCleanup} from '../services/change-source/common/init-cleanup.ts';\nimport type {PgChangeLogPurgeLocker} from '../services/change-source/common/replica-restore.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source-init.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport type {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {DEFAULT_READINESS_GRACE_PERIOD_MS} from '../services/change-streamer/restore-admission.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {ServerStartMetrics} from '../services/change-streamer/server-start-metrics.ts';\nimport {\n  PurgeLocker,\n  type PurgeLock,\n} from '../services/change-streamer/storer.ts';\nimport {\n  exitAfter,\n  runUntilKilled,\n  SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {deleteLitestreamMetaDir} from '../services/litestream/commands.ts';\nimport {\n  changeLogFileName,\n  deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n  replicationStatusError,\n  ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {warnIfPostgresUnpatched} from '../types/pg-patch.ts';\nimport {connectPgClient, type PostgresDB} from '../types/pg.ts';\nimport {\n  broadcastWorker,\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig, type ShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n// Errors thrown once the services are running are reported by the services\n// themselves (e.g. as 'Replicating' status events), and are thus not\n// published as 'Initializing' errors.\nlet running = false;\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'change-streamer');\n  const config = getNormalizedZeroConfig({env, argv});\n  const {\n    taskID,\n    changeStreamer: {\n      port,\n      address,\n      protocol,\n      startupDelayMs,\n      backPressureLimitHeapProportion,\n      flowControlConsensusTimeoutProportion,\n      flowControlSlowSubscriberGracePeriodSeconds,\n      maxConcurrentSwitchoverRestores,\n      pgChangeLogEnabled,\n      sqliteChangeLogMode,\n      sqliteChangeLogReadPercent,\n      sqliteChangeLogColdReadPercent,\n      sqliteChangeLogComparePercent,\n      sqliteChangeLogRetentionMs,\n      sqliteChangeLogReadBatchRows,\n      sqliteChangeLogPurgeBatchRows,\n      sqliteChangeLogBarrierTimeoutMs,\n    },\n    autoReset,\n    replicationLag,\n    litestream,\n    upstream,\n    change,\n    replica,\n    initialSync,\n    keepaliveTimeoutMs,\n    sqliteCorruptionChecks,\n  } = config;\n\n  startOtelAuto(\n    createLogContext(config, 'change-streamer', 0, false),\n    'change-streamer',\n    0,\n  );\n  lc = createLogContext(config, 'change-streamer');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'change-streamer replica',\n      dbPath: replica.file,\n    },\n    sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n  if (upstream.type === 'pg') {\n    warnIfPostgresUnpatched(lc);\n  }\n\n  const shard = getShardConfig(config);\n\n  // The change DB is only used by the PG change log. When it is disabled\n  // (RMv2), no connection to the change DB is made.\n  const changeDB = pgChangeLogEnabled\n    ? await connectChangeDB(lc, change.db, shard)\n    : null;\n\n  let purgeLock = null as PurgeLock | null;\n  let changeStreamer: ChangeStreamerService | undefined;\n  let backupMonitor: BackupMonitor | undefined;\n  let backupURL: string | undefined;\n\n  const context = getServerContext(config);\n  const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n  // The modes are cumulative, so `serve` implies `compare`.\n  const sqliteChangeLogComparing =\n    sqliteChangeLogMode === 'compare' || sqliteChangeLogMode === 'serve';\n  if (sqliteChangeLogEnabled) {\n    const changeLogFile = changeLogFileName(replica.file);\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: 'change-streamer change-log',\n        dbPath: changeLogFile,\n      },\n      sqliteCorruptionChecks,\n    );\n    // The log's bytes are accounted for in the process that writes them. This\n    // is local disk only — the log is excluded from the litestream backup — and\n    // it is a whole-database total, because a table that left the replica left\n    // through its wal: the replica's footprint is what shrank and this is where\n    // it went.\n    getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n      description:\n        `The SQLite change log's total on-disk footprint: its main db file ` +\n        `plus its wal sidecars.`,\n      unit: 'By',\n    }).addCallback(async o =>\n      o.observe(await sqliteFileBytes(lc, changeLogFile)),\n    );\n  }\n\n  const startMetrics = new ServerStartMetrics();\n  let waitForFirstBackupBeforeServing = false;\n  let newBackupLineage = false;\n  for (const first of [true, false]) {\n    // Resources acquired by an initialization attempt (e.g. a claimed\n    // replication slot, the purge lock) to be released if the attempt fails.\n    const cleanup = new InitCleanup(lc);\n    // When restoring from litestream, a lock is acquired to prevent change-log\n    // purges. This ensures that (this) change-streamer will be able to resume\n    // from the backup. The lock is acquired by the change source initialization\n    // just before restoring (i.e. after a replication slot is created, if\n    // applicable), and released once this change-streamer takes over the\n    // change-log.\n    const acquirePurgeLock: PgChangeLogPurgeLocker | undefined =\n      changeDB && litestream.backupURL && litestream.executable\n        ? async (slotWatermark?: string) => {\n            const lock = await new PurgeLocker(lc, shard, changeDB).acquire(\n              slotWatermark,\n            );\n            purgeLock = lock === 'behind-slot' ? null : lock;\n            if (purgeLock) {\n              const acquired = purgeLock;\n              cleanup.onFailure('purge lock', () => {\n                purgeLock = null;\n                return acquired.release();\n              });\n            }\n            return lock;\n          }\n        : undefined;\n\n    const restoreOptions = {litestream, acquirePurgeLock, cleanup};\n    try {\n      // Note: This performs initial sync of the replica if necessary.\n      const {\n        pgReplicationEpoch: epoch,\n        pgReplicationSlotPerReplica: slotPerReplica,\n        pgResumeOrphanedSlotGracePeriodMs: inactiveReplicaGracePeriodMs,\n      } = upstream;\n      const {\n        changeSource,\n        subscriptionState,\n        destinationBackupURL,\n        replicaID,\n        waitForBackupBeforeServing,\n        newBackupLineage: initNewBackupLineage,\n        pgChangeLogBehindSlot,\n        startTimings,\n      } = upstream.type === 'pg'\n        ? await initializePostgresChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            {\n              ...initialSync,\n              replicationSlotFailover: upstream.pgReplicationSlotFailover,\n              installPartialIndexTriggers: upstream.pgPartialIndexTriggers,\n            },\n            context,\n            replicationLag.reportIntervalMs,\n            restoreOptions,\n            {\n              epoch,\n              slotPerReplica,\n              inactiveReplicaGracePeriodMs,\n              backupV5: litestream.backupUsingV5,\n              pgChangeLog: pgChangeLogEnabled,\n            },\n            upstream.pgStreamInboundTimeoutMs,\n          )\n        : await initializeCustomChangeSource(\n            lc,\n            upstream.db,\n            shard,\n            replica.file,\n            context,\n            restoreOptions,\n          );\n\n      const replicationStatusPublisher =\n        ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n      // Created before the change-streamer, which tracks its watermarks (and\n      // requests prompt backups from it) from construction. It is run (and\n      // stopped) as a service below. It monitors the backup-replicator's\n      // destination (see `backupURLOverride` below).\n      backupMonitor = createBackupCleanupMonitor({\n        lc,\n        config: destinationBackupURL\n          ? {\n              ...config,\n              litestream: {\n                ...config.litestream,\n                backupURLOverride: destinationBackupURL,\n              },\n            }\n          : config,\n        replicaFile: replica.file,\n      });\n\n      changeStreamer = await initializeStreamer(\n        lc,\n        shard,\n        taskID,\n        address,\n        protocol,\n        changeDB,\n        changeSource,\n        replicationStatusPublisher,\n        subscriptionState,\n        destinationBackupURL\n          ? {\n              backupURL: destinationBackupURL,\n              litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n              replicaFile: replica.file,\n            }\n          : null,\n        purgeLock,\n        autoReset ?? false,\n        {\n          pgChangeLogBehindSlot: pgChangeLogBehindSlot ?? false,\n          backPressureLimitHeapProportion,\n          flowControlConsensusTimeoutProportion,\n          flowControlSlowSubscriberGracePeriodMs:\n            flowControlSlowSubscriberGracePeriodSeconds > 0\n              ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n              : undefined,\n          maxConcurrentSwitchoverRestores,\n          // When lag reporting is enabled, a restored view-syncer waits for\n          // the next lag report (sent at an interval with ~1 second of\n          // jitter) before becoming ready.\n          switchoverRestoreReadinessGracePeriodMs:\n            (replicationLag.reportIntervalMs > 0\n              ? replicationLag.reportIntervalMs + 1000\n              : 0) + DEFAULT_READINESS_GRACE_PERIOD_MS,\n          statementTimeoutMs: change.statementTimeoutMs,\n          changeLogBatchSize: change.logBatchSize,\n          sqliteCatchup: {\n            changeLogFile: changeLogFileName(replica.file),\n            readBatchRows: sqliteChangeLogReadBatchRows,\n            barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n          },\n          // The presence of these options is the writer's gate. This process\n          // performs the restore and the initial sync, so the replica's\n          // identity is in hand at the moment the log is opened.\n          sqliteChangeLogWriter: sqliteChangeLogEnabled\n            ? {\n                replicaFile: replica.file,\n                identity: {\n                  // RMv2 supplies the epoch; until then the generation and the\n                  // replica ID are the whole of the identity.\n                  epoch: null,\n                  generation: subscriptionState.replicaVersion,\n                  replicaID,\n                },\n              }\n            : undefined,\n          // The purge scheduler shares the writer's gate -- mode, not the\n          // read path -- because `write` mode, with no read selector at all,\n          // is the configuration it ships in.\n          sqliteChangeLogPurge: sqliteChangeLogEnabled\n            ? {\n                retentionMs: sqliteChangeLogRetentionMs,\n                batchRows: sqliteChangeLogPurgeBatchRows,\n              }\n            : undefined,\n          // Compare mode runs both advisory checks. Postgres remains authoritative.\n          sqliteChangeLogCompare:\n            pgChangeLogEnabled && sqliteChangeLogComparing\n              ? {\n                  replicaFile: replica.file,\n                  comparePercent: sqliteChangeLogComparePercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                  readBatchRows: sqliteChangeLogReadBatchRows,\n                }\n              : undefined,\n          // Slice 11 lands dark by default: serve mode constructs the stable\n          // router, while readPercent=0 keeps every catchup on PG and emits\n          // eligibility metrics before any canary traffic is enabled.\n          sqliteChangeLogServe:\n            sqliteChangeLogMode === 'serve'\n              ? {\n                  readPercent: sqliteChangeLogReadPercent,\n                  coldReadPercent: sqliteChangeLogColdReadPercent,\n                  retentionMs: sqliteChangeLogRetentionMs,\n                }\n              : undefined,\n          backupMonitor,\n        },\n        setTimeout,\n      );\n      backupURL = destinationBackupURL;\n      waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n      startMetrics.setKind(startTimings?.kind);\n      startMetrics.record('slot_create', startTimings?.slotCreateMs);\n      startMetrics.record('fork_wait', startTimings?.forkWaitMs);\n      startMetrics.record('restore', startTimings?.restoreMs);\n      newBackupLineage = initNewBackupLineage;\n      break;\n    } catch (e) {\n      // Release the resources acquired by the failed attempt, e.g.:\n      // * The claimed replication slot, which would otherwise remain active\n      //   indefinitely, preventing a retry from resuming (or forking) it.\n      // * The purge lock. This is safe because the purge lock exists to\n      //   preserve change-log entries so the new change-streamer can resume\n      //   from the backup replica's watermark. An AutoResetSignal means we\n      //   can't resume from the backup replica (e.g. its replication slot is\n      //   gone), so the change-log entries the lock was protecting are no\n      //   longer needed. The retry performs a fresh initial sync with a new\n      //   replication slot, independent of the old change-log. Releasing is\n      //   also necessary to avoid a self-deadlock when CHANGE_DB ==\n      //   UPSTREAM_DB: CREATE_REPLICATION_SLOT waits for all older\n      //   transactions to finish, including this lock's open transaction.\n      await cleanup.release();\n      // Not yet started (the change-streamer starts it), so nothing to wait on.\n      void backupMonitor?.stop();\n      backupMonitor = undefined;\n      if (first && e instanceof AutoResetSignal) {\n        lc.warn?.(`resetting replica ${replica.file}`, e);\n        // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n        //       a semantic wipe instead of a file delete.\n        deleteLiteDB(replica.file);\n        // The change log carries the identity of the replica it was written\n        // beside, and the retry performs a fresh initial sync with a new\n        // replicaVersion. Reconciliation would catch that on its own (as\n        // 'identity-mismatch') but only after the writer opened a file that is\n        // known here to be garbage.\n        deleteChangeLogDB(replica.file);\n        continue; // execute again with a fresh initial-sync\n      }\n      if (e instanceof DatabaseInitError) {\n        throw new Error(\n          `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n          {cause: e},\n        );\n      }\n      throw e;\n    }\n  }\n  // impossible: upstream must have advanced in order for replication to be stuck.\n  assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n  assert(backupMonitor, `backup monitor is created with the change-streamer`);\n\n  // The backup-replicator is stopped with the other services, i.e. after the\n  // ChangeStreamerHttpServer has drained its subscribers.\n  const processes = new SupportingProcesses(lc, parent);\n  const profileSubWorkers: Worker[] = [];\n  const {promise: replicatorReady, resolve} = resolver();\n  if (backupURL) {\n    const catchUpStart = performance.now();\n    void replicatorReady.then(() =>\n      startMetrics.record('catch_up', performance.now() - catchUpStart),\n    );\n  }\n  if (!backupURL) {\n    resolve(); // No backup replicator to wait for, so resolve immediately.\n  } else {\n    lc.info?.('setting up backup to', backupURL);\n    config.litestream.backupURLOverride = backupURL;\n    if (newBackupLineage) {\n      deleteLitestreamMetaDir(replica.file);\n    }\n    // Start a backup replicator, which starts up the corresponding\n    // litestream backup process.\n    const backupReplicator = processes\n      .add(\n        childWorker(\n          REPLICATOR_URL,\n          {...env, ZERO_LITESTREAM_BACKUP_URL_OVERRIDE: backupURL},\n          'backup' satisfies ReplicaFileMode,\n        ),\n        'backup-replicator',\n      )\n      .onceMessageType('ready', () => resolve());\n    profileSubWorkers.push(backupReplicator);\n    // Relay profileResponse messages from backup-replicator up to parent\n    backupReplicator.onMessageType<ProfileResponseMessage>(\n      'profileResponse',\n      res => parent.send(['profileResponse', res]),\n    );\n  }\n\n  let backupReady = promiseVoid;\n  if (waitForFirstBackupBeforeServing) {\n    const start = performance.now();\n    lc.info?.(`awaiting initial backup ...`);\n\n    backupReady = backupMonitor.firstBackupReceived().then(() => {\n      const elapsed = performance.now() - start;\n      lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n      startMetrics.record('backup_confirm', elapsed);\n    });\n  }\n\n  // In RMv2, readiness additionally waits for the backup-replicator to catch\n  // up to the replication stream. This is not done in RMv1 (i.e. when the\n  // PG change-log is enabled), in which the change-streamer itself is only\n  // started after the readinessGate (plus a startup delay), and thus the\n  // backup-replicator cannot catch up before then.\n  const readinessGate = pgChangeLogEnabled\n    ? backupReady\n    : Promise.all([backupReady, replicatorReady]);\n\n  // Create the broadcast facade once: each broadcastWorker() adds permanent\n  // 'message' forwarders to every sub worker, so creating one per /profz\n  // request would leak a forwarder per request.\n  const profileWorker =\n    profileSubWorkers.length > 0\n      ? broadcastWorker(profileSubWorkers)\n      : undefined;\n  const getProfileWorker = profileWorker\n    ? () => Promise.resolve(profileWorker)\n    : undefined;\n\n  const changeStreamerWebServer = new ChangeStreamerHttpServer(\n    lc,\n    {\n      port,\n      keepaliveTimeoutMs,\n      // The startup delay is only relevant when taking over the PG change-log\n      // (RMv1 and RMv1.5), and is disabled for RMv2.\n      startupDelayMs: pgChangeLogEnabled ? startupDelayMs : 0,\n      readinessGate,\n      config,\n      getProfileWorker,\n    },\n    parent,\n    changeStreamer,\n  );\n\n  void readinessGate.then(() => {\n    startMetrics.ready();\n    parent.send(['ready', {ready: true}]);\n  });\n\n  // Note: The changeStreamer itself is not started here; it is started by the\n  //       changeStreamerWebServer after a delay to ensure that routing\n  //       routing elements have registered the server before the\n  //       change-streamer takes over the replication slot.\n  // TODO: Remove this delay and start the changeStreamer normally here once\n  //       transitioned to RMv2, since changeStreamer startup will no longer\n  //       disrupt an existing task.\n  running = true;\n  await runUntilKilled(\n    lc,\n    parent,\n    changeStreamerWebServer,\n    backupMonitor,\n    processes,\n  );\n}\n\n/**\n * Connects the startup-time client used for change-streamer initialization\n * and handoff / takeover, and ensures that the change DB schema is\n * initialized / up-to-date. Steady-state clients are managed by the\n * change-streamer (Storer) implementation.\n */\nasync function connectChangeDB(\n  lc: LogContext,\n  connectionURI: string,\n  shard: ShardConfig,\n): Promise<PostgresDB> {\n  const changeDB = await connectPgClient(\n    lc,\n    connectionURI,\n    'change-streamer-init',\n    {max: 5},\n    {sendStringAsJson: true},\n  );\n  void warmupConnections(lc, changeDB, 'change').catch(() => {});\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  return changeDB;\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () =>\n      runWorker(\n        must(parentWorker),\n        process.env,\n        ...process.argv.slice(2),\n      ).catch(async e => {\n        if (!running) {\n          await publishCriticalEvent(\n            lc,\n            replicationStatusError(lc, 'Initializing', e),\n          );\n        }\n        throw e;\n      }),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAKlD,IAAI,UAAU;AAEd,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,iBAAiB;CAC/C,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,iCACA,oBACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CACxB,IAAI,SAAS,SAAS,MACpB,wBAAwB,EAAE;CAG5B,MAAM,QAAQ,eAAe,MAAM;CAInC,MAAM,WAAW,qBACb,MAAM,gBAAgB,IAAI,OAAO,IAAI,KAAK,IAC1C;CAEJ,IAAI,YAAY;CAChB,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,CAAC,CAAC,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,MAAM,eAAe,IAAI,mBAAmB;CAC5C,IAAI,kCAAkC;CACtC,IAAI,mBAAmB;CACvB,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAAG;EAGjC,MAAM,UAAU,IAAI,YAAY,EAAE;EAyBlC,MAAM,iBAAiB;GAAC;GAAY,kBAjBlC,YAAY,WAAW,aAAa,WAAW,aAC3C,OAAO,kBAA2B;IAChC,MAAM,OAAO,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,QACtD,aACF;IACA,YAAY,SAAS,gBAAgB,OAAO;IAC5C,IAAI,WAAW;KACb,MAAM,WAAW;KACjB,QAAQ,UAAU,oBAAoB;MACpC,YAAY;MACZ,OAAO,SAAS,QAAQ;KAC1B,CAAC;IACH;IACA,OAAO;GACT,IACA,KAAA;GAEgD;EAAO;EAC7D,IAAI;GAEF,MAAM,EACJ,oBAAoB,OACpB,6BAA6B,gBAC7B,mCAAmC,iCACjC;GACJ,MAAM,EACJ,cACA,mBACA,sBACA,WACA,4BACA,kBAAkB,sBAClB,uBACA,iBACE,SAAS,SAAS,OAClB,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;IACE,GAAG;IACH,yBAAyB,SAAS;IAClC,6BAA6B,SAAS;GACxC,GACA,SACA,eAAe,kBACf,gBACA;IACE;IACA;IACA;IACA,UAAU,WAAW;IACrB,aAAa;GACf,GACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;GAEJ,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;GAMxD,gBAAgB,2BAA2B;IACzC;IACA,QAAQ,uBACJ;KACE,GAAG;KACH,YAAY;MACV,GAAG,OAAO;MACV,mBAAmB;KACrB;IACF,IACA;IACJ,aAAa,QAAQ;GACvB,CAAC;GAED,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;IACE,WAAW;IACX,mBAAmB,WAAW,gBAAgB,OAAO;IACrD,aAAa,QAAQ;GACvB,IACA,MACJ,WACA,aAAa,OACb;IACE,uBAAuB,yBAAyB;IAChD;IACA;IACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;IACN;IAIA,0CACG,eAAe,mBAAmB,IAC/B,eAAe,mBAAmB,MAClC,KAAK;IACX,oBAAoB,OAAO;IAC3B,oBAAoB,OAAO;IAC3B,eAAe;KACb,eAAe,kBAAkB,QAAQ,IAAI;KAC7C,eAAe;KACf,kBAAkB;IACpB;IAIA,uBAAuB,yBACnB;KACE,aAAa,QAAQ;KACrB,UAAU;MAGR,OAAO;MACP,YAAY,kBAAkB;MAC9B;KACF;IACF,IACA,KAAA;IAIJ,sBAAsB,yBAClB;KACE,aAAa;KACb,WAAW;IACb,IACA,KAAA;IAEJ,wBACE,sBAAsB,2BAClB;KACE,aAAa,QAAQ;KACrB,gBAAgB;KAChB,aAAa;KACb,eAAe;IACjB,IACA,KAAA;IAIN,sBACE,wBAAwB,UACpB;KACE,aAAa;KACb,iBAAiB;KACjB,aAAa;IACf,IACA,KAAA;IACN;GACF,GACA,UACF;GACA,YAAY;GACZ,kCAAkC;GAClC,aAAa,QAAQ,cAAc,IAAI;GACvC,aAAa,OAAO,eAAe,cAAc,YAAY;GAC7D,aAAa,OAAO,aAAa,cAAc,UAAU;GACzD,aAAa,OAAO,WAAW,cAAc,SAAS;GACtD,mBAAmB;GACnB;EACF,SAAS,GAAG;GAcV,MAAM,QAAQ,QAAQ;GAEtB,eAAoB,KAAK;GACzB,gBAAgB,KAAA;GAChB,IAAI,SAAS,aAAa,iBAAiB;IACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;IAGhD,aAAa,QAAQ,IAAI;IAMzB,kBAAkB,QAAQ,IAAI;IAC9B;GACF;GACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;GAEF,MAAM;EACR;CACF;CAEA,OAAO,gBAAgB,kDAAkD;CACzE,OAAO,eAAe,oDAAoD;CAI1E,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,MAAM,oBAA8B,CAAC;CACrC,MAAM,EAAC,SAAS,iBAAiB,YAAW,SAAS;CACrD,IAAI,WAAW;EACb,MAAM,eAAe,YAAY,IAAI;EACrC,gBAAqB,WACnB,aAAa,OAAO,YAAY,YAAY,IAAI,IAAI,YAAY,CAClE;CACF;CACA,IAAI,CAAC,WACH,QAAQ;MACH;EACL,GAAG,OAAO,wBAAwB,SAAS;EAC3C,OAAO,WAAW,oBAAoB;EACtC,IAAI,kBACF,wBAAwB,QAAQ,IAAI;EAItC,MAAM,mBAAmB,UACtB,IACC,YACE,gBACA;GAAC,GAAG;GAAK,qCAAqC;EAAS,GACvD,QACF,GACA,mBACF,CAAC,CACA,gBAAgB,eAAe,QAAQ,CAAC;EAC3C,kBAAkB,KAAK,gBAAgB;EAEvC,iBAAiB,cACf,oBACA,QAAO,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAC7C;CACF;CAEA,IAAI,cAAc;CAClB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAc,cAAc,oBAAoB,CAAC,CAAC,WAAW;GAC3D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,aAAa,OAAO,kBAAkB,OAAO;EAC/C,CAAC;CACH;CAOA,MAAM,gBAAgB,qBAClB,cACA,QAAQ,IAAI,CAAC,aAAa,eAAe,CAAC;CAK9C,MAAM,gBACJ,kBAAkB,SAAS,IACvB,gBAAgB,iBAAiB,IACjC,KAAA;CAKN,MAAM,0BAA0B,IAAI,yBAClC,IACA;EACE;EACA;EAGA,gBAAgB,qBAAqB,iBAAiB;EACtD;EACA;EACA,kBAdqB,sBACf,QAAQ,QAAQ,aAAa,IACnC,KAAA;CAaF,GACA,QACA,cACF;CAEA,cAAmB,WAAW;EAC5B,aAAa,MAAM;EACnB,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CASD,UAAU;CACV,MAAM,eACJ,IACA,QACA,yBACA,eACA,SACF;AACF;;;;;;;AAQA,eAAe,gBACb,IACA,eACA,OACqB;CACrB,MAAM,WAAW,MAAM,gBACrB,IACA,eACA,wBACA,EAAC,KAAK,EAAC,GACP,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC;CAC7D,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,OAAO;AACT;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,CAAC,CAAC,MAAM,OAAM,MAAK;CACjB,IAAI,CAAC,SACH,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CAEF,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/inspector-delegate.js.map

@@ -1 +1 @@
-{"version":3,"file":"inspector-delegate.js","names":["#globalMetrics","#perQueryHydrateMs","#perQueryUpdateMetrics","#queryIDToAST","#customQueryTransformer"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n  isServerMetric,\n  type MetricMap,\n  type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n  'query-materialization-server': TDigest;\n  'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n * The service that established a client group's authentication, i.e. the\n * ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.\n */\ntype AuthOwner = object;\n\nconst NO_OWNER: AuthOwner = {};\n\n/**\n * Authenticated client group IDs, each mapped to the owner that authenticated\n * it. We keep this outside of the class to share this state across all\n * instances of the InspectorDelegate.\n */\nconst authenticatedClientGroupIDs = new Map<ClientGroupID, AuthOwner>();\n\n/**\n * `query-update-server` is reported for every change pushed into every\n * pipeline: millions of values a minute while a bulk write fans out across\n * many client groups. Inserting each into a TDigest (two allocations, and a\n * sort every few hundred inserts) cost ~16% of a busy syncer's CPU. Instead,\n * every UPDATE_SAMPLE_RATE-th value of each query is recorded, weighted by\n * the rate, so the digests keep describing the distribution and, to within\n * one sample, the count.\n */\nexport const UPDATE_SAMPLE_RATE = 32;\n\ntype UpdateMetrics = {\n  readonly digest: TDigest;\n  /** Values reported for the query, recorded or not. */\n  reported: number;\n};\n\nfunction newUpdateMetrics(): UpdateMetrics {\n  return {digest: new TDigest(), reported: 0};\n}\n\nexport class InspectorDelegate implements MetricsDelegate {\n  readonly #globalMetrics: ServerMetrics = newMetrics();\n  readonly #perQueryHydrateMs = new Map<string, number>();\n  readonly #perQueryUpdateMetrics = new Map<string, UpdateMetrics>();\n  readonly #queryIDToAST: Map<string, AST> = new Map();\n  readonly #customQueryTransformer: CustomQueryTransformer | undefined;\n\n  constructor(customQueryTransformer: CustomQueryTransformer | undefined) {\n    this.#customQueryTransformer = customQueryTransformer;\n  }\n\n  addMetric<K extends keyof MetricMap>(\n    metric: K,\n    value: number,\n    ...args: MetricMap[K]\n  ): void {\n    assert(isServerMetric(metric), `Invalid server metric: ${metric}`);\n    const queryID = args[0];\n    if (metric === 'query-materialization-server') {\n      this.#perQueryHydrateMs.set(queryID, value);\n      this.#globalMetrics[metric].add(value);\n      return;\n    }\n    const update = getOrInsertComputed(\n      this.#perQueryUpdateMetrics,\n      queryID,\n      newUpdateMetrics,\n    );\n    if (update.reported++ % UPDATE_SAMPLE_RATE === 0) {\n      update.digest.add(value, UPDATE_SAMPLE_RATE);\n      this.#globalMetrics[metric].add(value, UPDATE_SAMPLE_RATE);\n    }\n  }\n\n  getMetricsJSONForQuery(queryID: string): QueryServerMetricsJSON | null {\n    const hydrateMs = this.#perQueryHydrateMs.get(queryID);\n    const updateMetrics = this.#perQueryUpdateMetrics.get(queryID);\n    if (hydrateMs === undefined && updateMetrics === undefined) {\n      return null;\n    }\n    return {\n      'query-hydration-server-ms': hydrateMs,\n      'query-update-server': (updateMetrics?.digest ?? new TDigest()).toJSON(),\n    };\n  }\n\n  getMetricsJSON() {\n    return mapValues(this.#globalMetrics, v => v.toJSON());\n  }\n\n  getASTForQuery(queryID: string): AST | undefined {\n    return this.#queryIDToAST.get(queryID);\n  }\n\n  removeQuery(queryID: string): void {\n    this.#perQueryHydrateMs.delete(queryID);\n    this.#perQueryUpdateMetrics.delete(queryID);\n    this.#queryIDToAST.delete(queryID);\n  }\n\n  addQuery(queryID: string, ast: AST): void {\n    this.#queryIDToAST.set(queryID, ast);\n  }\n\n  /**\n   * Check if the client is authenticated. We only require authentication once\n   * per \"worker\".\n   */\n  isAuthenticated(clientGroupID: ClientGroupID): boolean {\n    return (\n      isDevelopmentMode() || authenticatedClientGroupIDs.has(clientGroupID)\n    );\n  }\n\n  /**\n   * Marks the client group as authenticated on behalf of `owner`, the service\n   * (ViewSyncer) that handled the authentication.\n   */\n  setAuthenticated(\n    clientGroupID: ClientGroupID,\n    owner: AuthOwner = NO_OWNER,\n  ): void {\n    authenticatedClientGroupIDs.set(clientGroupID, owner);\n  }\n\n  /**\n   * Clears the client group's authentication. When `owner` is given, the\n   * authentication is only cleared if it was established by that owner: a\n   * replacement ViewSyncer can start for the same client group while the\n   * previous one is still shutting down, and the previous one's cleanup must\n   * not revoke what the replacement has since authenticated.\n   */\n  clearAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner) {\n    if (\n      owner === undefined ||\n      authenticatedClientGroupIDs.get(clientGroupID) === owner\n    ) {\n      authenticatedClientGroupIDs.delete(clientGroupID);\n    }\n  }\n\n  /**\n   * Transforms a single custom query by name and args using the configured\n   * CustomQueryTransformer. This is primarily used by the inspector to transform\n   * queries for analysis.\n   */\n  async transformCustomQuery(\n    name: string,\n    args: readonly ReadonlyJSONValue[],\n    ctx: ConnectionContext,\n  ): Promise<AST> {\n    assert(\n      this.#customQueryTransformer,\n      'Custom query transformation requested but no CustomQueryTransformer is configured',\n    );\n\n    // Create a fake CustomQueryRecord for the single query\n    const queryID = hashOfNameAndArgs(name, args);\n    const queries: CustomQueryRecord[] = [\n      {\n        id: queryID,\n        type: 'custom',\n        name,\n        args,\n        clientState: {},\n      },\n    ];\n\n    const results = await this.#customQueryTransformer.transform(ctx, queries);\n\n    if ('kind' in results.result) {\n      throw new ProtocolErrorWithLevel(results.result, 'warn');\n    }\n\n    const result = results.result[0];\n    if (!result) {\n      throw new Error('No transformation result returned');\n    }\n\n    if ('error' in result) {\n      const message =\n        result.message ?? 'Unknown application error from custom query';\n      throw new Error(\n        `Error transforming custom query ${name} (${result.error}): ${message} ${JSON.stringify(result.details)}`,\n      );\n    }\n\n    return result.transformedAst;\n  }\n}\n\nfunction newMetrics(): ServerMetrics {\n  return {\n    'query-materialization-server': new TDigest(),\n    'query-update-server': new TDigest(),\n  };\n}\n"],"mappings":";;;;;;;;;;AAqCA,IAAM,WAAsB,CAAC;;;;;;AAO7B,IAAM,8CAA8B,IAAI,IAA8B;AAmBtE,SAAS,mBAAkC;CACzC,OAAO;EAAC,QAAQ,IAAI,QAAQ;EAAG,UAAU;CAAC;AAC5C;AAEA,IAAa,oBAAb,MAA0D;CACxD,iBAAyC,WAAW;CACpD,qCAA8B,IAAI,IAAoB;CACtD,yCAAkC,IAAI,IAA2B;CACjE,gCAA2C,IAAI,IAAI;CACnD;CAEA,YAAY,wBAA4D;EACtE,KAAKI,0BAA0B;CACjC;CAEA,UACE,QACA,OACA,GAAG,MACG;EACN,OAAO,eAAe,MAAM,GAAG,0BAA0B,QAAQ;EACjE,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,gCAAgC;GAC7C,KAAKH,mBAAmB,IAAI,SAAS,KAAK;GAC1C,KAAKD,eAAe,QAAQ,IAAI,KAAK;GACrC;EACF;EACA,MAAM,SAAS,oBACb,KAAKE,wBACL,SACA,gBACF;EACA,IAAI,OAAO,aAAA,OAAoC,GAAG;GAChD,OAAO,OAAO,IAAI,OAAA,EAAyB;GAC3C,KAAKF,eAAe,QAAQ,IAAI,OAAA,EAAyB;EAC3D;CACF;CAEA,uBAAuB,SAAgD;EACrE,MAAM,YAAY,KAAKC,mBAAmB,IAAI,OAAO;EACrD,MAAM,gBAAgB,KAAKC,uBAAuB,IAAI,OAAO;EAC7D,IAAI,cAAc,KAAA,KAAa,kBAAkB,KAAA,GAC/C,OAAO;EAET,OAAO;GACL,6BAA6B;GAC7B,wBAAwB,eAAe,UAAU,IAAI,QAAQ,GAAG,OAAO;EACzE;CACF;CAEA,iBAAiB;EACf,OAAO,UAAU,KAAKF,iBAAgB,MAAK,EAAE,OAAO,CAAC;CACvD;CAEA,eAAe,SAAkC;EAC/C,OAAO,KAAKG,cAAc,IAAI,OAAO;CACvC;CAEA,YAAY,SAAuB;EACjC,KAAKF,mBAAmB,OAAO,OAAO;EACtC,KAAKC,uBAAuB,OAAO,OAAO;EAC1C,KAAKC,cAAc,OAAO,OAAO;CACnC;CAEA,SAAS,SAAiB,KAAgB;EACxC,KAAKA,cAAc,IAAI,SAAS,GAAG;CACrC;;;;;CAMA,gBAAgB,eAAuC;EACrD,OACE,kBAAkB,KAAK,4BAA4B,IAAI,aAAa;CAExE;;;;;CAMA,iBACE,eACA,QAAmB,UACb;EACN,4BAA4B,IAAI,eAAe,KAAK;CACtD;;;;;;;;CASA,mBAAmB,eAA8B,OAAmB;EAClE,IACE,UAAU,KAAA,KACV,4BAA4B,IAAI,aAAa,MAAM,OAEnD,4BAA4B,OAAO,aAAa;CAEpD;;;;;;CAOA,MAAM,qBACJ,MACA,MACA,KACc;EACd,OACE,KAAKC,yBACL,mFACF;EAIA,MAAM,UAA+B,CACnC;GACE,IAHY,gBAAkB,MAAM,IAGhC;GACJ,MAAM;GACN;GACA;GACA,aAAa,CAAC;EAChB,CACF;EAEA,MAAM,UAAU,MAAM,KAAKA,wBAAwB,UAAU,KAAK,OAAO;EAEzE,IAAI,UAAU,QAAQ,QACpB,MAAM,IAAI,uBAAuB,QAAQ,QAAQ,MAAM;EAGzD,MAAM,SAAS,QAAQ,OAAO;EAC9B,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,mCAAmC;EAGrD,IAAI,WAAW,QAAQ;GACrB,MAAM,UACJ,OAAO,WAAW;GACpB,MAAM,IAAI,MACR,mCAAmC,KAAK,IAAI,OAAO,MAAM,KAAK,QAAQ,GAAG,KAAK,UAAU,OAAO,OAAO,GACxG;EACF;EAEA,OAAO,OAAO;CAChB;AACF;AAEA,SAAS,aAA4B;CACnC,OAAO;EACL,gCAAgC,IAAI,QAAQ;EAC5C,uBAAuB,IAAI,QAAQ;CACrC;AACF"}
+{"version":3,"file":"inspector-delegate.js","names":["hashOfNameAndArgs"],"sources":["../../../../../zero-cache/src/server/inspector-delegate.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport type {ReadonlyJSONValue} from '../../../shared/src/json.ts';\nimport {getOrInsertComputed} from '../../../shared/src/map.ts';\nimport {mapValues} from '../../../shared/src/objects.ts';\nimport {TDigest} from '../../../shared/src/tdigest.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {QueryServerMetrics as QueryServerMetricsJSON} from '../../../zero-protocol/src/inspect-down.ts';\nimport {hashOfNameAndArgs} from '../../../zero-protocol/src/query-hash.ts';\nimport {\n  isServerMetric,\n  type MetricMap,\n  type MetricsDelegate,\n} from '../../../zql/src/query/metrics-delegate.ts';\nimport {isDevelopmentMode} from '../config/normalize.ts';\nimport type {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport type {ConnectionContext} from '../services/view-syncer/connection-context-manager.ts';\nimport type {CustomQueryRecord} from '../services/view-syncer/schema/types.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\n/**\n * Server-side metrics collected for queries during materialization and update.\n * These metrics are reported via the inspector and complement client-side metrics.\n * Used for the global aggregate (all queries combined).\n */\nexport type ServerMetrics = {\n  'query-materialization-server': TDigest;\n  'query-update-server': TDigest;\n};\n\ntype ClientGroupID = string;\n\n/**\n * The service that established a client group's authentication, i.e. the\n * ViewSyncer serving it. See {@link InspectorDelegate.clearAuthenticated}.\n */\ntype AuthOwner = object;\n\nconst NO_OWNER: AuthOwner = {};\n\n/**\n * Authenticated client group IDs, each mapped to the owner that authenticated\n * it. We keep this outside of the class to share this state across all\n * instances of the InspectorDelegate.\n */\nconst authenticatedClientGroupIDs = new Map<ClientGroupID, AuthOwner>();\n\n/**\n * `query-update-server` is reported for every change pushed into every\n * pipeline: millions of values a minute while a bulk write fans out across\n * many client groups. Inserting each into a TDigest (two allocations, and a\n * sort every few hundred inserts) cost ~16% of a busy syncer's CPU. Instead,\n * every UPDATE_SAMPLE_RATE-th value of each query is recorded, weighted by\n * the rate, so the digests keep describing the distribution and, to within\n * one sample, the count.\n */\nexport const UPDATE_SAMPLE_RATE = 32;\n\ntype UpdateMetrics = {\n  readonly digest: TDigest;\n  /** Values reported for the query, recorded or not. */\n  reported: number;\n};\n\nfunction newUpdateMetrics(): UpdateMetrics {\n  return {digest: new TDigest(), reported: 0};\n}\n\nexport class InspectorDelegate implements MetricsDelegate {\n  readonly #globalMetrics: ServerMetrics = newMetrics();\n  readonly #perQueryHydrateMs = new Map<string, number>();\n  readonly #perQueryUpdateMetrics = new Map<string, UpdateMetrics>();\n  readonly #queryIDToAST: Map<string, AST> = new Map();\n  readonly #customQueryTransformer: CustomQueryTransformer | undefined;\n\n  constructor(customQueryTransformer: CustomQueryTransformer | undefined) {\n    this.#customQueryTransformer = customQueryTransformer;\n  }\n\n  addMetric<K extends keyof MetricMap>(\n    metric: K,\n    value: number,\n    ...args: MetricMap[K]\n  ): void {\n    assert(isServerMetric(metric), `Invalid server metric: ${metric}`);\n    const queryID = args[0];\n    if (metric === 'query-materialization-server') {\n      this.#perQueryHydrateMs.set(queryID, value);\n      this.#globalMetrics[metric].add(value);\n      return;\n    }\n    const update = getOrInsertComputed(\n      this.#perQueryUpdateMetrics,\n      queryID,\n      newUpdateMetrics,\n    );\n    if (update.reported++ % UPDATE_SAMPLE_RATE === 0) {\n      update.digest.add(value, UPDATE_SAMPLE_RATE);\n      this.#globalMetrics[metric].add(value, UPDATE_SAMPLE_RATE);\n    }\n  }\n\n  getMetricsJSONForQuery(queryID: string): QueryServerMetricsJSON | null {\n    const hydrateMs = this.#perQueryHydrateMs.get(queryID);\n    const updateMetrics = this.#perQueryUpdateMetrics.get(queryID);\n    if (hydrateMs === undefined && updateMetrics === undefined) {\n      return null;\n    }\n    return {\n      'query-hydration-server-ms': hydrateMs,\n      'query-update-server': (updateMetrics?.digest ?? new TDigest()).toJSON(),\n    };\n  }\n\n  getMetricsJSON() {\n    return mapValues(this.#globalMetrics, v => v.toJSON());\n  }\n\n  getASTForQuery(queryID: string): AST | undefined {\n    return this.#queryIDToAST.get(queryID);\n  }\n\n  removeQuery(queryID: string): void {\n    this.#perQueryHydrateMs.delete(queryID);\n    this.#perQueryUpdateMetrics.delete(queryID);\n    this.#queryIDToAST.delete(queryID);\n  }\n\n  addQuery(queryID: string, ast: AST): void {\n    this.#queryIDToAST.set(queryID, ast);\n  }\n\n  /**\n   * Check if the client is authenticated. We only require authentication once\n   * per \"worker\".\n   */\n  isAuthenticated(clientGroupID: ClientGroupID): boolean {\n    return (\n      isDevelopmentMode() || authenticatedClientGroupIDs.has(clientGroupID)\n    );\n  }\n\n  /**\n   * Marks the client group as authenticated on behalf of `owner`, the service\n   * (ViewSyncer) that handled the authentication.\n   */\n  setAuthenticated(\n    clientGroupID: ClientGroupID,\n    owner: AuthOwner = NO_OWNER,\n  ): void {\n    authenticatedClientGroupIDs.set(clientGroupID, owner);\n  }\n\n  /**\n   * Clears the client group's authentication. When `owner` is given, the\n   * authentication is only cleared if it was established by that owner: a\n   * replacement ViewSyncer can start for the same client group while the\n   * previous one is still shutting down, and the previous one's cleanup must\n   * not revoke what the replacement has since authenticated.\n   */\n  clearAuthenticated(clientGroupID: ClientGroupID, owner?: AuthOwner) {\n    if (\n      owner === undefined ||\n      authenticatedClientGroupIDs.get(clientGroupID) === owner\n    ) {\n      authenticatedClientGroupIDs.delete(clientGroupID);\n    }\n  }\n\n  /**\n   * Transforms a single custom query by name and args using the configured\n   * CustomQueryTransformer. This is primarily used by the inspector to transform\n   * queries for analysis.\n   */\n  async transformCustomQuery(\n    name: string,\n    args: readonly ReadonlyJSONValue[],\n    ctx: ConnectionContext,\n  ): Promise<AST> {\n    assert(\n      this.#customQueryTransformer,\n      'Custom query transformation requested but no CustomQueryTransformer is configured',\n    );\n\n    // Create a fake CustomQueryRecord for the single query\n    const queryID = hashOfNameAndArgs(name, args);\n    const queries: CustomQueryRecord[] = [\n      {\n        id: queryID,\n        type: 'custom',\n        name,\n        args,\n        clientState: {},\n      },\n    ];\n\n    const results = await this.#customQueryTransformer.transform(ctx, queries);\n\n    if ('kind' in results.result) {\n      throw new ProtocolErrorWithLevel(results.result, 'warn');\n    }\n\n    const result = results.result[0];\n    if (!result) {\n      throw new Error('No transformation result returned');\n    }\n\n    if ('error' in result) {\n      const message =\n        result.message ?? 'Unknown application error from custom query';\n      throw new Error(\n        `Error transforming custom query ${name} (${result.error}): ${message} ${JSON.stringify(result.details)}`,\n      );\n    }\n\n    return result.transformedAst;\n  }\n}\n\nfunction newMetrics(): ServerMetrics {\n  return {\n    'query-materialization-server': new TDigest(),\n    'query-update-server': new TDigest(),\n  };\n}\n"],"mappings":";;;;;;;;;;AAqCA,IAAM,WAAsB,CAAC;;;;;;AAO7B,IAAM,8CAA8B,IAAI,IAA8B;AAmBtE,SAAS,mBAAkC;CACzC,OAAO;EAAC,QAAQ,IAAI,QAAQ;EAAG,UAAU;CAAC;AAC5C;AAEA,IAAa,oBAAb,MAA0D;CACxD,iBAAyC,WAAW;CACpD,qCAA8B,IAAI,IAAoB;CACtD,yCAAkC,IAAI,IAA2B;CACjE,gCAA2C,IAAI,IAAI;CACnD;CAEA,YAAY,wBAA4D;EACtE,KAAK,0BAA0B;CACjC;CAEA,UACE,QACA,OACA,GAAG,MACG;EACN,OAAO,eAAe,MAAM,GAAG,0BAA0B,QAAQ;EACjE,MAAM,UAAU,KAAK;EACrB,IAAI,WAAW,gCAAgC;GAC7C,KAAK,mBAAmB,IAAI,SAAS,KAAK;GAC1C,KAAK,eAAe,OAAO,CAAC,IAAI,KAAK;GACrC;EACF;EACA,MAAM,SAAS,oBACb,KAAK,wBACL,SACA,gBACF;EACA,IAAI,OAAO,aAAA,OAAoC,GAAG;GAChD,OAAO,OAAO,IAAI,OAAA,EAAyB;GAC3C,KAAK,eAAe,OAAO,CAAC,IAAI,OAAA,EAAyB;EAC3D;CACF;CAEA,uBAAuB,SAAgD;EACrE,MAAM,YAAY,KAAK,mBAAmB,IAAI,OAAO;EACrD,MAAM,gBAAgB,KAAK,uBAAuB,IAAI,OAAO;EAC7D,IAAI,cAAc,KAAA,KAAa,kBAAkB,KAAA,GAC/C,OAAO;EAET,OAAO;GACL,6BAA6B;GAC7B,wBAAwB,eAAe,UAAU,IAAI,QAAQ,EAAA,CAAG,OAAO;EACzE;CACF;CAEA,iBAAiB;EACf,OAAO,UAAU,KAAK,iBAAgB,MAAK,EAAE,OAAO,CAAC;CACvD;CAEA,eAAe,SAAkC;EAC/C,OAAO,KAAK,cAAc,IAAI,OAAO;CACvC;CAEA,YAAY,SAAuB;EACjC,KAAK,mBAAmB,OAAO,OAAO;EACtC,KAAK,uBAAuB,OAAO,OAAO;EAC1C,KAAK,cAAc,OAAO,OAAO;CACnC;CAEA,SAAS,SAAiB,KAAgB;EACxC,KAAK,cAAc,IAAI,SAAS,GAAG;CACrC;;;;;CAMA,gBAAgB,eAAuC;EACrD,OACE,kBAAkB,KAAK,4BAA4B,IAAI,aAAa;CAExE;;;;;CAMA,iBACE,eACA,QAAmB,UACb;EACN,4BAA4B,IAAI,eAAe,KAAK;CACtD;;;;;;;;CASA,mBAAmB,eAA8B,OAAmB;EAClE,IACE,UAAU,KAAA,KACV,4BAA4B,IAAI,aAAa,MAAM,OAEnD,4BAA4B,OAAO,aAAa;CAEpD;;;;;;CAOA,MAAM,qBACJ,MACA,MACA,KACc;EACd,OACE,KAAK,yBACL,mFACF;EAIA,MAAM,UAA+B,CACnC;GACE,IAHYA,gBAAkB,MAAM,IAGhC;GACJ,MAAM;GACN;GACA;GACA,aAAa,CAAC;EAChB,CACF;EAEA,MAAM,UAAU,MAAM,KAAK,wBAAwB,UAAU,KAAK,OAAO;EAEzE,IAAI,UAAU,QAAQ,QACpB,MAAM,IAAI,uBAAuB,QAAQ,QAAQ,MAAM;EAGzD,MAAM,SAAS,QAAQ,OAAO;EAC9B,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,mCAAmC;EAGrD,IAAI,WAAW,QAAQ;GACrB,MAAM,UACJ,OAAO,WAAW;GACpB,MAAM,IAAI,MACR,mCAAmC,KAAK,IAAI,OAAO,MAAM,KAAK,QAAQ,GAAG,KAAK,UAAU,OAAO,OAAO,GACxG;EACF;EAEA,OAAO,OAAO;CAChB;AACF;AAEA,SAAS,aAA4B;CACnC,OAAO;EACL,gCAAgC,IAAI,QAAQ;EAC5C,uBAAuB,IAAI,QAAQ;CACrC;AACF"}

package/out/zero-cache/src/server/logging.js

@@ -54,6 +54,6 @@ var CompositeLogSink = class {
 	}
 };
 //#endregion
-export { createLogContext };
+export { CompositeLogSink, createLogContext, logUncaughtException };
 
 //# sourceMappingURL=logging.js.map

package/out/zero-cache/src/server/logging.js.map

@@ -1 +1 @@
-{"version":3,"file":"logging.js","names":["#sinks"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n  createLogContext as createLogContextShared,\n  getLogSink,\n  type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\ntype UncaughtExceptionHandler = (\n  err: Error,\n  origin: NodeJS.UncaughtExceptionOrigin,\n) => Promise<void>;\n\nlet bootstrapUncaughtExceptionHandler: UncaughtExceptionHandler | undefined;\n\nexport function createLogContext(\n  {log}: {log: LogConfig},\n  worker: string,\n  workerIndex = 0,\n  includeOtel = true,\n): LogContext {\n  const logSink = createLogSink(log, includeOtel);\n  const lc = createLogContextShared({log}, {worker, workerIndex}, logSink);\n  const handleUncaughtException: UncaughtExceptionHandler = async (\n    err,\n    origin,\n  ) => {\n    // Workers create an includeOtel=false context while bootstrapping OTel,\n    // then a primary context. Run full corruption checks from the primary\n    // handler only.\n    try {\n      await logUncaughtException(lc, logSink, err, origin, includeOtel);\n    } finally {\n      process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n    }\n  };\n  if (bootstrapUncaughtExceptionHandler) {\n    process.off('uncaughtException', bootstrapUncaughtExceptionHandler);\n    bootstrapUncaughtExceptionHandler = undefined;\n  }\n  process.on('uncaughtException', handleUncaughtException);\n  if (!includeOtel) {\n    bootstrapUncaughtExceptionHandler = handleUncaughtException;\n  }\n  return lc;\n}\n\nexport async function logUncaughtException(\n  lc: LogContext,\n  logSink: LogSink,\n  err: unknown,\n  origin: string,\n  includeLastChanceDiagnostics = true,\n): Promise<void> {\n  lc.error?.(origin, err);\n  if (includeLastChanceDiagnostics) {\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, err);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n  }\n  await logSink.flush?.();\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n  const sink = getLogSink(config);\n  if (includeOtel && otelLogsEnabled()) {\n    const otelSink = new OtelLogSink();\n    return new CompositeLogSink([otelSink, sink]);\n  }\n  return sink;\n}\n\nexport class CompositeLogSink implements LogSink {\n  readonly #sinks: LogSink[];\n\n  constructor(sinks: LogSink[]) {\n    this.#sinks = sinks;\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    for (const sink of this.#sinks) {\n      sink.log(level, context, ...args);\n    }\n  }\n\n  async flush(): Promise<void> {\n    await Promise.all(this.#sinks.map(sink => sink.flush?.()));\n  }\n}\n"],"mappings":";;;;;;;AAiBA,IAAI;AAEJ,SAAgB,iBACd,EAAC,OACD,QACA,cAAc,GACd,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,WAAW;CAC9C,MAAM,KAAK,mBAAuB,EAAC,IAAG,GAAG;EAAC;EAAQ;CAAW,GAAG,OAAO;CACvE,MAAM,0BAAoD,OACxD,KACA,WACG;EAIH,IAAI;GACF,MAAM,qBAAqB,IAAI,SAAS,KAAK,QAAQ,WAAW;EAClE,UAAU;GACR,QAAQ,KAAA,EAAmC;EAC7C;CACF;CACA,IAAI,mCAAmC;EACrC,QAAQ,IAAI,qBAAqB,iCAAiC;EAClE,oCAAoC,KAAA;CACtC;CACA,QAAQ,GAAG,qBAAqB,uBAAuB;CACvD,IAAI,CAAC,aACH,oCAAoC;CAEtC,OAAO;AACT;AAEA,eAAsB,qBACpB,IACA,SACA,KACA,QACA,+BAA+B,MAChB;CACf,GAAG,QAAQ,QAAQ,GAAG;CACtB,IAAI,8BACF,IAAI;EACF,yCAAyC,IAAI,GAAG;CAClD,SAAS,iBAAiB;EACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;CACH;CAEF,MAAM,QAAQ,QAAQ;AACxB;AAEA,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,MAAM;CAC9B,IAAI,eAAe,gBAAgB,GAEjC,OAAO,IAAI,iBAAiB,CAAC,IADR,YACQ,GAAU,IAAI,CAAC;CAE9C,OAAO;AACT;AAEA,IAAa,mBAAb,MAAiD;CAC/C;CAEA,YAAY,OAAkB;EAC5B,KAAKA,SAAS;CAChB;CAEA,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,KAAK,MAAM,QAAQ,KAAKA,QACtB,KAAK,IAAI,OAAO,SAAS,GAAG,IAAI;CAEpC;CAEA,MAAM,QAAuB;EAC3B,MAAM,QAAQ,IAAI,KAAKA,OAAO,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC;CAC3D;AACF"}
+{"version":3,"file":"logging.js","names":["createLogContextShared"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n  createLogContext as createLogContextShared,\n  getLogSink,\n  type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {logLastChanceSQLiteCorruptionDiagnostics} from '../db/sqlite-corruption.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\ntype UncaughtExceptionHandler = (\n  err: Error,\n  origin: NodeJS.UncaughtExceptionOrigin,\n) => Promise<void>;\n\nlet bootstrapUncaughtExceptionHandler: UncaughtExceptionHandler | undefined;\n\nexport function createLogContext(\n  {log}: {log: LogConfig},\n  worker: string,\n  workerIndex = 0,\n  includeOtel = true,\n): LogContext {\n  const logSink = createLogSink(log, includeOtel);\n  const lc = createLogContextShared({log}, {worker, workerIndex}, logSink);\n  const handleUncaughtException: UncaughtExceptionHandler = async (\n    err,\n    origin,\n  ) => {\n    // Workers create an includeOtel=false context while bootstrapping OTel,\n    // then a primary context. Run full corruption checks from the primary\n    // handler only.\n    try {\n      await logUncaughtException(lc, logSink, err, origin, includeOtel);\n    } finally {\n      process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n    }\n  };\n  if (bootstrapUncaughtExceptionHandler) {\n    process.off('uncaughtException', bootstrapUncaughtExceptionHandler);\n    bootstrapUncaughtExceptionHandler = undefined;\n  }\n  process.on('uncaughtException', handleUncaughtException);\n  if (!includeOtel) {\n    bootstrapUncaughtExceptionHandler = handleUncaughtException;\n  }\n  return lc;\n}\n\nexport async function logUncaughtException(\n  lc: LogContext,\n  logSink: LogSink,\n  err: unknown,\n  origin: string,\n  includeLastChanceDiagnostics = true,\n): Promise<void> {\n  lc.error?.(origin, err);\n  if (includeLastChanceDiagnostics) {\n    try {\n      logLastChanceSQLiteCorruptionDiagnostics(lc, err);\n    } catch (diagnosticError) {\n      lc.error?.('SQLite corruption last-chance diagnostic failed', {\n        error: diagnosticError,\n      });\n    }\n  }\n  await logSink.flush?.();\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n  const sink = getLogSink(config);\n  if (includeOtel && otelLogsEnabled()) {\n    const otelSink = new OtelLogSink();\n    return new CompositeLogSink([otelSink, sink]);\n  }\n  return sink;\n}\n\nexport class CompositeLogSink implements LogSink {\n  readonly #sinks: LogSink[];\n\n  constructor(sinks: LogSink[]) {\n    this.#sinks = sinks;\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    for (const sink of this.#sinks) {\n      sink.log(level, context, ...args);\n    }\n  }\n\n  async flush(): Promise<void> {\n    await Promise.all(this.#sinks.map(sink => sink.flush?.()));\n  }\n}\n"],"mappings":";;;;;;;AAiBA,IAAI;AAEJ,SAAgB,iBACd,EAAC,OACD,QACA,cAAc,GACd,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,WAAW;CAC9C,MAAM,KAAKA,mBAAuB,EAAC,IAAG,GAAG;EAAC;EAAQ;CAAW,GAAG,OAAO;CACvE,MAAM,0BAAoD,OACxD,KACA,WACG;EAIH,IAAI;GACF,MAAM,qBAAqB,IAAI,SAAS,KAAK,QAAQ,WAAW;EAClE,UAAU;GACR,QAAQ,KAAA,EAAmC;EAC7C;CACF;CACA,IAAI,mCAAmC;EACrC,QAAQ,IAAI,qBAAqB,iCAAiC;EAClE,oCAAoC,KAAA;CACtC;CACA,QAAQ,GAAG,qBAAqB,uBAAuB;CACvD,IAAI,CAAC,aACH,oCAAoC;CAEtC,OAAO;AACT;AAEA,eAAsB,qBACpB,IACA,SACA,KACA,QACA,+BAA+B,MAChB;CACf,GAAG,QAAQ,QAAQ,GAAG;CACtB,IAAI,8BACF,IAAI;EACF,yCAAyC,IAAI,GAAG;CAClD,SAAS,iBAAiB;EACxB,GAAG,QAAQ,mDAAmD,EAC5D,OAAO,gBACT,CAAC;CACH;CAEF,MAAM,QAAQ,QAAQ;AACxB;AAEA,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,MAAM;CAC9B,IAAI,eAAe,gBAAgB,GAEjC,OAAO,IAAI,iBAAiB,CAAC,IADR,YACQ,GAAU,IAAI,CAAC;CAE9C,OAAO;AACT;AAEA,IAAa,mBAAb,MAAiD;CAC/C;CAEA,YAAY,OAAkB;EAC5B,KAAK,SAAS;CAChB;CAEA,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,KAAK,MAAM,QAAQ,KAAK,QACtB,KAAK,IAAI,OAAO,SAAS,GAAG,IAAI;CAEpC;CAEA,MAAM,QAAuB;EAC3B,MAAM,QAAQ,IAAI,KAAK,OAAO,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC;CAC3D;AACF"}

package/out/zero-cache/src/server/main.js

@@ -1,3 +1,4 @@
+import { resolver } from "../../../shared/src/resolver.js";
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { runsChangeStreamer } from "../config/normalize.js";
@@ -6,11 +7,10 @@ import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruptio
 import { ProcessManager, exitAfter, recordStartupDurationMs, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { CHANGE_STREAMER_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL } from "./worker-urls.js";
+import { WorkerDispatcher } from "./worker-dispatcher.js";
 import { initEventSink } from "../observability/events.js";
 import { startOtelAuto } from "./otel-start.js";
 import { createNotifierFrom, handleSubscriptionsFrom, subscribeTo } from "../workers/replicator.js";
-import { WorkerDispatcher } from "./worker-dispatcher.js";
-import { resolver } from "@rocicorp/resolver";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 import path from "node:path";
 //#region ../zero-cache/src/server/main.ts

package/out/zero-cache/src/server/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  subscribeTo,\n  type ReplicaFileMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    // In a dedicated replication-manager (i.e. no syncers), the\n    // change-streamer is the user-facing worker: it receives the SIGTERM\n    // first so that it can drain subscribers before other supporting\n    // workers are stopped. When syncers run in the same task, the\n    // change-streamer supports them and thus must outlive them.\n    changeStreamer = loadWorker(\n      CHANGE_STREAMER_URL,\n      numSyncers > 0 ? 'supporting' : 'user-facing',\n    ).once('message', changeStreamerStarted);\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    // Relay drain requests to the runner, which coordinates the SelfDrain.\n    replicator.onMessageType<DrainRequestedMessage>('drainRequested', msg => {\n      lc.info?.(`drain requested by the replicator`);\n      parent.send<DrainRequestedMessage>(['drainRequested', msg]);\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EAMX,iBAAiB,WACf,qBACA,aAAa,IAAI,eAAe,aAClC,EAAE,KAAK,WAAW,qBAAqB;EAGvC,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EAED,WAAW,cAAqC,mBAAkB,QAAO;GACvE,GAAG,OAAO,mCAAmC;GAC7C,OAAO,KAA4B,CAAC,kBAAkB,GAAG,CAAC;EAC5D,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}
+{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {resolver} from '../../../shared/src/resolver.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n  exitAfter,\n  ProcessManager,\n  recordStartupDurationMs,\n  runUntilKilled,\n  type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n  childWorker,\n  parentWorker,\n  singleProcessMode,\n  type DrainRequestedMessage,\n  type ProfileMessage,\n  type ProfileResponseMessage,\n  type Worker,\n} from '../types/processes.ts';\nimport {\n  createNotifierFrom,\n  handleSubscriptionsFrom,\n  subscribeTo,\n  type ReplicaFileMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n  CHANGE_STREAMER_URL,\n  MUTATOR_URL,\n  REAPER_URL,\n  REPLICATOR_URL,\n  SHADOW_SYNCER_URL,\n  SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  const startMs = Date.now();\n  const config = getNormalizedZeroConfig({env});\n\n  startOtelAuto(\n    createLogContext(config, 'dispatcher', 0, false),\n    'dispatcher',\n    0,\n  );\n  lc = createLogContext(config, 'dispatcher');\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: 'dispatcher replica',\n      dbPath: config.replica.file,\n    },\n    config.sqliteCorruptionChecks,\n  );\n  initEventSink(lc, config);\n\n  const processes = new ProcessManager(lc, parent);\n\n  const {numSyncWorkers: numSyncers} = config;\n  if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n  if (config.cvr.maxConns < numSyncers) {\n    throw new Error(\n      `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n        `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n    );\n  }\n\n  const internalFlags: string[] =\n    numSyncers === 0\n      ? []\n      : [\n          '--upstream-max-conns-per-worker',\n          String(Math.floor(config.upstream.maxConns / numSyncers)),\n          '--cvr-max-conns-per-worker',\n          String(Math.floor(config.cvr.maxConns / numSyncers)),\n        ];\n\n  const allSubWorkers: Worker[] = [];\n  function loadWorker(\n    moduleUrl: URL,\n    type: WorkerType,\n    id?: string | number,\n    ...args: string[]\n  ): Worker {\n    const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n    const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n    const w = processes.addWorker(worker, type, name);\n    allSubWorkers.push(w);\n    w.onMessageType<ProfileResponseMessage>('profileResponse', res =>\n      parent.send(['profileResponse', res]),\n    );\n    return w;\n  }\n\n  const {taskID, litestream} = config;\n  const runChangeStreamer = runsChangeStreamer(config);\n  const sqliteChangeLogEnabled =\n    config.changeStreamer.sqliteChangeLogMode !== 'off';\n  if (sqliteChangeLogEnabled && !runChangeStreamer) {\n    // A multi-node deployment configures every task from one environment, so\n    // the change log's options reach the view-syncers too. The log lives in\n    // the change-streamer, and a task that connects to one simply never writes\n    // a log. Warning rather than refusing to start is what makes the\n    // fleet-wide environment -- often the only knob an operator has -- able to\n    // express the rollout at all.\n    lc.warn?.(\n      `ignoring --change-streamer-sqlite-change-log-mode=` +\n        `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n        `a change-streamer rather than running one, and the SQLite change log ` +\n        `lives in the change-streamer`,\n    );\n  }\n\n  let changeStreamer: Worker | undefined;\n\n  if (runChangeStreamer) {\n    const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n      resolver();\n    // In a dedicated replication-manager (i.e. no syncers), the\n    // change-streamer is the user-facing worker: it receives the SIGTERM\n    // first so that it can drain subscribers before other supporting\n    // workers are stopped. When syncers run in the same task, the\n    // change-streamer supports them and thus must outlive them.\n    changeStreamer = loadWorker(\n      CHANGE_STREAMER_URL,\n      numSyncers > 0 ? 'supporting' : 'user-facing',\n    ).once('message', changeStreamerStarted);\n    // Wait for the change-streamer to be ready to guarantee that a replica\n    // file is present.\n    await changeStreamerReady;\n  }\n\n  if (numSyncers > 0) {\n    const {promise: reaperReady, resolve: reaperStarted} = resolver();\n    loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n    // Before starting the view-syncers, ensure that the reaper has started\n    // up, indicating that any CVR db migrations have been performed.\n    await reaperReady;\n  }\n\n  // Only run the shadow-sync canary on the replication-manager (or in\n  // single-node mode, where it also owns upstream). Running on every\n  // view-syncer would hammer the upstream with N redundant canaries.\n  if (config.shadowSync.enabled && runChangeStreamer) {\n    const {promise: shadowReady, resolve: shadowStarted} = resolver();\n    loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n    await shadowReady;\n  }\n\n  const syncers: Worker[] = [];\n  if (numSyncers) {\n    const mode: ReplicaFileMode =\n      runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n    const {promise: replicaReady, resolve} = resolver();\n    const replicator = loadWorker(\n      REPLICATOR_URL,\n      'supporting',\n      mode,\n      mode,\n    ).once('message', () => {\n      subscribeTo(lc, replicator);\n      resolve();\n    });\n    // Relay drain requests to the runner, which coordinates the SelfDrain.\n    replicator.onMessageType<DrainRequestedMessage>('drainRequested', msg => {\n      lc.info?.(`drain requested by the replicator`);\n      parent.send<DrainRequestedMessage>(['drainRequested', msg]);\n    });\n    await replicaReady;\n\n    const notifier = createNotifierFrom(lc, replicator);\n    for (let i = 0; i < numSyncers; i++) {\n      syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n    }\n    syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n  }\n  let mutator: Worker | undefined;\n  if (clientConnectionBifurcated) {\n    mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n  }\n\n  lc.info?.('waiting for workers to be ready ...');\n  const logWaiting = setInterval(\n    () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n    10_000,\n  );\n  await processes.allWorkersReady();\n  clearInterval(logWaiting);\n  const startupDurationMs = Date.now() - startMs;\n  lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n  recordStartupDurationMs(startupDurationMs);\n\n  parent.send(['ready', {ready: true}]);\n  parent.onMessageType<ProfileMessage>('profile', req => {\n    for (const w of allSubWorkers) {\n      w.send(['profile', req]);\n    }\n  });\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent,\n      new WorkerDispatcher(\n        lc,\n        taskID,\n        parent,\n        syncers,\n        mutator,\n        changeStreamer,\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'dispatcher');\n  } finally {\n    await processes.shutdown();\n  }\n}\n\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA6CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,MAAM,gBAA0B,CAAC;CACjC,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,MAAM,IAAI,UAAU,UAAU,QAAQ,MAAM,IAAI;EAChD,cAAc,KAAK,CAAC;EACpB,EAAE,cAAsC,oBAAmB,QACzD,OAAO,KAAK,CAAC,mBAAmB,GAAG,CAAC,CACtC;EACA,OAAO;CACT;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EAMX,iBAAiB,WACf,qBACA,aAAa,IAAI,eAAe,aAClC,CAAC,CAAC,KAAK,WAAW,qBAAqB;EAGvC,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,CAAC,CAAC,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,CAAC,CAAC,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EAED,WAAW,cAAqC,mBAAkB,QAAO;GACvE,GAAG,OAAO,mCAAmC;GAC7C,OAAO,KAA4B,CAAC,kBAAkB,GAAG,CAAC;EAC5D,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,CAAC,CAAC,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACpC,OAAO,cAA8B,YAAW,QAAO;EACrD,KAAK,MAAM,KAAK,eACd,EAAE,KAAK,CAAC,WAAW,GAAG,CAAC;CAE3B,CAAC;CAED,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}

package/out/zero-cache/src/server/otel-log-sink.js.map

@@ -1 +1 @@
-{"version":3,"file":"otel-log-sink.js","names":["#logger","#loggerProvider"],"sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"sourcesContent":["import {\n  logs,\n  SeverityNumber,\n  type AnyValueMap,\n  type Logger,\n  type LoggerProvider,\n  type LogRecord,\n} from '@opentelemetry/api-logs';\nimport type {Context, LogLevel, LogSink} from '@rocicorp/logger';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {errorOrObject} from '../../../shared/src/logging.ts';\n\ntype FlushableLoggerProvider = LoggerProvider & {\n  forceFlush?: (() => Promise<void>) | undefined;\n};\n\nexport class OtelLogSink implements LogSink {\n  readonly #logger: Logger;\n  readonly #loggerProvider: FlushableLoggerProvider;\n\n  constructor(\n    loggerProvider = logs.getLoggerProvider() as FlushableLoggerProvider,\n  ) {\n    this.#loggerProvider = loggerProvider;\n    this.#logger = loggerProvider.getLogger('zero-cache');\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    const lastObj = errorOrObject(args.at(-1));\n    if (lastObj) {\n      args.pop();\n    }\n\n    let message = args.length\n      ? args.map(s => (typeof s === 'string' ? s : stringify(s))).join(' ')\n      : '';\n\n    if (lastObj) {\n      message += ` ${stringify(lastObj)}`;\n    }\n\n    const payload: LogRecord = {\n      severityText: level,\n      severityNumber: toErrorNum(level),\n      body: message,\n    };\n    if (context) {\n      payload.attributes = context as AnyValueMap;\n    }\n    this.#logger.emit(payload);\n  }\n\n  async flush(): Promise<void> {\n    await this.#loggerProvider.forceFlush?.();\n  }\n}\n\nfunction toErrorNum(level: LogLevel): SeverityNumber {\n  switch (level) {\n    case 'error':\n      return SeverityNumber.ERROR;\n    case 'warn':\n      return SeverityNumber.WARN;\n    case 'info':\n      return SeverityNumber.INFO;\n    case 'debug':\n      return SeverityNumber.DEBUG;\n    default:\n      throw new Error(`Unknown log level: ${level}`);\n  }\n}\n"],"mappings":";;;;AAgBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CAEA,YACE,iBAAiB,KAAK,kBAAkB,GACxC;EACA,KAAKC,kBAAkB;EACvB,KAAKD,UAAU,eAAe,UAAU,YAAY;CACtD;CAEA,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,MAAM,UAAU,cAAc,KAAK,GAAG,EAAE,CAAC;EACzC,IAAI,SACF,KAAK,IAAI;EAGX,IAAI,UAAU,KAAK,SACf,KAAK,KAAI,MAAM,OAAO,MAAM,WAAW,IAAI,UAAU,CAAC,CAAE,EAAE,KAAK,GAAG,IAClE;EAEJ,IAAI,SACF,WAAW,IAAI,UAAU,OAAO;EAGlC,MAAM,UAAqB;GACzB,cAAc;GACd,gBAAgB,WAAW,KAAK;GAChC,MAAM;EACR;EACA,IAAI,SACF,QAAQ,aAAa;EAEvB,KAAKA,QAAQ,KAAK,OAAO;CAC3B;CAEA,MAAM,QAAuB;EAC3B,MAAM,KAAKC,gBAAgB,aAAa;CAC1C;AACF;AAEA,SAAS,WAAW,OAAiC;CACnD,QAAQ,OAAR;EACE,KAAK,SACH,OAAO,eAAe;EACxB,KAAK,QACH,OAAO,eAAe;EACxB,KAAK,QACH,OAAO,eAAe;EACxB,KAAK,SACH,OAAO,eAAe;EACxB,SACE,MAAM,IAAI,MAAM,sBAAsB,OAAO;CACjD;AACF"}
+{"version":3,"file":"otel-log-sink.js","names":[],"sources":["../../../../../zero-cache/src/server/otel-log-sink.ts"],"sourcesContent":["import {\n  logs,\n  SeverityNumber,\n  type AnyValueMap,\n  type Logger,\n  type LoggerProvider,\n  type LogRecord,\n} from '@opentelemetry/api-logs';\nimport type {Context, LogLevel, LogSink} from '@rocicorp/logger';\nimport {stringify} from '../../../shared/src/bigint-json.ts';\nimport {errorOrObject} from '../../../shared/src/logging.ts';\n\ntype FlushableLoggerProvider = LoggerProvider & {\n  forceFlush?: (() => Promise<void>) | undefined;\n};\n\nexport class OtelLogSink implements LogSink {\n  readonly #logger: Logger;\n  readonly #loggerProvider: FlushableLoggerProvider;\n\n  constructor(\n    loggerProvider = logs.getLoggerProvider() as FlushableLoggerProvider,\n  ) {\n    this.#loggerProvider = loggerProvider;\n    this.#logger = loggerProvider.getLogger('zero-cache');\n  }\n\n  log(level: LogLevel, context: Context | undefined, ...args: unknown[]): void {\n    const lastObj = errorOrObject(args.at(-1));\n    if (lastObj) {\n      args.pop();\n    }\n\n    let message = args.length\n      ? args.map(s => (typeof s === 'string' ? s : stringify(s))).join(' ')\n      : '';\n\n    if (lastObj) {\n      message += ` ${stringify(lastObj)}`;\n    }\n\n    const payload: LogRecord = {\n      severityText: level,\n      severityNumber: toErrorNum(level),\n      body: message,\n    };\n    if (context) {\n      payload.attributes = context as AnyValueMap;\n    }\n    this.#logger.emit(payload);\n  }\n\n  async flush(): Promise<void> {\n    await this.#loggerProvider.forceFlush?.();\n  }\n}\n\nfunction toErrorNum(level: LogLevel): SeverityNumber {\n  switch (level) {\n    case 'error':\n      return SeverityNumber.ERROR;\n    case 'warn':\n      return SeverityNumber.WARN;\n    case 'info':\n      return SeverityNumber.INFO;\n    case 'debug':\n      return SeverityNumber.DEBUG;\n    default:\n      throw new Error(`Unknown log level: ${level}`);\n  }\n}\n"],"mappings":";;;;AAgBA,IAAa,cAAb,MAA4C;CAC1C;CACA;CAEA,YACE,iBAAiB,KAAK,kBAAkB,GACxC;EACA,KAAK,kBAAkB;EACvB,KAAK,UAAU,eAAe,UAAU,YAAY;CACtD;CAEA,IAAI,OAAiB,SAA8B,GAAG,MAAuB;EAC3E,MAAM,UAAU,cAAc,KAAK,GAAG,EAAE,CAAC;EACzC,IAAI,SACF,KAAK,IAAI;EAGX,IAAI,UAAU,KAAK,SACf,KAAK,KAAI,MAAM,OAAO,MAAM,WAAW,IAAI,UAAU,CAAC,CAAE,CAAC,CAAC,KAAK,GAAG,IAClE;EAEJ,IAAI,SACF,WAAW,IAAI,UAAU,OAAO;EAGlC,MAAM,UAAqB;GACzB,cAAc;GACd,gBAAgB,WAAW,KAAK;GAChC,MAAM;EACR;EACA,IAAI,SACF,QAAQ,aAAa;EAEvB,KAAK,QAAQ,KAAK,OAAO;CAC3B;CAEA,MAAM,QAAuB;EAC3B,MAAM,KAAK,gBAAgB,aAAa;CAC1C;AACF;AAEA,SAAS,WAAW,OAAiC;CACnD,QAAQ,OAAR;EACE,KAAK,SACH,OAAO,eAAe;EACxB,KAAK,QACH,OAAO,eAAe;EACxB,KAAK,QACH,OAAO,eAAe;EACxB,KAAK,SACH,OAAO,eAAe;EACxB,SACE,MAAM,IAAI,MAAM,sBAAsB,OAAO;CACjD;AACF"}

package/out/zero-cache/src/server/otel-start.js.map

@@ -1 +1 @@
-{"version":3,"file":"otel-start.js","names":["#instance","#started"],"sources":["../../../../../zero-cache/src/server/otel-start.ts"],"sourcesContent":["import {logs} from '@opentelemetry/api-logs';\nimport {getNodeAutoInstrumentations} from '@opentelemetry/auto-instrumentations-node';\nimport {resourceFromAttributes} from '@opentelemetry/resources';\nimport {AggregationType, InstrumentType} from '@opentelemetry/sdk-metrics';\nimport {NodeSDK} from '@opentelemetry/sdk-node';\nimport {ATTR_SERVICE_VERSION} from '@opentelemetry/semantic-conventions';\nimport type {LogContext} from '@rocicorp/logger';\nimport {\n  otelEnabled,\n  otelLogsEnabled,\n  otelMetricsEnabled,\n  otelTracesEnabled,\n} from '../../../otel/src/enabled.ts';\nimport {NATIVE_HISTOGRAM_INSTRUMENT_NAMES} from '../observability/metrics.ts';\nimport {setupOtelDiagnosticLogger} from './otel-diag-logger.ts';\n\nclass OtelManager {\n  static #instance: OtelManager;\n  #started = false;\n\n  private constructor() {}\n\n  static getInstance(): OtelManager {\n    if (!OtelManager.#instance) {\n      OtelManager.#instance = new OtelManager();\n    }\n    return OtelManager.#instance;\n  }\n\n  startOtelAuto(\n    lc: LogContext | undefined,\n    workerName: string,\n    workerIndex: number,\n  ) {\n    if (this.#started || !otelEnabled()) {\n      return;\n    }\n    this.#started = true;\n\n    // Store and temporarily remove OTEL_LOG_LEVEL to prevent NodeSDK from setting its own logger\n    const otelLogLevel = process.env.OTEL_LOG_LEVEL;\n    delete process.env.OTEL_LOG_LEVEL;\n\n    // Use exponential histograms by default to reduce cardinality from auto-instrumentation\n    // This affects HTTP server/client and other auto-instrumented histogram metrics\n    // Exponential histograms automatically adjust bucket boundaries and use fewer buckets\n    process.env.OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION ??=\n      'base2_exponential_bucket_histogram';\n\n    const resource = resourceFromAttributes({\n      [ATTR_SERVICE_VERSION]: process.env.ZERO_SERVER_VERSION ?? 'unknown',\n      // Tag every metric/trace/log with the worker name and index so each\n      // worker process in a multi-worker pod is distinguishable. Without\n      // this, N syncer workers sharing the same pod labels clobber each\n      // other in the OTel collector on every scrape interval.\n      // These mirror the 'worker' and 'workerIndex' keys in every log\n      // context so logs and metrics can be correlated on the same fields.\n      // Using a stable index instead of PID avoids label churn in Prometheus.\n      'process.worker': workerName,\n      'process.worker_index': workerIndex,\n    });\n\n    // Set defaults to be backwards compatible with the previously\n    // hard-coded exporters\n    process.env.OTEL_EXPORTER_OTLP_PROTOCOL ??= 'http/json';\n    process.env.OTEL_METRICS_EXPORTER ??= otelMetricsEnabled()\n      ? 'otlp'\n      : 'none';\n    process.env.OTEL_TRACES_EXPORTER ??= otelTracesEnabled() ? 'otlp' : 'none';\n    process.env.OTEL_LOGS_EXPORTER ??= otelLogsEnabled() ? 'otlp' : 'none';\n\n    const sdk = new NodeSDK({\n      resource,\n      autoDetectResources: true,\n      views: NATIVE_HISTOGRAM_INSTRUMENT_NAMES.map(instrumentName => ({\n        instrumentName,\n        instrumentType: InstrumentType.HISTOGRAM,\n        aggregation: {type: AggregationType.EXPONENTIAL_HISTOGRAM},\n      })),\n      instrumentations:\n        process.env.OTEL_NODE_ENABLED_INSTRUMENTATIONS ||\n        process.env.OTEL_NODE_DISABLED_INSTRUMENTATIONS\n          ? [getNodeAutoInstrumentations()]\n          : [],\n    });\n\n    try {\n      sdk.start();\n    } finally {\n      if (otelLogLevel) {\n        process.env.OTEL_LOG_LEVEL = otelLogLevel;\n      }\n    }\n    setupOtelDiagnosticLogger(lc, true);\n\n    logs.getLogger('zero-cache').emit({\n      severityText: 'INFO',\n      body: 'OpenTelemetry SDK started successfully',\n    });\n  }\n}\n\nexport const startOtelAuto = (\n  lc: LogContext | undefined,\n  workerName: string,\n  workerIndex: number,\n) => OtelManager.getInstance().startOtelAuto(lc, workerName, workerIndex);\n"],"mappings":";;;;;;;;;;AAgBA,IAAM,cAAN,MAAM,YAAY;CAChB,OAAOA;CACP,WAAW;CAEX,cAAsB,CAAC;CAEvB,OAAO,cAA2B;EAChC,IAAI,CAAC,YAAYA,WACf,YAAYA,YAAY,IAAI,YAAY;EAE1C,OAAO,YAAYA;CACrB;CAEA,cACE,IACA,YACA,aACA;EACA,IAAI,KAAKC,YAAY,CAAC,YAAY,GAChC;EAEF,KAAKA,WAAW;EAGhB,MAAM,eAAe,QAAQ,IAAI;EACjC,OAAO,QAAQ,IAAI;EAKnB,QAAQ,IAAI,6DACV;EAEF,MAAM,WAAW,uBAAuB;IACrC,uBAAuB,QAAQ,IAAI,uBAAuB;GAQ3D,kBAAkB;GAClB,wBAAwB;EAC1B,CAAC;EAID,QAAQ,IAAI,gCAAgC;EAC5C,QAAQ,IAAI,0BAA0B,mBAAmB,IACrD,SACA;EACJ,QAAQ,IAAI,yBAAyB,kBAAkB,IAAI,SAAS;EACpE,QAAQ,IAAI,uBAAuB,gBAAgB,IAAI,SAAS;EAEhE,MAAM,MAAM,IAAI,QAAQ;GACtB;GACA,qBAAqB;GACrB,OAAO,kCAAkC,KAAI,oBAAmB;IAC9D;IACA,gBAAgB,eAAe;IAC/B,aAAa,EAAC,MAAM,gBAAgB,sBAAqB;GAC3D,EAAE;GACF,kBACE,QAAQ,IAAI,sCACZ,QAAQ,IAAI,sCACR,CAAC,4BAA4B,CAAC,IAC9B,CAAC;EACT,CAAC;EAED,IAAI;GACF,IAAI,MAAM;EACZ,UAAU;GACR,IAAI,cACF,QAAQ,IAAI,iBAAiB;EAEjC;EACA,0BAA0B,IAAI,IAAI;EAElC,KAAK,UAAU,YAAY,EAAE,KAAK;GAChC,cAAc;GACd,MAAM;EACR,CAAC;CACH;AACF;AAEA,IAAa,iBACX,IACA,YACA,gBACG,YAAY,YAAY,EAAE,cAAc,IAAI,YAAY,WAAW"}
+{"version":3,"file":"otel-start.js","names":[],"sources":["../../../../../zero-cache/src/server/otel-start.ts"],"sourcesContent":["import {logs} from '@opentelemetry/api-logs';\nimport {getNodeAutoInstrumentations} from '@opentelemetry/auto-instrumentations-node';\nimport {resourceFromAttributes} from '@opentelemetry/resources';\nimport {AggregationType, InstrumentType} from '@opentelemetry/sdk-metrics';\nimport {NodeSDK} from '@opentelemetry/sdk-node';\nimport {ATTR_SERVICE_VERSION} from '@opentelemetry/semantic-conventions';\nimport type {LogContext} from '@rocicorp/logger';\nimport {\n  otelEnabled,\n  otelLogsEnabled,\n  otelMetricsEnabled,\n  otelTracesEnabled,\n} from '../../../otel/src/enabled.ts';\nimport {NATIVE_HISTOGRAM_INSTRUMENT_NAMES} from '../observability/metrics.ts';\nimport {setupOtelDiagnosticLogger} from './otel-diag-logger.ts';\n\nclass OtelManager {\n  static #instance: OtelManager;\n  #started = false;\n\n  private constructor() {}\n\n  static getInstance(): OtelManager {\n    if (!OtelManager.#instance) {\n      OtelManager.#instance = new OtelManager();\n    }\n    return OtelManager.#instance;\n  }\n\n  startOtelAuto(\n    lc: LogContext | undefined,\n    workerName: string,\n    workerIndex: number,\n  ) {\n    if (this.#started || !otelEnabled()) {\n      return;\n    }\n    this.#started = true;\n\n    // Store and temporarily remove OTEL_LOG_LEVEL to prevent NodeSDK from setting its own logger\n    const otelLogLevel = process.env.OTEL_LOG_LEVEL;\n    delete process.env.OTEL_LOG_LEVEL;\n\n    // Use exponential histograms by default to reduce cardinality from auto-instrumentation\n    // This affects HTTP server/client and other auto-instrumented histogram metrics\n    // Exponential histograms automatically adjust bucket boundaries and use fewer buckets\n    process.env.OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION ??=\n      'base2_exponential_bucket_histogram';\n\n    const resource = resourceFromAttributes({\n      [ATTR_SERVICE_VERSION]: process.env.ZERO_SERVER_VERSION ?? 'unknown',\n      // Tag every metric/trace/log with the worker name and index so each\n      // worker process in a multi-worker pod is distinguishable. Without\n      // this, N syncer workers sharing the same pod labels clobber each\n      // other in the OTel collector on every scrape interval.\n      // These mirror the 'worker' and 'workerIndex' keys in every log\n      // context so logs and metrics can be correlated on the same fields.\n      // Using a stable index instead of PID avoids label churn in Prometheus.\n      'process.worker': workerName,\n      'process.worker_index': workerIndex,\n    });\n\n    // Set defaults to be backwards compatible with the previously\n    // hard-coded exporters\n    process.env.OTEL_EXPORTER_OTLP_PROTOCOL ??= 'http/json';\n    process.env.OTEL_METRICS_EXPORTER ??= otelMetricsEnabled()\n      ? 'otlp'\n      : 'none';\n    process.env.OTEL_TRACES_EXPORTER ??= otelTracesEnabled() ? 'otlp' : 'none';\n    process.env.OTEL_LOGS_EXPORTER ??= otelLogsEnabled() ? 'otlp' : 'none';\n\n    const sdk = new NodeSDK({\n      resource,\n      autoDetectResources: true,\n      views: NATIVE_HISTOGRAM_INSTRUMENT_NAMES.map(instrumentName => ({\n        instrumentName,\n        instrumentType: InstrumentType.HISTOGRAM,\n        aggregation: {type: AggregationType.EXPONENTIAL_HISTOGRAM},\n      })),\n      instrumentations:\n        process.env.OTEL_NODE_ENABLED_INSTRUMENTATIONS ||\n        process.env.OTEL_NODE_DISABLED_INSTRUMENTATIONS\n          ? [getNodeAutoInstrumentations()]\n          : [],\n    });\n\n    try {\n      sdk.start();\n    } finally {\n      if (otelLogLevel) {\n        process.env.OTEL_LOG_LEVEL = otelLogLevel;\n      }\n    }\n    setupOtelDiagnosticLogger(lc, true);\n\n    logs.getLogger('zero-cache').emit({\n      severityText: 'INFO',\n      body: 'OpenTelemetry SDK started successfully',\n    });\n  }\n}\n\nexport const startOtelAuto = (\n  lc: LogContext | undefined,\n  workerName: string,\n  workerIndex: number,\n) => OtelManager.getInstance().startOtelAuto(lc, workerName, workerIndex);\n"],"mappings":";;;;;;;;;;AAgBA,IAAM,cAAN,MAAM,YAAY;CAChB,OAAO;CACP,WAAW;CAEX,cAAsB,CAAC;CAEvB,OAAO,cAA2B;EAChC,IAAI,CAAC,YAAY,WACf,YAAY,YAAY,IAAI,YAAY;EAE1C,OAAO,YAAY;CACrB;CAEA,cACE,IACA,YACA,aACA;EACA,IAAI,KAAK,YAAY,CAAC,YAAY,GAChC;EAEF,KAAK,WAAW;EAGhB,MAAM,eAAe,QAAQ,IAAI;EACjC,OAAO,QAAQ,IAAI;EAKnB,QAAQ,IAAI,6DACV;EAEF,MAAM,WAAW,uBAAuB;IACrC,uBAAuB,QAAQ,IAAI,uBAAuB;GAQ3D,kBAAkB;GAClB,wBAAwB;EAC1B,CAAC;EAID,QAAQ,IAAI,gCAAgC;EAC5C,QAAQ,IAAI,0BAA0B,mBAAmB,IACrD,SACA;EACJ,QAAQ,IAAI,yBAAyB,kBAAkB,IAAI,SAAS;EACpE,QAAQ,IAAI,uBAAuB,gBAAgB,IAAI,SAAS;EAEhE,MAAM,MAAM,IAAI,QAAQ;GACtB;GACA,qBAAqB;GACrB,OAAO,kCAAkC,KAAI,oBAAmB;IAC9D;IACA,gBAAgB,eAAe;IAC/B,aAAa,EAAC,MAAM,gBAAgB,sBAAqB;GAC3D,EAAE;GACF,kBACE,QAAQ,IAAI,sCACZ,QAAQ,IAAI,sCACR,CAAC,4BAA4B,CAAC,IAC9B,CAAC;EACT,CAAC;EAED,IAAI;GACF,IAAI,MAAM;EACZ,UAAU;GACR,IAAI,cACF,QAAQ,IAAI,iBAAiB;EAEjC;EACA,0BAA0B,IAAI,IAAI;EAElC,KAAK,UAAU,YAAY,CAAC,CAAC,KAAK;GAChC,cAAc;GACd,MAAM;EACR,CAAC;CACH;AACF;AAEA,IAAa,iBACX,IACA,YACA,gBACG,YAAY,YAAY,CAAC,CAAC,cAAc,IAAI,YAAY,WAAW"}

package/out/zero-cache/src/server/reaper.js

@@ -14,7 +14,7 @@ import { CVRPurger } from "../services/view-syncer/cvr-purger.js";
 import { initViewSyncerSchema } from "../services/view-syncer/schema/init.js";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 //#region ../zero-cache/src/server/reaper.ts
-var MS_PER_HOUR = 1e3 * 60 * 60;
+var MS_PER_HOUR = 36e5;
 var lc = new LogContext("info", {}, consoleLogSink);
 async function runWorker(parent, env, ...argv) {
 	installProfileHandler(parent, "reaper");
@@ -35,7 +35,7 @@ async function runWorker(parent, env, ...argv) {
 		inactivityThresholdMs: cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,
 		initialBatchSize: cvr.garbageCollectionInitialBatchSize,
 		initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1e3
-	}), new ActiveUsersGauge(lc, cvrDB, shard, { updateIntervalMs: 600 * 1e3 }));
+	}), new ActiveUsersGauge(lc, cvrDB, shard, { updateIntervalMs: 6e5 }));
 }
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
 //#endregion

package/out/zero-cache/src/server/reaper.js.map

@@ -1 +1 @@
-{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'reaper');\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAoBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,QAAQ;CACtC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,IAC9B,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...argv: string[]\n): Promise<void> {\n  installProfileHandler(parent, 'reaper');\n  const config = getNormalizedZeroConfig({env, argv});\n\n  startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n  lc = createLogContext(config, 'reaper');\n  initEventSink(lc, config);\n  startAnonymousTelemetry(lc, config);\n\n  const {cvr} = config;\n  const shard = getShardID(config);\n  const cvrDB = await connectPgClient(lc, cvr.db, `sync-cvr-purger`, {\n    max: 1,\n  });\n  await initViewSyncerSchema(lc, cvrDB, shard);\n  parent.send(['ready', {ready: true}]);\n\n  return runUntilKilled(\n    lc,\n    parent,\n    new CVRPurger(lc, cvrDB, shard, {\n      inactivityThresholdMs:\n        cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n      initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n      initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n    }),\n    // Periodically computes and exports active users gauge to anonymous telemetry\n    new ActiveUsersGauge(lc, cvrDB, shard, {\n      // Default 10minutes refresh; can be made configurable later if needed\n      updateIntervalMs: 10 * 60 * 1000,\n    }),\n  );\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAoBA,IAAM,cAAc;AAGpB,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,sBAAsB,QAAQ,QAAQ;CACtC,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAElD,cAAc,iBAAiB,QAAQ,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC;CACvE,KAAK,iBAAiB,QAAQ,QAAQ;CACtC,cAAc,IAAI,MAAM;CACxB,wBAAwB,IAAI,MAAM;CAElC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,MAAM;CAC/B,MAAM,QAAQ,MAAM,gBAAgB,IAAI,IAAI,IAAI,mBAAmB,EACjE,KAAK,EACP,CAAC;CACD,MAAM,qBAAqB,IAAI,OAAO,KAAK;CAC3C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,OAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;CACnE,CAAC,GAED,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,IACpB,CAAC,CACH;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/replicator.js

@@ -6,8 +6,8 @@ import { parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { Database } from "../../../zqlite/src/db.js";
-import { getOrCreateGauge } from "../observability/metrics.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
+import { getOrCreateGauge } from "../observability/metrics.js";
 import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { installProfileHandler } from "../types/profiler.js";