@rocicorp/zero 1.9.1-canary.1 → 1.10.0-canary.1

package/out/zero-cache/src/server/replicator.js

@@ -1,4 +1,5 @@
 import { assert } from "../../../shared/src/asserts.js";
+import { sleep } from "../../../shared/src/sleep.js";
 import { parse } from "../../../shared/src/valita.js";
 import { must } from "../../../shared/src/must.js";
 import { parentWorker, singleProcessMode } from "../types/processes.js";
@@ -10,10 +11,14 @@ import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
 import { createLogContext } from "./logging.js";
 import { initEventSink } from "../observability/events.js";
 import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
+import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../services/litestream/metrics.js";
+import { tryRestore } from "../services/litestream/commands.js";
+import { reserveAndGetSnapshotStatus } from "../services/change-streamer/snapshot.js";
 import { ChangeStreamerHttpClient } from "../services/change-streamer/change-streamer-http.js";
+import { sqliteFileBytes } from "../services/replicator/sqlite-change-log-observability.js";
 import { startOtelAuto } from "./otel-start.js";
 import { ThreadWriteWorkerClient } from "../services/replicator/write-worker-client.js";
-import { getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
+import { deleteStaleChangeLog, getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
 import { ReplicatorService } from "../services/replicator/replicator.js";
 import { LogContext, consoleLogSink } from "@rocicorp/logger";
 import { pid } from "node:process";
@@ -28,6 +33,7 @@ async function runWorker(parent, env, ...args) {
 		argv: args.slice(1)
 	});
 	const mode = fileMode === "backup" ? "backup" : "serving";
+	const runningLocalChangeStreamer = config.changeStreamer.mode === "dedicated" && !config.changeStreamer.uri;
 	const workerName = `${mode}-replicator`;
 	startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);
 	lc = createLogContext(config, workerName);
@@ -36,21 +42,22 @@ async function runWorker(parent, env, ...args) {
 		dbPath: config.replica.file
 	}, config.sqliteCorruptionChecks);
 	initEventSink(lc, config);
+	if (fileMode === "serving" && (config.litestream.executable || config.litestream.executableV5)) await restoreReplica(lc, config);
 	const { file: dbPath, walMode } = await setupReplica(lc, fileMode, config.replica);
 	unregisterInitialCorruptionDiagnosticTarget();
 	registerSQLiteCorruptionDiagnosticTarget({
 		debugName: `${workerName} replica`,
 		dbPath
 	}, config.sqliteCorruptionChecks);
+	deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);
 	setupMetrics(lc, dbPath, walMode);
 	const pragmas = getPragmaConfig(fileMode);
 	const workerClient = new ThreadWriteWorkerClient();
 	await workerClient.init(dbPath, mode, pragmas, config.log);
-	const runningLocalChangeStreamer = config.changeStreamer.mode === "dedicated" && !config.changeStreamer.uri;
 	const shard = getShardConfig(config);
 	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
 	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
-	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
+	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, dbPath, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
 	setUpMessageHandlers(lc, replicator, parent);
 	const running = runUntilKilled(lc, parent, replicator);
 	for await (const _ of replicator.subscribe()) {
@@ -72,6 +79,10 @@ function setupMetrics(lc, file, walMode) {
 		description: `The size of the replica's wal2 file`,
 		unit: "bytes"
 	}).addCallback(observeFileSize(lc, `${file}-wal2`));
+	getOrCreateGauge("replica", "file_bytes", {
+		description: "The replica's total on-disk footprint: its main db file plus its wal sidecars.",
+		unit: "By"
+	}).addCallback(observeSQLiteFileBytes(lc, file));
 }
 function observeFileSize(lc, file) {
 	return async (o) => {
@@ -83,6 +94,38 @@ function observeFileSize(lc, file) {
 		}
 	};
 }
+function observeSQLiteFileBytes(lc, file) {
+	return async (o) => o.observe(await sqliteFileBytes(lc, file));
+}
+var RETRY_INTERVAL_MS = 3e3;
+async function restoreReplica(lc, config) {
+	const start = performance.now();
+	let backupURL;
+	let result;
+	try {
+		for (;;) {
+			const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);
+			({backupURL} = snapshotStatus);
+			const attempt = await tryRestore(lc, {
+				...config.litestream,
+				backupURL
+			}, config.replica.file, snapshotStatus, "view_syncer");
+			if (attempt.restored) {
+				result = attempt.result;
+				return;
+			}
+			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
+			await sleep(RETRY_INTERVAL_MS);
+		}
+	} finally {
+		const labels = {
+			...litestreamRestoreMetricAttrs(config.litestream, "view_syncer", backupURL),
+			result: result ?? "error"
+		};
+		litestreamRestoreRuns().add(1, labels);
+		litestreamRestoreDuration().recordMs(performance.now() - start, labels);
+	}
+}
 if (!singleProcessMode()) exitAfter(() => lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
 //#endregion
 export { runWorker as default };

package/out/zero-cache/src/server/replicator.js.map

@@ -1 +1 @@
-{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  const {file: dbPath, walMode} = await setupReplica(\n    lc,\n    fileMode,\n    config.replica,\n  );\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes.\n  const pragmas = getPragmaConfig(fileMode);\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log);\n\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAEA,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;AAEtD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
+{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n  LitestreamConfig,\n  NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n  tryRestore,\n  type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n  ReplicatorService,\n  type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n  deleteStaleChangeLog,\n  getPragmaConfig,\n  replicaFileModeSchema,\n  setUpMessageHandlers,\n  setupReplica,\n  type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n  const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n  const runningLocalChangeStreamer =\n    config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n  const workerName = `${mode}-replicator`;\n  startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n  lc = createLogContext(config, workerName);\n  const unregisterInitialCorruptionDiagnosticTarget =\n    registerSQLiteCorruptionDiagnosticTarget(\n      {\n        debugName: `${workerName} replica`,\n        dbPath: config.replica.file,\n      },\n      config.sqliteCorruptionChecks,\n    );\n  initEventSink(lc, config);\n\n  if (\n    fileMode === 'serving' &&\n    (config.litestream.executable || config.litestream.executableV5)\n  ) {\n    await restoreReplica(lc, config);\n  }\n\n  const {file: dbPath, walMode} = await setupReplica(\n    lc,\n    fileMode,\n    config.replica,\n  );\n  unregisterInitialCorruptionDiagnosticTarget();\n  registerSQLiteCorruptionDiagnosticTarget(\n    {\n      debugName: `${workerName} replica`,\n      dbPath,\n    },\n    config.sqliteCorruptionChecks,\n  );\n\n  // A file left behind by a run with the writer enabled would otherwise hold\n  // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n  // this is keyed on the config flag and nothing else.\n  deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n  setupMetrics(lc, dbPath, walMode);\n\n  // Create the write worker for async SQLite writes.\n  const pragmas = getPragmaConfig(fileMode);\n  const workerClient = new ThreadWriteWorkerClient();\n  await workerClient.init(dbPath, mode, pragmas, config.log);\n\n  const shard = getShardConfig(config);\n  const {\n    taskID,\n    change,\n    changeStreamer: {\n      port,\n      uri: changeStreamerURI = runningLocalChangeStreamer\n        ? `http://localhost:${port}/`\n        : undefined,\n    },\n  } = config;\n  const changeStreamer = new ChangeStreamerHttpClient(\n    lc,\n    shard,\n    change.db,\n    changeStreamerURI,\n  );\n\n  const replicator = new ReplicatorService(\n    lc,\n    taskID,\n    `${workerName}-${pid}`,\n    mode,\n    dbPath,\n    changeStreamer,\n    workerClient,\n    runningLocalChangeStreamer\n      ? // publish ReplicationStatusEvents from backup-replicator only\n        ReplicationStatusPublisher.forReplicaFile(dbPath)\n      : null,\n  );\n\n  setUpMessageHandlers(lc, replicator, parent);\n\n  const running = runUntilKilled(lc, parent, replicator);\n\n  // Signal readiness once the first ReplicaVersionReady notification is received.\n  for await (const _ of replicator.subscribe()) {\n    parent.send(['ready', {ready: true}]);\n    break;\n  }\n\n  return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n  getOrCreateGauge('replica', 'db_size', {\n    description:\n      `The size of the replica's main db file, ` +\n      `which does not include the wal file(s)`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, file));\n\n  getOrCreateGauge('replica', 'wal_size', {\n    description: `The size of the replica's wal file`,\n    unit: 'bytes',\n  }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n  if (walMode === 'wal2') {\n    getOrCreateGauge('replica', 'wal2_size', {\n      description: `The size of the replica's wal2 file`,\n      unit: 'bytes',\n    }).addCallback(observeFileSize(lc, `${file}-wal2`));\n  }\n\n  // A whole-database total rather than a per-file one. Its counterpart,\n  // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n  // the process that writes the log.\n  getOrCreateGauge('replica', 'file_bytes', {\n    description:\n      `The replica's total on-disk footprint: its main db file plus its ` +\n      `wal sidecars.`,\n    unit: 'By',\n  }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n  return async o => {\n    try {\n      const stats = await stat(file);\n      o.observe(stats.size);\n    } catch (e) {\n      lc.warn?.(`unable to stat ${file} for size metrics`, e);\n    }\n  };\n}\n\nfunction observeSQLiteFileBytes(\n  lc: LogContext,\n  file: string,\n): ObservableCallback {\n  return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n  const start = performance.now();\n  let backupURL: string | undefined;\n  let result: RestoreResult | undefined;\n  try {\n    for (;;) {\n      const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n      // The backupURL comes from the replication-manager's snapshot response.\n      ({backupURL} = snapshotStatus);\n      const litestream: LitestreamConfig = {...config.litestream, backupURL};\n      const attempt = await tryRestore(\n        lc,\n        litestream,\n        config.replica.file,\n        snapshotStatus,\n        'view_syncer',\n      );\n      if (attempt.restored) {\n        result = attempt.result;\n        return;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(\n      config.litestream,\n      'view_syncer',\n      backupURL,\n    );\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(\n    () => lc,\n    () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,IACE,aAAa,cACZ,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,QACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/server/runner/run-worker.js

@@ -66,8 +66,9 @@ async function runWorker(parent, env) {
 		}, startZeroCache, () => printStartupMessage(env)));
 	} catch (err) {
 		processes.logErrorAndExit(err, "main");
+	} finally {
+		await processes.shutdown();
 	}
-	await processes.done();
 }
 //#endregion
 export { runWorker };

package/out/zero-cache/src/server/runner/run-worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {ProcessManager, runUntilKilled} from '../../services/life-cycle.ts';\nimport {childWorker, type Worker} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent ?? process,\n      new ZeroDispatcher(\n        config,\n        lc,\n        {port, keepaliveTimeoutMs},\n        startZeroCache,\n        () => printStartupMessage(env),\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  }\n\n  await processes.done();\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAGzC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,MADrB,UAAU,EAAE,CACsB;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,gBAAe;CAChD,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,EAChC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,EACA,KAAK,SAAS,EAAE,MAAM;GAEzB,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eACJ,IACA,UAAU,SACV,IAAI,eACF,QACA,IACA;GAAC;GAAM;EAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC;CAEA,MAAM,UAAU,KAAK;AACvB"}
+{"version":3,"file":"run-worker.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/run-worker.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {styleText} from 'node:util';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {colorConsole} from '../../../../shared/src/logging.ts';\nimport {PROTOCOL_VERSION} from '../../../../zero-protocol/src/protocol-version.ts';\nimport {normalizeZeroConfig} from '../../config/normalize.ts';\nimport {getServerVersion, getZeroConfig} from '../../config/zero-config.ts';\nimport {ProcessManager, runUntilKilled} from '../../services/life-cycle.ts';\nimport {childWorker, type Worker} from '../../types/processes.ts';\nimport {createLogContext} from '../logging.ts';\nimport {MAIN_URL} from '../worker-urls.ts';\nimport {getTaskID} from './runtime.ts';\nimport {ZeroDispatcher} from './zero-dispatcher.ts';\n\nconst startupMessageEnv = 'ZERO_ENABLE_STARTUP_MESSAGE';\n\nfunction printStartupMessage(env: NodeJS.ProcessEnv) {\n  if (env[startupMessageEnv] !== '1') {\n    return;\n  }\n\n  colorConsole.log(\n    `\\nBTW, ${styleText(['bold', 'cyan'], 'Cloud Zero')} ` +\n      'is now available - professional Zero hosting from the team that built it.\\n' +\n      `Get started now: ${styleText(['blue', 'underline'], 'https://zero.rocicorp.dev/cloud')}\\n\\n` +\n      styleText('dim', `Disable this message with ${startupMessageEnv}=0`) +\n      '\\n',\n  );\n}\n\n/**\n * Top-level `runner` entry point to the zero-cache. This layer is responsible for:\n * * runtime-based config normalization\n * * lazy startup\n * * serving /statsz\n * * auto-reset restarts (TODO)\n */\nexport async function runWorker(\n  parent: Worker | null,\n  env: NodeJS.ProcessEnv,\n): Promise<void> {\n  // Note: Deprecation warnings are only emitted at this top-level parse;\n  //       they are suppressed when parsed in subprocesses.\n  const cfg = getZeroConfig({env, emitDeprecationWarnings: true});\n  const lc = createLogContext(cfg, 'runner');\n\n  const defaultTaskID = await getTaskID(lc);\n  const config = normalizeZeroConfig(lc, cfg, env, defaultTaskID);\n  const processes = new ProcessManager(lc, parent ?? process);\n\n  const {port, keepaliveTimeoutMs, lazyStartup} = config;\n  const serverVersion = getServerVersion(config);\n  lc.info?.(`starting server${!serverVersion ? '' : `@${serverVersion}`} `, {\n    protocolVersion: PROTOCOL_VERSION,\n    taskID: config.taskID,\n    app: config.app,\n    shard: config.shard,\n    port: config.port,\n  });\n\n  let zeroCache: Resolver<Worker> | undefined;\n  function startZeroCache(): Promise<Worker> {\n    if (zeroCache === undefined) {\n      const startMs = performance.now();\n      lc.info?.('starting zero-cache');\n\n      const r = (zeroCache = resolver<Worker>());\n      const w = childWorker(MAIN_URL, env)\n        .once('message', () => {\n          r.resolve(w);\n          lc.info?.(`zero-cache ready (${performance.now() - startMs} ms)`);\n        })\n        .once('error', r.reject);\n\n      processes.addWorker(w, 'user-facing', 'zero-cache');\n    }\n    return zeroCache.promise;\n  }\n\n  // Eagerly start the zero-cache if it was not configured with --lazy-startup.\n  if (!lazyStartup) {\n    void startZeroCache();\n  }\n\n  await processes.allWorkersReady();\n  parent?.send(['ready', {ready: true}]);\n\n  try {\n    await runUntilKilled(\n      lc,\n      parent ?? process,\n      new ZeroDispatcher(\n        config,\n        lc,\n        {port, keepaliveTimeoutMs},\n        startZeroCache,\n        () => printStartupMessage(env),\n      ),\n    );\n  } catch (err) {\n    processes.logErrorAndExit(err, 'main');\n  } finally {\n    await processes.shutdown();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAeA,IAAM,oBAAoB;AAE1B,SAAS,oBAAoB,KAAwB;CACnD,IAAI,IAAI,uBAAuB,KAC7B;CAGF,aAAa,IACX,UAAU,UAAU,CAAC,QAAQ,MAAM,GAAG,YAAY,EAAE;mBAE9B,UAAU,CAAC,QAAQ,WAAW,GAAG,iCAAiC,EAAE,QACxF,UAAU,OAAO,6BAA6B,kBAAkB,GAAG,IACnE,IACJ;AACF;;;;;;;;AASA,eAAsB,UACpB,QACA,KACe;CAGf,MAAM,MAAM,cAAc;EAAC;EAAK,yBAAyB;CAAI,CAAC;CAC9D,MAAM,KAAK,iBAAiB,KAAK,QAAQ;CAGzC,MAAM,SAAS,oBAAoB,IAAI,KAAK,KAAK,MADrB,UAAU,EAAE,CACsB;CAC9D,MAAM,YAAY,IAAI,eAAe,IAAI,UAAU,OAAO;CAE1D,MAAM,EAAC,MAAM,oBAAoB,gBAAe;CAChD,MAAM,gBAAgB,iBAAiB,MAAM;CAC7C,GAAG,OAAO,kBAAkB,CAAC,gBAAgB,KAAK,IAAI,gBAAgB,IAAI;EACxE,iBAAA;EACA,QAAQ,OAAO;EACf,KAAK,OAAO;EACZ,OAAO,OAAO;EACd,MAAM,OAAO;CACf,CAAC;CAED,IAAI;CACJ,SAAS,iBAAkC;EACzC,IAAI,cAAc,KAAA,GAAW;GAC3B,MAAM,UAAU,YAAY,IAAI;GAChC,GAAG,OAAO,qBAAqB;GAE/B,MAAM,IAAK,YAAY,SAAiB;GACxC,MAAM,IAAI,YAAY,UAAU,GAAG,EAChC,KAAK,iBAAiB;IACrB,EAAE,QAAQ,CAAC;IACX,GAAG,OAAO,qBAAqB,YAAY,IAAI,IAAI,QAAQ,KAAK;GAClE,CAAC,EACA,KAAK,SAAS,EAAE,MAAM;GAEzB,UAAU,UAAU,GAAG,eAAe,YAAY;EACpD;EACA,OAAO,UAAU;CACnB;CAGA,IAAI,CAAC,aACH,eAAoB;CAGtB,MAAM,UAAU,gBAAgB;CAChC,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAErC,IAAI;EACF,MAAM,eACJ,IACA,UAAU,SACV,IAAI,eACF,QACA,IACA;GAAC;GAAM;EAAkB,GACzB,sBACM,oBAAoB,GAAG,CAC/B,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,MAAM;CACvC,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF"}

package/out/zero-cache/src/server/worker-urls.d.ts

@@ -1,5 +1,5 @@
 export declare const CHANGE_STREAMER_URL: URL;
-export declare const BACKUP_WATERMARK_READER_URL: URL;
+export declare const BACKUP_WATERMARK_POLLER_URL: URL;
 export declare const MAIN_URL: URL;
 export declare const MUTATOR_URL: URL;
 export declare const REAPER_URL: URL;

package/out/zero-cache/src/server/worker-urls.js

@@ -6,7 +6,7 @@ function resolve(path) {
 	return new URL(path, url);
 }
 var CHANGE_STREAMER_URL = resolve("./change-streamer.ts");
-var BACKUP_WATERMARK_READER_URL = resolve("./backup-watermark-reader.ts");
+var BACKUP_WATERMARK_POLLER_URL = resolve("./backup-watermark-poller.ts");
 var MAIN_URL = resolve("./main.ts");
 resolve("./mutator.ts");
 var REAPER_URL = resolve("./reaper.ts");
@@ -15,6 +15,6 @@ var SHADOW_SYNCER_URL = resolve("./shadow-syncer.ts");
 var SYNCER_URL = resolve("./syncer.ts");
 var WRITE_WORKER_URL = resolve("./write-worker.ts");
 //#endregion
-export { BACKUP_WATERMARK_READER_URL, CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
+export { BACKUP_WATERMARK_POLLER_URL, CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SYNCER_URL, WRITE_WORKER_URL };
 
 //# sourceMappingURL=worker-urls.js.map

package/out/zero-cache/src/server/worker-urls.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const BACKUP_WATERMARK_READER_URL = resolve(\n  './backup-watermark-reader.ts',\n);\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,8BAA8B,QACzC,8BACF;AACA,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
+{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n  const {url} = import.meta;\n  if (url.endsWith('.js')) {\n    // When compiled, change .ts to .js\n    path = path.replace(tsRe, '.js');\n  }\n  return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const BACKUP_WATERMARK_POLLER_URL = resolve(\n  './backup-watermark-poller.ts',\n);\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;CACrB,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,8BAA8B,QACzC,8BACF;AACA,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}

package/out/zero-cache/src/services/change-source/change-source.d.ts

@@ -11,10 +11,12 @@ export type ChangeStream = {
 }; /** Encapsulates an upstream-specific implementation of a stream of Changes. */
 export interface ChangeSource {
     /**
-     * Starts a replication lag reporter, returning the send time of the next
-     * expected report, or `null` if lag reporting is not supported / enabled.
+     * Starts a replication lag reporter, returning the commit time of the first
+     * report, and the send time of the next expected report, or `null` if lag
+     * reporting is not supported / enabled.
      */
     startLagReporter(): Promise<{
+        firstCommitTimeMs: number;
         nextSendTimeMs: number;
     } | null> | null;
     /**

package/out/zero-cache/src/services/change-source/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEpE;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEjB;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts

@@ -0,0 +1,35 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { LitestreamConfig } from '../../../config/normalize.ts';
+import { type ReplicaConstraints } from '../../litestream/commands.ts';
+import type { SubscriptionState } from '../../replicator/schema/replication-state.ts';
+import type { ChangeSource } from '../change-source.ts';
+export type RestoreOptions = {
+    litestream?: LitestreamConfig;
+    constraints?: ReplicaConstraints | undefined;
+};
+export type InitializeResult = {
+    subscriptionState: SubscriptionState;
+    changeSource: ChangeSource;
+    destinationBackupURL: string | undefined;
+    /**
+     * The replica this change stream belongs to, from the upstream `replicas`
+     * table. It is part of the identity the SQLite change log records, because a
+     * generation (i.e. `replicaVersion`) is shared by every sibling of a forked
+     * replica and so cannot distinguish two siblings' logs. `null` when the
+     * change source has no upstream table to identify itself from.
+     */
+    replicaID: string | null;
+    /**
+     * Whether server readiness should be gated on the first litestream backup,
+     * in order to ensure that a newly connecting subscriber can restore from
+     * this change-streamer's backup.
+     *
+     * This should be true when the backup destination differs from where the
+     * backup was restored from, which is the case for (1) initial-sync and
+     * (2) most initialization cases in RMv2, with the exception being when
+     * an abandoned replica is resumed.
+     */
+    waitForBackupBeforeServing: boolean;
+};
+export declare function restoreReplica(lc: LogContext, config: LitestreamConfig, replicaFile: string, replicaConstraints: ReplicaConstraints | undefined): Promise<void>;
+//# sourceMappingURL=replica-restore.d.ts.map

package/out/zero-cache/src/services/change-source/common/replica-restore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-restore.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,8BAA8B,CAAC;AAMtC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AACpF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAEtD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,EAAE,YAAY,CAAC;IAC3B,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;;;;;;;;OASG;IACH,0BAA0B,EAAE,OAAO,CAAC;CACrC,CAAC;AAOF,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,GACjD,OAAO,CAAC,IAAI,CAAC,CAyCf"}

package/out/zero-cache/src/services/change-source/common/replica-restore.js

@@ -0,0 +1,34 @@
+import { sleep } from "../../../../../shared/src/sleep.js";
+import { litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreRuns } from "../../litestream/metrics.js";
+import { tryRestore } from "../../litestream/commands.js";
+//#region ../zero-cache/src/services/change-source/common/replica-restore.ts
+var MAX_RESTORE_ATTEMPTS = 3;
+var RESTORE_RETRY_DELAY_MS = 5e3;
+async function restoreReplica(lc, config, replicaFile, replicaConstraints) {
+	const start = performance.now();
+	let result;
+	try {
+		for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) try {
+			result = (await tryRestore(lc, config, replicaFile, replicaConstraints, "replication_manager")).result;
+			return;
+		} catch (e) {
+			if (attemptNum === MAX_RESTORE_ATTEMPTS) {
+				lc.error?.(`litestream restore failed after ${attemptNum} attempts; resyncing the replica`, e);
+				return;
+			}
+			lc.warn?.(`litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`, e);
+			await sleep(RESTORE_RETRY_DELAY_MS);
+		}
+	} finally {
+		const labels = {
+			...litestreamRestoreMetricAttrs(config, "replication_manager"),
+			result: result ?? "error"
+		};
+		litestreamRestoreRuns().add(1, labels);
+		litestreamRestoreDuration().recordMs(performance.now() - start, labels);
+	}
+}
+//#endregion
+export { restoreReplica };
+
+//# sourceMappingURL=replica-restore.js.map

package/out/zero-cache/src/services/change-source/common/replica-restore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replica-restore.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-restore.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport type {LitestreamConfig} from '../../../config/normalize.ts';\nimport {\n  tryRestore,\n  type ReplicaConstraints,\n  type RestoreResult,\n} from '../../litestream/commands.ts';\nimport {\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreRuns,\n} from '../../litestream/metrics.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource} from '../change-source.ts';\n\nexport type RestoreOptions = {\n  litestream?: LitestreamConfig;\n  constraints?: ReplicaConstraints | undefined;\n};\n\nexport type InitializeResult = {\n  subscriptionState: SubscriptionState;\n  changeSource: ChangeSource;\n  destinationBackupURL: string | undefined;\n  /**\n   * The replica this change stream belongs to, from the upstream `replicas`\n   * table. It is part of the identity the SQLite change log records, because a\n   * generation (i.e. `replicaVersion`) is shared by every sibling of a forked\n   * replica and so cannot distinguish two siblings' logs. `null` when the\n   * change source has no upstream table to identify itself from.\n   */\n  replicaID: string | null;\n\n  /**\n   * Whether server readiness should be gated on the first litestream backup,\n   * in order to ensure that a newly connecting subscriber can restore from\n   * this change-streamer's backup.\n   *\n   * This should be true when the backup destination differs from where the\n   * backup was restored from, which is the case for (1) initial-sync and\n   * (2) most initialization cases in RMv2, with the exception being when\n   * an abandoned replica is resumed.\n   */\n  waitForBackupBeforeServing: boolean;\n};\n\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\nexport async function restoreReplica(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n): Promise<void> {\n  const start = performance.now();\n  let result: RestoreResult | undefined;\n  try {\n    // `tryRestore` returns ordinary restore outcomes (no backup or an invalid\n    // replica) rather than throwing; preserve their existing immediate\n    // initial-sync fallback. Retry every thrown restore error twice before\n    // falling back to the initial Postgres sync.\n    for (let attemptNum = 1; attemptNum <= MAX_RESTORE_ATTEMPTS; attemptNum++) {\n      try {\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaFile,\n          replicaConstraints,\n          'replication_manager',\n        );\n        result = attempt.result;\n        return;\n      } catch (e) {\n        if (attemptNum === MAX_RESTORE_ATTEMPTS) {\n          lc.error?.(\n            `litestream restore failed after ${attemptNum} attempts; resyncing the replica`,\n            e,\n          );\n          return;\n        }\n\n        lc.warn?.(\n          `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n          e,\n        );\n        await sleep(RESTORE_RETRY_DELAY_MS);\n      }\n    }\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, 'replication_manager');\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n"],"mappings":";;;;AAiDA,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAE/B,eAAsB,eACpB,IACA,QACA,aACA,oBACe;CACf,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;EAKF,KAAK,IAAI,aAAa,GAAG,cAAc,sBAAsB,cAC3D,IAAI;GAQF,UAAS,MAPa,WACpB,IACA,QACA,aACA,oBACA,qBACF,GACiB;GACjB;EACF,SAAS,GAAG;GACV,IAAI,eAAe,sBAAsB;IACvC,GAAG,QACD,mCAAmC,WAAW,mCAC9C,CACF;IACA;GACF;GAEA,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;GACA,MAAM,MAAM,sBAAsB;EACpC;CAEJ,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,qBAChC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF"}

package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts

@@ -6,6 +6,8 @@ export declare function upgradeReplica(log: LogContext, debugName: string, dbPat
 export declare const CREATE_V6_COLUMN_METADATA_TABLE = "\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n";
 export declare const CREATE_V7_CHANGE_LOG = "\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n";
 export declare const CREATE_V9_TABLE_METADATA_TABLE = "\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n";
+export declare const V14_CHANGE_LOG_STREAM_TABLE = "_zero.changeLogStream";
+export declare const CREATE_V14_CHANGE_LOG_STREAM = "\n  CREATE TABLE \"_zero.changeLogStream\" (\n    \"watermark\"   TEXT NOT NULL,\n    \"pos\"         INTEGER NOT NULL,\n    \"change\"      TEXT NOT NULL,\n    \"precommit\"   TEXT,\n    \"writeTimeMs\" INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"_zero.changeLogStream_writeTimeMs\"\n    ON \"_zero.changeLogStream\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n";
 export declare const schemaVersionMigrationMap: IncrementalMigrationMap;
 export declare const CURRENT_SCHEMA_VERSION: number;
 //# sourceMappingURL=replica-schema.d.ts.map

package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAEL,KAAK,uBAAuB,EAE7B,MAAM,+BAA+B,CAAC;AAYvC,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,iBAoBf;AAkBD,eAAO,MAAM,+BAA+B,0UAW3C,CAAC;AAEF,eAAO,MAAM,oBAAoB,8XAUhC,CAAC;AAEF,eAAO,MAAM,8BAA8B,sMAO1C,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,uBA8HvC,CAAC;AAGF,eAAO,MAAM,sBAAsB,QAEwB,CAAC"}
+{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAE9D,OAAO,EAEL,KAAK,uBAAuB,EAE7B,MAAM,+BAA+B,CAAC;AAgBvC,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,iBAoBf;AAkBD,eAAO,MAAM,+BAA+B,0UAW3C,CAAC;AAEF,eAAO,MAAM,oBAAoB,8XAUhC,CAAC;AAEF,eAAO,MAAM,8BAA8B,sMAO1C,CAAC;AAKF,eAAO,MAAM,2BAA2B,0BAA0B,CAAC;AAQnE,eAAO,MAAM,4BAA4B,8aAaxC,CAAC;AA4BF,eAAO,MAAM,yBAAyB,EAAE,uBA+MvC,CAAC;AAGF,eAAO,MAAM,sBAAsB,QAEwB,CAAC"}

package/out/zero-cache/src/services/change-source/common/replica-schema.js

@@ -1,9 +1,11 @@
+import { assert } from "../../../../../shared/src/asserts.js";
 import { populateFromExistingTables } from "../../replicator/schema/column-metadata.js";
 import { listTables } from "../../../db/lite-tables.js";
+import { CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
 import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
 import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
-import { runSchemaMigrations } from "../../../db/migration-lite.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import { runSchemaMigrations } from "../../../db/migration-lite.js";
 //#region ../zero-cache/src/services/change-source/common/replica-schema.ts
 async function initReplica(log, debugName, dbPath, initialSync) {
 	const setupMigration = {
@@ -63,6 +65,38 @@ var CREATE_V9_TABLE_METADATA_TABLE = `
     PRIMARY KEY ("schema", "table")
   );
 `;
+var V14_CHANGE_LOG_STREAM_TABLE = "_zero.changeLogStream";
+var V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX = "_zero.changeLogStream_writeTimeMs";
+var CREATE_V14_CHANGE_LOG_STREAM = `
+  CREATE TABLE "${V14_CHANGE_LOG_STREAM_TABLE}" (
+    "watermark"   TEXT NOT NULL,
+    "pos"         INTEGER NOT NULL,
+    "change"      TEXT NOT NULL,
+    "precommit"   TEXT,
+    "writeTimeMs" INTEGER,
+    PRIMARY KEY ("watermark", "pos")
+  );
+
+  CREATE INDEX "${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}"
+    ON "${V14_CHANGE_LOG_STREAM_TABLE}" ("writeTimeMs", "watermark")
+    WHERE "writeTimeMs" IS NOT NULL;
+`;
+var SEED_V14_CHANGE_LOG_STREAM = `
+  INSERT INTO "${V14_CHANGE_LOG_STREAM_TABLE}"
+    ("watermark", "pos", "change", "precommit", "writeTimeMs")
+  VALUES
+    (@stateVersion, 0, '{"tag":"begin"}', NULL, NULL),
+    (@stateVersion, 1, '{"tag":"commit"}', @stateVersion, @writeTimeMs)
+  ON CONFLICT ("watermark", "pos") DO NOTHING
+`;
+function seedV14ChangeLogStream(db) {
+	const state = db.prepare(`
+      SELECT "stateVersion", "writeTimeMs" FROM "_zero.replicationState"
+    `).get();
+	assert(state !== void 0, "replication state must be initialized");
+	assert(state.writeTimeMs !== null, "replication state writeTimeMs must be initialized");
+	db.prepare(SEED_V14_CHANGE_LOG_STREAM).run(state);
+}
 var schemaVersionMigrationMap = {
 	4: {
 		migrateSchema: () => {
@@ -145,6 +179,44 @@ var schemaVersionMigrationMap = {
         UPDATE "_zero.replicationState" 
           SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);
 		}
+	},
+	14: {
+		migrateSchema: (_, db) => {
+			db.exec(CREATE_V14_CHANGE_LOG_STREAM);
+		},
+		migrateData: (_, db) => {
+			seedV14ChangeLogStream(db);
+		}
+	},
+	15: { migrateSchema: (_, db) => {
+		db.exec(`
+        CREATE TABLE "_zero.replicationState2" (
+          stateVersion TEXT NOT NULL,
+          writeTimeMs INTEGER NOT NULL,
+          lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)
+        );
+
+        INSERT INTO "_zero.replicationState2" (stateVersion, writeTimeMs, lock)
+          SELECT stateVersion, writeTimeMs, lock FROM "_zero.replicationState";
+
+        DROP TABLE "_zero.replicationState";
+
+        ALTER TABLE "_zero.replicationState2" RENAME TO "_zero.replicationState";
+      `);
+	} },
+	16: { migrateSchema: (_, db) => {
+		db.exec(`
+        DROP INDEX IF EXISTS "${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}";
+        DROP TABLE IF EXISTS "${V14_CHANGE_LOG_STREAM_TABLE}";
+      `);
+	} },
+	17: {
+		migrateSchema: (_, db) => {
+			db.exec(CREATE_BACKFILLING_TABLE);
+		},
+		migrateData: (lc, db) => {
+			populateBackfillingFromColumnMetadata(lc, db);
+		}
 	}
 };
 Object.keys(schemaVersionMigrationMap).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);

package/out/zero-cache/src/services/change-source/common/replica-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nfunction throwAutoResetForCorruption(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  e: unknown,\n): never {\n  if (isSQLiteCorruption(e)) {\n    logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;AAmBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAStD,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
+{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n  runSchemaMigrations,\n  type IncrementalMigrationMap,\n  type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  CREATE_BACKFILLING_TABLE,\n  populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n  CREATE_RUNTIME_EVENTS_TABLE,\n  recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n  const setupMigration: Migration = {\n    migrateSchema: (log, tx) => initialSync(log, tx),\n    minSafeVersion: 1,\n  };\n\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      setupMigration,\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nexport async function upgradeReplica(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n) {\n  try {\n    await runSchemaMigrations(\n      log,\n      debugName,\n      dbPath,\n      // setupMigration should never be invoked\n      {\n        migrateSchema: () => {\n          throw new Error(\n            'This should only be called for already synced replicas',\n          );\n        },\n      },\n      schemaVersionMigrationMap,\n    );\n  } catch (e) {\n    throwAutoResetForCorruption(log, debugName, dbPath, e);\n  }\n}\n\nfunction throwAutoResetForCorruption(\n  log: LogContext,\n  debugName: string,\n  dbPath: string,\n  e: unknown,\n): never {\n  if (isSQLiteCorruption(e)) {\n    logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n    throw new AutoResetSignal(\n      `replica database appears corrupt: ${String(e)}`,\n      {cause: e},\n    );\n  }\n  throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.column_metadata\" (\n    table_name TEXT NOT NULL,\n    column_name TEXT NOT NULL,\n    upstream_type TEXT NOT NULL,\n    is_not_null INTEGER NOT NULL,\n    is_enum INTEGER NOT NULL,\n    is_array INTEGER NOT NULL,\n    character_max_length INTEGER,\n    PRIMARY KEY (table_name, column_name)\n  );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n  CREATE TABLE \"_zero.changeLog2\" (\n    \"stateVersion\"              TEXT NOT NULL,\n    \"pos\"                       INT  NOT NULL,\n    \"table\"                     TEXT NOT NULL,\n    \"rowKey\"                    TEXT NOT NULL,\n    \"op\"                        TEXT NOT NULL,\n    PRIMARY KEY(\"stateVersion\", \"pos\"),\n    UNIQUE(\"table\", \"rowKey\")\n  );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n  CREATE TABLE \"_zero.tableMetadata\" (\n    \"schema\"    TEXT NOT NULL,\n    \"table\"     TEXT NOT NULL,\n    \"metadata\"  TEXT NOT NULL,\n    PRIMARY KEY (\"schema\", \"table\")\n  );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// are the v14 replica's, that module's are the change-log database's, and the\n// two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n  '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table, as migration 14 created it and migration 16\n// drops it. Frozen at its v14 shape: a migration is history, so this must not\n// be kept in sync with the live definition in `replicator/change-log-db.ts`,\n// which now describes a table in a different database.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /*sql*/ `\n  CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n    \"watermark\"   TEXT NOT NULL,\n    \"pos\"         INTEGER NOT NULL,\n    \"change\"      TEXT NOT NULL,\n    \"precommit\"   TEXT,\n    \"writeTimeMs\" INTEGER,\n    PRIMARY KEY (\"watermark\", \"pos\")\n  );\n\n  CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n    ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n    WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n// Idempotent because migrateData may run again after a rollback followed by a\n// roll-forward, and both rows go in with one statement so a partial seed is\n// not reachable.\nconst SEED_V14_CHANGE_LOG_STREAM = /*sql*/ `\n  INSERT INTO \"${V14_CHANGE_LOG_STREAM_TABLE}\"\n    (\"watermark\", \"pos\", \"change\", \"precommit\", \"writeTimeMs\")\n  VALUES\n    (@stateVersion, 0, '{\"tag\":\"begin\"}', NULL, NULL),\n    (@stateVersion, 1, '{\"tag\":\"commit\"}', @stateVersion, @writeTimeMs)\n  ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nfunction seedV14ChangeLogStream(db: Database): void {\n  const state = db\n    .prepare(/*sql*/ `\n      SELECT \"stateVersion\", \"writeTimeMs\" FROM \"_zero.replicationState\"\n    `)\n    .get<{stateVersion: string; writeTimeMs: number | null} | undefined>();\n  assert(state !== undefined, 'replication state must be initialized');\n  assert(\n    state.writeTimeMs !== null,\n    'replication state writeTimeMs must be initialized',\n  );\n  db.prepare(SEED_V14_CHANGE_LOG_STREAM).run(state);\n}\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n  // There's no incremental migration from v1. Just reset the replica.\n  4: {\n    migrateSchema: () => {\n      throw new AutoResetSignal('upgrading replica to new schema');\n    },\n    minSafeVersion: 3,\n  },\n\n  5: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n    },\n    migrateData: (_, db) => {\n      recordEvent(db, 'upgrade');\n    },\n  },\n\n  // Revised in the migration to v8 because the v6 code was incomplete.\n  6: {},\n\n  7: {\n    migrateSchema: (_, db) => {\n      // Note: The original \"changeLog\" table is kept so that the replica file\n      // is compatible with older zero-caches. However, it is truncated for\n      // space savings (since historic changes were never read).\n      db.exec(`DELETE FROM \"_zero.changeLog\"`);\n      // First version of changeLog2\n      db.exec(CREATE_V7_CHANGE_LOG);\n    },\n  },\n\n  8: {\n    migrateSchema: (_, db) => {\n      const tableExists = db\n        .prepare(\n          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n        )\n        .get();\n\n      if (!tableExists) {\n        db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n      }\n    },\n    migrateData: (_, db) => {\n      // Re-populate the ColumnMetadataStore; the original migration\n      // at v6 was incomplete, as covered replicas migrated from earlier\n      // versions but did not initialize the table for new replicas.\n      db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n      const tables = listTables(db, false, false);\n      populateFromExistingTables(db, tables);\n    },\n  },\n\n  9: {\n    migrateSchema: (_, db) => {\n      db.exec(\n        /*sql*/ `\n        ALTER TABLE \"_zero.changeLog2\" \n          ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n        ALTER TABLE \"_zero.column_metadata\"\n          ADD COLUMN backfill TEXT;\n      ` + CREATE_V9_TABLE_METADATA_TABLE,\n      );\n    },\n  },\n\n  10: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationConfig\" \n          ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n      `);\n    },\n  },\n\n  11: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n        -- Removing the NOT NULL constraint from \"metadata\" requires copying\n        -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"upstreamMetadata\" TEXT;\n        UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n        ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n      `);\n    },\n  },\n\n  12: {\n    migrateSchema: (_, db) => {\n      // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n      // It is needed for backwards compatibility.\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.tableMetadata\"\n          ADD COLUMN \"metadata\" TEXT;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      // For rollback then roll forward, re-copy anything written to metadata.\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.tableMetadata\" \n          SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n              \"metadata\" = NULL;\n      `);\n    },\n  },\n\n  13: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n      `);\n    },\n\n    migrateData: (_, db) => {\n      db.exec(/*sql*/ `\n        UPDATE \"_zero.replicationState\" \n          SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n    },\n  },\n\n  // Deliberately left as it shipped, even though v16 immediately undoes it.\n  // Rewriting it to a no-op would strand replicas already at v14: they roll\n  // back to v14 code, which expects the table this created.\n  14: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_V14_CHANGE_LOG_STREAM);\n    },\n\n    migrateData: (_, db) => {\n      seedV14ChangeLogStream(db);\n    },\n  },\n\n  15: {\n    migrateSchema: (_, db) => {\n      // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n      // create a new table, copy the data over, delete the old table, and\n      // rename. This is cheap because it's a single row table.\n      db.exec(/*sql*/ `\n        CREATE TABLE \"_zero.replicationState2\" (\n          stateVersion TEXT NOT NULL,\n          writeTimeMs INTEGER NOT NULL,\n          lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n        );\n\n        INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n          SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n        DROP TABLE \"_zero.replicationState\";\n\n        ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n      `);\n    },\n  },\n\n  // The change log moved out of the replica and into its own database\n  // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n  // every file the backup ships and every follower downloads. Nothing reads it\n  // as of this version.\n  //\n  // `IF EXISTS` because a fresh replica never runs the incremental migrations\n  // at all — its setup migration creates the current schema directly, which no\n  // longer includes this table — and because a rollback to v14 followed by a\n  // roll-forward re-runs `migrateData` while skipping `migrateSchema`.\n  //\n  // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n  // trigger in `workers/replicator.ts`; no special handling here.\n  //\n  // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n  // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n  // table. Bumping the rollback limit would turn every rollback into a hard\n  // startup failure to protect a configuration that is not enabled anywhere.\n  16: {\n    migrateSchema: (_, db) => {\n      db.exec(/*sql*/ `\n        DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n        DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n      `);\n    },\n  },\n\n  // `_zero.column_metadata.backfill` records which columns are backfilling but\n  // is keyed by lite table name, which has no inverse: it cannot be turned back\n  // into the `BackfillRequest`s that a change log initialized from this replica\n  // would have to send. This table carries the upstream identity alongside the\n  // same state. See `replicator/schema/backfilling.ts`.\n  //\n  // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n  // since nothing at v16 reads the new table. Its writes to\n  // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n  // forward again, so a rollback costs nothing but the re-seed.\n  17: {\n    migrateSchema: (_, db) => {\n      db.exec(CREATE_BACKFILLING_TABLE);\n    },\n\n    migrateData: (lc, db) => {\n      populateBackfillingFromColumnMetadata(lc, db);\n    },\n  },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n  schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;AAwBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAYtD,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAMF,IAAa,+BAAuC;kBAClC,4BAA4B;;;;;;;;;kBAS5B,uCAAuC;UAC/C,4BAA4B;;;AAOtC,IAAM,6BAAqC;iBAC1B,4BAA4B;;;;;;;AAQ7C,SAAS,uBAAuB,IAAoB;CAClD,MAAM,QAAQ,GACX,QAAgB;;KAEhB,EACA,IAAoE;CACvE,OAAO,UAAU,KAAA,GAAW,uCAAuC;CACnE,OACE,MAAM,gBAAgB,MACtB,mDACF;CACA,GAAG,QAAQ,0BAA0B,EAAE,IAAI,KAAK;AAClD;AAEA,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAKA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,4BAA4B;EACtC;EAEA,cAAc,GAAG,OAAO;GACtB,uBAAuB,EAAE;EAC3B;CACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAmBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,wBAAwB;EAClC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts

@@ -2,18 +2,14 @@ import type { LogContext } from '@rocicorp/logger';
 import { type JSONObject } from '../../../../../shared/src/bigint-json.ts';
 import { Database } from '../../../../../zqlite/src/db.ts';
 import type { ShardConfig } from '../../../types/shards.ts';
-import { type SubscriptionState } from '../../replicator/schema/replication-state.ts';
-import type { ChangeSource } from '../change-source.ts';
+import { type InitializeResult, type RestoreOptions } from '../common/replica-restore.ts';
 /** Server context to store with the initial sync metadata for debugging. */
 export type ServerContext = JSONObject;
 /**
  * Initializes a Custom change source before streaming changes from the
  * corresponding logical replication stream.
  */
-export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext): Promise<{
-    subscriptionState: SubscriptionState;
-    changeSource: ChangeSource;
-}>;
+export declare function initializeCustomChangeSource(lc: LogContext, upstreamURI: string, shard: ShardConfig, replicaDbFile: string, context: ServerContext, { litestream, constraints }?: RestoreOptions): Promise<InitializeResult>;
 /**
  * Initial sync for a custom change source makes a request to the
  * change source endpoint with no `replicaVersion` or `lastWatermark`.

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAQnE,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAQpE,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CA+B7E;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAcnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAQtC,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,EACtB,EAAC,UAAU,EAAE,WAAW,EAAC,GAAE,cAAmB,GAC7C,OAAO,CAAC,gBAAgB,CAAC,CAwD3B;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -7,19 +7,25 @@ import { Database } from "../../../../../zqlite/src/db.js";
 import { computeZqlSpecs } from "../../../db/lite-tables.js";
 import { createReplicationStateTables, getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
 import { StatementRunner } from "../../../db/statements.js";
-import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
-import { initReplica } from "../common/replica-schema.js";
 import { stream } from "../../../types/streams.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { ChangeProcessor } from "../../replicator/change-processor.js";
 import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
-import { WebSocket as WebSocket$1 } from "ws";
+import { restoreReplica } from "../common/replica-restore.js";
+import { initReplica } from "../common/replica-schema.js";
+import { WebSocket } from "ws";
 //#region ../zero-cache/src/services/change-source/custom/change-source.ts
 /**
 * Initializes a Custom change source before streaming changes from the
 * corresponding logical replication stream.
 */
-async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context) {
-	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => initialSync(log, shard, tx, upstreamURI, context));
+async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFile, context, { litestream, constraints } = {}) {
+	if (litestream?.backupURL) await restoreReplica(lc, litestream, replicaDbFile, constraints);
+	let initialSynced = false;
+	await initReplica(lc, `replica-${shard.appID}-${shard.shardNum}`, replicaDbFile, (log, tx) => {
+		initialSynced = true;
+		return initialSync(log, shard, tx, upstreamURI, context);
+	});
 	const replica = new Database(lc, replicaDbFile);
 	const subscriptionState = getSubscriptionState(new StatementRunner(replica));
 	replica.close();
@@ -30,7 +36,10 @@ async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFil
 	}
 	return {
 		subscriptionState,
-		changeSource: new CustomChangeSource(lc, upstreamURI, shard, subscriptionState)
+		changeSource: new CustomChangeSource(lc, upstreamURI, shard, subscriptionState),
+		destinationBackupURL: litestream?.backupURL,
+		replicaID: null,
+		waitForBackupBeforeServing: initialSynced
 	};
 }
 var CustomChangeSource = class {
@@ -69,7 +78,7 @@ var CustomChangeSource = class {
 			url.searchParams.set("lastWatermark", clientWatermark);
 			url.searchParams.set("replicaVersion", replicaVersion);
 		}
-		const ws = new WebSocket$1(url);
+		const ws = new WebSocket(url);
 		const { instream, outstream } = stream(this.#lc, ws, changeStreamMessageSchema, { coalesce: (curr) => curr });
 		return {
 			changes: instream,