npm - @rocicorp/zero - Versions diffs - 1.9.0-canary.0 → 1.9.0-canary.10 - Mend

@rocicorp/zero 1.9.0-canary.0 → 1.9.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (126) hide show

package/out/zero-cache/src/services/litestream/commands.js CHANGED Viewed

@@ -11,9 +11,14 @@ import { ChangeStreamerHttpClient } from "../change-streamer/change-streamer-htt
 import { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoreWaitDuration, litestreamRestoredDbBytes } from "./metrics.js";
 import { resolver } from "@rocicorp/resolver";
 import { spawn } from "node:child_process";
-import { existsSync, statSync } from "node:fs";
+import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
 //#region ../zero-cache/src/services/litestream/commands.ts
 var RETRY_INTERVAL_MS = 3e3;
+var MAX_RESTORE_ATTEMPTS = 3;
+var RESTORE_RETRY_DELAY_MS = 5e3;
+var MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;
+var LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;
 var BackupNotFoundException = class extends Error {
 	static name = "BackupNotFoundException";
 	constructor(backupURL) {
@@ -31,28 +36,25 @@ async function restoreReplica(lc, config, replicaConstraints) {
 	const role = replicaConstraints ? "replication_manager" : "view_syncer";
 	let backupURL = config.litestream.backupURL;
 	let result;
-	let consecutiveErrors = 0;
 	try {
-		for (;;) {
+		for (let attemptNum = 1;; attemptNum++) {
+			let attempt;
 			try {
-				const attempt = await tryRestore(lc, config, replicaConstraints, role, consecutiveErrors >= 1);
-				backupURL = attempt.backupURL;
-				if (attempt.restored) {
-					result = attempt.result;
-					return;
-				}
-				consecutiveErrors = 0;
-				if (replicaConstraints) {
-					result = attempt.result;
-					throw new BackupNotFoundException(config.litestream.backupURL);
-				}
+				attempt = await tryRestore(lc, config, replicaConstraints, role);
 			} catch (e) {
-				if (e instanceof BackupNotFoundException) throw e;
-				if (++consecutiveErrors <= 1) {
-					lc.warn?.(`restore attempt failed. retrying once`, e);
-					continue;
-				}
-				throw e;
+				if (!replicaConstraints || attemptNum === MAX_RESTORE_ATTEMPTS) throw e;
+				lc.warn?.(`litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`, e);
+				await sleep(RESTORE_RETRY_DELAY_MS);
+				continue;
+			}
+			backupURL = attempt.backupURL;
+			if (attempt.restored) {
+				result = attempt.result;
+				return;
+			}
+			if (replicaConstraints) {
+				result = attempt.result;
+				throw new BackupNotFoundException(config.litestream.backupURL);
 			}
 			lc.info?.(`replica not found. retrying in ${RETRY_INTERVAL_MS / 1e3} seconds`);
 			await sleep(RETRY_INTERVAL_MS);
@@ -71,7 +73,6 @@ async function restoreReplica(lc, config, replicaConstraints) {
 }
 function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 	const { executable, executableV5, restoreUsingV5, backupURL, logLevel, configPath, endpoint, region, port = config.port + 2, checkpointThresholdMB, minCheckpointPageCount = checkpointThresholdMB * 250, maxCheckpointPageCount = minCheckpointPageCount * 10, incrementalBackupIntervalMinutes, snapshotBackupIntervalHours, multipartConcurrency, multipartSize } = config.litestream;
-	const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;
 	return {
 		litestream: (mode === "restore" && restoreUsingV5 ? executableV5 : executable) ?? must(executable, `Missing --litestream-executable`),
 		env: {
@@ -82,7 +83,8 @@ function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 			["ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT"]: String(maxCheckpointPageCount),
 			["ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES"]: String(incrementalBackupIntervalMinutes),
 			["ZERO_LITESTREAM_LOG_LEVEL"]: logLevelOverride ?? logLevel,
-			["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES"]: String(snapshotBackupIntervalMinutes),
+			["ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS"]: String(snapshotBackupIntervalHours),
+			["ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS"]: String(snapshotBackupIntervalHours + 6),
 			["ZERO_LITESTREAM_MULTIPART_CONCURRENCY"]: String(multipartConcurrency),
 			["ZERO_LITESTREAM_MULTIPART_SIZE"]: String(multipartSize),
 			["ZERO_LOG_FORMAT"]: config.log.format,
@@ -93,7 +95,7 @@ function getLitestream(mode, config, logLevelOverride, backupURLOverride) {
 		}
 	};
 }
-async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt) {
+async function tryRestore(lc, config, replicaConstraints, role) {
 	let snapshotStatus;
 	if (!replicaConstraints) {
 		const waitStart = performance.now();
@@ -104,9 +106,12 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 	}
 	const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;
 	const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);
+	const replicaFile = config.replica.file;
 	let result = "error";
 	try {
-		const replicaExistedBeforeRestore = existsSync(config.replica.file);
+		logRestoreDirectoryContents(lc, replicaFile, "before-temp-cleanup");
+		deleteRestoreTempFiles(replicaFile);
+		const replicaExistedBeforeRestore = existsSync(replicaFile);
 		const { litestream, env } = getLitestream("restore", config, "debug", snapshotStatus?.backupURL);
 		const { restoreParallelism: parallelism, multipartConcurrency, multipartSize } = config.litestream;
 		lc.info?.(`starting litestream restore`, {
@@ -157,10 +162,8 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 				...attrs,
 				result: "error"
 			});
-			const output = (stdout + stderr).trim();
-			if (isFinalAttempt) lc.error?.(`litestream restore failed after retry`, output || e);
-			else lc.error?.(`litestream restore attempt failed, will retry`, output || e);
-			throw e;
+			const output = [stdout, stderr].map((value) => value.trim()).filter(Boolean).join("\n");
+			throw new Error(output ? `${String(e)}\n${output}` : String(e), { cause: e });
 		}
 		if (!existsSync(config.replica.file)) {
 			result = "no_backup";
@@ -197,12 +200,58 @@ async function tryRestore(lc, config, replicaConstraints, role, isFinalAttempt)
 			result
 		};
 	} finally {
+		try {
+			deleteRestoreTempFiles(replicaFile);
+		} catch (e) {
+			lc.warn?.("Unable to clean up Litestream restore temporary files", {
+				replicaFile,
+				error: String(e)
+			});
+		}
+		logRestoreDirectoryContents(lc, replicaFile, "after-temp-cleanup");
 		litestreamRestoreAttempts().add(1, {
 			...attrs,
 			result
 		});
 	}
 }
+function deleteRestoreTempFiles(replicaFile) {
+	const temporaryReplicaFile = `${replicaFile}.tmp`;
+	deleteLiteDB(temporaryReplicaFile);
+	const directory = dirname(temporaryReplicaFile);
+	if (!existsSync(directory)) return;
+	const prefix = `${basename(temporaryReplicaFile)}-`;
+	for (const name of readdirSync(directory)) {
+		if (!name.startsWith(prefix) || !name.endsWith("-wal")) continue;
+		const index = name.slice(prefix.length, -4);
+		if (LEGACY_RESTORE_WAL_INDEX.test(index)) rmSync(join(directory, name), { force: true });
+	}
+}
+function logRestoreDirectoryContents(lc, replicaFile, phase) {
+	const directory = dirname(replicaFile);
+	try {
+		const names = readdirSync(directory).sort();
+		const entries = names.slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES).map((name) => ({
+			name,
+			sizeBytes: statSync(join(directory, name)).size
+		}));
+		lc.info?.("Litestream restore directory contents", {
+			phase,
+			directory,
+			replicaFile,
+			entryCount: names.length,
+			omittedEntryCount: Math.max(0, names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES),
+			entries
+		});
+	} catch (e) {
+		lc.warn?.("Unable to inspect Litestream restore directory", {
+			phase,
+			directory,
+			replicaFile,
+			error: String(e)
+		});
+	}
+}
 function replicaIsValid(lc, replica, constraints) {
 	let db;
 	try {

package/out/zero-cache/src/services/litestream/commands.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, statSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreRuns,\n litestreamRestoreValidationDuration,\n litestreamRestoreWaitDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\ntype RestoreResult = 'success' \| 'no_backup' \| 'invalid_replica' \| 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n};\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\n/*\n @param replicaConstraints The constraints of the restored backup when\n * restoring for the change-streamer (replication-manager). For the\n * view-syncer, this should be unspecified so that the constraints are\n * retrieved from the replication-manager via the snapshot protocol.\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n) {\n const start = performance.now();\n const role: LitestreamRole = replicaConstraints\n ? 'replication_manager'\n : 'view_syncer';\n let backupURL = config.litestream.backupURL;\n let result: RestoreResult \| undefined;\n // View-syncers (no replicaConstraints) wait indefinitely for the\n // replication-manager to publish a restorable backup. On a fresh stack the\n // first backup is not durable until the initial sync completes and litestream\n // uploads the initial snapshot, which can take many minutes for a large\n // replica. The platform's startup probe budget (which scales with replica\n // size) is the backstop, so restoreReplica must not impose its own shorter\n // cap and self-terminate while the backup is still being produced.\n //\n // `consecutiveErrors` distinguishes a transient restore error* (e.g. the\n // `replicate` process compacting a snapshot mid-restore) — which is retried\n // once — from a \"backup not found\" result, which is expected while waiting.\n let consecutiveErrors = 0;\n try {\n for (;;) {\n try {\n // `consecutiveErrors` is 0 on a fresh attempt and 1 on the retry, so\n // `consecutiveErrors >= 1` identifies the final attempt of a streak.\n // tryRestore surfaces litestream's failure output with a different\n // message on the final attempt (which the log classifier routes to a\n // paging alert) than on a retriable earlier attempt (routed to a\n // non-paging warning). See INC-961.\n const attempt = await tryRestore(\n lc,\n config,\n replicaConstraints,\n role,\n consecutiveErrors >= 1,\n );\n backupURL = attempt.backupURL;\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n consecutiveErrors = 0;\n if (replicaConstraints) {\n // The replication-manager restores against explicit constraints, so a\n // missing backup is fatal (e.g. the litestream URL was purposefully\n // changed to force a resync). Only the view-syncer (no constraints)\n // waits for a backup to appear.\n result = attempt.result;\n throw new BackupNotFoundException(config.litestream.backupURL);\n }\n } catch (e) {\n if (e instanceof BackupNotFoundException) {\n throw e;\n }\n if (++consecutiveErrors <= 1) {\n // A restore will fail if the `replicate` process creates a new\n // snapshot (and compacts old files) at the same time. Snapshots are\n // infrequent (e.g. once every 12 hours), and the scenario is\n // recoverable with a retry.\n lc.warn?.(`restore attempt failed. retrying once`, e);\n continue;\n }\n // If it fails again on the retry, though, bail.\n throw e;\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } catch (e) {\n if (e instanceof BackupNotFoundException && result === undefined) {\n result = 'no_backup';\n }\n throw e;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n region,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n // Set the snapshot interval to something smaller than x hours so that\n // the hourly check triggers on the hour, rather than the hour after.\n const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n const litestream =\n // The v0.5.8+ litestream executable can restore from either the new LTX\n // format or the legacy WAL format, allowing forwards-compatibility /\n // rollback safety with zero-cache versions that backup to LTX.\n (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n snapshotBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nasync function tryRestore(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n role: LitestreamRole,\n isFinalAttempt: boolean,\n): Promise<RestoreAttempt> {\n let snapshotStatus: SnapshotStatus \| undefined;\n if (!replicaConstraints) {\n // view-syncers fetch replica constraints from the replication-manager\n // via the snapshot protocol.\n const waitStart = performance.now();\n snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n litestreamRestoreWaitDuration().recordMs(\n performance.now() - waitStart,\n litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n );\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n replicaConstraints = snapshotStatus;\n }\n\n const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n const replicaExistedBeforeRestore = existsSync(config.replica.file);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided whether the failure is retriable. The captured output is\n // re-surfaced through `lc` below with a distinct message per attempt, so a\n // retriable failure is routed to a non-paging warning and only a post-retry\n // failure pages. See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.stderr.on('data', chunk => (stderr += chunk));\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n // A successful restore does not emit ERROR-level output; forward\n // litestream's own (minimal, debug-level) restore logging to the pod's\n // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n if (stdout) {\n process.stdout.write(stdout);\n }\n if (stderr) {\n process.stderr.write(stderr);\n }\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n // litestream logs its restore failure at ERROR. Re-surface that captured\n // output through our logger with a distinct message per attempt, so the\n // cloudzero log classifier can route them differently: the retriable\n // earlier attempt to a non-paging warning, and the post-retry failure to\n // a paging critical. Both are emitted at ERROR because the log scraper\n // only ingests error-level lines, and both carry our `worker` context,\n // distinguishing them from litestream's untagged raw output. See INC-961.\n const output = (stdout + stderr).trim();\n if (isFinalAttempt) {\n lc.error?.(`litestream restore failed after retry`, output \|\| e);\n } else {\n lc.error?.(\n `litestream restore attempt failed, will retry`,\n output \|\| e,\n );\n }\n throw e;\n }\n if (!existsSync(config.replica.file)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n litestreamRestoreAttempts().add(1, {\n ...attrs,\n result,\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints,\n) {\n let db: Database \| undefined;\n try {\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/*\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, 'snapshots'),\n listBackupCreatedTimes(lc, config, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: ZeroConfig,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config);\n const proc = spawn(litestream, [command, config.replica.file], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown;\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // backup monitor returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyCA,IAAM,oBAAoB;AAe1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAYJ,IAAI,oBAAoB;CACxB,IAAI;EACF,SAAS;GACP,IAAI;IAOF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,oBACA,MACA,qBAAqB,CACvB;IACA,YAAY,QAAQ;IACpB,IAAI,QAAQ,UAAU;KACpB,SAAS,QAAQ;KACjB;IACF;IACA,oBAAoB;IACpB,IAAI,oBAAoB;KAKtB,SAAS,QAAQ;KACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;IAC/D;GACF,SAAS,GAAG;IACV,IAAI,aAAa,yBACf,MAAM;IAER,IAAI,EAAE,qBAAqB,GAAG;KAK5B,GAAG,OAAO,yCAAyC,CAAC;KACpD;IACF;IAEA,MAAM;GACR;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACA,gBACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,MAAM,8BAA8B,WAAW,OAAO,QAAQ,IAAI;EAClE,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAQD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GAQA,MAAM,UAAU,SAAS,QAAQ,KAAK;GACtC,IAAI,gBACF,GAAG,QAAQ,yCAAyC,UAAU,CAAC;QAE/D,GAAG,QACD,iDACA,UAAU,CACZ;GAEF,MAAM;EACR;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,0BAA0B,EAAE,IAAI,GAAG;GACjC,GAAG;GACH;EACF,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
1	+ {"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreRuns,\n litestreamRestoreValidationDuration,\n litestreamRestoreWaitDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n// A short retry is much cheaper than an initial Postgres sync, while keeping a\n// persistent failure from delaying startup appreciably.\nconst MAX_RESTORE_ATTEMPTS = 3;\nconst RESTORE_RETRY_DELAY_MS = 5_000;\n\ntype ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\ntype RestoreResult = 'success' \| 'no_backup' \| 'invalid_replica' \| 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string \| undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string \| undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\n/*\n @param replicaConstraints The constraints of the restored backup when\n * restoring for the change-streamer (replication-manager). For the\n * view-syncer, this should be unspecified so that the constraints are\n * retrieved from the replication-manager via the snapshot protocol.\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n) {\n const start = performance.now();\n const role: LitestreamRole = replicaConstraints\n ? 'replication_manager'\n : 'view_syncer';\n let backupURL = config.litestream.backupURL;\n let result: RestoreResult \| undefined;\n // View-syncers (no replicaConstraints) wait indefinitely for the\n // replication-manager to publish a restorable backup. On a fresh stack the\n // first backup is not durable until the initial sync completes and litestream\n // uploads the initial snapshot, which can take many minutes for a large\n // replica. The platform's startup probe budget (which scales with replica\n // size) is the backstop, so restoreReplica must not impose its own shorter\n // cap and self-terminate while the backup is still being produced.\n try {\n for (let attemptNum = 1; ; attemptNum++) {\n let attempt: RestoreAttempt;\n try {\n attempt = await tryRestore(lc, config, replicaConstraints, role);\n } catch (e) {\n // View-syncers retain their existing startup behavior. For replication\n // managers, retry a transient restore failure twice before falling back\n // to the existing initial-sync recovery path.\n if (!replicaConstraints \|\| attemptNum === MAX_RESTORE_ATTEMPTS) {\n throw e;\n }\n lc.warn?.(\n `litestream restore attempt ${attemptNum} failed; retrying in ${RESTORE_RETRY_DELAY_MS}ms (attempt ${attemptNum + 1} of ${MAX_RESTORE_ATTEMPTS})`,\n e,\n );\n await sleep(RESTORE_RETRY_DELAY_MS);\n continue;\n }\n backupURL = attempt.backupURL;\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n if (replicaConstraints) {\n // The replication-manager restores against explicit constraints, so a\n // missing backup is fatal (e.g. the litestream URL was purposefully\n // changed to force a resync). Only the view-syncer (no constraints)\n // waits for a backup to appear.\n result = attempt.result;\n throw new BackupNotFoundException(config.litestream.backupURL);\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } catch (e) {\n if (e instanceof BackupNotFoundException && result === undefined) {\n result = 'no_backup';\n }\n throw e;\n } finally {\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' \| 'replicate',\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n region,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n const litestream =\n // The v0.5.8+ litestream executable can restore from either the new LTX\n // format or the legacy WAL format, allowing forwards-compatibility /\n // rollback safety with zero-cache versions that backup to LTX.\n (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nasync function tryRestore(\n lc: LogContext,\n config: ZeroConfig,\n replicaConstraints: ReplicaConstraints \| null,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n let snapshotStatus: SnapshotStatus \| undefined;\n if (!replicaConstraints) {\n // view-syncers fetch replica constraints from the replication-manager\n // via the snapshot protocol.\n const waitStart = performance.now();\n snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n litestreamRestoreWaitDuration().recordMs(\n performance.now() - waitStart,\n litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n );\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n replicaConstraints = snapshotStatus;\n }\n\n const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n const replicaFile = config.replica.file;\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.stderr.on('data', chunk => (stderr += chunk));\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n // A successful restore does not emit ERROR-level output; forward\n // litestream's own (minimal, debug-level) restore logging to the pod's\n // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n if (stdout) {\n process.stdout.write(stdout);\n }\n if (stderr) {\n process.stderr.write(stderr);\n }\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(config.replica.file)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) \|\| !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' \| 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints,\n) {\n let db: Database \| undefined;\n try {\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' \| 'error' \| 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/*\n Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n \n This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * believes an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n \n Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n /\nexport async function getLastBackupTime(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, 'snapshots'),\n listBackupCreatedTimes(lc, config, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/\n Runs `litestream <snapshots\|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n \n ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n /\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: ZeroConfig,\n command: 'snapshots' \| 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' \| 'empty' \| 'timeout' \| 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config);\n const proc = spawn(litestream, [command, config.replica.file], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/\n Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n \n Exported for testing.\n /\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' \| 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 \|\|\n created === undefined \|\|\n created === 'created' / header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown;\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // backup monitor returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA0CA,IAAM,oBAAoB;AAG1B,IAAM,uBAAuB;AAC7B,IAAM,yBAAyB;AAe/B,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAEjC,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAQJ,IAAI;EACF,KAAK,IAAI,aAAa,IAAK,cAAc;GACvC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WAAW,IAAI,QAAQ,oBAAoB,IAAI;GACjE,SAAS,GAAG;IAIV,IAAI,CAAC,sBAAsB,eAAe,sBACxC,MAAM;IAER,GAAG,OACD,8BAA8B,WAAW,uBAAuB,uBAAuB,cAAc,aAAa,EAAE,MAAM,qBAAqB,IAC/I,CACF;IACA,MAAM,MAAM,sBAAsB;IAClC;GACF;GACA,YAAY,QAAQ;GACpB,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,IAAI,oBAAoB;IAKtB,SAAS,QAAQ;IACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;GAC/D;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAQX,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,MAAM,cAAc,OAAO,QAAQ;CACnC,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/services/litestream/config.yml CHANGED Viewed

@@ -8,8 +8,9 @@ dbs:
       - url: ${ZERO_LITESTREAM_BACKUP_URL}
         endpoint: ${ZERO_LITESTREAM_ENDPOINT}
         region: ${ZERO_LITESTREAM_REGION}
-        retention: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES}m
-        retention-check-interval: 1h
+        snapshot-interval: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS}h
+        retention: ${ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS}h
+        retention-check-interval: 15m
         sync-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES}m
         multipart-concurrency: ${ZERO_LITESTREAM_MULTIPART_CONCURRENCY}
         multipart-size: ${ZERO_LITESTREAM_MULTIPART_SIZE}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
 import { Database } from "../../../../zqlite/src/db.js";
-import Sqlite3Database from "@rocicorp/zero-sqlite3";
+import SQLite3Database from "@rocicorp/zero-sqlite3";
 //#region ../zero-cache/src/services/litestream/vfs-watermark-reader.ts
 var DEFAULT_VFS_DATABASE_URI = "file:zero-backup.db?vfs=litestream&mode=ro";
 var DEFAULT_VFS_LOG_LEVEL = "info";
@@ -27,7 +27,7 @@ var VfsBackupWatermarkReader = class {
 		env["LITESTREAM_REPLICA_URL"] = options.replicaURL;
 		env["LITESTREAM_LOG_LEVEL"] = (options.logLevel ?? DEFAULT_VFS_LOG_LEVEL).toUpperCase();
 		setOptionalEnv(env, "LITESTREAM_LOG_FILE", options.logFile);
-		const loaderFactory = deps.loaderFactory ?? (() => new Sqlite3Database(":memory:"));
+		const loaderFactory = deps.loaderFactory ?? (() => new SQLite3Database(":memory:"));
 		const databaseFactory = deps.databaseFactory ?? ((lc, uri) => new Database(lc, uri, { readonly: true }));
 		this.#loader = loaderFactory();
 		this.#loader.loadExtension(options.extensionPath);

package/out/zero-cache/src/services/replicator/change-processor.d.ts CHANGED Viewed

@@ -9,6 +9,12 @@ export type CommitResult = {
     completedBackfill: DownloadStatus | undefined;
     schemaUpdated: boolean;
     changeLogUpdated: boolean;
+    /**
+     * Millisecond epoch at which the transaction committed upstream, if the
+     * ChangeSource reported one. Propagated to ViewSyncers as the origin
+     * timestamp of the end-to-end serving lag measurement.
+     */
+    upstreamCommitTimeMs?: number | undefined;
 };
 /**
  * The ChangeProcessor partitions the stream of messages into transactions

package/out/zero-cache/src/services/replicator/change-processor.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"~~AAAA~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAiB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAiC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AASpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;~~AAElE~~,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;~~CAC3B~~,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAiBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAgCrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;CAyJvB"}
1	+ {"version":3,"file":"change-processor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-processor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAiB1E,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,wBAAwB,CAAC;AAiC5D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAC;AASpD,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG,cAAc,CAAC;AASlE,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,cAAc,GAAG,SAAS,CAAC;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF;;;;;;;;;;GAUG;AACH,qBAAa,eAAe;;gBAiBxB,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,mBAAmB,EACzB,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,KAAK,IAAI;IAgCrD,KAAK,CAAC,EAAE,EAAE,UAAU;IAIpB,8CAA8C;IAC9C,cAAc,CACZ,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,gBAAgB,GAC3B,YAAY,GAAG,IAAI;CAyJvB"}

package/out/zero-cache/src/services/replicator/change-processor.js CHANGED Viewed

@@ -15,7 +15,10 @@ import { updateReplicationWatermark } from "./schema/replication-state.js";
 import { id } from "../../types/sql.js";
 import { createLiteIndexStatement, createLiteTableStatement, liteColumnDef } from "../../db/create.js";
 import { SqliteError } from "@rocicorp/zero-sqlite3";
+import { constants } from "node:buffer";
 //#region ../zero-cache/src/services/replicator/change-processor.ts
+var BIND_VALUE_TOO_BIG_ERROR = "The bound string, buffer, or bigint is too big";
+var MAX_SQLITE_BIND_BYTES = Math.min(constants.MAX_LENGTH, constants.MAX_STRING_LENGTH);
 /**
 * The ChangeProcessor partitions the stream of messages into transactions
 * by creating a {@link TransactionProcessor} when a transaction begins, and dispatching
@@ -201,7 +204,7 @@ var TransactionProcessor = class {
 		this.#jsonFormat = jsonFormat;
 		switch (mode) {
 			case "serving":
-				db.beginConcurrent();
+				db.beginImmediate();
 				break;
 			case "backup":
 				db.beginImmediate();
@@ -274,12 +277,21 @@ var TransactionProcessor = class {
 		const currKey = oldKey ?? newKey;
 		const conds = Object.keys(currKey).map((col) => `${id(col)}=?`);
 		const setExprs = Object.keys(row).map((col) => `${id(col)}=?`);
-		const { changes } = this.#db.run(`
-      UPDATE ${id(table)}
-        SET ${setExprs.join(",")}
-        WHERE ${conds.join(" AND ")}
-      `, [...Object.values(row), ...Object.values(currKey)]);
-		if (changes === 0) this.#upsert(table, row);
+		try {
+			const { changes } = this.#db.run(`
+        UPDATE ${id(table)}
+          SET ${setExprs.join(",")}
+          WHERE ${conds.join(" AND ")}
+        `, [...Object.values(row), ...Object.values(currKey)]);
+			if (changes === 0) this.#upsert(table, row);
+		} catch (e) {
+			if (!(e instanceof RangeError) || e.message !== BIND_VALUE_TOO_BIG_ERROR) throw e;
+			const binding = [...Object.entries(row), ...Object.entries(currKey)].map(([column, value]) => [column, oversizedValueDescription(value)]).find(([, description]) => description);
+			const relationOid = "relationOid" in update.relation && typeof update.relation.relationOid === "number" ? ` relationOid=${update.relation.relationOid}` : "";
+			const error = new Error(`Oversized SQLite update binding: tx=${this.#version}${relationOid} table=${update.relation.schema}.${update.relation.name} column=${binding?.[0] ?? "unknown"}${binding?.[1] ? ` ${binding[1]}` : ""}`, { cause: e });
+			error.name = "OversizedUpdateBindingError";
+			throw error;
+		}
 	}
 	processDelete(del) {
 		const table = liteTableName(del.relation);
@@ -500,7 +512,8 @@ var TransactionProcessor = class {
 			watermark,
 			completedBackfill: this.#completedBackfill,
 			schemaUpdated: this.#schemaChanged,
-			changeLogUpdated: this.#numChangeLogEntries > 0
+			changeLogUpdated: this.#numChangeLogEntries > 0,
+			upstreamCommitTimeMs: commit.commitTimeMs
 		};
 	}
 	abort(lc) {
@@ -512,6 +525,11 @@ function getBackfilledColumns(row, { backfilling }) {
 	if (!backfilling?.length) return;
 	return backfilling.filter((col) => col in row);
 }
+function oversizedValueDescription(value) {
+	if (typeof value === "bigint") return BigInt.asIntN(64, value) !== value ? "valueType=bigint fitsInt64=false" : void 0;
+	const sizeBytes = typeof value === "string" ? Buffer.byteLength(value) : value instanceof Uint8Array ? value.byteLength : void 0;
+	if (sizeBytes !== void 0 && sizeBytes > MAX_SQLITE_BIND_BYTES) return `valueType=${typeof value === "string" ? "string" : "buffer"} sizeBytes=${sizeBytes} limitBytes=${MAX_SQLITE_BIND_BYTES}`;
+}
 function ensureError(err) {
 	if (err instanceof Error) return err;
 	const error = /* @__PURE__ */ new Error();