npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.1 → 1.10.0-canary.10 - Mend

@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (215) hide show

package/out/zero-cache/src/db/transaction-pool.js CHANGED Viewed

@@ -22,16 +22,14 @@ var TransactionPool = class {
 	#workers = [];
 	#initialWorkers;
 	#maxWorkers;
-	#responseTimeout;
 	#timeoutTask;
 	#numWorkers;
 	#numWorking = 0;
 	#db;
 	#done = false;
 	#failure;
-	#orphanedQueryCheckInterval;
 	constructor(lc, opts, timeoutTasks = TIMEOUT_TASKS) {
-		const { mode, init, cleanup, initialWorkers = 1, maxWorkers = initialWorkers, statementResponseTimeout } = opts;
+		const { mode, init, cleanup, initialWorkers = 1, maxWorkers = initialWorkers } = opts;
 		assert(initialWorkers > 0, "initialWorkers must be positive");
 		assert(maxWorkers >= initialWorkers, "maxWorkers must be >= initialWorkers");
 		this.#lc = lc;
@@ -42,8 +40,6 @@ var TransactionPool = class {
 		this.#numWorkers = initialWorkers;
 		this.#maxWorkers = maxWorkers;
 		this.#timeoutTask = timeoutTasks;
-		this.#responseTimeout = statementResponseTimeout;
-		this.#orphanedQueryCheckInterval = setInterval(this.#checkForOrphanedQueries, Math.min(5e3, statementResponseTimeout ?? 5e3));
 	}
 	/**
 	* Starts the pool of workers to process Tasks with transactions opened from the
@@ -159,9 +155,6 @@ var TransactionPool = class {
 	}
 	#start = performance.now();
 	#stmts = 0;
-	#latestDoneIndex = 0;
-	#latestDoneTime = 0;
-	#pending = /* @__PURE__ */ new Map();
 	/**
 	* Implements the semantics specified in {@link process()}.
 	*
@@ -188,69 +181,18 @@ var TransactionPool = class {
 					r.resolve();
 					return { pending: null };
 				}
-				const time = Date.now();
 				return { pending: stmts.reduce((_, stmt) => {
 					const query = stmt;
 					const index = ++this.#stmts;
-					const { promise: aborted, reject: abort } = resolver();
-					aborted.catch(() => {});
-					this.#pending.set(query, {
-						index,
-						time,
-						abort
-					});
-					const responded = stmt.execute().then(() => {
+					return stmt.execute().then(() => {
 						if (index % 1e3 === 0) lc[index % 1e4 === 0 ? "info" : "debug"]?.(`executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`, { statement: query.string });
-					}).catch((e) => this.fail(e)).finally(() => {
-						this.#pending.delete(query);
-						if (index > this.#latestDoneIndex) {
-							this.#latestDoneIndex = index;
-							this.#latestDoneTime = Date.now();
-						}
-					});
-					const done = Promise.race([responded, aborted]);
-					done.catch(() => {});
-					return done;
+					}).catch((e) => this.fail(e));
 				}, promiseVoid).finally(r.resolve) };
 			},
 			rejected: r.resolve
 		};
 	}
 	/**
-	* Periodically checks the map of `#pending` queries for which Promises
-	* have yet to be resolved. Checks for pathological scenarios that have
-	* been observed:
-	*
-	* * A promise does not resolve, even though the transaction is otherwise
-	*   healthy with keepalives being sent.
-	* * A transaction never "finishes" at the postgres.js level after the
-	*   worker exits, implying that postgres.js is awaiting the `last` promise
-	*   returned. In this case Postgres disconnected the application with an
-	*   idle-in-transaction timeout, suggesting that all statements did in
-	*   fact complete.
-	*/
-	#checkForOrphanedQueries = () => {
-		if (this.#pending.size === 0) return;
-		const now = Date.now();
-		for (const [query, { index, time, abort }] of this.#pending.entries()) {
-			const statement = query.string;
-			const abortWith = (msg) => {
-				this.#lc.warn?.(msg, { statement });
-				const err = new ResponseTimeoutError(msg);
-				abort(err);
-				this.fail(err);
-				this.#pending.delete(query);
-			};
-			if (index < this.#latestDoneIndex) {
-				const elapsed = now - this.#latestDoneTime;
-				if (this.#responseTimeout && elapsed > this.#responseTimeout) abortWith(`statement ${this.#latestDoneIndex} completed ${elapsed} ms ago, but statement ${index} has not.`);
-			} else {
-				const elapsed = now - time;
-				if (this.#responseTimeout && elapsed > this.#responseTimeout) abortWith(`response for statement timed out after ${elapsed} ms`);
-			}
-		}
-	};
-	/**
 	* Processes and returns the result of executing the {@link ReadTask} from
 	* within the transaction. An error thrown by the task will result in
 	* rejecting the returned Promise, but will not affect the transaction pool
@@ -317,7 +259,6 @@ var TransactionPool = class {
 		assert(!this.#done, "already set done");
 		this.#done = true;
 		for (let i = 0; i < this.#numWorkers; i++) this.#tasks.enqueue("done");
-		Promise.allSettled(this.#workers).then(() => clearInterval(this.#orphanedQueryCheckInterval));
 	}
 	isRunning() {
 		return this.#db !== void 0 && !this.#done && this.#failure === void 0;
@@ -331,19 +272,20 @@ var TransactionPool = class {
 			const level = this.#failure instanceof ControlFlowError ? "debug" : "info";
 			this.#lc[level]?.(this.#failure);
 			for (let i = 0; i < this.#numWorkers; i++) this.#tasks.enqueue(this.#failure);
-			Promise.allSettled(this.#workers).then(() => clearInterval(this.#orphanedQueryCheckInterval));
 		}
 	}
 };
 /**
-* @returns An `init` Task for importing a snapshot from another transaction.
+* @returns An `init` Task for importing a snapshot from another transaction,
+*          as well as an `imported` Queue which resolves or rejects when each
+*          worker is initialized.
 */
 function importSnapshot(snapshotID) {
-	const { promise: imported, resolve, reject } = resolver();
+	const imported = new Queue();
 	return {
 		init: (tx) => {
 			const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);
-			stmt.then(() => resolve(), reject);
+			stmt.then(() => imported.enqueue(), (err) => imported.enqueueRejection(err));
 			return [stmt];
 		},
 		imported
@@ -397,12 +339,6 @@ var TIMEOUT_TASKS = {
 		task: "done"
 	}
 };
-var ResponseTimeoutError = class extends Error {
-	static name = "ResponseTimeoutError";
-	constructor(msg) {
-		super(msg);
-	}
-};
 //#endregion
 export { TransactionPool, importSnapshot };

package/out/zero-cache/src/db/transaction-pool.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#responseTimeout","#timeoutTask","#lc","#stmtRunner","#numWorkers","#orphanedQueryCheckInterval","#checkForOrphanedQueries","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#pending","#latestDoneIndex","#latestDoneTime","#readRunner"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> \| T;\n\nexport type Statement =\n \| postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n \| postgres.PendingQuery<postgres.Row[]>;\n\n/*\n A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n \n /\nexport type Task = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/*\n A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n /\nexport type ReadTask<T> = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<T>;\n\nexport type Options = {\n mode: Mode;\n\n /\n A {@link Task} that is run in each Transaction before it begins\n * processing general tasks. This can be used to to set the transaction\n * mode, export/set snapshots, etc. This will be run even if\n * {@link fail} has been called on the pool.\n /\n init?: Task \| undefined;\n\n /\n A {@link Task} that is run in each Transaction before it closes.\n * This will be run even if {@link fail} has been called, or if a\n * preceding Task threw an Error.\n /\n cleanup?: Task \| undefined;\n\n /\n The initial number of transaction workers to process tasks.\n * This is the steady state number of workers that will be kept\n * alive if the TransactionPool is long lived.\n * This must be greater than 0. Defaults to 1.\n /\n initialWorkers?: number;\n\n /\n When specified, allows the pool to grow to `maxWorkers`. This\n * must be greater than or equal to `initialWorkers`. On-demand\n * workers will be shut down after an idle timeout of 5 seconds.\n /\n maxWorkers?: number;\n\n /\n Aborts the transaction if the response for a statement is not received\n * within the specified timeout. Note that this is different from a Postgres\n * `statement_timeout` or `lock_timeout` in that it is specifically intended\n * to detect situations in which Postgres either never received the\n * statement, or the application never got the response for the executed\n * statement.\n /\n statementResponseTimeout?: number;\n};\n\n/\n A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n \n This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n /\nexport class TransactionPool {\n #lc: LogContext;\n readonly #mode: Mode;\n readonly #init: TaskRunner \| undefined;\n readonly #cleanup: TaskRunner \| undefined;\n readonly #tasks = new Queue<TaskRunner \| Error \| 'done'>();\n readonly #workers: Promise<unknown>[] = [];\n readonly #initialWorkers: number;\n readonly #maxWorkers: number;\n readonly #responseTimeout: number \| undefined;\n readonly #timeoutTask: TimeoutTasks;\n #numWorkers: number;\n #numWorking = 0;\n #db: PostgresDB \| undefined; // set when running. stored to allow adaptive pool sizing.\n\n #done = false;\n #failure: Error \| undefined;\n #orphanedQueryCheckInterval: NodeJS.Timeout;\n\n constructor(\n lc: LogContext,\n opts: Options,\n timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n ) {\n const {\n mode,\n init,\n cleanup,\n initialWorkers = 1,\n maxWorkers = initialWorkers,\n statementResponseTimeout,\n } = opts;\n assert(initialWorkers > 0, 'initialWorkers must be positive');\n assert(\n maxWorkers >= initialWorkers,\n 'maxWorkers must be >= initialWorkers',\n );\n\n this.#lc = lc;\n this.#mode = mode;\n this.#init = init ? this.#stmtRunner(init) : undefined;\n this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n this.#initialWorkers = initialWorkers;\n this.#numWorkers = initialWorkers;\n this.#maxWorkers = maxWorkers;\n this.#timeoutTask = timeoutTasks;\n this.#responseTimeout = statementResponseTimeout;\n\n this.#orphanedQueryCheckInterval = setInterval(\n this.#checkForOrphanedQueries,\n Math.min(5_000, statementResponseTimeout ?? 5_000),\n );\n }\n\n /\n Starts the pool of workers to process Tasks with transactions opened from the\n * specified {@link db}.\n /\n run(db: PostgresDB): this {\n assert(!this.#db, 'already running');\n this.#db = db;\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#addWorker(db);\n }\n return this;\n }\n\n /\n Adds context parameters to internal LogContext. This is useful for context values that\n * are not known when the TransactionPool is constructed (e.g. determined after a database\n * call when the pool is running).\n \n Returns an object that can be used to add more parameters.\n /\n addLoggingContext(key: string, value: string) {\n this.#lc = this.#lc.withContext(key, value);\n\n return {\n addLoggingContext: (key: string, value: string) =>\n this.addLoggingContext(key, value),\n };\n }\n\n /\n Returns a promise that:\n \n * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n * to a 0 ref count), once all added tasks have been processed and all transactions have been\n * committed or closed.\n \n * rejects if processing was aborted with {@link fail} or if processing any of\n * the tasks resulted in an error. All uncommitted transactions will have been\n * rolled back.\n \n Note that partial failures are possible if processing writes with multiple workers\n * (e.g. `setDone` is called, allowing some workers to commit, after which other\n * workers encounter errors). Using a TransactionPool in this manner does not make\n * sense in terms of transactional semantics, and is thus not recommended.\n \n For reads, however, multiple workers is useful for performing parallel reads\n * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n * Resolves or rejects when all workers are done or failed.\n /\n async done() {\n const numWorkers = this.#workers.length;\n await Promise.all(this.#workers);\n\n if (numWorkers < this.#workers.length) {\n // If workers were added after the initial set, they must be awaited to ensure\n // that the results (i.e. rejections) of all workers are accounted for. This only\n // needs to be re-done once, because the fact that the first `await` completed\n // guarantees that the pool is in a terminal state and no new workers can be added.\n await Promise.all(this.#workers);\n }\n this.#lc.debug?.('transaction pool done');\n\n const elapsed = performance.now() - this.#start;\n if (elapsed > 60_000) {\n if (this.#stmts > 0) {\n this.#lc.warn?.(\n `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n );\n } else {\n this.#lc.warn?.(\n `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n );\n }\n }\n }\n\n #addWorker(db: PostgresDB) {\n const id = this.#workers.length + 1;\n const lc = this.#lc.withContext('tx', id);\n\n const tt: TimeoutTask =\n this.#workers.length < this.#initialWorkers\n ? this.#timeoutTask.forInitialWorkers\n : this.#timeoutTask.forExtraWorkers;\n const {timeoutMs} = tt;\n const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n const worker = async (tx: PostgresTransaction) => {\n const start = performance.now();\n try {\n lc.debug?.('started transaction');\n\n let last: Promise<void> = promiseVoid;\n\n const executeTask = async (runner: TaskRunner) => {\n runner !== this.#init && this.#numWorking++;\n const {pending} = await runner.run(tx, lc, () => {\n runner !== this.#init && this.#numWorking--;\n });\n last = pending ?? last;\n };\n\n let task: TaskRunner \| Error \| 'done' =\n this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n try {\n while (task !== 'done') {\n if (\n task instanceof Error \|\|\n (task !== this.#init && this.#failure)\n ) {\n throw this.#failure ?? task;\n }\n await executeTask(task);\n\n // await the next task.\n task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n }\n } finally {\n // Execute the cleanup task even on failure.\n if (this.#cleanup) {\n await executeTask(this.#cleanup);\n }\n }\n\n const elapsed = performance.now() - start;\n lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n // Given the semantics of a Postgres transaction, the last statement\n // will only succeed if all of the preceding statements succeeded.\n return last;\n } catch (e) {\n if (e !== this.#failure) {\n this.fail(e); // A failure in any worker should fail the pool.\n }\n throw e;\n }\n };\n\n const workerTx = runTx(db, worker, {mode: this.#mode})\n .catch(e => {\n if (e instanceof RollbackSignal) {\n // A RollbackSignal is used to gracefully rollback the postgres.js\n // transaction block. It should not be thrown up to the application.\n lc.debug?.('aborted transaction');\n } else {\n throw e;\n }\n })\n .finally(() => this.#numWorkers--);\n\n // Attach a rejection handler immediately to prevent unhandledRejections.\n // The application will handle errors when it awaits processReadTask()\n // or done().\n workerTx.catch(() => {});\n\n this.#workers.push(workerTx);\n\n // After adding the worker, enqueue a terminal signal if we are in either of the\n // terminal states (both of which prevent more tasks from being enqueued), to ensure\n // that the added worker eventually exits.\n if (this.#done) {\n this.#tasks.enqueue('done');\n }\n if (this.#failure) {\n this.#tasks.enqueue(this.#failure);\n }\n }\n\n /\n Processes the statements produced by the specified {@link Task},\n * returning a Promise that resolves when the statements are either processed\n * by the database or rejected.\n \n Note that statement failures will result in failing the entire\n * TransactionPool (per transaction semantics). However, the returned Promise\n * itself will resolve rather than reject. As such, it is fine to ignore\n * returned Promises in order to pipeline requests to the database. It is\n * recommended to occasionally await them (e.g. after some threshold) in\n * order to avoid memory blowup in the case of database slowness.\n /\n process(task: Task): Promise<void> {\n const r = resolver<void>();\n this.#process(this.#stmtRunner(task, r));\n return r.promise;\n }\n\n readonly #start = performance.now();\n #stmts = 0;\n #latestDoneIndex = 0;\n #latestDoneTime = 0;\n\n readonly #pending = new Map<\n Query,\n {index: number; time: number; abort: (e: ResponseTimeoutError) => void}\n >();\n\n /\n Implements the semantics specified in {@link process()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the statements are produced,\n * allowing them to be pipelined to the database.\n * * Statement errors result in failing the transaction pool.\n * * The client-supplied Resolver resolves on success or failure;\n * it is never rejected.\n /\n #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let stmts: Statement[];\n try {\n stmts = await task(tx, lc);\n } catch (e) {\n r.resolve();\n throw e;\n } finally {\n freeWorker();\n }\n\n if (stmts.length === 0) {\n r.resolve();\n return {pending: null};\n }\n\n // Execute the statements (i.e. send to the db) immediately.\n // The last result is returned for the worker to await before\n // closing the transaction.\n const time = Date.now();\n const last = stmts.reduce((_, stmt) => {\n const query = stmt as unknown as Query;\n const index = ++this.#stmts;\n const {promise: aborted, reject: abort} = resolver();\n aborted.catch(() => {}); // always consider it \"handled\"\n\n this.#pending.set(query, {index, time, abort});\n const responded = stmt\n .execute()\n .then(() => {\n if (index % 1000 === 0) {\n const log = index % 10000 === 0 ? 'info' : 'debug';\n lc[log]?.(\n `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n {statement: query.string},\n );\n }\n })\n .catch(e => this.fail(e))\n .finally(() => {\n this.#pending.delete(query);\n if (index > this.#latestDoneIndex) {\n this.#latestDoneIndex = index;\n this.#latestDoneTime = Date.now();\n }\n });\n\n const done = Promise.race([responded, aborted]);\n done.catch(() => {});\n\n return done;\n }, promiseVoid);\n return {pending: last.finally(r.resolve)};\n },\n rejected: r.resolve,\n };\n }\n\n /\n Periodically checks the map of `#pending` queries for which Promises\n * have yet to be resolved. Checks for pathological scenarios that have\n * been observed:\n \n * A promise does not resolve, even though the transaction is otherwise\n * healthy with keepalives being sent.\n * * A transaction never \"finishes\" at the postgres.js level after the\n * worker exits, implying that postgres.js is awaiting the `last` promise\n * returned. In this case Postgres disconnected the application with an\n * idle-in-transaction timeout, suggesting that all statements did in\n * fact complete.\n /\n readonly #checkForOrphanedQueries = () => {\n if (this.#pending.size === 0) {\n return;\n }\n const now = Date.now();\n for (const [query, {index, time, abort}] of this.#pending.entries()) {\n const statement = query.string;\n const abortWith = (msg: string) => {\n this.#lc.warn?.(msg, {statement});\n const err = new ResponseTimeoutError(msg);\n abort(err);\n this.fail(err);\n this.#pending.delete(query);\n };\n\n if (index < this.#latestDoneIndex) {\n const elapsed = now - this.#latestDoneTime;\n if (this.#responseTimeout && elapsed > this.#responseTimeout) {\n abortWith(\n `statement ${this.#latestDoneIndex} completed ${elapsed} ms ago, but statement ${index} has not.`,\n );\n }\n } else {\n // Nothing out of order, but checking for a response timeout.\n const elapsed = now - time;\n if (this.#responseTimeout && elapsed > this.#responseTimeout) {\n abortWith(`response for statement timed out after ${elapsed} ms`);\n }\n }\n }\n };\n\n /\n Processes and returns the result of executing the {@link ReadTask} from\n * within the transaction. An error thrown by the task will result in\n * rejecting the returned Promise, but will not affect the transaction pool\n * itself.\n /\n processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n const r = resolver<T>();\n this.#process(this.#readRunner(readTask, r));\n return r.promise;\n }\n\n /\n Implements the semantics specified in {@link processReadTask()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the result is produced,\n * before resolving the client-supplied Resolver.\n * * Errors result in rejecting the client-supplied Resolver but\n * do not affect transaction pool.\n /\n #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let result: T;\n try {\n result = await readTask(tx, lc);\n freeWorker();\n r.resolve(result);\n } catch (e) {\n freeWorker();\n r.reject(e);\n }\n return {pending: null};\n },\n rejected: r.reject,\n };\n }\n\n #process(runner: TaskRunner): void {\n assert(!this.#done, 'already set done');\n if (this.#failure) {\n runner.rejected(this.#failure);\n return;\n }\n\n this.#tasks.enqueue(runner);\n\n // Check if the pool size can and should be increased.\n if (this.#numWorkers < this.#maxWorkers) {\n const outstanding = this.#tasks.size();\n\n if (outstanding > this.#numWorkers - this.#numWorking) {\n this.#db && this.#addWorker(this.#db);\n this.#numWorkers++;\n this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n }\n }\n }\n\n /\n Ends all workers with a ROLLBACK. Throws if the pool is already done\n * or aborted.\n /\n abort() {\n this.fail(new RollbackSignal());\n }\n\n /\n Signals to all workers to end their transaction once all pending tasks have\n * been completed. Throws if the pool is already done or aborted.\n /\n setDone() {\n assert(!this.#done, 'already set done');\n this.#done = true;\n\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#tasks.enqueue('done');\n }\n void Promise.allSettled(this.#workers).then(() =>\n clearInterval(this.#orphanedQueryCheckInterval),\n );\n }\n\n isRunning(): boolean {\n return this.#db !== undefined && !this.#done && this.#failure === undefined;\n }\n\n /\n Signals all workers to fail their transactions with the given {@link err}.\n /\n fail(err: unknown) {\n if (!this.#done && !this.#failure) {\n this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n // Logged for informational purposes. It is the responsibility of\n // higher level logic to classify and handle the exception.\n const level =\n this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n this.#lc[level]?.(this.#failure);\n\n for (let i = 0; i < this.#numWorkers; i++) {\n // Enqueue the Error to terminate any workers waiting for tasks.\n this.#tasks.enqueue(this.#failure);\n }\n void Promise.allSettled(this.#workers).then(() =>\n clearInterval(this.#orphanedQueryCheckInterval),\n );\n }\n }\n}\n\ntype SynchronizeSnapshotTasks = {\n /\n The `init` Task for the TransactionPool from which the snapshot originates.\n * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n * first worker will export a snapshot that the others set.\n /\n exportSnapshot: Task;\n\n /\n The `cleanup` Task for the TransactionPool from which the snapshot\n * originates. This Task will wait for the follower pool to `setSnapshot`\n * to ensure that the snapshot is successfully shared before the originating\n * transaction is closed.\n /\n cleanupExport: Task;\n\n /\n The `init` Task for the TransactionPool in which workers will\n * consequently see the same snapshot as that of the first pool. The pool\n * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n /\n setSnapshot: Task;\n\n /* The ID of the shared snapshot. /\n snapshotID: Promise<string>;\n};\n\n/\n Init Tasks for Postgres snapshot synchronization across transactions.\n \n https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n /\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n const {\n promise: snapshotCaptured,\n resolve: captureSnapshot,\n reject: failCapture,\n } = resolver<unknown>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n let firstWorkerRun = false;\n\n // Note: Neither init task should `await`, as processing in each pool can proceed\n // as soon as the statements have been sent to the db. However, the `cleanupExport`\n // task must `await` the result of `setSnapshot` to ensure that exporting transaction\n // does not close before the snapshot has been captured.\n return {\n exportSnapshot: tx => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n const stmt =\n tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n tx`SET TRANSACTION READ ONLY`.simple(),\n ]);\n },\n\n setSnapshot: tx =>\n snapshotExported.then(snapshotID => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n // Intercept the promise to propagate the information to `cleanupExport`.\n stmt.then(captureSnapshot, failCapture);\n return [stmt];\n }),\n\n cleanupExport: async () => {\n await snapshotCaptured;\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n /\nexport function sharedSnapshot(): {\n init: Task;\n cleanup: Task;\n snapshotID: Promise<string>;\n} {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot.\n let firstWorkerRun = false;\n\n // The LogContext of the exporting worker, used to identify its cleanup call.\n // Each worker receives a unique lc instance (via withContext('tx', id)), so\n // reference equality reliably identifies the exporting worker.\n let exporterLc: LogContext \| undefined;\n\n // Set when the exporting worker's cleanup runs, signalling that the snapshot\n // is no longer needed and any subsequently spawned workers should skip their\n // initTask.\n let firstWorkerDone = false;\n\n return {\n init: (tx, lc) => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n exporterLc = lc; // Remember which worker is the exporter.\n const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n if (!firstWorkerDone) {\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n ]);\n }\n lc.debug?.('All work is done. No need to set snapshot');\n return [];\n },\n\n cleanup: (_tx, lc) => {\n // Only the exporting worker's cleanup should disable snapshot-setting.\n // Non-exporter workers may finish early; letting them flip this flag\n // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n // and read a newer database view, violating snapshot isolation.\n if (lc === exporterLc) {\n firstWorkerDone = true;\n }\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n @returns An `init` Task for importing a snapshot from another transaction.\n /\nexport function importSnapshot(snapshotID: string): {\n init: Task;\n imported: Promise<void>;\n} {\n const {promise: imported, resolve, reject} = resolver<void>();\n\n return {\n init: tx => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n stmt.then(() => resolve(), reject);\n return [stmt];\n },\n\n imported,\n };\n}\n\n/\n A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n /\nexport class ControlFlowError extends Error {\n constructor(cause?: unknown) {\n super();\n this.cause = cause;\n }\n}\n\n/\n Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n \n Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n /\nclass RollbackSignal extends ControlFlowError {\n readonly name = 'RollbackSignal';\n readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n\ninterface TaskRunner {\n /\n Manages the running of a Task or ReadTask in two phases:\n \n - If the task involves blocking, this is done in the worker. Once the\n * blocking is done, `freeWorker()` is invoked to signal that the worker\n * is available to run another task. Note that this should be invoked\n * before resolving the result to the calling thread so that a\n * subsequent task can reuse the same worker.\n \n - Task statements are executed on the database asynchronously. The final\n * result of this processing is encapsulated in the returned `pending`\n * Promise. The worker will await the last pending Promise before closing\n * the transaction.\n \n @param freeWorker should be called as soon as all blocking operations are\n * completed in order to return the transaction to the pool.\n * @returns A `pending` Promise indicating when the statements have been\n * processed by the database, allowing the transaction to be closed.\n * This should be `null` if there are no transaction-dependent\n * statements to await.\n /\n run(\n tx: PostgresTransaction,\n lc: LogContext,\n freeWorker: () => void,\n ): Promise<{pending: Promise<void> \| null}>;\n\n /\n Invoked if the TransactionPool is already in a failed state when the task\n * is requested.\n */\n rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n lc.debug?.(`sending tx keepalive`);\n return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n timeoutMs: number;\n task: Task \| 'done';\n};\n\ntype TimeoutTasks = {\n forInitialWorkers: TimeoutTask;\n forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n forInitialWorkers: {\n timeoutMs: KEEPALIVE_TIMEOUT_MS,\n task: KEEPALIVE_TASK,\n },\n forExtraWorkers: {\n timeoutMs: IDLE_TIMEOUT_MS,\n task: 'done',\n },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n\nexport class ResponseTimeoutError extends Error {\n static readonly name = 'ResponseTimeoutError';\n\n constructor(msg: string) {\n super(msg);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4FA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAmC;CACzD,WAAwC,CAAC;CACzC;CACA;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,QAAQ;CACR;CACA;CAEA,YACE,IACA,MACA,eAAe,eACf;EACA,MAAM,EACJ,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,gBACb,6BACE;EACJ,OAAO,iBAAiB,GAAG,iCAAiC;EAC5D,OACE,cAAc,gBACd,sCACF;EAEA,KAAKS,MAAM;EACX,KAAKT,QAAQ;EACb,KAAKC,QAAQ,OAAO,KAAKS,YAAY,IAAI,IAAI,KAAA;EAC7C,KAAKR,WAAW,UAAU,KAAKQ,YAAY,OAAO,IAAI,KAAA;EACtD,KAAKL,kBAAkB;EACvB,KAAKM,cAAc;EACnB,KAAKL,cAAc;EACnB,KAAKE,eAAe;EACpB,KAAKD,mBAAmB;EAExB,KAAKK,8BAA8B,YACjC,KAAKC,0BACL,KAAK,IAAI,KAAO,4BAA4B,GAAK,CACnD;CACF;;;;;CAMA,IAAI,IAAsB;EACxB,OAAO,CAAC,KAAKC,KAAK,iBAAiB;EACnC,KAAKA,MAAM;EACX,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKH,aAAa,KACpC,KAAKI,WAAW,EAAE;EAEpB,OAAO;CACT;;;;;;;;CASA,kBAAkB,KAAa,OAAe;EAC5C,KAAKN,MAAM,KAAKA,IAAI,YAAY,KAAK,KAAK;EAE1C,OAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,KAAK,EACrC;CACF;;;;;;;;;;;;;;;;;;;;;CAsBA,MAAM,OAAO;EACX,MAAM,aAAa,KAAKL,SAAS;EACjC,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAE/B,IAAI,aAAa,KAAKA,SAAS,QAK7B,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAEjC,KAAKK,IAAI,QAAQ,uBAAuB;EAExC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKO;EACzC,IAAI,UAAU,KACZ,IAAI,KAAKC,SAAS,GAChB,KAAKR,IAAI,OACP,kCAAkC,KAAKQ,OAAO,eAAe,QAAQ,QAAQ,CAAC,EAAE,KAClF;OAEA,KAAKR,IAAI,OACP,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,KACxD;CAGN;CAEA,WAAW,IAAgB;EACzB,MAAM,KAAK,KAAKL,SAAS,SAAS;EAClC,MAAM,KAAK,KAAKK,IAAI,YAAY,MAAM,EAAE;EAExC,MAAM,KACJ,KAAKL,SAAS,SAAS,KAAKC,kBACxB,KAAKG,aAAa,oBAClB,KAAKA,aAAa;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,KAAKE,YAAY,GAAG,IAAI;EAE1E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI;IACF,GAAG,QAAQ,qBAAqB;IAEhC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;KAChD,WAAW,KAAKT,SAAS,KAAKiB;KAC9B,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;MAC/C,WAAW,KAAKjB,SAAS,KAAKiB;KAChC,CAAC;KACD,OAAO,WAAW;IACpB;IAEA,IAAI,OACF,KAAKjB,SAAU,MAAM,KAAKE,OAAO,QAAQ,aAAa,SAAS;IAEjE,IAAI;KACF,OAAO,SAAS,QAAQ;MACtB,IACE,gBAAgB,SACf,SAAS,KAAKF,SAAS,KAAKkB,UAE7B,MAAM,KAAKA,YAAY;MAEzB,MAAM,YAAY,IAAI;MAGtB,OAAO,MAAM,KAAKhB,OAAO,QAAQ,aAAa,SAAS;KACzD;IACF,UAAU;KAER,IAAI,KAAKD,UACP,MAAM,YAAY,KAAKA,QAAQ;IAEnC;IAEA,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,GAAG,QAAQ,wBAAwB,QAAQ,QAAQ,CAAC,EAAE,KAAK;IAG3D,OAAO;GACT,SAAS,GAAG;IACV,IAAI,MAAM,KAAKiB,UACb,KAAK,KAAK,CAAC;IAEb,MAAM;GACR;EACF;EAEA,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,KAAKnB,MAAK,CAAC,EAClD,OAAM,MAAK;GACV,IAAI,aAAa,gBAGf,GAAG,QAAQ,qBAAqB;QAEhC,MAAM;EAEV,CAAC,EACA,cAAc,KAAKW,aAAa;EAKnC,SAAS,YAAY,CAAC,CAAC;EAEvB,KAAKP,SAAS,KAAK,QAAQ;EAK3B,IAAI,KAAKgB,OACP,KAAKjB,OAAO,QAAQ,MAAM;EAE5B,IAAI,KAAKgB,UACP,KAAKhB,OAAO,QAAQ,KAAKgB,QAAQ;CAErC;;;;;;;;;;;;;CAcA,QAAQ,MAA2B;EACjC,MAAM,IAAI,SAAe;EACzB,KAAKE,SAAS,KAAKX,YAAY,MAAM,CAAC,CAAC;EACvC,OAAO,EAAE;CACX;CAEA,SAAkB,YAAY,IAAI;CAClC,SAAS;CACT,mBAAmB;CACnB,kBAAkB;CAElB,2BAAoB,IAAI,IAGtB;;;;;;;;;;;CAYF,YAAY,MAAY,IAA2B,SAAS,GAAe;EACzE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,QAAQ,MAAM,KAAK,IAAI,EAAE;IAC3B,SAAS,GAAG;KACV,EAAE,QAAQ;KACV,MAAM;IACR,UAAU;KACR,WAAW;IACb;IAEA,IAAI,MAAM,WAAW,GAAG;KACtB,EAAE,QAAQ;KACV,OAAO,EAAC,SAAS,KAAI;IACvB;IAKA,MAAM,OAAO,KAAK,IAAI;IAiCtB,OAAO,EAAC,SAhCK,MAAM,QAAQ,GAAG,SAAS;KACrC,MAAM,QAAQ;KACd,MAAM,QAAQ,EAAE,KAAKO;KACrB,MAAM,EAAC,SAAS,SAAS,QAAQ,UAAS,SAAS;KACnD,QAAQ,YAAY,CAAC,CAAC;KAEtB,KAAKK,SAAS,IAAI,OAAO;MAAC;MAAO;MAAM;KAAK,CAAC;KAC7C,MAAM,YAAY,KACf,QAAQ,EACR,WAAW;MACV,IAAI,QAAQ,QAAS,GAEnB,GADY,QAAQ,QAAU,IAAI,SAAS,WAEzC,YAAY,KAAKL,OAAO,iBAAiB,YAAY,IAAI,IAAI,KAAKD,QAAQ,QAAQ,CAAC,EAAE,OACrF,EAAC,WAAW,MAAM,OAAM,CAC1B;KAEJ,CAAC,EACA,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC,EACvB,cAAc;MACb,KAAKM,SAAS,OAAO,KAAK;MAC1B,IAAI,QAAQ,KAAKC,kBAAkB;OACjC,KAAKA,mBAAmB;OACxB,KAAKC,kBAAkB,KAAK,IAAI;MAClC;KACF,CAAC;KAEH,MAAM,OAAO,QAAQ,KAAK,CAAC,WAAW,OAAO,CAAC;KAC9C,KAAK,YAAY,CAAC,CAAC;KAEnB,OAAO;IACT,GAAG,WACc,EAAK,QAAQ,EAAE,OAAO,EAAC;GAC1C;GACA,UAAU,EAAE;EACd;CACF;;;;;;;;;;;;;;CAeA,iCAA0C;EACxC,IAAI,KAAKF,SAAS,SAAS,GACzB;EAEF,MAAM,MAAM,KAAK,IAAI;EACrB,KAAK,MAAM,CAAC,OAAO,EAAC,OAAO,MAAM,YAAW,KAAKA,SAAS,QAAQ,GAAG;GACnE,MAAM,YAAY,MAAM;GACxB,MAAM,aAAa,QAAgB;IACjC,KAAKb,IAAI,OAAO,KAAK,EAAC,UAAS,CAAC;IAChC,MAAM,MAAM,IAAI,qBAAqB,GAAG;IACxC,MAAM,GAAG;IACT,KAAK,KAAK,GAAG;IACb,KAAKa,SAAS,OAAO,KAAK;GAC5B;GAEA,IAAI,QAAQ,KAAKC,kBAAkB;IACjC,MAAM,UAAU,MAAM,KAAKC;IAC3B,IAAI,KAAKjB,oBAAoB,UAAU,KAAKA,kBAC1C,UACE,aAAa,KAAKgB,iBAAiB,aAAa,QAAQ,yBAAyB,MAAM,UACzF;GAEJ,OAAO;IAEL,MAAM,UAAU,MAAM;IACtB,IAAI,KAAKhB,oBAAoB,UAAU,KAAKA,kBAC1C,UAAU,0CAA0C,QAAQ,IAAI;GAEpE;EACF;CACF;;;;;;;CAQA,gBAAmB,UAAmC;EACpD,MAAM,IAAI,SAAY;EACtB,KAAKc,SAAS,KAAKI,YAAY,UAAU,CAAC,CAAC;EAC3C,OAAO,EAAE;CACX;;;;;;;;;;CAWA,YAAe,UAAuB,GAA4B;EAChE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,SAAS,MAAM,SAAS,IAAI,EAAE;KAC9B,WAAW;KACX,EAAE,QAAQ,MAAM;IAClB,SAAS,GAAG;KACV,WAAW;KACX,EAAE,OAAO,CAAC;IACZ;IACA,OAAO,EAAC,SAAS,KAAI;GACvB;GACA,UAAU,EAAE;EACd;CACF;CAEA,SAAS,QAA0B;EACjC,OAAO,CAAC,KAAKL,OAAO,kBAAkB;EACtC,IAAI,KAAKD,UAAU;GACjB,OAAO,SAAS,KAAKA,QAAQ;GAC7B;EACF;EAEA,KAAKhB,OAAO,QAAQ,MAAM;EAG1B,IAAI,KAAKQ,cAAc,KAAKL;OACN,KAAKH,OAAO,KAE5B,IAAc,KAAKQ,cAAc,KAAKO,aAAa;IACrD,KAAKJ,OAAO,KAAKC,WAAW,KAAKD,GAAG;IACpC,KAAKH;IACL,KAAKF,IAAI,QAAQ,0BAA0B,KAAKE,aAAa;GAC/D;;CAEJ;;;;;CAMA,QAAQ;EACN,KAAK,KAAK,IAAI,eAAe,CAAC;CAChC;;;;;CAMA,UAAU;EACR,OAAO,CAAC,KAAKS,OAAO,kBAAkB;EACtC,KAAKA,QAAQ;EAEb,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKT,aAAa,KACpC,KAAKR,OAAO,QAAQ,MAAM;EAE5B,QAAa,WAAW,KAAKC,QAAQ,EAAE,WACrC,cAAc,KAAKQ,2BAA2B,CAChD;CACF;CAEA,YAAqB;EACnB,OAAO,KAAKE,QAAQ,KAAA,KAAa,CAAC,KAAKM,SAAS,KAAKD,aAAa,KAAA;CACpE;;;;CAKA,KAAK,KAAc;EACjB,IAAI,CAAC,KAAKC,SAAS,CAAC,KAAKD,UAAU;GACjC,KAAKA,WAAW,YAAY,GAAG;GAG/B,MAAM,QACJ,KAAKA,oBAAoB,mBAAmB,UAAU;GACxD,KAAKV,IAAI,SAAS,KAAKU,QAAQ;GAE/B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKR,aAAa,KAEpC,KAAKR,OAAO,QAAQ,KAAKgB,QAAQ;GAEnC,QAAa,WAAW,KAAKf,QAAQ,EAAE,WACrC,cAAc,KAAKQ,2BAA2B,CAChD;EACF;CACF;AACF;;;;AA4JA,SAAgB,eAAe,YAG7B;CACA,MAAM,EAAC,SAAS,UAAU,SAAS,WAAU,SAAe;CAE5D,OAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,EAAE;GACjE,KAAK,WAAW,QAAQ,GAAG,MAAM;GACjC,OAAO,CAAC,IAAI;EACd;EAEA;CACF;AACF;;;;;;;AAQA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;EAC3B,MAAM;EACN,KAAK,QAAQ;CACf;AACF;;;;;;;;;;;AAYA,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;AACrB;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT;AAqCA,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,MACpD;AAEA,IAAM,kBAAwB,IAAI,OAAO;CACvC,GAAG,QAAQ,sBAAsB;CACjC,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC;AAC/B;AAaA,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;CACR;CACA,iBAAiB;EACf,WAAW;EACX,MAAM;CACR;AACF;AAMA,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB,OAAO;CAEvB,YAAY,KAAa;EACvB,MAAM,GAAG;CACX;AACF"}
1	+ {"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#timeoutTask","#lc","#stmtRunner","#numWorkers","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#readRunner"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> \| T;\n\nexport type Statement =\n \| postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n \| postgres.PendingQuery<postgres.Row[]>;\n\n/*\n A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n \n /\nexport type Task = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/*\n A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n /\nexport type ReadTask<T> = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<T>;\n\nexport type Options = {\n mode: Mode;\n\n /\n A {@link Task} that is run in each Transaction before it begins\n * processing general tasks. This can be used to to set the transaction\n * mode, export/set snapshots, etc. This will be run even if\n * {@link fail} has been called on the pool.\n /\n init?: Task \| undefined;\n\n /\n A {@link Task} that is run in each Transaction before it closes.\n * This will be run even if {@link fail} has been called, or if a\n * preceding Task threw an Error.\n /\n cleanup?: Task \| undefined;\n\n /\n The initial number of transaction workers to process tasks.\n * This is the steady state number of workers that will be kept\n * alive if the TransactionPool is long lived.\n * This must be greater than 0. Defaults to 1.\n /\n initialWorkers?: number;\n\n /\n When specified, allows the pool to grow to `maxWorkers`. This\n * must be greater than or equal to `initialWorkers`. On-demand\n * workers will be shut down after an idle timeout of 5 seconds.\n /\n maxWorkers?: number;\n};\n\n/\n A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n \n This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n /\nexport class TransactionPool {\n #lc: LogContext;\n readonly #mode: Mode;\n readonly #init: TaskRunner \| undefined;\n readonly #cleanup: TaskRunner \| undefined;\n readonly #tasks = new Queue<TaskRunner \| Error \| 'done'>();\n readonly #workers: Promise<unknown>[] = [];\n readonly #initialWorkers: number;\n readonly #maxWorkers: number;\n readonly #timeoutTask: TimeoutTasks;\n #numWorkers: number;\n #numWorking = 0;\n #db: PostgresDB \| undefined; // set when running. stored to allow adaptive pool sizing.\n\n #done = false;\n #failure: Error \| undefined;\n\n constructor(\n lc: LogContext,\n opts: Options,\n timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n ) {\n const {\n mode,\n init,\n cleanup,\n initialWorkers = 1,\n maxWorkers = initialWorkers,\n } = opts;\n assert(initialWorkers > 0, 'initialWorkers must be positive');\n assert(\n maxWorkers >= initialWorkers,\n 'maxWorkers must be >= initialWorkers',\n );\n\n this.#lc = lc;\n this.#mode = mode;\n this.#init = init ? this.#stmtRunner(init) : undefined;\n this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n this.#initialWorkers = initialWorkers;\n this.#numWorkers = initialWorkers;\n this.#maxWorkers = maxWorkers;\n this.#timeoutTask = timeoutTasks;\n }\n\n /\n Starts the pool of workers to process Tasks with transactions opened from the\n * specified {@link db}.\n /\n run(db: PostgresDB): this {\n assert(!this.#db, 'already running');\n this.#db = db;\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#addWorker(db);\n }\n return this;\n }\n\n /\n Adds context parameters to internal LogContext. This is useful for context values that\n * are not known when the TransactionPool is constructed (e.g. determined after a database\n * call when the pool is running).\n \n Returns an object that can be used to add more parameters.\n /\n addLoggingContext(key: string, value: string) {\n this.#lc = this.#lc.withContext(key, value);\n\n return {\n addLoggingContext: (key: string, value: string) =>\n this.addLoggingContext(key, value),\n };\n }\n\n /\n Returns a promise that:\n \n * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n * to a 0 ref count), once all added tasks have been processed and all transactions have been\n * committed or closed.\n \n * rejects if processing was aborted with {@link fail} or if processing any of\n * the tasks resulted in an error. All uncommitted transactions will have been\n * rolled back.\n \n Note that partial failures are possible if processing writes with multiple workers\n * (e.g. `setDone` is called, allowing some workers to commit, after which other\n * workers encounter errors). Using a TransactionPool in this manner does not make\n * sense in terms of transactional semantics, and is thus not recommended.\n \n For reads, however, multiple workers is useful for performing parallel reads\n * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n * Resolves or rejects when all workers are done or failed.\n /\n async done() {\n const numWorkers = this.#workers.length;\n await Promise.all(this.#workers);\n\n if (numWorkers < this.#workers.length) {\n // If workers were added after the initial set, they must be awaited to ensure\n // that the results (i.e. rejections) of all workers are accounted for. This only\n // needs to be re-done once, because the fact that the first `await` completed\n // guarantees that the pool is in a terminal state and no new workers can be added.\n await Promise.all(this.#workers);\n }\n this.#lc.debug?.('transaction pool done');\n\n const elapsed = performance.now() - this.#start;\n if (elapsed > 60_000) {\n if (this.#stmts > 0) {\n this.#lc.warn?.(\n `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n );\n } else {\n this.#lc.warn?.(\n `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n );\n }\n }\n }\n\n #addWorker(db: PostgresDB) {\n const id = this.#workers.length + 1;\n const lc = this.#lc.withContext('tx', id);\n\n const tt: TimeoutTask =\n this.#workers.length < this.#initialWorkers\n ? this.#timeoutTask.forInitialWorkers\n : this.#timeoutTask.forExtraWorkers;\n const {timeoutMs} = tt;\n const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n const worker = async (tx: PostgresTransaction) => {\n const start = performance.now();\n try {\n lc.debug?.('started transaction');\n\n let last: Promise<void> = promiseVoid;\n\n const executeTask = async (runner: TaskRunner) => {\n runner !== this.#init && this.#numWorking++;\n const {pending} = await runner.run(tx, lc, () => {\n runner !== this.#init && this.#numWorking--;\n });\n last = pending ?? last;\n };\n\n let task: TaskRunner \| Error \| 'done' =\n this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n try {\n while (task !== 'done') {\n if (\n task instanceof Error \|\|\n (task !== this.#init && this.#failure)\n ) {\n throw this.#failure ?? task;\n }\n await executeTask(task);\n\n // await the next task.\n task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n }\n } finally {\n // Execute the cleanup task even on failure.\n if (this.#cleanup) {\n await executeTask(this.#cleanup);\n }\n }\n\n const elapsed = performance.now() - start;\n lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n // Given the semantics of a Postgres transaction, the last statement\n // will only succeed if all of the preceding statements succeeded.\n return last;\n } catch (e) {\n if (e !== this.#failure) {\n this.fail(e); // A failure in any worker should fail the pool.\n }\n throw e;\n }\n };\n\n const workerTx = runTx(db, worker, {mode: this.#mode})\n .catch(e => {\n if (e instanceof RollbackSignal) {\n // A RollbackSignal is used to gracefully rollback the postgres.js\n // transaction block. It should not be thrown up to the application.\n lc.debug?.('aborted transaction');\n } else {\n throw e;\n }\n })\n .finally(() => this.#numWorkers--);\n\n // Attach a rejection handler immediately to prevent unhandledRejections.\n // The application will handle errors when it awaits processReadTask()\n // or done().\n workerTx.catch(() => {});\n\n this.#workers.push(workerTx);\n\n // After adding the worker, enqueue a terminal signal if we are in either of the\n // terminal states (both of which prevent more tasks from being enqueued), to ensure\n // that the added worker eventually exits.\n if (this.#done) {\n this.#tasks.enqueue('done');\n }\n if (this.#failure) {\n this.#tasks.enqueue(this.#failure);\n }\n }\n\n /\n Processes the statements produced by the specified {@link Task},\n * returning a Promise that resolves when the statements are either processed\n * by the database or rejected.\n \n Note that statement failures will result in failing the entire\n * TransactionPool (per transaction semantics). However, the returned Promise\n * itself will resolve rather than reject. As such, it is fine to ignore\n * returned Promises in order to pipeline requests to the database. It is\n * recommended to occasionally await them (e.g. after some threshold) in\n * order to avoid memory blowup in the case of database slowness.\n /\n process(task: Task): Promise<void> {\n const r = resolver<void>();\n this.#process(this.#stmtRunner(task, r));\n return r.promise;\n }\n\n readonly #start = performance.now();\n #stmts = 0;\n\n /\n Implements the semantics specified in {@link process()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the statements are produced,\n * allowing them to be pipelined to the database.\n * * Statement errors result in failing the transaction pool.\n * * The client-supplied Resolver resolves on success or failure;\n * it is never rejected.\n /\n #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let stmts: Statement[];\n try {\n stmts = await task(tx, lc);\n } catch (e) {\n r.resolve();\n throw e;\n } finally {\n freeWorker();\n }\n\n if (stmts.length === 0) {\n r.resolve();\n return {pending: null};\n }\n\n // Execute the statements (i.e. send to the db) immediately.\n // The last result is returned for the worker to await before\n // closing the transaction.\n const last = stmts.reduce((_, stmt) => {\n const query = stmt as unknown as Query;\n const index = ++this.#stmts;\n return stmt\n .execute()\n .then(() => {\n if (index % 1000 === 0) {\n const log = index % 10000 === 0 ? 'info' : 'debug';\n lc[log]?.(\n `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n {statement: query.string},\n );\n }\n })\n .catch(e => this.fail(e));\n }, promiseVoid);\n return {pending: last.finally(r.resolve)};\n },\n rejected: r.resolve,\n };\n }\n\n /\n Processes and returns the result of executing the {@link ReadTask} from\n * within the transaction. An error thrown by the task will result in\n * rejecting the returned Promise, but will not affect the transaction pool\n * itself.\n /\n processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n const r = resolver<T>();\n this.#process(this.#readRunner(readTask, r));\n return r.promise;\n }\n\n /\n Implements the semantics specified in {@link processReadTask()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the result is produced,\n * before resolving the client-supplied Resolver.\n * * Errors result in rejecting the client-supplied Resolver but\n * do not affect transaction pool.\n /\n #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let result: T;\n try {\n result = await readTask(tx, lc);\n freeWorker();\n r.resolve(result);\n } catch (e) {\n freeWorker();\n r.reject(e);\n }\n return {pending: null};\n },\n rejected: r.reject,\n };\n }\n\n #process(runner: TaskRunner): void {\n assert(!this.#done, 'already set done');\n if (this.#failure) {\n runner.rejected(this.#failure);\n return;\n }\n\n this.#tasks.enqueue(runner);\n\n // Check if the pool size can and should be increased.\n if (this.#numWorkers < this.#maxWorkers) {\n const outstanding = this.#tasks.size();\n\n if (outstanding > this.#numWorkers - this.#numWorking) {\n this.#db && this.#addWorker(this.#db);\n this.#numWorkers++;\n this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n }\n }\n }\n\n /\n Ends all workers with a ROLLBACK. Throws if the pool is already done\n * or aborted.\n /\n abort() {\n this.fail(new RollbackSignal());\n }\n\n /\n Signals to all workers to end their transaction once all pending tasks have\n * been completed. Throws if the pool is already done or aborted.\n /\n setDone() {\n assert(!this.#done, 'already set done');\n this.#done = true;\n\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#tasks.enqueue('done');\n }\n }\n\n isRunning(): boolean {\n return this.#db !== undefined && !this.#done && this.#failure === undefined;\n }\n\n /\n Signals all workers to fail their transactions with the given {@link err}.\n /\n fail(err: unknown) {\n if (!this.#done && !this.#failure) {\n this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n // Logged for informational purposes. It is the responsibility of\n // higher level logic to classify and handle the exception.\n const level =\n this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n this.#lc[level]?.(this.#failure);\n\n for (let i = 0; i < this.#numWorkers; i++) {\n // Enqueue the Error to terminate any workers waiting for tasks.\n this.#tasks.enqueue(this.#failure);\n }\n }\n }\n}\n\ntype SynchronizeSnapshotTasks = {\n /\n The `init` Task for the TransactionPool from which the snapshot originates.\n * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n * first worker will export a snapshot that the others set.\n /\n exportSnapshot: Task;\n\n /\n The `cleanup` Task for the TransactionPool from which the snapshot\n * originates. This Task will wait for the follower pool to `setSnapshot`\n * to ensure that the snapshot is successfully shared before the originating\n * transaction is closed.\n /\n cleanupExport: Task;\n\n /\n The `init` Task for the TransactionPool in which workers will\n * consequently see the same snapshot as that of the first pool. The pool\n * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n /\n setSnapshot: Task;\n\n /* The ID of the shared snapshot. /\n snapshotID: Promise<string>;\n};\n\n/\n Init Tasks for Postgres snapshot synchronization across transactions.\n \n https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n /\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n const {\n promise: snapshotCaptured,\n resolve: captureSnapshot,\n reject: failCapture,\n } = resolver<unknown>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n let firstWorkerRun = false;\n\n // Note: Neither init task should `await`, as processing in each pool can proceed\n // as soon as the statements have been sent to the db. However, the `cleanupExport`\n // task must `await` the result of `setSnapshot` to ensure that exporting transaction\n // does not close before the snapshot has been captured.\n return {\n exportSnapshot: tx => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n const stmt =\n tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n tx`SET TRANSACTION READ ONLY`.simple(),\n ]);\n },\n\n setSnapshot: tx =>\n snapshotExported.then(snapshotID => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n // Intercept the promise to propagate the information to `cleanupExport`.\n stmt.then(captureSnapshot, failCapture);\n return [stmt];\n }),\n\n cleanupExport: async () => {\n await snapshotCaptured;\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n /\nexport function sharedSnapshot(): {\n init: Task;\n cleanup: Task;\n snapshotID: Promise<string>;\n} {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot.\n let firstWorkerRun = false;\n\n // The LogContext of the exporting worker, used to identify its cleanup call.\n // Each worker receives a unique lc instance (via withContext('tx', id)), so\n // reference equality reliably identifies the exporting worker.\n let exporterLc: LogContext \| undefined;\n\n // Set when the exporting worker's cleanup runs, signalling that the snapshot\n // is no longer needed and any subsequently spawned workers should skip their\n // initTask.\n let firstWorkerDone = false;\n\n return {\n init: (tx, lc) => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n exporterLc = lc; // Remember which worker is the exporter.\n const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n if (!firstWorkerDone) {\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n ]);\n }\n lc.debug?.('All work is done. No need to set snapshot');\n return [];\n },\n\n cleanup: (_tx, lc) => {\n // Only the exporting worker's cleanup should disable snapshot-setting.\n // Non-exporter workers may finish early; letting them flip this flag\n // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n // and read a newer database view, violating snapshot isolation.\n if (lc === exporterLc) {\n firstWorkerDone = true;\n }\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n @returns An `init` Task for importing a snapshot from another transaction,\n * as well as an `imported` Queue which resolves or rejects when each\n * worker is initialized.\n /\nexport function importSnapshot(snapshotID: string): {\n init: Task;\n imported: Queue<void>;\n} {\n const imported = new Queue<void>();\n\n return {\n init: tx => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n stmt.then(\n () => imported.enqueue(),\n err => imported.enqueueRejection(err),\n );\n return [stmt];\n },\n\n imported,\n };\n}\n\n/\n A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n /\nexport class ControlFlowError extends Error {\n constructor(cause?: unknown) {\n super();\n this.cause = cause;\n }\n}\n\n/\n Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n \n Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n /\nclass RollbackSignal extends ControlFlowError {\n readonly name = 'RollbackSignal';\n readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n\ninterface TaskRunner {\n /\n Manages the running of a Task or ReadTask in two phases:\n \n - If the task involves blocking, this is done in the worker. Once the\n * blocking is done, `freeWorker()` is invoked to signal that the worker\n * is available to run another task. Note that this should be invoked\n * before resolving the result to the calling thread so that a\n * subsequent task can reuse the same worker.\n \n - Task statements are executed on the database asynchronously. The final\n * result of this processing is encapsulated in the returned `pending`\n * Promise. The worker will await the last pending Promise before closing\n * the transaction.\n \n @param freeWorker should be called as soon as all blocking operations are\n * completed in order to return the transaction to the pool.\n * @returns A `pending` Promise indicating when the statements have been\n * processed by the database, allowing the transaction to be closed.\n * This should be `null` if there are no transaction-dependent\n * statements to await.\n /\n run(\n tx: PostgresTransaction,\n lc: LogContext,\n freeWorker: () => void,\n ): Promise<{pending: Promise<void> \| null}>;\n\n /\n Invoked if the TransactionPool is already in a failed state when the task\n * is requested.\n */\n rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n lc.debug?.(`sending tx keepalive`);\n return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n timeoutMs: number;\n task: Task \| 'done';\n};\n\ntype TimeoutTasks = {\n forInitialWorkers: TimeoutTask;\n forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n forInitialWorkers: {\n timeoutMs: KEEPALIVE_TIMEOUT_MS,\n task: KEEPALIVE_TASK,\n },\n forExtraWorkers: {\n timeoutMs: IDLE_TIMEOUT_MS,\n task: 'done',\n },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;AAkFA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,MAAmC;CACzD,WAAwC,CAAC;CACzC;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,QAAQ;CACR;CAEA,YACE,IACA,MACA,eAAe,eACf;EACA,MAAM,EACJ,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,mBACX;EACJ,OAAO,iBAAiB,GAAG,iCAAiC;EAC5D,OACE,cAAc,gBACd,sCACF;EAEA,KAAKQ,MAAM;EACX,KAAKR,QAAQ;EACb,KAAKC,QAAQ,OAAO,KAAKQ,YAAY,IAAI,IAAI,KAAA;EAC7C,KAAKP,WAAW,UAAU,KAAKO,YAAY,OAAO,IAAI,KAAA;EACtD,KAAKJ,kBAAkB;EACvB,KAAKK,cAAc;EACnB,KAAKJ,cAAc;EACnB,KAAKC,eAAe;CACtB;;;;;CAMA,IAAI,IAAsB;EACxB,OAAO,CAAC,KAAKI,KAAK,iBAAiB;EACnC,KAAKA,MAAM;EACX,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKD,aAAa,KACpC,KAAKE,WAAW,EAAE;EAEpB,OAAO;CACT;;;;;;;;CASA,kBAAkB,KAAa,OAAe;EAC5C,KAAKJ,MAAM,KAAKA,IAAI,YAAY,KAAK,KAAK;EAE1C,OAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,KAAK,EACrC;CACF;;;;;;;;;;;;;;;;;;;;;CAsBA,MAAM,OAAO;EACX,MAAM,aAAa,KAAKJ,SAAS;EACjC,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAE/B,IAAI,aAAa,KAAKA,SAAS,QAK7B,MAAM,QAAQ,IAAI,KAAKA,QAAQ;EAEjC,KAAKI,IAAI,QAAQ,uBAAuB;EAExC,MAAM,UAAU,YAAY,IAAI,IAAI,KAAKK;EACzC,IAAI,UAAU,KACZ,IAAI,KAAKC,SAAS,GAChB,KAAKN,IAAI,OACP,kCAAkC,KAAKM,OAAO,eAAe,QAAQ,QAAQ,CAAC,EAAE,KAClF;OAEA,KAAKN,IAAI,OACP,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,KACxD;CAGN;CAEA,WAAW,IAAgB;EACzB,MAAM,KAAK,KAAKJ,SAAS,SAAS;EAClC,MAAM,KAAK,KAAKI,IAAI,YAAY,MAAM,EAAE;EAExC,MAAM,KACJ,KAAKJ,SAAS,SAAS,KAAKC,kBACxB,KAAKE,aAAa,oBAClB,KAAKA,aAAa;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,KAAKE,YAAY,GAAG,IAAI;EAE1E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI;IACF,GAAG,QAAQ,qBAAqB;IAEhC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;KAChD,WAAW,KAAKR,SAAS,KAAKc;KAC9B,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;MAC/C,WAAW,KAAKd,SAAS,KAAKc;KAChC,CAAC;KACD,OAAO,WAAW;IACpB;IAEA,IAAI,OACF,KAAKd,SAAU,MAAM,KAAKE,OAAO,QAAQ,aAAa,SAAS;IAEjE,IAAI;KACF,OAAO,SAAS,QAAQ;MACtB,IACE,gBAAgB,SACf,SAAS,KAAKF,SAAS,KAAKe,UAE7B,MAAM,KAAKA,YAAY;MAEzB,MAAM,YAAY,IAAI;MAGtB,OAAO,MAAM,KAAKb,OAAO,QAAQ,aAAa,SAAS;KACzD;IACF,UAAU;KAER,IAAI,KAAKD,UACP,MAAM,YAAY,KAAKA,QAAQ;IAEnC;IAEA,MAAM,UAAU,YAAY,IAAI,IAAI;IACpC,GAAG,QAAQ,wBAAwB,QAAQ,QAAQ,CAAC,EAAE,KAAK;IAG3D,OAAO;GACT,SAAS,GAAG;IACV,IAAI,MAAM,KAAKc,UACb,KAAK,KAAK,CAAC;IAEb,MAAM;GACR;EACF;EAEA,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,KAAKhB,MAAK,CAAC,EAClD,OAAM,MAAK;GACV,IAAI,aAAa,gBAGf,GAAG,QAAQ,qBAAqB;QAEhC,MAAM;EAEV,CAAC,EACA,cAAc,KAAKU,aAAa;EAKnC,SAAS,YAAY,CAAC,CAAC;EAEvB,KAAKN,SAAS,KAAK,QAAQ;EAK3B,IAAI,KAAKa,OACP,KAAKd,OAAO,QAAQ,MAAM;EAE5B,IAAI,KAAKa,UACP,KAAKb,OAAO,QAAQ,KAAKa,QAAQ;CAErC;;;;;;;;;;;;;CAcA,QAAQ,MAA2B;EACjC,MAAM,IAAI,SAAe;EACzB,KAAKE,SAAS,KAAKT,YAAY,MAAM,CAAC,CAAC;EACvC,OAAO,EAAE;CACX;CAEA,SAAkB,YAAY,IAAI;CAClC,SAAS;;;;;;;;;;;CAYT,YAAY,MAAY,IAA2B,SAAS,GAAe;EACzE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,QAAQ,MAAM,KAAK,IAAI,EAAE;IAC3B,SAAS,GAAG;KACV,EAAE,QAAQ;KACV,MAAM;IACR,UAAU;KACR,WAAW;IACb;IAEA,IAAI,MAAM,WAAW,GAAG;KACtB,EAAE,QAAQ;KACV,OAAO,EAAC,SAAS,KAAI;IACvB;IAqBA,OAAO,EAAC,SAhBK,MAAM,QAAQ,GAAG,SAAS;KACrC,MAAM,QAAQ;KACd,MAAM,QAAQ,EAAE,KAAKK;KACrB,OAAO,KACJ,QAAQ,EACR,WAAW;MACV,IAAI,QAAQ,QAAS,GAEnB,GADY,QAAQ,QAAU,IAAI,SAAS,WAEzC,YAAY,KAAKA,OAAO,iBAAiB,YAAY,IAAI,IAAI,KAAKD,QAAQ,QAAQ,CAAC,EAAE,OACrF,EAAC,WAAW,MAAM,OAAM,CAC1B;KAEJ,CAAC,EACA,OAAM,MAAK,KAAK,KAAK,CAAC,CAAC;IAC5B,GAAG,WACc,EAAK,QAAQ,EAAE,OAAO,EAAC;GAC1C;GACA,UAAU,EAAE;EACd;CACF;;;;;;;CAQA,gBAAmB,UAAmC;EACpD,MAAM,IAAI,SAAY;EACtB,KAAKK,SAAS,KAAKC,YAAY,UAAU,CAAC,CAAC;EAC3C,OAAO,EAAE;CACX;;;;;;;;;;CAWA,YAAe,UAAuB,GAA4B;EAChE,OAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;IACJ,IAAI;KACF,SAAS,MAAM,SAAS,IAAI,EAAE;KAC9B,WAAW;KACX,EAAE,QAAQ,MAAM;IAClB,SAAS,GAAG;KACV,WAAW;KACX,EAAE,OAAO,CAAC;IACZ;IACA,OAAO,EAAC,SAAS,KAAI;GACvB;GACA,UAAU,EAAE;EACd;CACF;CAEA,SAAS,QAA0B;EACjC,OAAO,CAAC,KAAKF,OAAO,kBAAkB;EACtC,IAAI,KAAKD,UAAU;GACjB,OAAO,SAAS,KAAKA,QAAQ;GAC7B;EACF;EAEA,KAAKb,OAAO,QAAQ,MAAM;EAG1B,IAAI,KAAKO,cAAc,KAAKJ;OACN,KAAKH,OAAO,KAE5B,IAAc,KAAKO,cAAc,KAAKK,aAAa;IACrD,KAAKJ,OAAO,KAAKC,WAAW,KAAKD,GAAG;IACpC,KAAKD;IACL,KAAKF,IAAI,QAAQ,0BAA0B,KAAKE,aAAa;GAC/D;;CAEJ;;;;;CAMA,QAAQ;EACN,KAAK,KAAK,IAAI,eAAe,CAAC;CAChC;;;;;CAMA,UAAU;EACR,OAAO,CAAC,KAAKO,OAAO,kBAAkB;EACtC,KAAKA,QAAQ;EAEb,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKP,aAAa,KACpC,KAAKP,OAAO,QAAQ,MAAM;CAE9B;CAEA,YAAqB;EACnB,OAAO,KAAKQ,QAAQ,KAAA,KAAa,CAAC,KAAKM,SAAS,KAAKD,aAAa,KAAA;CACpE;;;;CAKA,KAAK,KAAc;EACjB,IAAI,CAAC,KAAKC,SAAS,CAAC,KAAKD,UAAU;GACjC,KAAKA,WAAW,YAAY,GAAG;GAG/B,MAAM,QACJ,KAAKA,oBAAoB,mBAAmB,UAAU;GACxD,KAAKR,IAAI,SAAS,KAAKQ,QAAQ;GAE/B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAKN,aAAa,KAEpC,KAAKP,OAAO,QAAQ,KAAKa,QAAQ;EAErC;CACF;AACF;;;;;;AA8JA,SAAgB,eAAe,YAG7B;CACA,MAAM,WAAW,IAAI,MAAY;CAEjC,OAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,EAAE;GACjE,KAAK,WACG,SAAS,QAAQ,IACvB,QAAO,SAAS,iBAAiB,GAAG,CACtC;GACA,OAAO,CAAC,IAAI;EACd;EAEA;CACF;AACF;;;;;;;AAQA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;EAC3B,MAAM;EACN,KAAK,QAAQ;CACf;AACF;;;;;;;;;;;AAYA,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;AACrB;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,eAAe,OACjB,OAAO;CAET,MAAM,wBAAQ,IAAI,MAAM;CACxB,MAAM,QAAQ;CACd,OAAO;AACT;AAqCA,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,MACpD;AAEA,IAAM,kBAAwB,IAAI,OAAO;CACvC,GAAG,QAAQ,sBAAsB;CACjC,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC;AAC/B;AAaA,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;CACR;CACA,iBAAiB;EACf,WAAW;EACX,MAAM;CACR;AACF"}

package/out/zero-cache/src/server/change-streamer.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"~~AAqCA~~,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAU/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,~~CA6Tf~~"}
1	+ {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAsCA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAU/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA4Uf"}

package/out/zero-cache/src/server/change-streamer.js CHANGED Viewed

@@ -1,4 +1,5 @@
 import { assert } from "../../../shared/src/asserts.js";
+import { promiseVoid } from "../../../shared/src/resolved-promises.js";
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getShardConfig } from "../types/shards.js";
@@ -36,7 +37,7 @@ async function runWorker(parent, env, ...argv) {
 		env,
 		argv
 	});
-	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
+	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusTimeoutProportion, flowControlSlowSubscriberGracePeriodSeconds, sqliteChangeLogMode, sqliteChangeLogReadPercent, sqliteChangeLogColdReadPercent, sqliteChangeLogComparePercent, sqliteChangeLogRetentionMs, sqliteChangeLogReadBatchRows, sqliteChangeLogPurgeBatchRows, sqliteChangeLogBarrierTimeoutMs }, autoReset, replicationLag, litestream, upstream, change, replica, initialSync, keepaliveTimeoutMs, sqliteCorruptionChecks } = config;
 	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
 	lc = createLogContext(config, "change-streamer");
 	registerSQLiteCorruptionDiagnosticTarget({
@@ -102,7 +103,17 @@ async function runWorker(parent, env, ...argv) {
 				retentionMs: sqliteChangeLogRetentionMs,
 				batchRows: sqliteChangeLogPurgeBatchRows
 			} : void 0,
-			sqliteChangeLogCompare: sqliteChangeLogComparing ? { replicaFile: replica.file } : void 0
+			sqliteChangeLogCompare: sqliteChangeLogComparing ? {
+				replicaFile: replica.file,
+				comparePercent: sqliteChangeLogComparePercent,
+				retentionMs: sqliteChangeLogRetentionMs,
+				readBatchRows: sqliteChangeLogReadBatchRows
+			} : void 0,
+			sqliteChangeLogServe: sqliteChangeLogMode === "serve" ? {
+				readPercent: sqliteChangeLogReadPercent,
+				coldReadPercent: sqliteChangeLogColdReadPercent,
+				retentionMs: sqliteChangeLogRetentionMs
+			} : void 0
 		}, setTimeout);
 		backupURL = destinationBackupURL;
 		waitForFirstBackupBeforeServing = waitForBackupBeforeServing;
@@ -133,26 +144,26 @@ async function runWorker(parent, env, ...argv) {
 	}
 	const backupMonitor = createBackupCleanupMonitor({
 		lc,
-		processes,
 		config,
 		replicaFile: replica.file,
-		changeStreamer,
-		env
+		changeStreamer
 	});
-	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
-		port,
-		keepaliveTimeoutMs,
-		startupDelayMs
-	}, parent, changeStreamer);
+	let readinessGate = promiseVoid;
 	if (waitForFirstBackupBeforeServing) {
 		const start = performance.now();
 		lc.info?.(`awaiting initial backup ...`);
-		backupMonitor.firstBackupReceived().then(() => {
+		readinessGate = backupMonitor.firstBackupReceived().then(() => {
 			const elapsed = performance.now() - start;
 			lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);
-			parent.send(["ready", { ready: true }]);
 		});
-	} else parent.send(["ready", { ready: true }]);
+	}
+	const changeStreamerWebServer = new ChangeStreamerHttpServer(lc, {
+		port,
+		keepaliveTimeoutMs,
+		startupDelayMs,
+		readinessGate
+	}, parent, changeStreamer);
+	readinessGate.then(() => parent.send(["ready", { ready: true }]));
 	try {
 		await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);
 	} catch (err) {

package/out/zero-cache/src/server/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n sqliteChangeLogMode,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer',\n {max: change.maxConns},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService \| undefined;\n let backupURL: string \| undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' \|\| sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {backupV5: litestream.backupUsingV5},\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // `compare` and above. The replica-derived initialization path runs\n // at full rate and is checked against Postgres, which stays\n // authoritative. Flipping authority to the replica comes later.\n sqliteChangeLogCompare: sqliteChangeLogComparing\n ? {replicaFile: replica.file}\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n processes,\n config,\n replicaFile: replica.file,\n changeStreamer,\n env,\n });\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs},\n parent,\n changeStreamer,\n );\n\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n void backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n parent.send(['ready', {ready: true}]);\n });\n } else {\n parent.send(['ready', {ready: true}]);\n }\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EAAC,KAAK,OAAO,SAAQ,GACrB,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAIJ,wBAAwB,2BACpB,EAAC,aAAa,QAAQ,KAAI,IAC1B,KAAA;EACN,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA;EACA,aAAa,QAAQ;EACrB;EACA;CACF,CAAC;CAED,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;CAAc,GACzC,QACA,cACF;CAEA,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,cAAmB,oBAAoB,EAAE,WAAW;GAClD,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;GAClE,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACtC,CAAC;CACH,OACE,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAUtC,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}
1	+ {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {createBackupCleanupMonitor} from '../services/change-streamer/backup-cleanup-monitor-factory.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n} from '../services/life-cycle.ts';\nimport {startReplicaBackupProcess} from '../services/litestream/commands.ts';\nimport {\n changeLogFileName,\n deleteChangeLogDB,\n} from '../services/replicator/change-log-db.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {connectPgClient} from '../types/pg.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport type {ReplicaFileMode} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {REPLICATOR_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodSeconds,\n sqliteChangeLogMode,\n sqliteChangeLogReadPercent,\n sqliteChangeLogColdReadPercent,\n sqliteChangeLogComparePercent,\n sqliteChangeLogRetentionMs,\n sqliteChangeLogReadBatchRows,\n sqliteChangeLogPurgeBatchRows,\n sqliteChangeLogBarrierTimeoutMs,\n },\n autoReset,\n replicationLag,\n litestream,\n upstream,\n change,\n replica,\n initialSync,\n keepaliveTimeoutMs,\n sqliteCorruptionChecks,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n lc = createLogContext(config, 'change-streamer');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer replica',\n dbPath: replica.file,\n },\n sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = await connectPgClient(\n lc,\n change.db,\n 'change-streamer',\n {max: change.maxConns},\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date.\n await initChangeStreamerSchema(lc, changeDB, shard);\n\n // When restoring from litestream, acquire a lock to prevent change-log\n // purges. This ensures that (this) change-streamer will be able to resume\n // from the backup.\n let purgeLock =\n litestream.backupURL && litestream.executable\n ? await new PurgeLocker(lc, shard, changeDB).acquire()\n : null;\n const restoreOptions = {litestream, constraints: purgeLock ?? undefined};\n\n let changeStreamer: ChangeStreamerService \| undefined;\n let backupURL: string \| undefined;\n\n const context = getServerContext(config);\n const sqliteChangeLogEnabled = sqliteChangeLogMode !== 'off';\n // The modes are cumulative, so `serve` implies `compare`.\n const sqliteChangeLogComparing =\n sqliteChangeLogMode === 'compare' \|\| sqliteChangeLogMode === 'serve';\n if (sqliteChangeLogEnabled) {\n const changeLogFile = changeLogFileName(replica.file);\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'change-streamer change-log',\n dbPath: changeLogFile,\n },\n sqliteCorruptionChecks,\n );\n // The log's bytes are accounted for in the process that writes them. This\n // is local disk only — the log is excluded from the litestream backup — and\n // it is a whole-database total, because a table that left the replica left\n // through its wal: the replica's footprint is what shrank and this is where\n // it went.\n getOrCreateGauge('replica', 'sqlite_change_log.file_bytes', {\n description:\n `The SQLite change log's total on-disk footprint: its main db file ` +\n `plus its wal sidecars.`,\n unit: 'By',\n }).addCallback(async o =>\n o.observe(await sqliteFileBytes(lc, changeLogFile)),\n );\n }\n\n let waitForFirstBackupBeforeServing = false;\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {\n changeSource,\n subscriptionState,\n destinationBackupURL,\n replicaID,\n waitForBackupBeforeServing,\n } =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n restoreOptions,\n {backupV5: litestream.backupUsingV5},\n purgeLock,\n upstream.pgStreamInboundTimeoutMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n restoreOptions,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n destinationBackupURL\n ? {\n backupURL: destinationBackupURL,\n litestreamVersion: litestream.backupUsingV5 ? 'v5' : 'legacy',\n }\n : null,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusTimeoutProportion,\n flowControlSlowSubscriberGracePeriodMs:\n flowControlSlowSubscriberGracePeriodSeconds > 0\n ? flowControlSlowSubscriberGracePeriodSeconds * 1000\n : undefined,\n statementTimeoutMs: change.statementTimeoutMs,\n changeLogBatchSize: change.logBatchSize,\n sqliteCatchup: {\n changeLogFile: changeLogFileName(replica.file),\n readBatchRows: sqliteChangeLogReadBatchRows,\n barrierTimeoutMs: sqliteChangeLogBarrierTimeoutMs,\n },\n // The presence of these options is the writer's gate. This process\n // performs the restore and the initial sync, so the replica's\n // identity is in hand at the moment the log is opened.\n sqliteChangeLogWriter: sqliteChangeLogEnabled\n ? {\n replicaFile: replica.file,\n identity: {\n // RMv2 supplies the epoch; until then the generation and the\n // replica ID are the whole of the identity.\n epoch: null,\n generation: subscriptionState.replicaVersion,\n replicaID,\n },\n }\n : undefined,\n // The purge scheduler shares the writer's gate -- mode, not the\n // read path -- because `write` mode, with no read selector at all,\n // is the configuration it ships in.\n sqliteChangeLogPurge: sqliteChangeLogEnabled\n ? {\n retentionMs: sqliteChangeLogRetentionMs,\n batchRows: sqliteChangeLogPurgeBatchRows,\n }\n : undefined,\n // Compare mode runs both advisory checks. Postgres remains authoritative.\n sqliteChangeLogCompare: sqliteChangeLogComparing\n ? {\n replicaFile: replica.file,\n comparePercent: sqliteChangeLogComparePercent,\n retentionMs: sqliteChangeLogRetentionMs,\n readBatchRows: sqliteChangeLogReadBatchRows,\n }\n : undefined,\n // Slice 11 lands dark by default: serve mode constructs the stable\n // router, while readPercent=0 keeps every catchup on PG and emits\n // eligibility metrics before any canary traffic is enabled.\n sqliteChangeLogServe:\n sqliteChangeLogMode === 'serve'\n ? {\n readPercent: sqliteChangeLogReadPercent,\n coldReadPercent: sqliteChangeLogColdReadPercent,\n retentionMs: sqliteChangeLogRetentionMs,\n }\n : undefined,\n },\n setTimeout,\n );\n backupURL = destinationBackupURL;\n waitForFirstBackupBeforeServing = waitForBackupBeforeServing;\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n // The change log carries the identity of the replica it was written\n // beside, and the retry performs a fresh initial sync with a new\n // replicaVersion. Reconciliation would catch that on its own (as\n // 'identity-mismatch') but only after the writer opened a file that is\n // known here to be garbage.\n deleteChangeLogDB(replica.file);\n // Release the purge lock before retrying. This is safe because the\n // purge lock exists to preserve change-log entries so the new\n // change-streamer can resume from the backup replica's watermark.\n // An AutoResetSignal means we cant resume from the backup replica\n // (e.g. its replication slot is gone), so the change-log entries the lock\n // was protecting are no longer needed. The retry performs a fresh\n // initial sync with a new replication slot, independent of the old\n // change-log. Releasing is also necessary to avoid a\n // self-deadlock when CHANGE_DB == UPSTREAM_DB:\n // CREATE_REPLICATION_SLOT waits for all older transactions to\n // finish, including this lock's open transaction.\n await purgeLock?.release();\n purgeLock = null;\n continue; // execute again with a fresh initial-sync\n }\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const processes = new ProcessManager(lc, parent);\n if (backupURL) {\n lc.info?.('setting up backup to', backupURL);\n litestream.backupURL = backupURL;\n const {promise: backupStarted, resolve} = resolver();\n\n // Start a backup replicator and corresponding litestream backup process.\n processes\n .addWorker(\n childWorker(REPLICATOR_URL, env, 'backup' satisfies ReplicaFileMode),\n 'supporting',\n 'backup-replicator',\n )\n // Wait for the replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the sqlite lock\n // when the replicator first prepares the db file.\n .once('message', () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, litestream, replica.file),\n 'supporting',\n 'litestream',\n );\n resolve();\n });\n await backupStarted;\n }\n\n const backupMonitor = createBackupCleanupMonitor({\n lc,\n config,\n replicaFile: replica.file,\n changeStreamer,\n });\n\n let readinessGate = promiseVoid;\n if (waitForFirstBackupBeforeServing) {\n const start = performance.now();\n lc.info?.(`awaiting initial backup ...`);\n\n readinessGate = backupMonitor.firstBackupReceived().then(() => {\n const elapsed = performance.now() - start;\n lc.info?.(`initial backup confirmed after ${elapsed.toFixed(2)}ms`);\n });\n }\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n {port, keepaliveTimeoutMs, startupDelayMs, readinessGate},\n parent,\n changeStreamer,\n );\n\n void readinessGate.then(() => parent.send(['ready', {ready: true}]));\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer after a delay to ensure that routing\n // routing elements have registered the server before the\n // change-streamer takes over the replication slot.\n // TODO: Remove this delay and start the changeStreamer normally here once\n // transitioned to RMv2, since changeStreamer startup will no longer\n // disrupt an existing task.\n try {\n await runUntilKilled(lc, parent, changeStreamerWebServer, backupMonitor);\n } catch (err) {\n processes.logErrorAndExit(err, 'change-streamer');\n } finally {\n await processes.shutdown();\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () =>\n runWorker(\n must(parentWorker),\n process.env,\n ...process.argv.slice(2),\n ).catch(async e => {\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n throw e;\n }),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;CAAI,CAAC;CAClD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,uCACA,6CACA,qBACA,4BACA,gCACA,+BACA,4BACA,8BACA,+BACA,mCAEF,WACA,gBACA,YACA,UACA,QACA,SACA,aACA,oBACA,2BACE;CAEJ,cACE,iBAAiB,QAAQ,mBAAmB,GAAG,KAAK,GACpD,mBACA,CACF;CACA,KAAK,iBAAiB,QAAQ,iBAAiB;CAC/C,yCACE;EACE,WAAW;EACX,QAAQ,QAAQ;CAClB,GACA,sBACF;CACA,cAAc,IAAI,MAAM;CAGxB,MAAM,WAAW,MAAM,gBACrB,IACA,OAAO,IACP,mBACA,EAAC,KAAK,OAAO,SAAQ,GACrB,EAAC,kBAAkB,KAAI,CACzB;CACA,kBAAuB,IAAI,UAAU,QAAQ,EAAE,YAAY,CAAC,CAAC;CAE7D,MAAM,QAAQ,eAAe,MAAM;CAGnC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAKlD,IAAI,YACF,WAAW,aAAa,WAAW,aAC/B,MAAM,IAAI,YAAY,IAAI,OAAO,QAAQ,EAAE,QAAQ,IACnD;CACN,MAAM,iBAAiB;EAAC;EAAY,aAAa,aAAa,KAAA;CAAS;CAEvE,IAAI;CACJ,IAAI;CAEJ,MAAM,UAAU,iBAAiB,MAAM;CACvC,MAAM,yBAAyB,wBAAwB;CAEvD,MAAM,2BACJ,wBAAwB,aAAa,wBAAwB;CAC/D,IAAI,wBAAwB;EAC1B,MAAM,gBAAgB,kBAAkB,QAAQ,IAAI;EACpD,yCACE;GACE,WAAW;GACX,QAAQ;EACV,GACA,sBACF;EAMA,iBAAiB,WAAW,gCAAgC;GAC1D,aACE;GAEF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MACnB,EAAE,QAAQ,MAAM,gBAAgB,IAAI,aAAa,CAAC,CACpD;CACF;CAEA,IAAI,kCAAkC;CACtC,KAAK,MAAM,SAAS,CAAC,MAAM,KAAK,GAC9B,IAAI;EAEF,MAAM,EACJ,cACA,mBACA,sBACA,WACA,+BAEA,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;EACpC,GACA,SACA,eAAe,kBACf,gBACA,EAAC,UAAU,WAAW,cAAa,GACnC,WACA,SAAS,wBACX,IACA,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,SACA,cACF;EAEN,MAAM,6BACJ,2BAA2B,eAAe,QAAQ,IAAI;EAExD,iBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cACA,4BACA,mBACA,uBACI;GACE,WAAW;GACX,mBAAmB,WAAW,gBAAgB,OAAO;EACvD,IACA,MACJ,WACA,aAAa,OACb;GACE;GACA;GACA,wCACE,8CAA8C,IAC1C,8CAA8C,MAC9C,KAAA;GACN,oBAAoB,OAAO;GAC3B,oBAAoB,OAAO;GAC3B,eAAe;IACb,eAAe,kBAAkB,QAAQ,IAAI;IAC7C,eAAe;IACf,kBAAkB;GACpB;GAIA,uBAAuB,yBACnB;IACE,aAAa,QAAQ;IACrB,UAAU;KAGR,OAAO;KACP,YAAY,kBAAkB;KAC9B;IACF;GACF,IACA,KAAA;GAIJ,sBAAsB,yBAClB;IACE,aAAa;IACb,WAAW;GACb,IACA,KAAA;GAEJ,wBAAwB,2BACpB;IACE,aAAa,QAAQ;IACrB,gBAAgB;IAChB,aAAa;IACb,eAAe;GACjB,IACA,KAAA;GAIJ,sBACE,wBAAwB,UACpB;IACE,aAAa;IACb,iBAAiB;IACjB,aAAa;GACf,IACA,KAAA;EACR,GACA,UACF;EACA,YAAY;EACZ,kCAAkC;EAClC;CACF,SAAS,GAAG;EACV,IAAI,SAAS,aAAa,iBAAiB;GACzC,GAAG,OAAO,qBAAqB,QAAQ,QAAQ,CAAC;GAGhD,aAAa,QAAQ,IAAI;GAMzB,kBAAkB,QAAQ,IAAI;GAY9B,MAAM,WAAW,QAAQ;GACzB,YAAY;GACZ;EACF;EACA,IAAI,aAAa,mBACf,MAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,EAAC,CACX;EAEF,MAAM;CACR;CAGF,OAAO,gBAAgB,kDAAkD;CAEzE,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAC/C,IAAI,WAAW;EACb,GAAG,OAAO,wBAAwB,SAAS;EAC3C,WAAW,YAAY;EACvB,MAAM,EAAC,SAAS,eAAe,YAAW,SAAS;EAGnD,UACG,UACC,YAAY,gBAAgB,KAAK,QAAkC,GACnE,cACA,mBACF,EAIC,KAAK,iBAAiB;GACrB,UAAU,cACR,0BAA0B,IAAI,YAAY,QAAQ,IAAI,GACtD,cACA,YACF;GACA,QAAQ;EACV,CAAC;EACH,MAAM;CACR;CAEA,MAAM,gBAAgB,2BAA2B;EAC/C;EACA;EACA,aAAa,QAAQ;EACrB;CACF,CAAC;CAED,IAAI,gBAAgB;CACpB,IAAI,iCAAiC;EACnC,MAAM,QAAQ,YAAY,IAAI;EAC9B,GAAG,OAAO,6BAA6B;EAEvC,gBAAgB,cAAc,oBAAoB,EAAE,WAAW;GAC7D,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,CAAC,EAAE,GAAG;EACpE,CAAC;CACH;CAEA,MAAM,0BAA0B,IAAI,yBAClC,IACA;EAAC;EAAM;EAAoB;EAAgB;CAAa,GACxD,QACA,cACF;CAEA,cAAmB,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC,CAAC;CASnE,IAAI;EACF,MAAM,eAAe,IAAI,QAAQ,yBAAyB,aAAa;CACzE,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,iBAAiB;CAClD,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UAEJ,UACE,KAAK,YAAY,GACjB,QAAQ,KACR,GAAG,QAAQ,KAAK,MAAM,CAAC,CACzB,EAAE,MAAM,OAAM,MAAK;CACjB,MAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,CAAC,CAC9C;CACA,MAAM;AACR,CAAC,CACL"}

package/out/zero-cache/src/server/main.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAwB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,~~CA+Jf~~"}
1	+ {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAwB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAoKf"}

package/out/zero-cache/src/server/main.js CHANGED Viewed

@@ -1,6 +1,6 @@
-import { assert } from "../../../shared/src/asserts.js";
 import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
+import { runsChangeStreamer } from "../config/normalize.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruption.js";
 import { ProcessManager, exitAfter, recordStartupDurationMs, runUntilKilled } from "../services/life-cycle.js";
@@ -40,9 +40,9 @@ async function runWorker(parent, env) {
 		const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : "");
 		return processes.addWorker(worker, type, name);
 	}
-	const { taskID, changeStreamer: { mode: changeStreamerMode, uri: changeStreamerURI }, litestream } = config;
-	const runChangeStreamer = changeStreamerMode === "dedicated" && changeStreamerURI === void 0;
-	assert(!(config.changeStreamer.sqliteChangeLogMode !== "off") || runChangeStreamer, "SQLite change-log writing requires this process tree to run the change-streamer, which is where the writer lives");
+	const { taskID, litestream } = config;
+	const runChangeStreamer = runsChangeStreamer(config);
+	if (config.changeStreamer.sqliteChangeLogMode !== "off" && !runChangeStreamer) lc.warn?.(`ignoring --change-streamer-sqlite-change-log-mode=${config.changeStreamer.sqliteChangeLogMode}: this task connects to a change-streamer rather than running one, and the SQLite change log lives in the change-streamer`);
 	let changeStreamer;
 	if (runChangeStreamer) {
 		const { promise: changeStreamerReady, resolve: changeStreamerStarted } = resolver();

package/out/zero-cache/src/server/main.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {~~assert~~} from '../../../shared/src/~~asserts~~.ts';\nimport {~~must~~} from '~~../../../shared~~/~~src/must~~.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n recordStartupDurationMs,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SHADOW_SYNCER_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(\n createLogContext(config, 'dispatcher', 0, false),\n 'dispatcher',\n 0,\n );\n lc = createLogContext(config, 'dispatcher');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'dispatcher replica',\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID~~,\n changeStreamer: {mode: changeStreamerMode~~, ~~uri: changeStreamerURI},\n~~ litestream~~,\n~~ } = config;\n const runChangeStreamer ~~=\n changeStreamerMode~~ ~~=== 'dedicated' && changeStreamerURI === undefined~~;\n const sqliteChangeLogEnabled =\n config.changeStreamer.sqliteChangeLogMode !== 'off';\n ~~assert~~(\n ~~!sqliteChangeLogEnabled~~ \|\| ~~runChangeStreamer,\~~n '~~SQLite~~ change-log ~~writing~~ ~~requires~~ ~~this~~ ~~process~~ ~~tree~~ to ~~run~~ the change-streamer, ~~which~~ is ~~where~~ the ~~writer~~ lives~~',\~~n );\n\n let changeStreamer: Worker \| undefined;\n\n if (runChangeStreamer) {\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n 'message',\n changeStreamerStarted,\n );\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n }\n\n if (numSyncers > 0) {\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n }\n\n // Only run the shadow-sync canary on the replication-manager (or in\n // single-node mode, where it also owns upstream). Running on every\n // view-syncer would hammer the upstream with N redundant canaries.\n if (config.shadowSync.enabled && runChangeStreamer) {\n const {promise: shadowReady, resolve: shadowStarted} = resolver();\n loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n await shadowReady;\n }\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n const startupDurationMs = Date.now() - startMs;\n lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n recordStartupDurationMs(startupDurationMs);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n } finally {\n await processes.shutdown();\n }\n}\n\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA0CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,OAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;CAC/C;CAEA,MAAM,~~EACJ~~,~~QACA~~,~~gBAAgB~~,~~EAAC,~~MAAM,oBAAoB,~~KAAK~~,~~qBAChD,eACE~~;~~CACJ~~,~~MAAM~~,~~oBACJ~~,~~uBAAuB,~~eAAe,~~sBAAsB~~,~~KAAA;CAG9D~~,~~OACE~~,~~EAFA~~,OAAO,eAAe,~~wBAAwB~~,~~UAEnB,mBAC3B,kHACF~~;~~CAEA~~,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EACX,iBAAiB,WAAW,qBAAqB,YAAY,EAAE,KAC7D,WACA,qBACF;EAGA,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}
1	+ {"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {runsChangeStreamer} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n recordStartupDurationMs,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SHADOW_SYNCER_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(\n createLogContext(config, 'dispatcher', 0, false),\n 'dispatcher',\n 0,\n );\n lc = createLogContext(config, 'dispatcher');\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: 'dispatcher replica',\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {taskID, litestream} = config;\n const runChangeStreamer = runsChangeStreamer(config);\n const sqliteChangeLogEnabled =\n config.changeStreamer.sqliteChangeLogMode !== 'off';\n if (sqliteChangeLogEnabled && !runChangeStreamer) {\n // A multi-node deployment configures every task from one environment, so\n // the change log's options reach the view-syncers too. The log lives in\n // the change-streamer, and a task that connects to one simply never writes\n // a log. Warning rather than refusing to start is what makes the\n // fleet-wide environment -- often the only knob an operator has -- able to\n // express the rollout at all.\n lc.warn?.(\n `ignoring --change-streamer-sqlite-change-log-mode=` +\n `${config.changeStreamer.sqliteChangeLogMode}: this task connects to ` +\n `a change-streamer rather than running one, and the SQLite change log ` +\n `lives in the change-streamer`,\n );\n }\n\n let changeStreamer: Worker \| undefined;\n\n if (runChangeStreamer) {\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n 'message',\n changeStreamerStarted,\n );\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n }\n\n if (numSyncers > 0) {\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n }\n\n // Only run the shadow-sync canary on the replication-manager (or in\n // single-node mode, where it also owns upstream). Running on every\n // view-syncer would hammer the upstream with N redundant canaries.\n if (config.shadowSync.enabled && runChangeStreamer) {\n const {promise: shadowReady, resolve: shadowStarted} = resolver();\n loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n await shadowReady;\n }\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n const startupDurationMs = Date.now() - startMs;\n lc.info?.(`all workers ready (${startupDurationMs} ms)`);\n recordStartupDurationMs(startupDurationMs);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n } finally {\n await processes.shutdown();\n }\n}\n\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA0CA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,wBAAwB,EAAC,IAAG,CAAC;CAE5C,cACE,iBAAiB,QAAQ,cAAc,GAAG,KAAK,GAC/C,cACA,CACF;CACA,KAAK,iBAAiB,QAAQ,YAAY;CAC1C,yCACE;EACE,WAAW;EACX,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACA,cAAc,IAAI,MAAM;CAExB,MAAM,YAAY,IAAI,eAAe,IAAI,MAAM;CAE/C,MAAM,EAAC,gBAAgB,eAAc;CACrC,IAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,YAC3D,MAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,iHAEpF;CAEF,IAAI,OAAO,IAAI,WAAW,YACxB,MAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,4GAE1E;CAGF,MAAM,gBACJ,eAAe,IACX,CAAC,IACD;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;EACxD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;CACrD;CAEN,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;EACpE,MAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,GAAG,KAAK;EACpE,OAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;CAC/C;CAEA,MAAM,EAAC,QAAQ,eAAc;CAC7B,MAAM,oBAAoB,mBAAmB,MAAM;CAGnD,IADE,OAAO,eAAe,wBAAwB,SAClB,CAAC,mBAO7B,GAAG,OACD,qDACK,OAAO,eAAe,oBAAoB,0HAGjD;CAGF,IAAI;CAEJ,IAAI,mBAAmB;EACrB,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,SAAS;EACX,iBAAiB,WAAW,qBAAqB,YAAY,EAAE,KAC7D,WACA,qBACF;EAGA,MAAM;CACR;CAEA,IAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;EAGlE,MAAM;CACR;CAKA,IAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,SAAS;EAChE,WAAW,mBAAmB,YAAY,EAAE,KAAK,WAAW,aAAa;EACzE,MAAM;CACR;CAEA,MAAM,UAAoB,CAAC;CAC3B,IAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,SAAS;EAClD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,IACF,EAAE,KAAK,iBAAiB;GACtB,YAAY,IAAI,UAAU;GAC1B,QAAQ;EACV,CAAC;EACD,MAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,UAAU;EAClD,KAAK,IAAI,IAAI,GAAG,IAAI,YAAY,KAC9B,QAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC;EAExE,QAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;CACzE;CACA,IAAI;CAKJ,GAAG,OAAO,qCAAqC;CAC/C,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,aAAa,EAAE,KAAK,IAAI,GAAG,GAC1E,GACF;CACA,MAAM,UAAU,gBAAgB;CAChC,cAAc,UAAU;CACxB,MAAM,oBAAoB,KAAK,IAAI,IAAI;CACvC,GAAG,OAAO,sBAAsB,kBAAkB,KAAK;CACvD,wBAAwB,iBAAiB;CAEzC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CAEpC,IAAI;EACF,MAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,cACF,CACF;CACF,SAAS,KAAK;EACZ,UAAU,gBAAgB,KAAK,YAAY;CAC7C,UAAU;EACR,MAAM,UAAU,SAAS;CAC3B;AACF;AAEA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CACjD"}

package/out/zero-cache/src/server/replicator.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,~~CAiGf~~"}
1	+ {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAqGf"}

package/out/zero-cache/src/server/replicator.js CHANGED Viewed

@@ -42,7 +42,7 @@ async function runWorker(parent, env, ...args) {
 		dbPath: config.replica.file
 	}, config.sqliteCorruptionChecks);
 	initEventSink(lc, config);
-	if (fileMode === "serving" && (config.litestream.executable || config.litestream.executableV5)) await restoreReplica(lc, config);
+	if (fileMode === "serving" && !runningLocalChangeStreamer && (config.litestream.executable || config.litestream.executableV5)) await restoreReplica(lc, config);
 	const { file: dbPath, walMode } = await setupReplica(lc, fileMode, config.replica);
 	unregisterInitialCorruptionDiagnosticTarget();
 	registerSQLiteCorruptionDiagnosticTarget({