npm - @rocicorp/zero - Versions diffs - 1.3.0-canary.2 → 1.3.0-canary.3 - Mend

@rocicorp/zero 1.3.0-canary.2 → 1.3.0-canary.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (64) hide show

package/out/zero-cache/src/db/transaction-pool.js CHANGED Viewed

@@ -22,30 +22,16 @@ var TransactionPool = class {
 	#workers = [];
 	#initialWorkers;
 	#maxWorkers;
+	#responseTimeout;
 	#timeoutTask;
 	#numWorkers;
 	#numWorking = 0;
 	#db;
-	#refCount = 1;
 	#done = false;
 	#failure;
-	/**
-	* @param init A {@link Task} that is run in each Transaction before it begins
-	*             processing general tasks. This can be used to to set the transaction
-	*             mode, export/set snapshots, etc. This will be run even if
-	*             {@link fail} has been called on the pool.
-	* @param cleanup A {@link Task} that is run in each Transaction before it closes.
-	*                This will be run even if {@link fail} has been called, or if a
-	*                preceding Task threw an Error.
-	* @param initialWorkers The initial number of transaction workers to process tasks.
-	*                       This is the steady state number of workers that will be kept
-	*                       alive if the TransactionPool is long lived.
-	*                       This must be greater than 0. Defaults to 1.
-	* @param maxWorkers When specified, allows the pool to grow to `maxWorkers`. This
-	*                   must be greater than or equal to `initialWorkers`. On-demand
-	*                   workers will be shut down after an idle timeout of 5 seconds.
-	*/
-	constructor(lc, mode, init, cleanup, initialWorkers = 1, maxWorkers = initialWorkers, timeoutTasks = TIMEOUT_TASKS) {
+	#orphanedQueryCheckInterval;
+	constructor(lc, opts, timeoutTasks = TIMEOUT_TASKS) {
+		const { mode, init, cleanup, initialWorkers = 1, maxWorkers = initialWorkers, statementResponseTimeout } = opts;
 		assert(initialWorkers > 0, "initialWorkers must be positive");
 		assert(maxWorkers >= initialWorkers, "maxWorkers must be >= initialWorkers");
 		this.#lc = lc;
@@ -56,6 +42,8 @@ var TransactionPool = class {
 		this.#numWorkers = initialWorkers;
 		this.#maxWorkers = maxWorkers;
 		this.#timeoutTask = timeoutTasks;
+		this.#responseTimeout = statementResponseTimeout;
+		this.#orphanedQueryCheckInterval = setInterval(this.#checkForOrphanedQueries, Math.min(5e3, statementResponseTimeout ?? 5e3));
 	}
 	/**
 	* Starts the pool of workers to process Tasks with transactions opened from the
@@ -171,6 +159,9 @@ var TransactionPool = class {
 	}
 	#start = performance.now();
 	#stmts = 0;
+	#latestDoneIndex = 0;
+	#latestDoneTime = 0;
+	#pending = /* @__PURE__ */ new Map();
 	/**
 	* Implements the semantics specified in {@link process()}.
 	*
@@ -197,18 +188,69 @@ var TransactionPool = class {
 					r.resolve();
 					return { pending: null };
 				}
-				return { pending: stmts.reduce((_, stmt) => stmt.execute().then(() => {
-					if (++this.#stmts % 1e3 === 0) {
-						const log = this.#stmts % 1e4 === 0 ? "info" : "debug";
-						const q = stmt;
-						lc[log]?.(`executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`, { statement: q.string });
-					}
-				}).catch((e) => this.fail(e)), promiseVoid).then(r.resolve) };
+				const time = Date.now();
+				return { pending: stmts.reduce((_, stmt) => {
+					const query = stmt;
+					const index = ++this.#stmts;
+					const { promise: aborted, reject: abort } = resolver();
+					aborted.catch(() => {});
+					this.#pending.set(query, {
+						index,
+						time,
+						abort
+					});
+					const responded = stmt.execute().then(() => {
+						if (index % 1e3 === 0) lc[index % 1e4 === 0 ? "info" : "debug"]?.(`executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`, { statement: query.string });
+					}).catch((e) => this.fail(e)).finally(() => {
+						this.#pending.delete(query);
+						if (index > this.#latestDoneIndex) {
+							this.#latestDoneIndex = index;
+							this.#latestDoneTime = Date.now();
+						}
+					});
+					const done = Promise.race([responded, aborted]);
+					done.catch(() => {});
+					return done;
+				}, promiseVoid).finally(r.resolve) };
 			},
 			rejected: r.resolve
 		};
 	}
 	/**
+	* Periodically checks the map of `#pending` queries for which Promises
+	* have yet to be resolved. Checks for pathological scenarios that have
+	* been observed:
+	*
+	* * A promise does not resolve, even though the transaction is otherwise
+	*   healthy with keepalives being sent.
+	* * A transaction never "finishes" at the postgres.js level after the
+	*   worker exits, implying that postgres.js is awaiting the `last` promise
+	*   returned. In this case Postgres disconnected the application with an
+	*   idle-in-transaction timeout, suggesting that all statements did in
+	*   fact complete.
+	*/
+	#checkForOrphanedQueries = () => {
+		if (this.#pending.size === 0) return;
+		const now = Date.now();
+		for (const [query, { index, time, abort }] of this.#pending.entries()) {
+			const statement = query.string;
+			const abortWith = (msg) => {
+				this.#lc.warn?.(msg, { statement });
+				const err = new ResponseTimeoutError(msg);
+				abort(err);
+				this.fail(err);
+				this.#pending.delete(query);
+			};
+			if (index < this.#latestDoneIndex) {
+				const elapsed = now - this.#latestDoneTime;
+				if (this.#responseTimeout && elapsed > this.#responseTimeout) abortWith(`statement ${this.#latestDoneIndex} completed ${elapsed} ms ago, but statement ${index} has not.`);
+			} else {
+				const elapsed = now - time;
+				if (this.#responseTimeout && elapsed > this.#responseTimeout) abortWith(`response for statement timed out after ${elapsed} ms`);
+			}
+		}
+	};
+	/**
 	* Processes and returns the result of executing the {@link ReadTask} from
 	* within the transaction. An error thrown by the task will result in
 	* rejecting the returned Promise, but will not affect the transaction pool
@@ -275,36 +317,7 @@ var TransactionPool = class {
 		assert(!this.#done, "already set done");
 		this.#done = true;
 		for (let i = 0; i < this.#numWorkers; i++) this.#tasks.enqueue("done");
-	}
-	/**
-	* An alternative to explicitly calling {@link setDone}, `ref()` increments an internal reference
-	* count, and {@link unref} decrements it. When the reference count reaches 0, {@link setDone} is
-	* automatically called. A TransactionPool is initialized with a reference count of 1.
-	*
-	* `ref()` should be called before sharing the pool with another component, and only after the
-	* pool has been started with {@link run()}. It must not be called on a TransactionPool that is
-	* already done (either via {@link unref()} or {@link setDone()}. (Doing so indicates a logical
-	* error in the code.)
-	*
-	* It follows that:
-	* * The creator of the TransactionPool is responsible for running it.
-	* * The TransactionPool should be ref'ed before being sharing.
-	* * The receiver of the TransactionPool is only responsible for unref'ing it.
-	*
-	* On the other hand, a transaction pool that fails with a runtime error can still be ref'ed;
-	* attempts to use the pool will result in the runtime error as expected.
-	*/
-	ref(count = 1) {
-		assert(this.#db !== void 0 && !this.#done, `Cannot ref() a TransactionPool that is not running`);
-		this.#refCount += count;
-	}
-	/**
-	* Decrements the internal reference count, automatically invoking {@link setDone} when it reaches 0.
-	*/
-	unref(count = 1) {
-		assert(count <= this.#refCount, () => `Cannot unref ${count} when refCount is ${this.#refCount}`);
-		this.#refCount -= count;
-		if (this.#refCount === 0) this.setDone();
+		Promise.allSettled(this.#workers).then(() => clearInterval(this.#orphanedQueryCheckInterval));
 	}
 	isRunning() {
 		return this.#db !== void 0 && !this.#done && this.#failure === void 0;
@@ -313,11 +326,12 @@ var TransactionPool = class {
 	* Signals all workers to fail their transactions with the given {@link err}.
 	*/
 	fail(err) {
-		if (!this.#failure) {
+		if (!this.#done && !this.#failure) {
 			this.#failure = ensureError(err);
 			const level = this.#failure instanceof ControlFlowError ? "debug" : "info";
 			this.#lc[level]?.(this.#failure);
 			for (let i = 0; i < this.#numWorkers; i++) this.#tasks.enqueue(this.#failure);
+			Promise.allSettled(this.#workers).then(() => clearInterval(this.#orphanedQueryCheckInterval));
 		}
 	}
 };
@@ -383,6 +397,12 @@ var TIMEOUT_TASKS = {
 		task: "done"
 	}
 };
+var ResponseTimeoutError = class extends Error {
+	static name = "ResponseTimeoutError";
+	constructor(msg) {
+		super(msg);
+	}
+};
 //#endregion
 export { TransactionPool, importSnapshot };

package/out/zero-cache/src/db/transaction-pool.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#timeoutTask","#lc","#stmtRunner","#numWorkers","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#readRunner","#refCount"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> \| T;\n\nexport type Statement =\n \| postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n \| postgres.PendingQuery<postgres.Row[]>;\n\n/*\n A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n \n /\nexport type Task = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/*\n A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n /\nexport type ReadTask<T> = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<T>;\n\n/\n A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n \n This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n /\nexport class TransactionPool {\n #lc: LogContext;\n readonly #mode: Mode;\n readonly #init: TaskRunner \| undefined;\n readonly #cleanup: TaskRunner \| undefined;\n readonly #tasks = new Queue<TaskRunner \| Error \| 'done'>();\n readonly #workers: Promise<unknown>[] = [];\n readonly #initialWorkers: number;\n readonly #maxWorkers: number;\n readonly #timeoutTask: TimeoutTasks;\n #numWorkers: number;\n #numWorking = 0;\n #db: PostgresDB \| undefined; // set when running. stored to allow adaptive pool sizing.\n\n #refCount = 1;\n #done = false;\n #failure: Error \| undefined;\n\n /\n @param init A {@link Task} that is run in each Transaction before it begins\n * processing general tasks. This can be used to to set the transaction\n * mode, export/set snapshots, etc. This will be run even if\n * {@link fail} has been called on the pool.\n * @param cleanup A {@link Task} that is run in each Transaction before it closes.\n * This will be run even if {@link fail} has been called, or if a\n * preceding Task threw an Error.\n * @param initialWorkers The initial number of transaction workers to process tasks.\n * This is the steady state number of workers that will be kept\n * alive if the TransactionPool is long lived.\n * This must be greater than 0. Defaults to 1.\n * @param maxWorkers When specified, allows the pool to grow to `maxWorkers`. This\n * must be greater than or equal to `initialWorkers`. On-demand\n * workers will be shut down after an idle timeout of 5 seconds.\n /\n constructor(\n lc: LogContext,\n mode: Mode,\n init?: Task,\n cleanup?: Task,\n initialWorkers = 1,\n maxWorkers = initialWorkers,\n timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n ) {\n assert(initialWorkers > 0, 'initialWorkers must be positive');\n assert(\n maxWorkers >= initialWorkers,\n 'maxWorkers must be >= initialWorkers',\n );\n\n this.#lc = lc;\n this.#mode = mode;\n this.#init = init ? this.#stmtRunner(init) : undefined;\n this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n this.#initialWorkers = initialWorkers;\n this.#numWorkers = initialWorkers;\n this.#maxWorkers = maxWorkers;\n this.#timeoutTask = timeoutTasks;\n }\n\n /\n Starts the pool of workers to process Tasks with transactions opened from the\n * specified {@link db}.\n /\n run(db: PostgresDB): this {\n assert(!this.#db, 'already running');\n this.#db = db;\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#addWorker(db);\n }\n return this;\n }\n\n /\n Adds context parameters to internal LogContext. This is useful for context values that\n * are not known when the TransactionPool is constructed (e.g. determined after a database\n * call when the pool is running).\n \n Returns an object that can be used to add more parameters.\n /\n addLoggingContext(key: string, value: string) {\n this.#lc = this.#lc.withContext(key, value);\n\n return {\n addLoggingContext: (key: string, value: string) =>\n this.addLoggingContext(key, value),\n };\n }\n\n /\n Returns a promise that:\n \n * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n * to a 0 ref count), once all added tasks have been processed and all transactions have been\n * committed or closed.\n \n * rejects if processing was aborted with {@link fail} or if processing any of\n * the tasks resulted in an error. All uncommitted transactions will have been\n * rolled back.\n \n Note that partial failures are possible if processing writes with multiple workers\n * (e.g. `setDone` is called, allowing some workers to commit, after which other\n * workers encounter errors). Using a TransactionPool in this manner does not make\n * sense in terms of transactional semantics, and is thus not recommended.\n \n For reads, however, multiple workers is useful for performing parallel reads\n * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n * Resolves or rejects when all workers are done or failed.\n /\n async done() {\n const numWorkers = this.#workers.length;\n await Promise.all(this.#workers);\n\n if (numWorkers < this.#workers.length) {\n // If workers were added after the initial set, they must be awaited to ensure\n // that the results (i.e. rejections) of all workers are accounted for. This only\n // needs to be re-done once, because the fact that the first `await` completed\n // guarantees that the pool is in a terminal state and no new workers can be added.\n await Promise.all(this.#workers);\n }\n this.#lc.debug?.('transaction pool done');\n\n const elapsed = performance.now() - this.#start;\n if (elapsed > 60_000) {\n if (this.#stmts > 0) {\n this.#lc.warn?.(\n `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n );\n } else {\n this.#lc.warn?.(\n `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n );\n }\n }\n }\n\n #addWorker(db: PostgresDB) {\n const id = this.#workers.length + 1;\n const lc = this.#lc.withContext('tx', id);\n\n const tt: TimeoutTask =\n this.#workers.length < this.#initialWorkers\n ? this.#timeoutTask.forInitialWorkers\n : this.#timeoutTask.forExtraWorkers;\n const {timeoutMs} = tt;\n const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n const worker = async (tx: PostgresTransaction) => {\n const start = performance.now();\n try {\n lc.debug?.('started transaction');\n\n let last: Promise<void> = promiseVoid;\n\n const executeTask = async (runner: TaskRunner) => {\n runner !== this.#init && this.#numWorking++;\n const {pending} = await runner.run(tx, lc, () => {\n runner !== this.#init && this.#numWorking--;\n });\n last = pending ?? last;\n };\n\n let task: TaskRunner \| Error \| 'done' =\n this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n try {\n while (task !== 'done') {\n if (\n task instanceof Error \|\|\n (task !== this.#init && this.#failure)\n ) {\n throw this.#failure ?? task;\n }\n await executeTask(task);\n\n // await the next task.\n task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n }\n } finally {\n // Execute the cleanup task even on failure.\n if (this.#cleanup) {\n await executeTask(this.#cleanup);\n }\n }\n\n const elapsed = performance.now() - start;\n lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n // Given the semantics of a Postgres transaction, the last statement\n // will only succeed if all of the preceding statements succeeded.\n return last;\n } catch (e) {\n if (e !== this.#failure) {\n this.fail(e); // A failure in any worker should fail the pool.\n }\n throw e;\n }\n };\n\n const workerTx = runTx(db, worker, {mode: this.#mode})\n .catch(e => {\n if (e instanceof RollbackSignal) {\n // A RollbackSignal is used to gracefully rollback the postgres.js\n // transaction block. It should not be thrown up to the application.\n lc.debug?.('aborted transaction');\n } else {\n throw e;\n }\n })\n .finally(() => this.#numWorkers--);\n\n // Attach a rejection handler immediately to prevent unhandledRejections.\n // The application will handle errors when it awaits processReadTask()\n // or done().\n workerTx.catch(() => {});\n\n this.#workers.push(workerTx);\n\n // After adding the worker, enqueue a terminal signal if we are in either of the\n // terminal states (both of which prevent more tasks from being enqueued), to ensure\n // that the added worker eventually exits.\n if (this.#done) {\n this.#tasks.enqueue('done');\n }\n if (this.#failure) {\n this.#tasks.enqueue(this.#failure);\n }\n }\n\n /\n Processes the statements produced by the specified {@link Task},\n * returning a Promise that resolves when the statements are either processed\n * by the database or rejected.\n \n Note that statement failures will result in failing the entire\n * TransactionPool (per transaction semantics). However, the returned Promise\n * itself will resolve rather than reject. As such, it is fine to ignore\n * returned Promises in order to pipeline requests to the database. It is\n * recommended to occasionally await them (e.g. after some threshold) in\n * order to avoid memory blowup in the case of database slowness.\n /\n process(task: Task): Promise<void> {\n const r = resolver<void>();\n this.#process(this.#stmtRunner(task, r));\n return r.promise;\n }\n\n readonly #start = performance.now();\n #stmts = 0;\n\n /\n Implements the semantics specified in {@link process()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the statements are produced,\n * allowing them to be pipelined to the database.\n * * Statement errors result in failing the transaction pool.\n * * The client-supplied Resolver resolves on success or failure;\n * it is never rejected.\n /\n #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let stmts: Statement[];\n try {\n stmts = await task(tx, lc);\n } catch (e) {\n r.resolve();\n throw e;\n } finally {\n freeWorker();\n }\n\n if (stmts.length === 0) {\n r.resolve();\n return {pending: null};\n }\n\n // Execute the statements (i.e. send to the db) immediately.\n // The last result is returned for the worker to await before\n // closing the transaction.\n const last = stmts.reduce(\n (_, stmt) =>\n stmt\n .execute()\n .then(() => {\n if (++this.#stmts % 1000 === 0) {\n const log = this.#stmts % 10000 === 0 ? 'info' : 'debug';\n const q = stmt as unknown as Query;\n lc[log]?.(\n `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n {statement: q.string},\n );\n }\n })\n .catch(e => this.fail(e)),\n promiseVoid,\n );\n return {pending: last.then(r.resolve)};\n },\n rejected: r.resolve,\n };\n }\n\n /\n Processes and returns the result of executing the {@link ReadTask} from\n * within the transaction. An error thrown by the task will result in\n * rejecting the returned Promise, but will not affect the transaction pool\n * itself.\n /\n processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n const r = resolver<T>();\n this.#process(this.#readRunner(readTask, r));\n return r.promise;\n }\n\n /\n Implements the semantics specified in {@link processReadTask()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the result is produced,\n * before resolving the client-supplied Resolver.\n * * Errors result in rejecting the client-supplied Resolver but\n * do not affect transaction pool.\n /\n #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let result: T;\n try {\n result = await readTask(tx, lc);\n freeWorker();\n r.resolve(result);\n } catch (e) {\n freeWorker();\n r.reject(e);\n }\n return {pending: null};\n },\n rejected: r.reject,\n };\n }\n\n #process(runner: TaskRunner): void {\n assert(!this.#done, 'already set done');\n if (this.#failure) {\n runner.rejected(this.#failure);\n return;\n }\n\n this.#tasks.enqueue(runner);\n\n // Check if the pool size can and should be increased.\n if (this.#numWorkers < this.#maxWorkers) {\n const outstanding = this.#tasks.size();\n\n if (outstanding > this.#numWorkers - this.#numWorking) {\n this.#db && this.#addWorker(this.#db);\n this.#numWorkers++;\n this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n }\n }\n }\n\n /\n Ends all workers with a ROLLBACK. Throws if the pool is already done\n * or aborted.\n /\n abort() {\n this.fail(new RollbackSignal());\n }\n\n /\n Signals to all workers to end their transaction once all pending tasks have\n * been completed. Throws if the pool is already done or aborted.\n /\n setDone() {\n assert(!this.#done, 'already set done');\n this.#done = true;\n\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#tasks.enqueue('done');\n }\n }\n\n /\n An alternative to explicitly calling {@link setDone}, `ref()` increments an internal reference\n * count, and {@link unref} decrements it. When the reference count reaches 0, {@link setDone} is\n * automatically called. A TransactionPool is initialized with a reference count of 1.\n \n `ref()` should be called before sharing the pool with another component, and only after the\n * pool has been started with {@link run()}. It must not be called on a TransactionPool that is\n * already done (either via {@link unref()} or {@link setDone()}. (Doing so indicates a logical\n * error in the code.)\n \n It follows that:\n * * The creator of the TransactionPool is responsible for running it.\n * * The TransactionPool should be ref'ed before being sharing.\n * * The receiver of the TransactionPool is only responsible for unref'ing it.\n \n On the other hand, a transaction pool that fails with a runtime error can still be ref'ed;\n * attempts to use the pool will result in the runtime error as expected.\n /\n // TODO: Get rid of the ref-counting stuff. It's no longer needed.\n ref(count = 1) {\n assert(\n this.#db !== undefined && !this.#done,\n `Cannot ref() a TransactionPool that is not running`,\n );\n this.#refCount += count;\n }\n\n /\n Decrements the internal reference count, automatically invoking {@link setDone} when it reaches 0.\n /\n unref(count = 1) {\n assert(\n count <= this.#refCount,\n () => `Cannot unref ${count} when refCount is ${this.#refCount}`,\n );\n\n this.#refCount -= count;\n if (this.#refCount === 0) {\n this.setDone();\n }\n }\n\n isRunning(): boolean {\n return this.#db !== undefined && !this.#done && this.#failure === undefined;\n }\n\n /\n Signals all workers to fail their transactions with the given {@link err}.\n /\n fail(err: unknown) {\n if (!this.#failure) {\n this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n // Logged for informational purposes. It is the responsibility of\n // higher level logic to classify and handle the exception.\n const level =\n this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n this.#lc[level]?.(this.#failure);\n\n for (let i = 0; i < this.#numWorkers; i++) {\n // Enqueue the Error to terminate any workers waiting for tasks.\n this.#tasks.enqueue(this.#failure);\n }\n }\n }\n}\n\ntype SynchronizeSnapshotTasks = {\n /\n The `init` Task for the TransactionPool from which the snapshot originates.\n * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n * first worker will export a snapshot that the others set.\n /\n exportSnapshot: Task;\n\n /\n The `cleanup` Task for the TransactionPool from which the snapshot\n * originates. This Task will wait for the follower pool to `setSnapshot`\n * to ensure that the snapshot is successfully shared before the originating\n * transaction is closed.\n /\n cleanupExport: Task;\n\n /\n The `init` Task for the TransactionPool in which workers will\n * consequently see the same snapshot as that of the first pool. The pool\n * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n /\n setSnapshot: Task;\n\n /* The ID of the shared snapshot. /\n snapshotID: Promise<string>;\n};\n\n/\n Init Tasks for Postgres snapshot synchronization across transactions.\n \n https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n /\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n const {\n promise: snapshotCaptured,\n resolve: captureSnapshot,\n reject: failCapture,\n } = resolver<unknown>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n let firstWorkerRun = false;\n\n // Note: Neither init task should `await`, as processing in each pool can proceed\n // as soon as the statements have been sent to the db. However, the `cleanupExport`\n // task must `await` the result of `setSnapshot` to ensure that exporting transaction\n // does not close before the snapshot has been captured.\n return {\n exportSnapshot: tx => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n const stmt =\n tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n tx`SET TRANSACTION READ ONLY`.simple(),\n ]);\n },\n\n setSnapshot: tx =>\n snapshotExported.then(snapshotID => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n // Intercept the promise to propagate the information to `cleanupExport`.\n stmt.then(captureSnapshot, failCapture);\n return [stmt];\n }),\n\n cleanupExport: async () => {\n await snapshotCaptured;\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n /\nexport function sharedSnapshot(): {\n init: Task;\n cleanup: Task;\n snapshotID: Promise<string>;\n} {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot.\n let firstWorkerRun = false;\n\n // The LogContext of the exporting worker, used to identify its cleanup call.\n // Each worker receives a unique lc instance (via withContext('tx', id)), so\n // reference equality reliably identifies the exporting worker.\n let exporterLc: LogContext \| undefined;\n\n // Set when the exporting worker's cleanup runs, signalling that the snapshot\n // is no longer needed and any subsequently spawned workers should skip their\n // initTask.\n let firstWorkerDone = false;\n\n return {\n init: (tx, lc) => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n exporterLc = lc; // Remember which worker is the exporter.\n const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n if (!firstWorkerDone) {\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n ]);\n }\n lc.debug?.('All work is done. No need to set snapshot');\n return [];\n },\n\n cleanup: (_tx, lc) => {\n // Only the exporting worker's cleanup should disable snapshot-setting.\n // Non-exporter workers may finish early; letting them flip this flag\n // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n // and read a newer database view, violating snapshot isolation.\n if (lc === exporterLc) {\n firstWorkerDone = true;\n }\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n @returns An `init` Task for importing a snapshot from another transaction.\n /\nexport function importSnapshot(snapshotID: string): {\n init: Task;\n imported: Promise<void>;\n} {\n const {promise: imported, resolve, reject} = resolver<void>();\n\n return {\n init: tx => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n stmt.then(() => resolve(), reject);\n return [stmt];\n },\n\n imported,\n };\n}\n\n/\n A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n /\nexport class ControlFlowError extends Error {\n constructor(cause?: unknown) {\n super();\n this.cause = cause;\n }\n}\n\n/\n Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n \n Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n /\nclass RollbackSignal extends ControlFlowError {\n readonly name = 'RollbackSignal';\n readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n\ninterface TaskRunner {\n /\n Manages the running of a Task or ReadTask in two phases:\n \n - If the task involves blocking, this is done in the worker. Once the\n * blocking is done, `freeWorker()` is invoked to signal that the worker\n * is available to run another task. Note that this should be invoked\n * before resolving the result to the calling thread so that a\n * subsequent task can reuse the same worker.\n \n - Task statements are executed on the database asynchronously. The final\n * result of this processing is encapsulated in the returned `pending`\n * Promise. The worker will await the last pending Promise before closing\n * the transaction.\n \n @param freeWorker should be called as soon as all blocking operations are\n * completed in order to return the transaction to the pool.\n * @returns A `pending` Promise indicating when the statements have been\n * processed by the database, allowing the transaction to be closed.\n * This should be `null` if there are no transaction-dependent\n * statements to await.\n /\n run(\n tx: PostgresTransaction,\n lc: LogContext,\n freeWorker: () => void,\n ): Promise<{pending: Promise<void> \| null}>;\n\n /\n Invoked if the TransactionPool is already in a failed state when the task\n * is requested.\n */\n rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n lc.debug?.(`sending tx keepalive`);\n return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n timeoutMs: number;\n task: Task \| 'done';\n};\n\ntype TimeoutTasks = {\n forInitialWorkers: TimeoutTask;\n forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n forInitialWorkers: {\n timeoutMs: KEEPALIVE_TIMEOUT_MS,\n task: KEEPALIVE_TASK,\n },\n forExtraWorkers: {\n timeoutMs: IDLE_TIMEOUT_MS,\n task: 'done',\n },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;AAgDA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAoC;CAC1D,WAAwC,EAAE;CAC1C;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,YAAY;CACZ,QAAQ;CACR;;;;;;;;;;;;;;;;;CAkBA,YACE,IACA,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,gBACb,eAAe,eACf;AACA,SAAO,iBAAiB,GAAG,kCAAkC;AAC7D,SACE,cAAc,gBACd,uCACD;AAED,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,OAAa,OAAO,MAAA,WAAiB,KAAK,GAAG,KAAA;AAC7C,QAAA,UAAgB,UAAU,MAAA,WAAiB,QAAQ,GAAG,KAAA;AACtD,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,aAAmB;AACnB,QAAA,cAAoB;;;;;;CAOtB,IAAI,IAAsB;AACxB,SAAO,CAAC,MAAA,IAAU,kBAAkB;AACpC,QAAA,KAAW;AACX,OAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IACpC,OAAA,UAAgB,GAAG;AAErB,SAAO;;;;;;;;;CAUT,kBAAkB,KAAa,OAAe;AAC5C,QAAA,KAAW,MAAA,GAAS,YAAY,KAAK,MAAM;AAE3C,SAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,MAAM,EACrC;;;;;;;;;;;;;;;;;;;;;;CAuBH,MAAM,OAAO;EACX,MAAM,aAAa,MAAA,QAAc;AACjC,QAAM,QAAQ,IAAI,MAAA,QAAc;AAEhC,MAAI,aAAa,MAAA,QAAc,OAK7B,OAAM,QAAQ,IAAI,MAAA,QAAc;AAElC,QAAA,GAAS,QAAQ,wBAAwB;EAEzC,MAAM,UAAU,YAAY,KAAK,GAAG,MAAA;AACpC,MAAI,UAAU,IACZ,KAAI,MAAA,QAAc,EAChB,OAAA,GAAS,OACP,kCAAkC,MAAA,MAAY,eAAe,QAAQ,QAAQ,EAAE,CAAC,MACjF;MAED,OAAA,GAAS,OACP,mCAAmC,QAAQ,QAAQ,EAAE,CAAC,MACvD;;CAKP,WAAW,IAAgB;EACzB,MAAM,KAAK,MAAA,QAAc,SAAS;EAClC,MAAM,KAAK,MAAA,GAAS,YAAY,MAAM,GAAG;EAEzC,MAAM,KACJ,MAAA,QAAc,SAAS,MAAA,iBACnB,MAAA,YAAkB,oBAClB,MAAA,YAAkB;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,MAAA,WAAiB,GAAG,KAAK;EAE3E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,KAAK;AAC/B,OAAI;AACF,OAAG,QAAQ,sBAAsB;IAEjC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;AAChD,gBAAW,MAAA,QAAc,MAAA;KACzB,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;AAC/C,iBAAW,MAAA,QAAc,MAAA;OACzB;AACF,YAAO,WAAW;;IAGpB,IAAI,OACF,MAAA,QAAe,MAAM,MAAA,MAAY,QAAQ,aAAa,UAAU;AAElE,QAAI;AACF,YAAO,SAAS,QAAQ;AACtB,UACE,gBAAgB,SACf,SAAS,MAAA,QAAc,MAAA,QAExB,OAAM,MAAA,WAAiB;AAEzB,YAAM,YAAY,KAAK;AAGvB,aAAO,MAAM,MAAA,MAAY,QAAQ,aAAa,UAAU;;cAElD;AAER,SAAI,MAAA,QACF,OAAM,YAAY,MAAA,QAAc;;IAIpC,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,OAAG,QAAQ,wBAAwB,QAAQ,QAAQ,EAAE,CAAC,MAAM;AAG5D,WAAO;YACA,GAAG;AACV,QAAI,MAAM,MAAA,QACR,MAAK,KAAK,EAAE;AAEd,UAAM;;;EAIV,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,MAAA,MAAW,CAAC,CACnD,OAAM,MAAK;AACV,OAAI,aAAa,eAGf,IAAG,QAAQ,sBAAsB;OAEjC,OAAM;IAER,CACD,cAAc,MAAA,aAAmB;AAKpC,WAAS,YAAY,GAAG;AAExB,QAAA,QAAc,KAAK,SAAS;AAK5B,MAAI,MAAA,KACF,OAAA,MAAY,QAAQ,OAAO;AAE7B,MAAI,MAAA,QACF,OAAA,MAAY,QAAQ,MAAA,QAAc;;;;;;;;;;;;;;CAgBtC,QAAQ,MAA2B;EACjC,MAAM,IAAI,UAAgB;AAC1B,QAAA,QAAc,MAAA,WAAiB,MAAM,EAAE,CAAC;AACxC,SAAO,EAAE;;CAGX,SAAkB,YAAY,KAAK;CACnC,SAAS;;;;;;;;;;;CAYT,YAAY,MAAY,IAA2B,UAAU,EAAc;AACzE,SAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;AACJ,QAAI;AACF,aAAQ,MAAM,KAAK,IAAI,GAAG;aACnB,GAAG;AACV,OAAE,SAAS;AACX,WAAM;cACE;AACR,iBAAY;;AAGd,QAAI,MAAM,WAAW,GAAG;AACtB,OAAE,SAAS;AACX,YAAO,EAAC,SAAS,MAAK;;AAuBxB,WAAO,EAAC,SAjBK,MAAM,QAChB,GAAG,SACF,KACG,SAAS,CACT,WAAW;AACV,SAAI,EAAE,MAAA,QAAc,QAAS,GAAG;MAC9B,MAAM,MAAM,MAAA,QAAc,QAAU,IAAI,SAAS;MACjD,MAAM,IAAI;AACV,SAAG,OACD,YAAY,MAAA,MAAY,iBAAiB,YAAY,KAAK,GAAG,MAAA,OAAa,QAAQ,EAAE,CAAC,OACrF,EAAC,WAAW,EAAE,QAAO,CACtB;;MAEH,CACD,OAAM,MAAK,KAAK,KAAK,EAAE,CAAC,EAC7B,YACD,CACqB,KAAK,EAAE,QAAQ,EAAC;;GAExC,UAAU,EAAE;GACb;;;;;;;;CASH,gBAAmB,UAAmC;EACpD,MAAM,IAAI,UAAa;AACvB,QAAA,QAAc,MAAA,WAAiB,UAAU,EAAE,CAAC;AAC5C,SAAO,EAAE;;;;;;;;;;;CAYX,YAAe,UAAuB,GAA4B;AAChE,SAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;AACJ,QAAI;AACF,cAAS,MAAM,SAAS,IAAI,GAAG;AAC/B,iBAAY;AACZ,OAAE,QAAQ,OAAO;aACV,GAAG;AACV,iBAAY;AACZ,OAAE,OAAO,EAAE;;AAEb,WAAO,EAAC,SAAS,MAAK;;GAExB,UAAU,EAAE;GACb;;CAGH,SAAS,QAA0B;AACjC,SAAO,CAAC,MAAA,MAAY,mBAAmB;AACvC,MAAI,MAAA,SAAe;AACjB,UAAO,SAAS,MAAA,QAAc;AAC9B;;AAGF,QAAA,MAAY,QAAQ,OAAO;AAG3B,MAAI,MAAA,aAAmB,MAAA;OACD,MAAA,MAAY,MAAM,GAEpB,MAAA,aAAmB,MAAA,YAAkB;AACrD,UAAA,MAAY,MAAA,UAAgB,MAAA,GAAS;AACrC,UAAA;AACA,UAAA,GAAS,QAAQ,0BAA0B,MAAA,aAAmB;;;;;;;;CASpE,QAAQ;AACN,OAAK,KAAK,IAAI,gBAAgB,CAAC;;;;;;CAOjC,UAAU;AACR,SAAO,CAAC,MAAA,MAAY,mBAAmB;AACvC,QAAA,OAAa;AAEb,OAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IACpC,OAAA,MAAY,QAAQ,OAAO;;;;;;;;;;;;;;;;;;;;CAuB/B,IAAI,QAAQ,GAAG;AACb,SACE,MAAA,OAAa,KAAA,KAAa,CAAC,MAAA,MAC3B,qDACD;AACD,QAAA,YAAkB;;;;;CAMpB,MAAM,QAAQ,GAAG;AACf,SACE,SAAS,MAAA,gBACH,gBAAgB,MAAM,oBAAoB,MAAA,WACjD;AAED,QAAA,YAAkB;AAClB,MAAI,MAAA,aAAmB,EACrB,MAAK,SAAS;;CAIlB,YAAqB;AACnB,SAAO,MAAA,OAAa,KAAA,KAAa,CAAC,MAAA,QAAc,MAAA,YAAkB,KAAA;;;;;CAMpE,KAAK,KAAc;AACjB,MAAI,CAAC,MAAA,SAAe;AAClB,SAAA,UAAgB,YAAY,IAAI;GAGhC,MAAM,QACJ,MAAA,mBAAyB,mBAAmB,UAAU;AACxD,SAAA,GAAS,SAAS,MAAA,QAAc;AAEhC,QAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IAEpC,OAAA,MAAY,QAAQ,MAAA,QAAc;;;;;;;AAgK1C,SAAgB,eAAe,YAG7B;CACA,MAAM,EAAC,SAAS,UAAU,SAAS,WAAU,UAAgB;AAE7D,QAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,GAAG;AAClE,QAAK,WAAW,SAAS,EAAE,OAAO;AAClC,UAAO,CAAC,KAAK;;EAGf;EACD;;;;;;;;AASH,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;AAC3B,SAAO;AACP,OAAK,QAAQ;;;;;;;;;;;;;AAcjB,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;;AAGrB,SAAS,YAAY,KAAqB;AACxC,KAAI,eAAe,MACjB,QAAO;CAET,MAAM,wBAAQ,IAAI,OAAO;AACzB,OAAM,QAAQ;AACd,QAAO;;AAsCT,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,OACnD;AAED,IAAM,kBAAwB,IAAI,OAAO;AACvC,IAAG,QAAQ,uBAAuB;AAClC,QAAO,CAAC,EAAE,WAAW,QAAQ,CAAC;;AAchC,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;EACP;CACD,iBAAiB;EACf,WAAW;EACX,MAAM;EACP;CACF"}
1	+ {"version":3,"file":"transaction-pool.js","names":["#mode","#init","#cleanup","#tasks","#workers","#initialWorkers","#maxWorkers","#responseTimeout","#timeoutTask","#lc","#stmtRunner","#numWorkers","#orphanedQueryCheckInterval","#checkForOrphanedQueries","#db","#addWorker","#start","#stmts","#numWorking","#failure","#done","#process","#pending","#latestDoneIndex","#latestDoneTime","#readRunner"],"sources":["../../../../../zero-cache/src/db/transaction-pool.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Resolver, resolver} from '@rocicorp/resolver';\nimport type postgres from 'postgres';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {Enum} from '../../../shared/src/enum.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../types/pg.ts';\nimport type * as Mode from './mode-enum.ts';\nimport {runTx} from './run-transaction.ts';\n\ntype Mode = Enum<typeof Mode>;\n\ntype MaybePromise<T> = Promise<T> \| T;\n\nexport type Statement =\n \| postgres.PendingQuery<(postgres.Row & Iterable<postgres.Row>)[]>\n \| postgres.PendingQuery<postgres.Row[]>;\n\n/*\n A {@link Task} is logic run from within a transaction in a {@link TransactionPool}.\n * It returns a list of `Statements` that the transaction executes asynchronously and\n * awaits when it receives the 'done' signal.\n \n /\nexport type Task = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<Statement[]>;\n\n/*\n A {@link ReadTask} is run from within a transaction, but unlike a {@link Task},\n * the results of a ReadTask are opaque to the TransactionPool and returned to the\n * caller of {@link TransactionPool.processReadTask}.\n /\nexport type ReadTask<T> = (\n tx: PostgresTransaction,\n lc: LogContext,\n) => MaybePromise<T>;\n\nexport type Options = {\n mode: Mode;\n\n /\n A {@link Task} that is run in each Transaction before it begins\n * processing general tasks. This can be used to to set the transaction\n * mode, export/set snapshots, etc. This will be run even if\n * {@link fail} has been called on the pool.\n /\n init?: Task \| undefined;\n\n /\n A {@link Task} that is run in each Transaction before it closes.\n * This will be run even if {@link fail} has been called, or if a\n * preceding Task threw an Error.\n /\n cleanup?: Task \| undefined;\n\n /\n The initial number of transaction workers to process tasks.\n * This is the steady state number of workers that will be kept\n * alive if the TransactionPool is long lived.\n * This must be greater than 0. Defaults to 1.\n /\n initialWorkers?: number;\n\n /\n When specified, allows the pool to grow to `maxWorkers`. This\n * must be greater than or equal to `initialWorkers`. On-demand\n * workers will be shut down after an idle timeout of 5 seconds.\n /\n maxWorkers?: number;\n\n /\n Aborts the transaction if the response for a statement is not received\n * within the specified timeout. Note that this is different from a Postgres\n * `statement_timeout` or `lock_timeout` in that it is specifically intended\n * to detect situations in which Postgres either never received the\n * statement, or the application never got the response for the executed\n * statement.\n /\n statementResponseTimeout?: number;\n};\n\n/\n A TransactionPool is a pool of one or more {@link postgres.TransactionSql}\n * objects that participate in processing a dynamic queue of tasks.\n \n This can be used for serializing a set of tasks that arrive asynchronously\n * to a single transaction (for writing) or performing parallel reads across\n * multiple connections at the same snapshot (e.g. read only snapshot transactions).\n /\nexport class TransactionPool {\n #lc: LogContext;\n readonly #mode: Mode;\n readonly #init: TaskRunner \| undefined;\n readonly #cleanup: TaskRunner \| undefined;\n readonly #tasks = new Queue<TaskRunner \| Error \| 'done'>();\n readonly #workers: Promise<unknown>[] = [];\n readonly #initialWorkers: number;\n readonly #maxWorkers: number;\n readonly #responseTimeout: number \| undefined;\n readonly #timeoutTask: TimeoutTasks;\n #numWorkers: number;\n #numWorking = 0;\n #db: PostgresDB \| undefined; // set when running. stored to allow adaptive pool sizing.\n\n #done = false;\n #failure: Error \| undefined;\n #orphanedQueryCheckInterval: NodeJS.Timeout;\n\n constructor(\n lc: LogContext,\n opts: Options,\n timeoutTasks = TIMEOUT_TASKS, // Overridden for tests.\n ) {\n const {\n mode,\n init,\n cleanup,\n initialWorkers = 1,\n maxWorkers = initialWorkers,\n statementResponseTimeout,\n } = opts;\n assert(initialWorkers > 0, 'initialWorkers must be positive');\n assert(\n maxWorkers >= initialWorkers,\n 'maxWorkers must be >= initialWorkers',\n );\n\n this.#lc = lc;\n this.#mode = mode;\n this.#init = init ? this.#stmtRunner(init) : undefined;\n this.#cleanup = cleanup ? this.#stmtRunner(cleanup) : undefined;\n this.#initialWorkers = initialWorkers;\n this.#numWorkers = initialWorkers;\n this.#maxWorkers = maxWorkers;\n this.#timeoutTask = timeoutTasks;\n this.#responseTimeout = statementResponseTimeout;\n\n this.#orphanedQueryCheckInterval = setInterval(\n this.#checkForOrphanedQueries,\n Math.min(5_000, statementResponseTimeout ?? 5_000),\n );\n }\n\n /\n Starts the pool of workers to process Tasks with transactions opened from the\n * specified {@link db}.\n /\n run(db: PostgresDB): this {\n assert(!this.#db, 'already running');\n this.#db = db;\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#addWorker(db);\n }\n return this;\n }\n\n /\n Adds context parameters to internal LogContext. This is useful for context values that\n * are not known when the TransactionPool is constructed (e.g. determined after a database\n * call when the pool is running).\n \n Returns an object that can be used to add more parameters.\n /\n addLoggingContext(key: string, value: string) {\n this.#lc = this.#lc.withContext(key, value);\n\n return {\n addLoggingContext: (key: string, value: string) =>\n this.addLoggingContext(key, value),\n };\n }\n\n /\n Returns a promise that:\n \n * resolves after {@link setDone} has been called (or the the pool as been {@link unref}ed\n * to a 0 ref count), once all added tasks have been processed and all transactions have been\n * committed or closed.\n \n * rejects if processing was aborted with {@link fail} or if processing any of\n * the tasks resulted in an error. All uncommitted transactions will have been\n * rolled back.\n \n Note that partial failures are possible if processing writes with multiple workers\n * (e.g. `setDone` is called, allowing some workers to commit, after which other\n * workers encounter errors). Using a TransactionPool in this manner does not make\n * sense in terms of transactional semantics, and is thus not recommended.\n \n For reads, however, multiple workers is useful for performing parallel reads\n * at the same snapshot. See {@link synchronizedSnapshots} for an example.\n * Resolves or rejects when all workers are done or failed.\n /\n async done() {\n const numWorkers = this.#workers.length;\n await Promise.all(this.#workers);\n\n if (numWorkers < this.#workers.length) {\n // If workers were added after the initial set, they must be awaited to ensure\n // that the results (i.e. rejections) of all workers are accounted for. This only\n // needs to be re-done once, because the fact that the first `await` completed\n // guarantees that the pool is in a terminal state and no new workers can be added.\n await Promise.all(this.#workers);\n }\n this.#lc.debug?.('transaction pool done');\n\n const elapsed = performance.now() - this.#start;\n if (elapsed > 60_000) {\n if (this.#stmts > 0) {\n this.#lc.warn?.(\n `finished long transaction with ${this.#stmts} statements (${elapsed.toFixed(3)} ms)`,\n );\n } else {\n this.#lc.warn?.(\n `finished long read transaction (${elapsed.toFixed(3)} ms)`,\n );\n }\n }\n }\n\n #addWorker(db: PostgresDB) {\n const id = this.#workers.length + 1;\n const lc = this.#lc.withContext('tx', id);\n\n const tt: TimeoutTask =\n this.#workers.length < this.#initialWorkers\n ? this.#timeoutTask.forInitialWorkers\n : this.#timeoutTask.forExtraWorkers;\n const {timeoutMs} = tt;\n const timeoutTask = tt.task === 'done' ? 'done' : this.#stmtRunner(tt.task);\n\n const worker = async (tx: PostgresTransaction) => {\n const start = performance.now();\n try {\n lc.debug?.('started transaction');\n\n let last: Promise<void> = promiseVoid;\n\n const executeTask = async (runner: TaskRunner) => {\n runner !== this.#init && this.#numWorking++;\n const {pending} = await runner.run(tx, lc, () => {\n runner !== this.#init && this.#numWorking--;\n });\n last = pending ?? last;\n };\n\n let task: TaskRunner \| Error \| 'done' =\n this.#init ?? (await this.#tasks.dequeue(timeoutTask, timeoutMs));\n\n try {\n while (task !== 'done') {\n if (\n task instanceof Error \|\|\n (task !== this.#init && this.#failure)\n ) {\n throw this.#failure ?? task;\n }\n await executeTask(task);\n\n // await the next task.\n task = await this.#tasks.dequeue(timeoutTask, timeoutMs);\n }\n } finally {\n // Execute the cleanup task even on failure.\n if (this.#cleanup) {\n await executeTask(this.#cleanup);\n }\n }\n\n const elapsed = performance.now() - start;\n lc.debug?.(`closing transaction (${elapsed.toFixed(3)} ms)`);\n // Given the semantics of a Postgres transaction, the last statement\n // will only succeed if all of the preceding statements succeeded.\n return last;\n } catch (e) {\n if (e !== this.#failure) {\n this.fail(e); // A failure in any worker should fail the pool.\n }\n throw e;\n }\n };\n\n const workerTx = runTx(db, worker, {mode: this.#mode})\n .catch(e => {\n if (e instanceof RollbackSignal) {\n // A RollbackSignal is used to gracefully rollback the postgres.js\n // transaction block. It should not be thrown up to the application.\n lc.debug?.('aborted transaction');\n } else {\n throw e;\n }\n })\n .finally(() => this.#numWorkers--);\n\n // Attach a rejection handler immediately to prevent unhandledRejections.\n // The application will handle errors when it awaits processReadTask()\n // or done().\n workerTx.catch(() => {});\n\n this.#workers.push(workerTx);\n\n // After adding the worker, enqueue a terminal signal if we are in either of the\n // terminal states (both of which prevent more tasks from being enqueued), to ensure\n // that the added worker eventually exits.\n if (this.#done) {\n this.#tasks.enqueue('done');\n }\n if (this.#failure) {\n this.#tasks.enqueue(this.#failure);\n }\n }\n\n /\n Processes the statements produced by the specified {@link Task},\n * returning a Promise that resolves when the statements are either processed\n * by the database or rejected.\n \n Note that statement failures will result in failing the entire\n * TransactionPool (per transaction semantics). However, the returned Promise\n * itself will resolve rather than reject. As such, it is fine to ignore\n * returned Promises in order to pipeline requests to the database. It is\n * recommended to occasionally await them (e.g. after some threshold) in\n * order to avoid memory blowup in the case of database slowness.\n /\n process(task: Task): Promise<void> {\n const r = resolver<void>();\n this.#process(this.#stmtRunner(task, r));\n return r.promise;\n }\n\n readonly #start = performance.now();\n #stmts = 0;\n #latestDoneIndex = 0;\n #latestDoneTime = 0;\n\n readonly #pending = new Map<\n Query,\n {index: number; time: number; abort: (e: ResponseTimeoutError) => void}\n >();\n\n /\n Implements the semantics specified in {@link process()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the statements are produced,\n * allowing them to be pipelined to the database.\n * * Statement errors result in failing the transaction pool.\n * * The client-supplied Resolver resolves on success or failure;\n * it is never rejected.\n /\n #stmtRunner(task: Task, r: {resolve: () => void} = resolver()): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let stmts: Statement[];\n try {\n stmts = await task(tx, lc);\n } catch (e) {\n r.resolve();\n throw e;\n } finally {\n freeWorker();\n }\n\n if (stmts.length === 0) {\n r.resolve();\n return {pending: null};\n }\n\n // Execute the statements (i.e. send to the db) immediately.\n // The last result is returned for the worker to await before\n // closing the transaction.\n const time = Date.now();\n const last = stmts.reduce((_, stmt) => {\n const query = stmt as unknown as Query;\n const index = ++this.#stmts;\n const {promise: aborted, reject: abort} = resolver();\n aborted.catch(() => {}); // always consider it \"handled\"\n\n this.#pending.set(query, {index, time, abort});\n const responded = stmt\n .execute()\n .then(() => {\n if (index % 1000 === 0) {\n const log = index % 10000 === 0 ? 'info' : 'debug';\n lc[log]?.(\n `executed ${this.#stmts}th statement (${(performance.now() - this.#start).toFixed(3)} ms)`,\n {statement: query.string},\n );\n }\n })\n .catch(e => this.fail(e))\n .finally(() => {\n this.#pending.delete(query);\n if (index > this.#latestDoneIndex) {\n this.#latestDoneIndex = index;\n this.#latestDoneTime = Date.now();\n }\n });\n\n const done = Promise.race([responded, aborted]);\n done.catch(() => {});\n\n return done;\n }, promiseVoid);\n return {pending: last.finally(r.resolve)};\n },\n rejected: r.resolve,\n };\n }\n\n /\n Periodically checks the map of `#pending` queries for which Promises\n * have yet to be resolved. Checks for pathological scenarios that have\n * been observed:\n \n * A promise does not resolve, even though the transaction is otherwise\n * healthy with keepalives being sent.\n * * A transaction never \"finishes\" at the postgres.js level after the\n * worker exits, implying that postgres.js is awaiting the `last` promise\n * returned. In this case Postgres disconnected the application with an\n * idle-in-transaction timeout, suggesting that all statements did in\n * fact complete.\n /\n readonly #checkForOrphanedQueries = () => {\n if (this.#pending.size === 0) {\n return;\n }\n const now = Date.now();\n for (const [query, {index, time, abort}] of this.#pending.entries()) {\n const statement = query.string;\n const abortWith = (msg: string) => {\n this.#lc.warn?.(msg, {statement});\n const err = new ResponseTimeoutError(msg);\n abort(err);\n this.fail(err);\n this.#pending.delete(query);\n };\n\n if (index < this.#latestDoneIndex) {\n const elapsed = now - this.#latestDoneTime;\n if (this.#responseTimeout && elapsed > this.#responseTimeout) {\n abortWith(\n `statement ${this.#latestDoneIndex} completed ${elapsed} ms ago, but statement ${index} has not.`,\n );\n }\n } else {\n // Nothing out of order, but checking for a response timeout.\n const elapsed = now - time;\n if (this.#responseTimeout && elapsed > this.#responseTimeout) {\n abortWith(`response for statement timed out after ${elapsed} ms`);\n }\n }\n }\n };\n\n /\n Processes and returns the result of executing the {@link ReadTask} from\n * within the transaction. An error thrown by the task will result in\n * rejecting the returned Promise, but will not affect the transaction pool\n * itself.\n /\n processReadTask<T>(readTask: ReadTask<T>): Promise<T> {\n const r = resolver<T>();\n this.#process(this.#readRunner(readTask, r));\n return r.promise;\n }\n\n /\n Implements the semantics specified in {@link processReadTask()}.\n \n Specifically:\n * * `freeWorker()` is called as soon as the result is produced,\n * before resolving the client-supplied Resolver.\n * * Errors result in rejecting the client-supplied Resolver but\n * do not affect transaction pool.\n /\n #readRunner<T>(readTask: ReadTask<T>, r: Resolver<T>): TaskRunner {\n return {\n run: async (tx, lc, freeWorker) => {\n let result: T;\n try {\n result = await readTask(tx, lc);\n freeWorker();\n r.resolve(result);\n } catch (e) {\n freeWorker();\n r.reject(e);\n }\n return {pending: null};\n },\n rejected: r.reject,\n };\n }\n\n #process(runner: TaskRunner): void {\n assert(!this.#done, 'already set done');\n if (this.#failure) {\n runner.rejected(this.#failure);\n return;\n }\n\n this.#tasks.enqueue(runner);\n\n // Check if the pool size can and should be increased.\n if (this.#numWorkers < this.#maxWorkers) {\n const outstanding = this.#tasks.size();\n\n if (outstanding > this.#numWorkers - this.#numWorking) {\n this.#db && this.#addWorker(this.#db);\n this.#numWorkers++;\n this.#lc.debug?.(`Increased pool size to ${this.#numWorkers}`);\n }\n }\n }\n\n /\n Ends all workers with a ROLLBACK. Throws if the pool is already done\n * or aborted.\n /\n abort() {\n this.fail(new RollbackSignal());\n }\n\n /\n Signals to all workers to end their transaction once all pending tasks have\n * been completed. Throws if the pool is already done or aborted.\n /\n setDone() {\n assert(!this.#done, 'already set done');\n this.#done = true;\n\n for (let i = 0; i < this.#numWorkers; i++) {\n this.#tasks.enqueue('done');\n }\n void Promise.allSettled(this.#workers).then(() =>\n clearInterval(this.#orphanedQueryCheckInterval),\n );\n }\n\n isRunning(): boolean {\n return this.#db !== undefined && !this.#done && this.#failure === undefined;\n }\n\n /\n Signals all workers to fail their transactions with the given {@link err}.\n /\n fail(err: unknown) {\n if (!this.#done && !this.#failure) {\n this.#failure = ensureError(err); // Fail fast: this is checked in the worker loop.\n // Logged for informational purposes. It is the responsibility of\n // higher level logic to classify and handle the exception.\n const level =\n this.#failure instanceof ControlFlowError ? 'debug' : 'info';\n this.#lc[level]?.(this.#failure);\n\n for (let i = 0; i < this.#numWorkers; i++) {\n // Enqueue the Error to terminate any workers waiting for tasks.\n this.#tasks.enqueue(this.#failure);\n }\n void Promise.allSettled(this.#workers).then(() =>\n clearInterval(this.#orphanedQueryCheckInterval),\n );\n }\n }\n}\n\ntype SynchronizeSnapshotTasks = {\n /\n The `init` Task for the TransactionPool from which the snapshot originates.\n * The pool must have Mode.SERIALIZABLE, and will be set to READ ONLY by the\n * `exportSnapshot` init task. If the TransactionPool has multiple workers, the\n * first worker will export a snapshot that the others set.\n /\n exportSnapshot: Task;\n\n /\n The `cleanup` Task for the TransactionPool from which the snapshot\n * originates. This Task will wait for the follower pool to `setSnapshot`\n * to ensure that the snapshot is successfully shared before the originating\n * transaction is closed.\n /\n cleanupExport: Task;\n\n /\n The `init` Task for the TransactionPool in which workers will\n * consequently see the same snapshot as that of the first pool. The pool\n * must have Mode.SERIALIZABLE, and will have the ability to perform writes.\n /\n setSnapshot: Task;\n\n /* The ID of the shared snapshot. /\n snapshotID: Promise<string>;\n};\n\n/\n Init Tasks for Postgres snapshot synchronization across transactions.\n \n https://www.postgresql.org/docs/9.3/functions-admin.html#:~:text=Snapshot%20Synchronization%20Functions,identical%20content%20in%20the%20database.\n /\nexport function synchronizedSnapshots(): SynchronizeSnapshotTasks {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n const {\n promise: snapshotCaptured,\n resolve: captureSnapshot,\n reject: failCapture,\n } = resolver<unknown>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot. TODO: Plumb the workerNum and use that instead.\n let firstWorkerRun = false;\n\n // Note: Neither init task should `await`, as processing in each pool can proceed\n // as soon as the statements have been sent to the db. However, the `cleanupExport`\n // task must `await` the result of `setSnapshot` to ensure that exporting transaction\n // does not close before the snapshot has been captured.\n return {\n exportSnapshot: tx => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n const stmt =\n tx`SELECT pg_export_snapshot() AS snapshot; SET TRANSACTION READ ONLY;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n tx`SET TRANSACTION READ ONLY`.simple(),\n ]);\n },\n\n setSnapshot: tx =>\n snapshotExported.then(snapshotID => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n // Intercept the promise to propagate the information to `cleanupExport`.\n stmt.then(captureSnapshot, failCapture);\n return [stmt];\n }),\n\n cleanupExport: async () => {\n await snapshotCaptured;\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n Returns `init` and `cleanup` {@link Task}s for a TransactionPool that ensure its workers\n * share a single view of the database. This is used for View Notifier and View Syncer logic\n * that allows multiple entities to perform parallel reads on the same snapshot of the database.\n /\nexport function sharedSnapshot(): {\n init: Task;\n cleanup: Task;\n snapshotID: Promise<string>;\n} {\n const {\n promise: snapshotExported,\n resolve: exportSnapshot,\n reject: failExport,\n } = resolver<string>();\n\n // Set by the first worker to run its initTask, who becomes responsible for\n // exporting the snapshot.\n let firstWorkerRun = false;\n\n // The LogContext of the exporting worker, used to identify its cleanup call.\n // Each worker receives a unique lc instance (via withContext('tx', id)), so\n // reference equality reliably identifies the exporting worker.\n let exporterLc: LogContext \| undefined;\n\n // Set when the exporting worker's cleanup runs, signalling that the snapshot\n // is no longer needed and any subsequently spawned workers should skip their\n // initTask.\n let firstWorkerDone = false;\n\n return {\n init: (tx, lc) => {\n if (!firstWorkerRun) {\n firstWorkerRun = true;\n exporterLc = lc; // Remember which worker is the exporter.\n const stmt = tx`SELECT pg_export_snapshot() AS snapshot;`.simple();\n // Intercept the promise to propagate the information to `snapshotExported`.\n stmt.then(result => exportSnapshot(result[0].snapshot), failExport);\n return [stmt]; // Also return the stmt so that it gets awaited (and errors handled).\n }\n if (!firstWorkerDone) {\n return snapshotExported.then(snapshotID => [\n tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`),\n ]);\n }\n lc.debug?.('All work is done. No need to set snapshot');\n return [];\n },\n\n cleanup: (_tx, lc) => {\n // Only the exporting worker's cleanup should disable snapshot-setting.\n // Non-exporter workers may finish early; letting them flip this flag\n // would cause subsequently spawned workers to skip SET TRANSACTION SNAPSHOT\n // and read a newer database view, violating snapshot isolation.\n if (lc === exporterLc) {\n firstWorkerDone = true;\n }\n return [];\n },\n\n snapshotID: snapshotExported,\n };\n}\n\n/\n @returns An `init` Task for importing a snapshot from another transaction.\n /\nexport function importSnapshot(snapshotID: string): {\n init: Task;\n imported: Promise<void>;\n} {\n const {promise: imported, resolve, reject} = resolver<void>();\n\n return {\n init: tx => {\n const stmt = tx.unsafe(`SET TRANSACTION SNAPSHOT '${snapshotID}'`);\n stmt.then(() => resolve(), reject);\n return [stmt];\n },\n\n imported,\n };\n}\n\n/\n A superclass of Errors used for control flow that is needed to handle\n * another Error but does not constitute an error condition itself (e.g.\n * aborting transactions after a previous one fails). Subclassing this Error\n * will result in lowering the log level from `error` to `debug`.\n /\nexport class ControlFlowError extends Error {\n constructor(cause?: unknown) {\n super();\n this.cause = cause;\n }\n}\n\n/\n Internal error used to rollback the worker transaction. This is used\n * instead of executing a `ROLLBACK` statement because the postgres.js\n * library will otherwise try to execute an extraneous `COMMIT`, which\n * results in outputting a \"no transaction in progress\" warning to the\n * database logs.\n \n Throwing an exception, on the other hand, executes the postgres.js\n * codepath that calls `ROLLBACK` instead.\n /\nclass RollbackSignal extends ControlFlowError {\n readonly name = 'RollbackSignal';\n readonly message = 'rolling back transaction';\n}\n\nfunction ensureError(err: unknown): Error {\n if (err instanceof Error) {\n return err;\n }\n const error = new Error();\n error.cause = err;\n return error;\n}\n\ninterface TaskRunner {\n /\n Manages the running of a Task or ReadTask in two phases:\n \n - If the task involves blocking, this is done in the worker. Once the\n * blocking is done, `freeWorker()` is invoked to signal that the worker\n * is available to run another task. Note that this should be invoked\n * before resolving the result to the calling thread so that a\n * subsequent task can reuse the same worker.\n \n - Task statements are executed on the database asynchronously. The final\n * result of this processing is encapsulated in the returned `pending`\n * Promise. The worker will await the last pending Promise before closing\n * the transaction.\n \n @param freeWorker should be called as soon as all blocking operations are\n * completed in order to return the transaction to the pool.\n * @returns A `pending` Promise indicating when the statements have been\n * processed by the database, allowing the transaction to be closed.\n * This should be `null` if there are no transaction-dependent\n * statements to await.\n /\n run(\n tx: PostgresTransaction,\n lc: LogContext,\n freeWorker: () => void,\n ): Promise<{pending: Promise<void> \| null}>;\n\n /\n Invoked if the TransactionPool is already in a failed state when the task\n * is requested.\n */\n rejected(reason: unknown): void;\n}\n\nconst IDLE_TIMEOUT_MS = 5_000;\n\n// The keepalive interval is settable by ZERO_TRANSACTION_POOL_KEEPALIVE_MS\n// as an emergency measure and is explicitly not made available as a server\n// option. This value is function of how the zero-cache uses transactions, and\n// should never need to be \"tuned\" or adjusted for different environments.\n//\n// Note that it must be shorter than IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS\n// with sufficient buffering to account for when the process is blocked by\n// synchronous calls (e.g. to the replica).\nconst KEEPALIVE_TIMEOUT_MS = parseInt(\n process.env.ZERO_TRANSACTION_POOL_KEEPALIVE_MS ?? '5000',\n);\n\nconst KEEPALIVE_TASK: Task = (tx, lc) => {\n lc.debug?.(`sending tx keepalive`);\n return [tx`SELECT 1`.simple()];\n};\n\ntype TimeoutTask = {\n timeoutMs: number;\n task: Task \| 'done';\n};\n\ntype TimeoutTasks = {\n forInitialWorkers: TimeoutTask;\n forExtraWorkers: TimeoutTask;\n};\n\n// Production timeout tasks. Overridden in tests.\nexport const TIMEOUT_TASKS: TimeoutTasks = {\n forInitialWorkers: {\n timeoutMs: KEEPALIVE_TIMEOUT_MS,\n task: KEEPALIVE_TASK,\n },\n forExtraWorkers: {\n timeoutMs: IDLE_TIMEOUT_MS,\n task: 'done',\n },\n};\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n\nexport class ResponseTimeoutError extends Error {\n static readonly name = 'ResponseTimeoutError';\n\n constructor(msg: string) {\n super(msg);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4FA,IAAa,kBAAb,MAA6B;CAC3B;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAoC;CAC1D,WAAwC,EAAE;CAC1C;CACA;CACA;CACA;CACA;CACA,cAAc;CACd;CAEA,QAAQ;CACR;CACA;CAEA,YACE,IACA,MACA,eAAe,eACf;EACA,MAAM,EACJ,MACA,MACA,SACA,iBAAiB,GACjB,aAAa,gBACb,6BACE;AACJ,SAAO,iBAAiB,GAAG,kCAAkC;AAC7D,SACE,cAAc,gBACd,uCACD;AAED,QAAA,KAAW;AACX,QAAA,OAAa;AACb,QAAA,OAAa,OAAO,MAAA,WAAiB,KAAK,GAAG,KAAA;AAC7C,QAAA,UAAgB,UAAU,MAAA,WAAiB,QAAQ,GAAG,KAAA;AACtD,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,aAAmB;AACnB,QAAA,cAAoB;AACpB,QAAA,kBAAwB;AAExB,QAAA,6BAAmC,YACjC,MAAA,yBACA,KAAK,IAAI,KAAO,4BAA4B,IAAM,CACnD;;;;;;CAOH,IAAI,IAAsB;AACxB,SAAO,CAAC,MAAA,IAAU,kBAAkB;AACpC,QAAA,KAAW;AACX,OAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IACpC,OAAA,UAAgB,GAAG;AAErB,SAAO;;;;;;;;;CAUT,kBAAkB,KAAa,OAAe;AAC5C,QAAA,KAAW,MAAA,GAAS,YAAY,KAAK,MAAM;AAE3C,SAAO,EACL,oBAAoB,KAAa,UAC/B,KAAK,kBAAkB,KAAK,MAAM,EACrC;;;;;;;;;;;;;;;;;;;;;;CAuBH,MAAM,OAAO;EACX,MAAM,aAAa,MAAA,QAAc;AACjC,QAAM,QAAQ,IAAI,MAAA,QAAc;AAEhC,MAAI,aAAa,MAAA,QAAc,OAK7B,OAAM,QAAQ,IAAI,MAAA,QAAc;AAElC,QAAA,GAAS,QAAQ,wBAAwB;EAEzC,MAAM,UAAU,YAAY,KAAK,GAAG,MAAA;AACpC,MAAI,UAAU,IACZ,KAAI,MAAA,QAAc,EAChB,OAAA,GAAS,OACP,kCAAkC,MAAA,MAAY,eAAe,QAAQ,QAAQ,EAAE,CAAC,MACjF;MAED,OAAA,GAAS,OACP,mCAAmC,QAAQ,QAAQ,EAAE,CAAC,MACvD;;CAKP,WAAW,IAAgB;EACzB,MAAM,KAAK,MAAA,QAAc,SAAS;EAClC,MAAM,KAAK,MAAA,GAAS,YAAY,MAAM,GAAG;EAEzC,MAAM,KACJ,MAAA,QAAc,SAAS,MAAA,iBACnB,MAAA,YAAkB,oBAClB,MAAA,YAAkB;EACxB,MAAM,EAAC,cAAa;EACpB,MAAM,cAAc,GAAG,SAAS,SAAS,SAAS,MAAA,WAAiB,GAAG,KAAK;EAE3E,MAAM,SAAS,OAAO,OAA4B;GAChD,MAAM,QAAQ,YAAY,KAAK;AAC/B,OAAI;AACF,OAAG,QAAQ,sBAAsB;IAEjC,IAAI,OAAsB;IAE1B,MAAM,cAAc,OAAO,WAAuB;AAChD,gBAAW,MAAA,QAAc,MAAA;KACzB,MAAM,EAAC,YAAW,MAAM,OAAO,IAAI,IAAI,UAAU;AAC/C,iBAAW,MAAA,QAAc,MAAA;OACzB;AACF,YAAO,WAAW;;IAGpB,IAAI,OACF,MAAA,QAAe,MAAM,MAAA,MAAY,QAAQ,aAAa,UAAU;AAElE,QAAI;AACF,YAAO,SAAS,QAAQ;AACtB,UACE,gBAAgB,SACf,SAAS,MAAA,QAAc,MAAA,QAExB,OAAM,MAAA,WAAiB;AAEzB,YAAM,YAAY,KAAK;AAGvB,aAAO,MAAM,MAAA,MAAY,QAAQ,aAAa,UAAU;;cAElD;AAER,SAAI,MAAA,QACF,OAAM,YAAY,MAAA,QAAc;;IAIpC,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,OAAG,QAAQ,wBAAwB,QAAQ,QAAQ,EAAE,CAAC,MAAM;AAG5D,WAAO;YACA,GAAG;AACV,QAAI,MAAM,MAAA,QACR,MAAK,KAAK,EAAE;AAEd,UAAM;;;EAIV,MAAM,WAAW,MAAM,IAAI,QAAQ,EAAC,MAAM,MAAA,MAAW,CAAC,CACnD,OAAM,MAAK;AACV,OAAI,aAAa,eAGf,IAAG,QAAQ,sBAAsB;OAEjC,OAAM;IAER,CACD,cAAc,MAAA,aAAmB;AAKpC,WAAS,YAAY,GAAG;AAExB,QAAA,QAAc,KAAK,SAAS;AAK5B,MAAI,MAAA,KACF,OAAA,MAAY,QAAQ,OAAO;AAE7B,MAAI,MAAA,QACF,OAAA,MAAY,QAAQ,MAAA,QAAc;;;;;;;;;;;;;;CAgBtC,QAAQ,MAA2B;EACjC,MAAM,IAAI,UAAgB;AAC1B,QAAA,QAAc,MAAA,WAAiB,MAAM,EAAE,CAAC;AACxC,SAAO,EAAE;;CAGX,SAAkB,YAAY,KAAK;CACnC,SAAS;CACT,mBAAmB;CACnB,kBAAkB;CAElB,2BAAoB,IAAI,KAGrB;;;;;;;;;;;CAYH,YAAY,MAAY,IAA2B,UAAU,EAAc;AACzE,SAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;AACJ,QAAI;AACF,aAAQ,MAAM,KAAK,IAAI,GAAG;aACnB,GAAG;AACV,OAAE,SAAS;AACX,WAAM;cACE;AACR,iBAAY;;AAGd,QAAI,MAAM,WAAW,GAAG;AACtB,OAAE,SAAS;AACX,YAAO,EAAC,SAAS,MAAK;;IAMxB,MAAM,OAAO,KAAK,KAAK;AAiCvB,WAAO,EAAC,SAhCK,MAAM,QAAQ,GAAG,SAAS;KACrC,MAAM,QAAQ;KACd,MAAM,QAAQ,EAAE,MAAA;KAChB,MAAM,EAAC,SAAS,SAAS,QAAQ,UAAS,UAAU;AACpD,aAAQ,YAAY,GAAG;AAEvB,WAAA,QAAc,IAAI,OAAO;MAAC;MAAO;MAAM;MAAM,CAAC;KAC9C,MAAM,YAAY,KACf,SAAS,CACT,WAAW;AACV,UAAI,QAAQ,QAAS,EAEnB,IADY,QAAQ,QAAU,IAAI,SAAS,WAEzC,YAAY,MAAA,MAAY,iBAAiB,YAAY,KAAK,GAAG,MAAA,OAAa,QAAQ,EAAE,CAAC,OACrF,EAAC,WAAW,MAAM,QAAO,CAC1B;OAEH,CACD,OAAM,MAAK,KAAK,KAAK,EAAE,CAAC,CACxB,cAAc;AACb,YAAA,QAAc,OAAO,MAAM;AAC3B,UAAI,QAAQ,MAAA,iBAAuB;AACjC,aAAA,kBAAwB;AACxB,aAAA,iBAAuB,KAAK,KAAK;;OAEnC;KAEJ,MAAM,OAAO,QAAQ,KAAK,CAAC,WAAW,QAAQ,CAAC;AAC/C,UAAK,YAAY,GAAG;AAEpB,YAAO;OACN,YAAY,CACO,QAAQ,EAAE,QAAQ,EAAC;;GAE3C,UAAU,EAAE;GACb;;;;;;;;;;;;;;;CAgBH,iCAA0C;AACxC,MAAI,MAAA,QAAc,SAAS,EACzB;EAEF,MAAM,MAAM,KAAK,KAAK;AACtB,OAAK,MAAM,CAAC,OAAO,EAAC,OAAO,MAAM,YAAW,MAAA,QAAc,SAAS,EAAE;GACnE,MAAM,YAAY,MAAM;GACxB,MAAM,aAAa,QAAgB;AACjC,UAAA,GAAS,OAAO,KAAK,EAAC,WAAU,CAAC;IACjC,MAAM,MAAM,IAAI,qBAAqB,IAAI;AACzC,UAAM,IAAI;AACV,SAAK,KAAK,IAAI;AACd,UAAA,QAAc,OAAO,MAAM;;AAG7B,OAAI,QAAQ,MAAA,iBAAuB;IACjC,MAAM,UAAU,MAAM,MAAA;AACtB,QAAI,MAAA,mBAAyB,UAAU,MAAA,gBACrC,WACE,aAAa,MAAA,gBAAsB,aAAa,QAAQ,yBAAyB,MAAM,WACxF;UAEE;IAEL,MAAM,UAAU,MAAM;AACtB,QAAI,MAAA,mBAAyB,UAAU,MAAA,gBACrC,WAAU,0CAA0C,QAAQ,KAAK;;;;;;;;;;CAYzE,gBAAmB,UAAmC;EACpD,MAAM,IAAI,UAAa;AACvB,QAAA,QAAc,MAAA,WAAiB,UAAU,EAAE,CAAC;AAC5C,SAAO,EAAE;;;;;;;;;;;CAYX,YAAe,UAAuB,GAA4B;AAChE,SAAO;GACL,KAAK,OAAO,IAAI,IAAI,eAAe;IACjC,IAAI;AACJ,QAAI;AACF,cAAS,MAAM,SAAS,IAAI,GAAG;AAC/B,iBAAY;AACZ,OAAE,QAAQ,OAAO;aACV,GAAG;AACV,iBAAY;AACZ,OAAE,OAAO,EAAE;;AAEb,WAAO,EAAC,SAAS,MAAK;;GAExB,UAAU,EAAE;GACb;;CAGH,SAAS,QAA0B;AACjC,SAAO,CAAC,MAAA,MAAY,mBAAmB;AACvC,MAAI,MAAA,SAAe;AACjB,UAAO,SAAS,MAAA,QAAc;AAC9B;;AAGF,QAAA,MAAY,QAAQ,OAAO;AAG3B,MAAI,MAAA,aAAmB,MAAA;OACD,MAAA,MAAY,MAAM,GAEpB,MAAA,aAAmB,MAAA,YAAkB;AACrD,UAAA,MAAY,MAAA,UAAgB,MAAA,GAAS;AACrC,UAAA;AACA,UAAA,GAAS,QAAQ,0BAA0B,MAAA,aAAmB;;;;;;;;CASpE,QAAQ;AACN,OAAK,KAAK,IAAI,gBAAgB,CAAC;;;;;;CAOjC,UAAU;AACR,SAAO,CAAC,MAAA,MAAY,mBAAmB;AACvC,QAAA,OAAa;AAEb,OAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IACpC,OAAA,MAAY,QAAQ,OAAO;AAExB,UAAQ,WAAW,MAAA,QAAc,CAAC,WACrC,cAAc,MAAA,2BAAiC,CAChD;;CAGH,YAAqB;AACnB,SAAO,MAAA,OAAa,KAAA,KAAa,CAAC,MAAA,QAAc,MAAA,YAAkB,KAAA;;;;;CAMpE,KAAK,KAAc;AACjB,MAAI,CAAC,MAAA,QAAc,CAAC,MAAA,SAAe;AACjC,SAAA,UAAgB,YAAY,IAAI;GAGhC,MAAM,QACJ,MAAA,mBAAyB,mBAAmB,UAAU;AACxD,SAAA,GAAS,SAAS,MAAA,QAAc;AAEhC,QAAK,IAAI,IAAI,GAAG,IAAI,MAAA,YAAkB,IAEpC,OAAA,MAAY,QAAQ,MAAA,QAAc;AAE/B,WAAQ,WAAW,MAAA,QAAc,CAAC,WACrC,cAAc,MAAA,2BAAiC,CAChD;;;;;;;AA+JP,SAAgB,eAAe,YAG7B;CACA,MAAM,EAAC,SAAS,UAAU,SAAS,WAAU,UAAgB;AAE7D,QAAO;EACL,OAAM,OAAM;GACV,MAAM,OAAO,GAAG,OAAO,6BAA6B,WAAW,GAAG;AAClE,QAAK,WAAW,SAAS,EAAE,OAAO;AAClC,UAAO,CAAC,KAAK;;EAGf;EACD;;;;;;;;AASH,IAAa,mBAAb,cAAsC,MAAM;CAC1C,YAAY,OAAiB;AAC3B,SAAO;AACP,OAAK,QAAQ;;;;;;;;;;;;;AAcjB,IAAM,iBAAN,cAA6B,iBAAiB;CAC5C,OAAgB;CAChB,UAAmB;;AAGrB,SAAS,YAAY,KAAqB;AACxC,KAAI,eAAe,MACjB,QAAO;CAET,MAAM,wBAAQ,IAAI,OAAO;AACzB,OAAM,QAAQ;AACd,QAAO;;AAsCT,IAAM,kBAAkB;AAUxB,IAAM,uBAAuB,SAC3B,QAAQ,IAAI,sCAAsC,OACnD;AAED,IAAM,kBAAwB,IAAI,OAAO;AACvC,IAAG,QAAQ,uBAAuB;AAClC,QAAO,CAAC,EAAE,WAAW,QAAQ,CAAC;;AAchC,IAAa,gBAA8B;CACzC,mBAAmB;EACjB,WAAW;EACX,MAAM;EACP;CACD,iBAAiB;EACf,WAAW;EACX,MAAM;EACP;CACF;AAMD,IAAa,uBAAb,cAA0C,MAAM;CAC9C,OAAgB,OAAO;CAEvB,YAAY,KAAa;AACvB,QAAM,IAAI"}

package/out/zero-cache/src/server/anonymous-otel-start.js CHANGED Viewed

@@ -3,8 +3,8 @@ import { getServerVersion, getZeroConfig } from "../config/zero-config.js";
 import { setupOtelDiagnosticLogger } from "./otel-diag-logger.js";
 import "@opentelemetry/api";
 import { homedir, platform } from "os";
-import { resourceFromAttributes } from "@opentelemetry/resources";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { resourceFromAttributes } from "@opentelemetry/resources";
 import { execSync } from "child_process";
 import { randomUUID } from "crypto";
 import { dirname, join } from "path";

package/out/zero-cache/src/server/change-streamer.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,~~CAmLf~~"}
1	+ {"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA0Lf"}

package/out/zero-cache/src/server/change-streamer.js CHANGED Viewed

@@ -6,7 +6,6 @@ import { getShardConfig } from "../types/shards.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { pgClient } from "../types/pg.js";
 import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
-import { startOtelAuto } from "./otel-start.js";
 import { createLogContext } from "./logging.js";
 import { getServerContext } from "../config/server-context.js";
 import { deleteLiteDB } from "../db/delete-lite-db.js";
@@ -24,6 +23,7 @@ import { PurgeLocker } from "../services/change-streamer/storer.js";
 import { initializeStreamer } from "../services/change-streamer/change-streamer-service.js";
 import { ReplicaMonitor } from "../services/change-streamer/replica-monitor.js";
 import { BackupNotFoundException, restoreReplica } from "../services/litestream/commands.js";
+import { startOtelAuto } from "./otel-start.js";
 //#region ../zero-cache/src/server/change-streamer.ts
 async function runWorker(parent, env, ...argv) {
 	const workerStartTime = Date.now();
@@ -32,8 +32,8 @@ async function runWorker(parent, env, ...argv) {
 		argv
 	});
 	const { taskID, changeStreamer: { port, address, protocol, startupDelayMs, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds }, upstream, change, replica, initialSync, litestream } = config;
-	startOtelAuto(createLogContext(config, { worker: "change-streamer" }, false));
-	const lc = createLogContext(config, { worker: "change-streamer" }, true);
+	startOtelAuto(createLogContext(config, "change-streamer", 0, false), "change-streamer", 0);
+	const lc = createLogContext(config, "change-streamer");
 	initEventSink(lc, config);
 	const changeDB = pgClient(lc, change.db, {
 		max: change.maxConns,
@@ -58,7 +58,11 @@ async function runWorker(parent, env, ...argv) {
 			...initialSync,
 			replicationSlotFailover: upstream.pgReplicationSlotFailover
 		}, context, replicationLag.reportIntervalMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context);
-		changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, ReplicationStatusPublisher.forReplicaFile(replica.file), subscriptionState, purgeLock, autoReset ?? false, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeout);
+		changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, ReplicationStatusPublisher.forReplicaFile(replica.file), subscriptionState, purgeLock, autoReset ?? false, {
+			backPressureLimitHeapProportion,
+			flowControlConsensusPaddingSeconds,
+			statementTimeoutMs: change.statementTimeoutMs
+		}, setTimeout);
 		break;
 	} catch (e) {
 		if (first && e instanceof AutoResetSignal) {

package/out/zero-cache/src/server/change-streamer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {\n AutoResetSignal,\n CHANGE_STREAMER_APP_NAME,\n} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n BackupNotFoundException,\n restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const workerStartTime = Date.now();\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n },\n upstream,\n change,\n replica,\n initialSync,\n litestream,\n } = config;\n\n startOtelAuto(createLogContext(config, {worker: 'change-streamer'}, false));\n const lc = createLogContext(config, {worker: 'change-streamer'}, true);\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = pgClient(\n lc,\n change.db,\n {\n max: change.maxConns,\n connection: {['application_name']: CHANGE_STREAMER_APP_NAME},\n },\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const {autoReset, replicationLag} = config;\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date, then acquire\n // a lock to prevent change-lock purges. This ensures that (this)\n // change-streamer will be able to resume from the backup.\n await initChangeStreamerSchema(lc, changeDB, shard);\n let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n // Restore from litestream if the change-log has entries.\n if (purgeLock) {\n try {\n await restoreReplica(lc, config, purgeLock);\n } catch (e) {\n // If the restore failed, e.g. due to a corrupt or missing backup, the\n // replication-manager recovers by re-syncing.\n const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n lc[log]?.(\n `error restoring backup. resyncing the replica: ${String(e)}`,\n e,\n );\n\n // The purgeLock must be released if the backup could not be restored,\n // or it will otherwise prevent the change-db update after the resync\n // completes.\n await purgeLock.release();\n purgeLock = null;\n }\n }\n\n let changeStreamer: ChangeStreamerService \| undefined;\n\n const context = getServerContext(config);\n\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {changeSource, subscriptionState} =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n purgeLock,\n autoReset ?? false,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeout,\n );\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n continue; // execute again with a fresh initial-sync\n }\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n // Perform any upgrades to the replica in case it was restored from an\n // earlier version. Note that this upgrade is done by the replicator worker\n // as well (in both the replication-manager and the view-syncer), but the\n // change-streamer independently reads the replica, and it is fine run the\n // upgrade logic redundantly since it is idempotent.\n await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n const {backupURL, port: metricsPort} = litestream;\n const monitor = backupURL\n ? new BackupMonitor(\n lc,\n replica.file,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n changeStreamer,\n // The time between when the zero-cache was started to when the\n // change-streamer is ready to start serves as the initial delay for\n // watermark cleanup (as it either includes a similar replica\n // restoration/preparation step, or an initial-sync, which\n // generally takes longer).\n //\n // Consider: Also account for permanent volumes?\n Date.now() - workerStartTime,\n )\n : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n config,\n {port, startupDelayMs},\n parent,\n changeStreamer,\n monitor instanceof BackupMonitor ? monitor : null,\n );\n\n parent.send(['ready', {ready: true}]);\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer.\n return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,KAAK;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;CACnD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,eACE;AAEJ,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,MAAM,CAAC;CAC3E,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,KAAK;AACtE,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP;EACE,KAAK,OAAO;EACZ,YAAY,GAAE,qBAAqB,0BAAyB;EAC7D,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;AAKpC,OAAM,yBAAyB,IAAI,UAAU,MAAM;CACnD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,SAAS,CAAC,SAAS;AAGpE,KAAI,UACF,KAAI;AACF,QAAM,eAAe,IAAI,QAAQ,UAAU;UACpC,GAAG;AAIV,KADY,aAAa,0BAA0B,SAAS,WAE1D,kDAAkD,OAAO,EAAE,IAC3D,EACD;AAKD,QAAM,UAAU,SAAS;AACzB,cAAY;;CAIhB,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,WACA,aAAa,OACb,iCACA,oCACA,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAC1B;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,gBACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAe,EACtB,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
1	+ {"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {initChangeStreamerSchema} from '../services/change-streamer/schema/init.ts';\nimport {\n AutoResetSignal,\n CHANGE_STREAMER_APP_NAME,\n} from '../services/change-streamer/schema/tables.ts';\nimport {PurgeLocker} from '../services/change-streamer/storer.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n BackupNotFoundException,\n restoreReplica,\n} from '../services/litestream/commands.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const workerStartTime = Date.now();\n const config = getNormalizedZeroConfig({env, argv});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n },\n upstream,\n change,\n replica,\n initialSync,\n litestream,\n } = config;\n\n startOtelAuto(\n createLogContext(config, 'change-streamer', 0, false),\n 'change-streamer',\n 0,\n );\n const lc = createLogContext(config, 'change-streamer');\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = pgClient(\n lc,\n change.db,\n {\n max: change.maxConns,\n connection: {['application_name']: CHANGE_STREAMER_APP_NAME},\n },\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change').catch(() => {});\n\n const {autoReset, replicationLag} = config;\n const shard = getShardConfig(config);\n\n // Ensure the change DB schema is initialized/up-to-date, then acquire\n // a lock to prevent change-lock purges. This ensures that (this)\n // change-streamer will be able to resume from the backup.\n await initChangeStreamerSchema(lc, changeDB, shard);\n let purgeLock = await new PurgeLocker(lc, shard, changeDB).acquire();\n\n // Restore from litestream if the change-log has entries.\n if (purgeLock) {\n try {\n await restoreReplica(lc, config, purgeLock);\n } catch (e) {\n // If the restore failed, e.g. due to a corrupt or missing backup, the\n // replication-manager recovers by re-syncing.\n const log = e instanceof BackupNotFoundException ? 'warn' : 'error';\n lc[log]?.(\n `error restoring backup. resyncing the replica: ${String(e)}`,\n e,\n );\n\n // The purgeLock must be released if the backup could not be restored,\n // or it will otherwise prevent the change-db update after the resync\n // completes.\n await purgeLock.release();\n purgeLock = null;\n }\n }\n\n let changeStreamer: ChangeStreamerService \| undefined;\n\n const context = getServerContext(config);\n\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {changeSource, subscriptionState} =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n {\n ...initialSync,\n replicationSlotFailover: upstream.pgReplicationSlotFailover,\n },\n context,\n replicationLag.reportIntervalMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n purgeLock,\n autoReset ?? false,\n {\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n statementTimeoutMs: change.statementTimeoutMs,\n },\n setTimeout,\n );\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n continue; // execute again with a fresh initial-sync\n }\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n // Perform any upgrades to the replica in case it was restored from an\n // earlier version. Note that this upgrade is done by the replicator worker\n // as well (in both the replication-manager and the view-syncer), but the\n // change-streamer independently reads the replica, and it is fine run the\n // upgrade logic redundantly since it is idempotent.\n await upgradeReplica(lc, 'change-streamer-init', replica.file);\n\n const {backupURL, port: metricsPort} = litestream;\n const monitor = backupURL\n ? new BackupMonitor(\n lc,\n replica.file,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n changeStreamer,\n // The time between when the zero-cache was started to when the\n // change-streamer is ready to start serves as the initial delay for\n // watermark cleanup (as it either includes a similar replica\n // restoration/preparation step, or an initial-sync, which\n // generally takes longer).\n //\n // Consider: Also account for permanent volumes?\n Date.now() - workerStartTime,\n )\n : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n config,\n {port, startupDelayMs},\n parent,\n changeStreamer,\n monitor instanceof BackupMonitor ? monitor : null,\n );\n\n parent.send(['ready', {ready: true}]);\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer.\n return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,kBAAkB,KAAK,KAAK;CAClC,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;CACnD,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,eACE;AAEJ,eACE,iBAAiB,QAAQ,mBAAmB,GAAG,MAAM,EACrD,mBACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,kBAAkB;AACtD,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP;EACE,KAAK,OAAO;EACZ,YAAY,GAAE,qBAAqB,0BAAyB;EAC7D,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS,CAAC,YAAY,GAAG;CAE9D,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;AAKpC,OAAM,yBAAyB,IAAI,UAAU,MAAM;CACnD,IAAI,YAAY,MAAM,IAAI,YAAY,IAAI,OAAO,SAAS,CAAC,SAAS;AAGpE,KAAI,UACF,KAAI;AACF,QAAM,eAAe,IAAI,QAAQ,UAAU;UACpC,GAAG;AAIV,KADY,aAAa,0BAA0B,SAAS,WAE1D,kDAAkD,OAAO,EAAE,IAC3D,EACD;AAKD,QAAM,UAAU,SAAS;AACzB,cAAY;;CAIhB,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR;GACE,GAAG;GACH,yBAAyB,SAAS;GACnC,EACD,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,WACA,aAAa,OACb;GACE;GACA;GACA,oBAAoB,OAAO;GAC5B,EACD,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAC1B;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;AAO1E,OAAM,eAAe,IAAI,wBAAwB,QAAQ,KAAK;CAE9D,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,QAAQ,MACR,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,gBACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAe,EACtB,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}

package/out/zero-cache/src/server/logging.d.ts CHANGED Viewed

@@ -2,7 +2,5 @@ import type { LogContext } from '@rocicorp/logger';
 import { type LogConfig } from '../../../shared/src/logging.ts';
 export declare function createLogContext({ log }: {
     log: LogConfig;
-}, context: {
-    worker: string;
-}, includeOtel?: boolean): LogContext;
+}, worker: string, workerIndex?: number, includeOtel?: boolean): LogContext;
 //# sourceMappingURL=logging.d.ts.map

package/out/zero-cache/src/server/logging.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/logging.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,gCAAgC,CAAC;AAIxC,wBAAgB,gBAAgB,CAC9B,EAAC,GAAG,EAAC,EAAE;IAAC,GAAG,EAAE,SAAS,CAAA;CAAC,EACvB,~~OAAO~~,EAAE~~;IAAC~~,MAAM,~~EAAE~~,~~MAAM~~,~~CAAA;CAAC~~,~~EACzB~~,WAAW,UAAO,GACjB,UAAU,CASZ"}
1	+ {"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/logging.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,gCAAgC,CAAC;AAIxC,wBAAgB,gBAAgB,CAC9B,EAAC,GAAG,EAAC,EAAE;IAAC,GAAG,EAAE,SAAS,CAAA;CAAC,EACvB,MAAM,EAAE,MAAM,EACd,WAAW,SAAI,EACf,WAAW,UAAO,GACjB,UAAU,CASZ"}

package/out/zero-cache/src/server/logging.js CHANGED Viewed

@@ -4,9 +4,12 @@ import { otelLogsEnabled } from "../../../otel/src/enabled.js";
 import { OtelLogSink } from "./otel-log-sink.js";
 import "@rocicorp/logger";
 //#region ../zero-cache/src/server/logging.ts
-function createLogContext({ log }, context, includeOtel = true) {
+function createLogContext({ log }, worker, workerIndex = 0, includeOtel = true) {
 	const logSink = createLogSink(log, includeOtel);
-	const lc = createLogContext$1({ log }, context, logSink);
+	const lc = createLogContext$1({ log }, {
+		worker,
+		workerIndex
+	}, logSink);
 	process.on("uncaughtException", async (err, origin) => {
 		lc.error?.(origin, err);
 		await logSink.flush?.();

package/out/zero-cache/src/server/logging.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"logging.js","names":["#sinks"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n createLogContext as createLogContextShared,\n getLogSink,\n type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\nexport function createLogContext(\n {log}: {log: LogConfig},\n ~~context: {~~worker: string},\n includeOtel = true,\n): LogContext {\n const logSink = createLogSink(log, includeOtel);\n const lc = createLogContextShared({log}, ~~context~~, logSink);\n process.on('uncaughtException', async (err, origin) => {\n lc.error?.(origin, err);\n await logSink.flush?.();\n process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n });\n return lc;\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n const sink = getLogSink(config);\n if (includeOtel && otelLogsEnabled()) {\n const otelSink = new OtelLogSink();\n return new CompositeLogSink([otelSink, sink]);\n }\n return sink;\n}\n\nclass CompositeLogSink implements LogSink {\n readonly #sinks: LogSink[];\n\n constructor(sinks: LogSink[]) {\n this.#sinks = sinks;\n }\n\n log(level: LogLevel, context: Context \| undefined, ...args: unknown[]): void {\n for (const sink of this.#sinks) {\n sink.log(level, context, ...args);\n }\n }\n}\n"],"mappings":";;;;;;AAWA,SAAgB,iBACd,EAAC,OACD,~~SACA~~,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,YAAY;CAC/C,MAAM,KAAK,mBAAuB,EAAC,KAAI,EAAE,~~SAAS~~,QAAQ;~~AAC1D~~,SAAQ,GAAG,qBAAqB,OAAO,KAAK,WAAW;AACrD,KAAG,QAAQ,QAAQ,IAAI;AACvB,QAAM,QAAQ,SAAS;AACvB,UAAQ,KAAA,GAAoC;GAC5C;AACF,QAAO;;AAGT,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,OAAO;AAC/B,KAAI,eAAe,iBAAiB,CAElC,QAAO,IAAI,iBAAiB,CADX,IAAI,aAAa,EACK,KAAK,CAAC;AAE/C,QAAO;;AAGT,IAAM,mBAAN,MAA0C;CACxC;CAEA,YAAY,OAAkB;AAC5B,QAAA,QAAc;;CAGhB,IAAI,OAAiB,SAA8B,GAAG,MAAuB;AAC3E,OAAK,MAAM,QAAQ,MAAA,MACjB,MAAK,IAAI,OAAO,SAAS,GAAG,KAAK"}
1	+ {"version":3,"file":"logging.js","names":["#sinks"],"sources":["../../../../../zero-cache/src/server/logging.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {type Context, type LogLevel, type LogSink} from '@rocicorp/logger';\nimport {otelLogsEnabled} from '../../../otel/src/enabled.ts';\nimport {\n createLogContext as createLogContextShared,\n getLogSink,\n type LogConfig,\n} from '../../../shared/src/logging.ts';\nimport {UNHANDLED_EXCEPTION_ERROR_CODE} from '../services/life-cycle.ts';\nimport {OtelLogSink} from './otel-log-sink.ts';\n\nexport function createLogContext(\n {log}: {log: LogConfig},\n worker: string,\n workerIndex = 0,\n includeOtel = true,\n): LogContext {\n const logSink = createLogSink(log, includeOtel);\n const lc = createLogContextShared({log}, {worker, workerIndex}, logSink);\n process.on('uncaughtException', async (err, origin) => {\n lc.error?.(origin, err);\n await logSink.flush?.();\n process.exit(UNHANDLED_EXCEPTION_ERROR_CODE);\n });\n return lc;\n}\n\nfunction createLogSink(config: LogConfig, includeOtel: boolean): LogSink {\n const sink = getLogSink(config);\n if (includeOtel && otelLogsEnabled()) {\n const otelSink = new OtelLogSink();\n return new CompositeLogSink([otelSink, sink]);\n }\n return sink;\n}\n\nclass CompositeLogSink implements LogSink {\n readonly #sinks: LogSink[];\n\n constructor(sinks: LogSink[]) {\n this.#sinks = sinks;\n }\n\n log(level: LogLevel, context: Context \| undefined, ...args: unknown[]): void {\n for (const sink of this.#sinks) {\n sink.log(level, context, ...args);\n }\n }\n}\n"],"mappings":";;;;;;AAWA,SAAgB,iBACd,EAAC,OACD,QACA,cAAc,GACd,cAAc,MACF;CACZ,MAAM,UAAU,cAAc,KAAK,YAAY;CAC/C,MAAM,KAAK,mBAAuB,EAAC,KAAI,EAAE;EAAC;EAAQ;EAAY,EAAE,QAAQ;AACxE,SAAQ,GAAG,qBAAqB,OAAO,KAAK,WAAW;AACrD,KAAG,QAAQ,QAAQ,IAAI;AACvB,QAAM,QAAQ,SAAS;AACvB,UAAQ,KAAA,GAAoC;GAC5C;AACF,QAAO;;AAGT,SAAS,cAAc,QAAmB,aAA+B;CACvE,MAAM,OAAO,WAAW,OAAO;AAC/B,KAAI,eAAe,iBAAiB,CAElC,QAAO,IAAI,iBAAiB,CADX,IAAI,aAAa,EACK,KAAK,CAAC;AAE/C,QAAO;;AAGT,IAAM,mBAAN,MAA0C;CACxC;CAEA,YAAY,OAAkB;AAC5B,QAAA,QAAc;;CAGhB,IAAI,OAAiB,SAA8B,GAAG,MAAuB;AAC3E,OAAK,MAAM,QAAQ,MAAA,MACjB,MAAK,IAAI,OAAO,SAAS,GAAG,KAAK"}

package/out/zero-cache/src/server/main.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,~~CAgKf~~"}
1	+ {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/main.ts"],"names":[],"mappings":"AAeA,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAoB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,OAAO,CAAC,IAAI,CAAC,CAoKf"}

package/out/zero-cache/src/server/main.js CHANGED Viewed

@@ -2,11 +2,11 @@ import { must } from "../../../shared/src/must.js";
 import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
 import { getNormalizedZeroConfig } from "../config/zero-config.js";
 import { ProcessManager, exitAfter, runUntilKilled } from "../services/life-cycle.js";
-import { startOtelAuto } from "./otel-start.js";
 import { createLogContext } from "./logging.js";
 import { CHANGE_STREAMER_URL, REAPER_URL, REPLICATOR_URL, SYNCER_URL } from "./worker-urls.js";
 import { initEventSink } from "../observability/events.js";
 import { restoreReplica, startReplicaBackupProcess } from "../services/litestream/commands.js";
+import { startOtelAuto } from "./otel-start.js";
 import { createNotifierFrom, handleSubscriptionsFrom, subscribeTo } from "../workers/replicator.js";
 import { WorkerDispatcher } from "./worker-dispatcher.js";
 import { resolver } from "@rocicorp/resolver";
@@ -15,8 +15,8 @@ import path from "node:path";
 async function runWorker(parent, env) {
 	const startMs = Date.now();
 	const config = getNormalizedZeroConfig({ env });
-	startOtelAuto(createLogContext(config, { worker: "dispatcher" }, false));
-	const lc = createLogContext(config, { worker: "dispatcher" }, true);
+	startOtelAuto(createLogContext(config, "dispatcher", 0, false), "dispatcher", 0);
+	const lc = createLogContext(config, "dispatcher");
 	initEventSink(lc, config);
 	const processes = new ProcessManager(lc, parent);
 	const { numSyncWorkers: numSyncers } = config;
@@ -68,7 +68,7 @@ async function runWorker(parent, env) {
 		});
 		await replicaReady;
 		const notifier = createNotifierFrom(lc, replicator);
-		for (let i = 0; i < numSyncers; i++) syncers.push(loadWorker(SYNCER_URL, "user-facing", i + 1, mode));
+		for (let i = 0; i < numSyncers; i++) syncers.push(loadWorker(SYNCER_URL, "user-facing", i, mode, String(i)));
 		syncers.forEach((syncer) => handleSubscriptionsFrom(lc, syncer, notifier));
 	}
 	let mutator;

package/out/zero-cache/src/server/main.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(createLogContext(config, ~~{worker:~~ 'dispatcher'}, false));\n const lc = createLogContext(config, ~~{worker:~~ 'dispatcher'~~}, true~~);\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID,\n changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n litestream,\n } = config;\n const runChangeStreamer =\n changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n let changeStreamer: Worker \| undefined;\n\n if (!runChangeStreamer) {\n changeStreamer = undefined;\n if (litestream.executable) {\n // For view-syncers, the backup is restored here. For the replication-manager,\n // the backup is restored in the change-streamer worker.\n await restoreReplica(lc, config, null);\n }\n } else {\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n 'message',\n changeStreamerStarted,\n );\n\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n\n if (litestream.backupURL) {\n // Start a backup replicator and corresponding litestream backup process.\n const {promise: backupReady, resolve} = resolver();\n const mode: ReplicaFileMode = 'backup';\n loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n // Wait for the Replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the lock when the\n // replicator first prepares the db file.\n 'message',\n () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, config),\n 'supporting',\n 'litestream',\n );\n resolve();\n },\n );\n await backupReady;\n }\n }\n\n if (numSyncers > 0) {\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n }\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i ~~+ 1~~, ~~mode~~));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n }\n\n await processes.done();\n}\n\nif (!singleProcessMode()) {\n void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"mappings":";;;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,KAAK;CAC1B,MAAM,SAAS,wBAAwB,EAAC,KAAI,CAAC;AAE7C,~~eAAc~~,iBAAiB,QAAQ,~~EAAC~~,~~QAAQ~~,~~cAAa~~,~~EAAE~~,~~MAAM~~,~~CAAC~~;~~CACtE~~,MAAM,KAAK,iBAAiB,QAAQ,~~EAAC,QAAQ,cAAa,EAAE,KAAK~~;~~AACjE~~,eAAc,IAAI,OAAO;CAEzB,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,EAAC,gBAAgB,eAAc;AACrC,KAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,WAC3D,OAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,kHAEnF;AAEH,KAAI,OAAO,IAAI,WAAW,WACxB,OAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,6GAEzE;CAGH,MAAM,gBACJ,eAAe,IACX,EAAE,GACF;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,WAAW,CAAC;EACzD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,WAAW,CAAC;EACrD;CAEP,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,cAAc;EACrE,MAAM,OAAO,KAAK,SAAS,UAAU,SAAS,IAAI,KAAK,KAAK,GAAG,KAAK;AACpE,SAAO,UAAU,UAAU,QAAQ,MAAM,KAAK;;CAGhD,MAAM,EACJ,QACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,qBAChD,eACE;CACJ,MAAM,oBACJ,uBAAuB,eAAe,sBAAsB,KAAA;CAE9D,IAAI;AAEJ,KAAI,CAAC,mBAAmB;AACtB,mBAAiB,KAAA;AACjB,MAAI,WAAW,WAGb,OAAM,eAAe,IAAI,QAAQ,KAAK;QAEnC;EACL,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,UAAU;AACZ,mBAAiB,WAAW,qBAAqB,aAAa,CAAC,KAC7D,WACA,sBACD;AAID,QAAM;AAEN,MAAI,WAAW,WAAW;GAExB,MAAM,EAAC,SAAS,aAAa,YAAW,UAAU;GAClD,MAAM,OAAwB;AAC9B,cAAW,gBAAgB,cAAc,MAAM,KAAK,CAAC,KAInD,iBACM;AACJ,cAAU,cACR,0BAA0B,IAAI,OAAO,EACrC,cACA,aACD;AACD,aAAS;KAEZ;AACD,SAAM;;;AAIV,KAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,aAAW,YAAY,aAAa,CAAC,KAAK,WAAW,cAAc;AAGnE,QAAM;;CAGR,MAAM,UAAoB,EAAE;AAC5B,KAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,UAAU;EACnD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,KACD,CAAC,KAAK,iBAAiB;AACtB,eAAY,IAAI,WAAW;AAC3B,YAAS;IACT;AACF,QAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,WAAW;AACnD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,IAC9B,SAAQ,KAAK,WAAW,YAAY,eAAe,~~IAAI,~~GAAG,~~KAAK~~,CAAC;~~AAElE~~,UAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,SAAS,CAAC;;CAE1E,IAAI;AAKJ,IAAG,OAAO,sCAAsC;CAChD,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,cAAc,CAAC,KAAK,KAAK,GAAG,EAC3E,IACD;AACD,OAAM,UAAU,iBAAiB;AACjC,eAAc,WAAW;AACzB,IAAG,OAAO,sBAAsB,KAAK,KAAK,GAAG,QAAQ,MAAM;AAE3D,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,KAAI;AACF,QAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,eACD,CACF;UACM,KAAK;AACZ,YAAU,gBAAgB,KAAK,aAAa;;AAG9C,OAAM,UAAU,MAAM;;AAGxB,IAAI,CAAC,mBAAmB,CACjB,iBAAgB,UAAU,KAAK,aAAa,EAAE,QAAQ,IAAI,CAAC"}
1	+ {"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(\n createLogContext(config, 'dispatcher', 0, false),\n 'dispatcher',\n 0,\n );\n const lc = createLogContext(config, 'dispatcher');\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID,\n changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n litestream,\n } = config;\n const runChangeStreamer =\n changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n let changeStreamer: Worker \| undefined;\n\n if (!runChangeStreamer) {\n changeStreamer = undefined;\n if (litestream.executable) {\n // For view-syncers, the backup is restored here. For the replication-manager,\n // the backup is restored in the change-streamer worker.\n await restoreReplica(lc, config, null);\n }\n } else {\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n 'message',\n changeStreamerStarted,\n );\n\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n\n if (litestream.backupURL) {\n // Start a backup replicator and corresponding litestream backup process.\n const {promise: backupReady, resolve} = resolver();\n const mode: ReplicaFileMode = 'backup';\n loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n // Wait for the Replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the lock when the\n // replicator first prepares the db file.\n 'message',\n () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, config),\n 'supporting',\n 'litestream',\n );\n resolve();\n },\n );\n await backupReady;\n }\n }\n\n if (numSyncers > 0) {\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n }\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n }\n\n await processes.done();\n}\n\nif (!singleProcessMode()) {\n void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"mappings":";;;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,KAAK;CAC1B,MAAM,SAAS,wBAAwB,EAAC,KAAI,CAAC;AAE7C,eACE,iBAAiB,QAAQ,cAAc,GAAG,MAAM,EAChD,cACA,EACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,aAAa;AACjD,eAAc,IAAI,OAAO;CAEzB,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,EAAC,gBAAgB,eAAc;AACrC,KAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,WAC3D,OAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,kHAEnF;AAEH,KAAI,OAAO,IAAI,WAAW,WACxB,OAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,6GAEzE;CAGH,MAAM,gBACJ,eAAe,IACX,EAAE,GACF;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,WAAW,CAAC;EACzD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,WAAW,CAAC;EACrD;CAEP,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,cAAc;EACrE,MAAM,OAAO,KAAK,SAAS,UAAU,SAAS,IAAI,KAAK,KAAK,GAAG,KAAK;AACpE,SAAO,UAAU,UAAU,QAAQ,MAAM,KAAK;;CAGhD,MAAM,EACJ,QACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,qBAChD,eACE;CACJ,MAAM,oBACJ,uBAAuB,eAAe,sBAAsB,KAAA;CAE9D,IAAI;AAEJ,KAAI,CAAC,mBAAmB;AACtB,mBAAiB,KAAA;AACjB,MAAI,WAAW,WAGb,OAAM,eAAe,IAAI,QAAQ,KAAK;QAEnC;EACL,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,UAAU;AACZ,mBAAiB,WAAW,qBAAqB,aAAa,CAAC,KAC7D,WACA,sBACD;AAID,QAAM;AAEN,MAAI,WAAW,WAAW;GAExB,MAAM,EAAC,SAAS,aAAa,YAAW,UAAU;GAClD,MAAM,OAAwB;AAC9B,cAAW,gBAAgB,cAAc,MAAM,KAAK,CAAC,KAInD,iBACM;AACJ,cAAU,cACR,0BAA0B,IAAI,OAAO,EACrC,cACA,aACD;AACD,aAAS;KAEZ;AACD,SAAM;;;AAIV,KAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,aAAW,YAAY,aAAa,CAAC,KAAK,WAAW,cAAc;AAGnE,QAAM;;CAGR,MAAM,UAAoB,EAAE;AAC5B,KAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,UAAU;EACnD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,KACD,CAAC,KAAK,iBAAiB;AACtB,eAAY,IAAI,WAAW;AAC3B,YAAS;IACT;AACF,QAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,WAAW;AACnD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,IAC9B,SAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,EAAE,CAAC,CAAC;AAEzE,UAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,SAAS,CAAC;;CAE1E,IAAI;AAKJ,IAAG,OAAO,sCAAsC;CAChD,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,cAAc,CAAC,KAAK,KAAK,GAAG,EAC3E,IACD;AACD,OAAM,UAAU,iBAAiB;AACjC,eAAc,WAAW;AACzB,IAAG,OAAO,sBAAsB,KAAK,KAAK,GAAG,QAAQ,MAAM;AAE3D,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,KAAI;AACF,QAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,eACD,CACF;UACM,KAAK;AACZ,YAAU,gBAAgB,KAAK,aAAa;;AAG9C,OAAM,UAAU,MAAM;;AAGxB,IAAI,CAAC,mBAAmB,CACjB,iBAAgB,UAAU,KAAK,aAAa,EAAE,QAAQ,IAAI,CAAC"}