npm - @rocicorp/zero - Versions diffs - 1.3.0 → 1.4.0-canary.1 - Mend

@rocicorp/zero 1.3.0 → 1.4.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (332) hide show

package/out/zero-cache/src/services/change-streamer/storer.js CHANGED Viewed

@@ -3,11 +3,11 @@ import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { AbortError } from "../../../../shared/src/abort-error.js";
 import { parse, valita_exports } from "../../../../shared/src/valita.js";
 import { BigIntJSON } from "../../../../shared/src/bigint-json.js";
+import { cdcSchema } from "../../types/shards.js";
 import { isDataChange, isSchemaChange } from "../change-source/protocol/current/data.js";
 import "../change-source/protocol/current/downstream.js";
 import { backfillRequestSchema } from "../change-source/protocol/current/upstream.js";
 import "../change-source/protocol/current.js";
-import { cdcSchema } from "../../types/shards.js";
 import "../../types/pg.js";
 import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
@@ -63,9 +63,10 @@ var Storer = class {
 	#onFatal;
 	#queue = new Queue();
 	#backPressureThresholdBytes;
+	#statementTimeoutMs;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, backPressureLimitHeapProportion) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
@@ -75,6 +76,7 @@ var Storer = class {
 		this.#replicaVersion = replicaVersion;
 		this.#onConsumed = onConsumed;
 		this.#onFatal = onFatal;
+		this.#statementTimeoutMs = statementTimeoutMs;
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -82,7 +84,7 @@ var Storer = class {
 	#cdc(table) {
 		return this.#db(`${cdcSchema(this.#shard)}.${table}`);
 	}
-	async assumeOwnership() {
+	async assumeOwnership(purgeLock) {
 		const db = this.#db;
 		const owner = this.#taskID;
 		const ownerAddress = this.#discoveryAddress;
@@ -96,6 +98,7 @@ var Storer = class {
 		})}`;
 		const elapsed = (performance.now() - start).toFixed(2);
 		this.#lc.info?.(`assumed ownership at ${addressWithProtocol} (${elapsed} ms)`);
+		if (purgeLock) purgeLock.release();
 	}
 	async getStartStreamInitializationParameters() {
 		const [[{ lastWatermark }], result] = await runTx(this.#db, (sql) => [sql`
@@ -125,6 +128,11 @@ var Storer = class {
 	}
 	purgeRecordsBefore(watermark) {
 		return runTx(this.#db, async (sql) => {
+			await sql`
+          SELECT watermark FROM ${this.#cdc("changeLog")}
+            ORDER BY watermark, pos LIMIT 1
+            FOR UPDATE NOWAIT
+        `;
 			const [{ deleted }] = await sql`
         WITH purged AS (
           DELETE FROM ${this.#cdc("changeLog")} WHERE watermark < ${watermark}
@@ -261,7 +269,10 @@ var Storer = class {
 					const { promise, resolve, reject } = resolver();
 					promise.catch(() => {});
 					tx = {
-						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), READ_COMMITTED),
+						pool: new TransactionPool(this.#lc.withContext("watermark", watermark), {
+							mode: READ_COMMITTED,
+							statementResponseTimeout: this.#statementTimeoutMs
+						}),
 						preCommitWatermark: watermark,
 						pos: 0,
 						startingReplicationState: promise,
@@ -317,7 +328,7 @@ var Storer = class {
 	}
 	async #startCatchup(subs) {
 		if (subs.length === 0) return;
-		const reader = new TransactionPool(this.#lc.withContext("pool", "catchup"), READONLY);
+		const reader = new TransactionPool(this.#lc.withContext("pool", "catchup"), { mode: READONLY });
 		reader.run(this.#db);
 		let lastWatermark;
 		try {
@@ -498,7 +509,60 @@ function toDownstream(entry) {
 		default: return [watermark, ["data", change]];
 	}
 }
+var PurgeLock = class {
+	#lc;
+	#tx;
+	replicaVersion;
+	minWatermark;
+	constructor(lc, tx, replicaVersion, watermark) {
+		this.#lc = lc;
+		this.#tx = tx;
+		this.replicaVersion = replicaVersion;
+		this.minWatermark = watermark;
+	}
+	#released = false;
+	async release() {
+		if (this.#released) return;
+		this.#released = true;
+		this.#tx.setDone();
+		await this.#tx.done().catch((e) => this.#lc.warn?.(`error from purge-lock release`, e));
+		this.#lc.info?.(`released purge lock on ${this.minWatermark}`);
+	}
+};
+var PurgeLocker = class {
+	#lc;
+	#shard;
+	#db;
+	constructor(lc, shard, db) {
+		this.#lc = lc.withContext("component", "purge-locker");
+		this.#shard = shard;
+		this.#db = db;
+	}
+	#cdc(table) {
+		return this.#db(`${cdcSchema(this.#shard)}.${table}`);
+	}
+	async acquire() {
+		const tx = new TransactionPool(this.#lc, { mode: READ_COMMITTED }).run(this.#db);
+		const row = await tx.processReadTask((sql) => sql`
+      SELECT watermark FROM ${this.#cdc("changeLog")}
+        ORDER BY watermark, pos LIMIT 1
+        FOR SHARE
+    `);
+		if (row.length === 0) {
+			this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);
+			tx.setDone();
+			await tx.done();
+			return null;
+		}
+		const [{ watermark }] = row;
+		const [{ replicaVersion }] = await tx.processReadTask((sql) => sql`
+        SELECT "replicaVersion" FROM ${this.#cdc("replicationConfig")}
+      `);
+		this.#lc.info?.(`locked watermark ${watermark} from being purged from replica@${replicaVersion}`);
+		return new PurgeLock(this.#lc, tx, replicaVersion, watermark);
+	}
+};
 //#endregion
-export { Storer };
+export { PurgeLocker, Storer };
 //# sourceMappingURL=storer.js.map

package/out/zero-cache/src/services/change-streamer/storer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {getHeapStatistics} from 'node:v8';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n DownstreamStatusMessage,\n UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type ReplicationState,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n \| [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> \| null, // null for DataChanges\n ]\n \| ['ready', callback: () => void]\n \| ['subscriber', SubscriberAndMode]\n \| DownstreamStatusMessage\n \| ['abort']\n \| 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationState>;\n ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\n/*\n Handles the storage of changes and the catchup of subscribers\n * that are behind.\n \n In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n \n Maintained invariant: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n \n Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must not be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n \n Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n /\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onConsumed: (c: Commit \| UpstreamStatusMessage) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onConsumed: (c: Commit \| UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n backPressureLimitHeapProportion: number,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onConsumed = onConsumed;\n this.#onFatal = onFatal;\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) \n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async assumeOwnership() {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n }\n\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n }> {\n const [[{lastWatermark}], result] = await runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT \n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\") \n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n ],\n {mode: Mode.READONLY},\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string \| null> {\n const [{minWatermark}] = await this.#db<{minWatermark: string \| null}[]>\n /sql/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n return minWatermark;\n }\n\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n WITH purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n RETURNING watermark, pos\n ) SELECT COUNT() as deleted FROM purged;`;\n\n // Before committing the purge, check that this process is still the\n // owner. This is done after the DELETE to minimize the amount of time\n // that writes to the changeLog are delayed.\n const [{owner}] = await sql<ReplicationState[]>`\n SELECT FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /*\n @returns The size of the serialized entry, for memory / I/O estimations.\n /\n store(entry: WatermarkedChange) {\n const [watermark, [_tag, change]] = entry;\n // Eagerly stringify the JSON object so that the memory usage can be\n // more accurately measured (i.e. without an extra object traversal and\n // ad hoc memory counting heuristics).\n //\n // This essentially moves the stringify() computation out of the pg client,\n // which is instead configured to pass `string` objects directly as JSON\n // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n const json = BigIntJSON.stringify(change);\n this.#approximateQueuedBytes += json.length;\n\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json.length;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n status(s: DownstreamStatusMessage) {\n this.#queue.enqueue(s);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> \| null = null;\n\n readyForMore(): Promise<void> \| undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 * 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') \|\| '.' \|\| (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT() AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (\n this.#readyForMore !== null &&\n // Wait for at least 20% of the threshold to free up.\n this.#approximateQueuedBytes < this.#backPressureThresholdBytes 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n\n #stopped = promiseVoid;\n\n /\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n /\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction \| null = null;\n let msg: QueueEntry \| false;\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await this.#queue.dequeue()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status':\n this.#onConsumed(msg);\n continue;\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationState>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n Mode.READ_COMMITTED,\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationState[]> /sql/ `\n SELECT FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n change: json,\n };\n\n const processed = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...(change !== null && isSchemaChange(change)\n ? this.#trackBackfillMetadata(sql, change)\n : []),\n ]);\n\n if (tx.pos % 100 === 0) {\n // Backpressure is exerted on commit when awaiting tx.pool.done().\n // However, backpressure checks need to be regularly done for\n // very large transactions in order to avoid memory blowup.\n await processed;\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onConsumed(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const reader = new TransactionPool(\n this.#lc.withContext('pool', 'catchup'),\n Mode.READONLY,\n );\n reader.run(this.#db);\n\n let lastWatermark: string \| undefined;\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<ReplicationState[]>`\n SELECT * FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n try {\n await reader.processReadTask(async tx => {\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> \| undefined;\n\n for await (const entries of tx<ChangeEntry[]> /sql/ `\n SELECT watermark, change FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from after the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(toDownstream(entry));\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n this.#lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n }\n if (watermarkFound) {\n await lastBatchConsumed;\n this.#lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n this.#lc.warn?.(\n `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n );\n }\n // Flushes the backlog of messages buffered during catchup and\n // allows the subscription to forward subsequent messages immediately.\n sub.setCaughtUp();\n });\n } catch (err) {\n this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n }\n }\n\n /*\n Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n /\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n const stmts: PendingQuery<Row[]>[] = [];\n\n switch (change.tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n break;\n }\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n if (metadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n }\n if (backfill) {\n Object.entries(backfill).forEach(([col, backfill]) => {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n );\n });\n }\n break;\n }\n\n case 'rename-table': {\n const {old} = change;\n const row = {schema: change.new.schema, table: change.new.name};\n stmts.push(\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n );\n break;\n }\n\n case 'drop-table': {\n const {\n id: {schema, name},\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n );\n break;\n }\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n if (tableMetadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n }\n if (backfill) {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n );\n }\n break;\n }\n\n case 'update-column': {\n const {\n table: {schema, name: table},\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n stmts.push(\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n );\n }\n break;\n }\n\n case 'drop-column': {\n const {\n table: {schema, name},\n column,\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n );\n break;\n }\n\n case 'backfill-completed': {\n const {\n relation: {schema, name: table, rowKey},\n columns,\n } = change;\n const cols = [...rowKey.columns, ...columns];\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n );\n }\n }\n return stmts;\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /\n Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n stop() {\n if (this.#running) {\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.#queue.enqueue('stop');\n }\n return this.#stopped;\n }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n const {watermark, change} = entry;\n switch (change.tag) {\n case 'begin':\n return [watermark, ['begin', change, {commitWatermark: watermark}]];\n case 'commit':\n return [watermark, ['commit', change, {watermark}]];\n case 'rollback':\n return [watermark, ['rollback', change]];\n default:\n return [watermark, ['data', change]];\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwEA,IAAM,yBAAyB,eAAE,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgC7D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAmB;CACzC;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,iCACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,aAAa;AACpD,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;AAC1B,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,UAAgB;EAEhB,MAAM,YAAY,mBAAmB;AACrC,QAAA,8BACG,UAAU,kBAAkB,UAAU,kBACvC;AAEF,QAAA,GAAS,OACP,gBAAgB,MAAA,6BAAmC,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAE/E,EAAC,WAAU,CACZ;;CAIH,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,kBAAkB;EACtB,MAAM,KAAK,MAAA;EACX,MAAM,QAAQ,MAAA;EACd,MAAM,eAAe,MAAA;EACrB,MAAM,gBAAgB,MAAA;EAEtB,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;AAC5B,QAAA,GAAS,OAAO,yBAAyB,sBAAsB;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,QAAM,EAAE,UAAU,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG;GAAC;GAAO,cAAc;GAAoB,CAAC;EACrG,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,QAAA,GAAS,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,MACzD;;CAGH,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,MAAA,KACA,QAAO,CACL,GAA8B;sCACA,MAAA,IAAU,mBAAmB,IAK3D,GAAG;;;;;;;;;iBASM,MAAA,IAAU,cAAc,CAAC;sBACpB,MAAA,IAAU,gBAAgB,CAAC;;;UAI1C,EACD,EAAC,MAAM,UAAc,CACtB;AAED,SAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,uBAAuB;GAC1D;;CAGH,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,MAAA,EACvB;qDACyC,MAAA,IAAU,YAAY;AACvE,SAAO;;CAGT,mBAAmB,WAAoC;AACrD,SAAO,MAAM,MAAA,IAAU,OAAM,QAAO;GAClC,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;wBAEhC,MAAA,IAAU,YAAY,CAAC,qBAAqB,UAAU;;;GAOxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;wBAC7B,MAAA,IAAU,mBAAmB,CAAC;AAChD,OAAI,UAAU,MAAA,OACZ,OAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,QACjF;AAEH,UAAO,OAAO,QAAQ;IACtB;;;;;CAMJ,MAAM,OAA0B;EAC9B,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW;EAQpC,MAAM,OAAO,WAAW,UAAU,OAAO;AACzC,QAAA,0BAAgC,KAAK;AAErC,QAAA,MAAY,QAAQ;GAClB;GACA;GACA;GACA,aAAa,OAAO,GAAG,OAAO;GAC/B,CAAC;AAEF,SAAO,KAAK;;CAGd,QAAQ;AACN,QAAA,MAAY,QAAQ,CAAC,QAAQ,CAAC;;CAGhC,OAAO,GAA4B;AACjC,QAAA,MAAY,QAAQ,EAAE;;CAGxB,QAAQ,YAAwB,MAAsB;AACpD,QAAA,MAAY,QAAQ,CAAC,cAAc;GAAC;GAAY;GAAK,CAAC,CAAC;;CAGzD,gBAAuC;CAEvC,eAA0C;AACxC,MAAI,CAAC,MAAA,QACH;AAEF,MACE,MAAA,iBAAuB,QACvB,MAAA,yBAA+B,MAAA,4BAC/B;AACA,SAAA,GAAS,OACP,+BAA+B,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,4SAW7H;AACD,SAAA,eAAqB,UAAU;;AAEjC,SAAO,MAAA,cAAoB;;CAG7B,4BAA4B;AAC1B,MACE,MAAA,iBAAuB,QAEvB,MAAA,yBAA+B,MAAA,6BAAmC,IAClE;AACA,SAAA,GAAS,OACP,gCAAgC,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,MAC9H;AACD,SAAA,aAAmB,SAAS;AAC5B,SAAA,eAAqB;;;CAIzB,WAAW;;;;;CAMX,MAAM,MAAM;AACV,SAAO,CAAC,MAAA,SAAe,4BAA4B;EAEnD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,UAAU;AAC7D,QAAA,UAAgB;AAChB,QAAA,UAAgB;AAEhB,QAAA,GAAS,OAAO,kBAAkB;EAClC,IAAI;AACJ,MAAI;AACF,SAAM,MAAA,cAAoB;WACnB,GAAG;AACV,SAAM;AACN,SAAM;YACE;AAER,OAAI,MAAA,iBAAuB,MAAM;AAC/B,UAAA,aAAmB,SAAS;AAC5B,UAAA,eAAqB;;AAEvB,SAAA,mBACE,MAAA,MAAY,OAAO,CAAC,QAAO,UAAS,UAAU,KAAA,EAAU,EACxD,IACD;AACD,SAAA,UAAgB;AAChB,kBAAe;AACf,SAAA,GAAS,OAAO,iBAAiB;;;CAIrC,oBAAoB,OAAqB,GAAY;AACnD,MAAI,MAAM,WAAW,EACnB;AAEF,QAAA,GAAS,OACP,aAAa,MAAM,OAAO,mCAC3B;EACD,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,uBAAuB;AAC3E,OAAK,MAAM,SAAS,OAAO;AACzB,OAAI,UAAU,OACZ;AAGF,WADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;AAC3B,WAAA,GAAS,OAAO,iBAAiB,WAAW,KAAK;AACjD,gBAAW,KAAK,IAAI;AACpB;;;;;CAMR,OAAA,eAAsB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,EAAE;AAC5C,MAAI;AACF,WAAQ,MAAM,MAAM,MAAA,MAAY,SAAS,MAAM,QAAQ;IACrD,MAAM,CAAC,WAAW;AAClB,YAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;AACxB,mBAAa;AACb;;KAEF,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;AACvB,UAAI,GACF,cAAa,KAAK,WAAW;UAE7B,OAAM,MAAA,aAAmB,CAAC,WAAW,CAAC;AAExC;;KAEF,KAAK;AACH,YAAA,WAAiB,IAAI;AACrB;KACF,KAAK;AACH,UAAI,IAAI;AACN,UAAG,KAAK,OAAO;AACf,aAAM,GAAG,KAAK,MAAM;AACpB,YAAK;;AAEP;;IAIJ,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;AACpB,UAAA,0BAAgC,KAAK;AAErC,QAAI,QAAQ,SAAS;AACnB,YAAO,CAAC,IAAI,gDAAgD;KAC5D,MAAM,EAAC,SAAS,SAAS,WAAU,UAA4B;AAC1D,aAAQ,YAAY,GAAG;AAC5B,UAAK;MACH,MAAM,IAAI,gBACR,MAAA,GAAS,YAAY,aAAa,UAAU,EAC5C,eACD;MACD,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACd;AACD,QAAG,KAAK,IAAI,MAAA,GAAS;AAGhB,QAAG,KAAK,SAAQ,OAAM;AACzB,QAA+B;0BACjB,MAAA,IAAU,mBAAmB,CAAC,aAAa,MACtD,CAAC,YAAY,QAAQ,OAAO,EAC7B,OACD;AACD,aAAO,EAAE;OACT;WACG;AACL,YAAO,UAAU,2CAA2C,OAAO;AACnE,QAAG;;IAGL,MAAM,QAAQ;KACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KACR,QAAQ;KACT;IAED,MAAM,YAAY,GAAG,KAAK,SAAQ,QAAO,CACvC,GAAG,eAAe,MAAA,IAAU,YAAY,CAAC,GAAG,IAAI,MAAM,IACtD,GAAI,WAAW,QAAQ,eAAe,OAAO,GACzC,MAAA,sBAA4B,KAAK,OAAO,GACxC,EAAE,CACP,CAAC;AAEF,QAAI,GAAG,MAAM,QAAQ,EAInB,OAAM;AAER,UAAA,0BAAgC;AAEhC,QAAI,QAAQ,UAAU;KACpB,MAAM,EAAC,UAAS,MAAM,GAAG;AACzB,SAAI,UAAU,MAAA,OAEZ,IAAG,KAAK,KACN,IAAI,WACF,2CAA2C,QAC5C,CACF;UACI;MAEL,MAAM,gBAAgB;AACjB,SAAG,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG,EAAC,eAAc,CAAC,GAChE,CAAC;AACF,SAAG,KAAK,SAAS;;AAGnB,WAAM,GAAG,KAAK,MAAM;AAGpB,SAAI,GAAG,IACL,OAAA,WAAiB;MAAC;MAAU;MAAQ,EAAC,WAAU;MAAC,CAAC;AAEnD,UAAK;AAIL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;eACvC,QAAQ,YAAY;AAG7B,QAAG,KAAK,OAAO;AACf,WAAM,GAAG,KAAK,MAAM;AACpB,UAAK;AAEL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;;;WAG7C,GAAG;AACV,gBAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC1D,SAAM;;;CAIV,OAAA,aAAoB,MAA2B;AAC7C,MAAI,KAAK,WAAW,EAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,MAAA,GAAS,YAAY,QAAQ,UAAU,EACvC,SACD;AACD,SAAO,IAAI,MAAA,GAAS;EAEpB,IAAI;AACJ,MAAI;AAKF,IAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAAuB;wBACd,MAAA,IAAU,mBAAmB,CAAC;QAE/C;WACM,GAAG;AACV,QAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC9C,SAAM;;AAKH,UAAQ,IACX,KAAK,KAAI,QAAO,MAAA,QAAc,KAAK,eAAe,OAAO,CAAC,CAC3D,CAAC,cAAc,OAAO,SAAS,CAAC;;CAGnC,OAAA,QACE,EAAC,YAAY,KAAK,QAClB,eACA,QACA;AACA,MAAI;AACF,SAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,KAAK;IAIxB,IAAI,iBAAiB,IAAI,cAAc,MAAA;IACvC,IAAI,QAAQ;IACZ,IAAI;AAEJ,eAAW,MAAM,WAAW,EAA0B;0CACpB,MAAA,IAAU,YAAY,CAAC;gCACjC,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,IAAK,EAAE;KAOxC,MAAM,QAAQ,YAAY,KAAK;AAC/B,WAAM;KACN,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAI,kBACF,EAAC,UAAU,MAAM,MAAA,GAAS,OAAO,MAAA,GAAS,SACxC,UAAU,QAAQ,QAAQ,EAAE,CAAC,UAAU,IAAI,GAAG,2CAC/C;AAGH,UAAK,MAAM,SAAS,QAClB,KAAI,MAAM,cAAc,IAAI,UAG1B,kBAAiB;cACR,gBAAgB;AACzB,0BAAoB,IAAI,QAAQ,aAAa,MAAM,CAAC;AACpD;gBACS,SAAS,SAClB,OAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,GACtF;UACI;AACL,YAAA,GAAS,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,GAC5F;AACD,UAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,GAChF;AACD;;;AAIN,QAAI,gBAAgB;AAClB,WAAM;AACN,WAAA,GAAS,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,KAAK,GAAG,MACd,MACF;UAED,OAAA,GAAS,OACP,2BAA2B,IAAI,UAAU,+BAC1C;AAIH,QAAI,aAAa;KACjB;WACK,KAAK;AACZ,SAAA,GAAS,QAAQ,sCAAsC,IAAI,MAAM,IAAI;AACrE,OAAI,eAAe,iBAAiB;AAClC,UAAM,kBAAkB,MAAA,IAAU,MAAA,MAAY;AAC9C,UAAA,QAAc,IAAI;;AAEpB,OAAI,KAAK,IAAI;;;;;;;CAQjB,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,EAAE;AAEvC,UAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;AAC/B,UAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,SAAS,CAAC;AAC/D;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;AACnC,QAAI,SACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,MAAM,SAAS,CAAC;AAEhE,QAAI,SACF,QAAO,QAAQ,SAAS,CAAC,SAAS,CAAC,KAAK,cAAc;AACpD,WAAM,KACJ,MAAA,yBAA+B,KAAK,MAAM,KAAK,SAAS,CACzD;MACD;AAEJ;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;KAAK;AAC/D,UAAM,KACJ,GAAG,UAAU,MAAA,IAAU,gBAAgB,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,OAC1D;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,gBAAgB,CAAC;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,OAClD;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,QAAI,cACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,cAAc,CAAC;AAEtE,QAAI,SACF,OAAM,KACJ,MAAA,yBAA+B,KAAK,OAAO,OAAO,MAAM,SAAS,CAClE;AAEH;;GAGF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,QACd,OAAM,KACJ,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,UACxE;AAEH;;GAGF,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,SACzE;AACD;;GAGF,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,QAAQ;AAC5C,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,KAAK,GACpF;;;AAGL,SAAO;;CAGT,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;GAAS;AACvD,SAAO,GAAG;sBACQ,MAAA,IAAU,gBAAgB,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEnC,IAAI,IAAI,CAAC;;;CAIjC,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;GAAS;AAChE,SAAO,GAAG;sBACQ,MAAA,IAAU,cAAc,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEjC,IAAI,IAAI,CAAC;;;;;;;CAQjC,MAAM,eAAe;AACnB,MAAI,MAAA,SAAe;GACjB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,SAAA,MAAY,QAAQ,CAAC,SAAS,QAAQ,CAAC;AACvC,SAAM;;;CAIV,OAAO;AACL,MAAI,MAAA,SAAe;AACjB,SAAA,GAAS,OAAO,YAAY,MAAA,MAAY,MAAM,CAAC,oBAAoB;AACnE,SAAA,MAAY,QAAQ,OAAO;;AAE7B,SAAO,MAAA;;;AAIX,SAAS,aAAa,OAAuC;CAC3D,MAAM,EAAC,WAAW,WAAU;AAC5B,SAAQ,OAAO,KAAf;EACE,KAAK,QACH,QAAO,CAAC,WAAW;GAAC;GAAS;GAAQ,EAAC,iBAAiB,WAAU;GAAC,CAAC;EACrE,KAAK,SACH,QAAO,CAAC,WAAW;GAAC;GAAU;GAAQ,EAAC,WAAU;GAAC,CAAC;EACrD,KAAK,WACH,QAAO,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC;EAC1C,QACE,QAAO,CAAC,WAAW,CAAC,QAAQ,OAAO,CAAC"}
1	+ {"version":3,"file":"storer.js","names":["#lc","#shard","#taskID","#discoveryAddress","#discoveryProtocol","#db","#replicaVersion","#onConsumed","#onFatal","#queue","#backPressureThresholdBytes","#statementTimeoutMs","#cdc","#approximateQueuedBytes","#running","#readyForMore","#stopped","#processQueue","#cancelQueueEntries","#startCatchup","#trackBackfillMetadata","#maybeReleaseBackPressure","#catchup","#upsertTableMetadataStmt","#upsertColumnBackfillStmt","#tx","#released"],"sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"sourcesContent":["import {getHeapStatistics} from 'node:v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver, type Resolver} from '@rocicorp/resolver';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../../shared/src/queue.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {TransactionPool} from '../../db/transaction-pool.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../types/shards.ts';\nimport {\n backfillRequestSchema,\n isDataChange,\n isSchemaChange,\n type BackfillID,\n type BackfillRequest,\n type Change,\n type DataChange,\n type Identifier,\n type SchemaChange,\n type TableMetadata,\n} from '../change-source/protocol/current.ts';\nimport {type Commit} from '../change-source/protocol/current/downstream.ts';\nimport type {\n DownstreamStatusMessage,\n UpstreamStatusMessage,\n} from '../change-source/protocol/current/status.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport type {Service} from '../service.ts';\nimport type {WatermarkedChange} from './change-streamer-service.ts';\nimport {type ChangeEntry} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {\n AutoResetSignal,\n markResetRequired,\n type BackfillingColumn,\n type ReplicationState,\n type TableMetadataRow,\n} from './schema/tables.ts';\nimport type {Subscriber} from './subscriber.ts';\n\ntype SubscriberAndMode = {\n subscriber: Subscriber;\n mode: ReplicatorMode;\n};\n\ntype QueueEntry =\n \| [\n 'change',\n watermark: string,\n json: string,\n orig: Exclude<Change, DataChange> \| null, // null for DataChanges\n ]\n \| ['ready', callback: () => void]\n \| ['subscriber', SubscriberAndMode]\n \| DownstreamStatusMessage\n \| ['abort']\n \| 'stop';\n\ntype PendingTransaction = {\n pool: TransactionPool;\n preCommitWatermark: string;\n pos: number;\n startingReplicationState: Promise<ReplicationState>;\n ack: boolean;\n};\n\nconst backfillRequestsSchema = v.array(backfillRequestSchema);\n\nexport type TuningOptions = {\n backPressureLimitHeapProportion: number;\n statementTimeoutMs: number;\n};\n\n/*\n Handles the storage of changes and the catchup of subscribers\n * that are behind.\n \n In the context of catchup and cleanup, it is the responsibility of the\n * Storer to decide whether a client can be caught up, or whether the\n * changes needed to catch a client up have been purged.\n \n Maintained invariant: The Change DB is only empty for a\n * completely new replica (i.e. initial-sync with no changes from the\n * replication stream).\n * * In this case, all new subscribers are expected start from the\n * `replicaVersion`, which is the version at which initial sync\n * was performed, and any attempts to catchup from a different\n * point fail.\n \n Conversely, if non-initial changes have flowed through the system\n * (i.e. via the replication stream), the ChangeDB must not be empty,\n * and the earliest change in the `changeLog` represents the earliest\n * \"commit\" from (after) which a subscriber can be caught up.\n * * Any attempts to catchup from an earlier point must fail with\n * a `WatermarkTooOld` error.\n * * Failure to do so could result in streaming changes to the\n * subscriber such that there is a gap in its replication history.\n \n Note: Subscribers (i.e. `incremental-syncer`) consider an \"error\" signal\n * an unrecoverable error and shut down in response. This allows the\n * production system to replace it with a new task and fresh copy of the\n * replica backup.\n /\nexport class Storer implements Service {\n readonly id = 'storer';\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #taskID: string;\n readonly #discoveryAddress: string;\n readonly #discoveryProtocol: string;\n readonly #db: PostgresDB;\n readonly #replicaVersion: string;\n readonly #onConsumed: (c: Commit \| UpstreamStatusMessage) => void;\n readonly #onFatal: (err: Error) => void;\n readonly #queue = new Queue<QueueEntry>();\n readonly #backPressureThresholdBytes: number;\n readonly #statementTimeoutMs: number;\n\n #approximateQueuedBytes = 0;\n #running = false;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n db: PostgresDB,\n replicaVersion: string,\n onConsumed: (c: Commit \| UpstreamStatusMessage) => void,\n onFatal: (err: Error) => void,\n {backPressureLimitHeapProportion, statementTimeoutMs}: TuningOptions,\n ) {\n this.#lc = lc.withContext('component', 'change-log');\n this.#shard = shard;\n this.#taskID = taskID;\n this.#discoveryAddress = discoveryAddress;\n this.#discoveryProtocol = discoveryProtocol;\n this.#db = db;\n this.#replicaVersion = replicaVersion;\n this.#onConsumed = onConsumed;\n this.#onFatal = onFatal;\n this.#statementTimeoutMs = statementTimeoutMs;\n\n const heapStats = getHeapStatistics();\n this.#backPressureThresholdBytes =\n (heapStats.heap_size_limit - heapStats.used_heap_size) \n backPressureLimitHeapProportion;\n\n this.#lc.info?.(\n `Using up to ${(this.#backPressureThresholdBytes / 1024 2).toFixed(2)} MB of ` +\n `--max-old-space-size (~${(heapStats.heap_size_limit / 1024 2).toFixed(2)} MB) ` +\n `to absorb upstream spikes`,\n {heapStats},\n );\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async assumeOwnership(purgeLock?: PurgeLock \| null) {\n const db = this.#db;\n const owner = this.#taskID;\n const ownerAddress = this.#discoveryAddress;\n const ownerProtocol = this.#discoveryProtocol;\n // we omit `ws://` so that old view syncer versions that are not expecting the protocol continue to not get it\n const addressWithProtocol =\n ownerProtocol === 'ws'\n ? ownerAddress\n : `${ownerProtocol}://${ownerAddress}`;\n this.#lc.info?.(`assuming ownership at ${addressWithProtocol}`);\n const start = performance.now();\n await db`UPDATE ${this.#cdc('replicationState')} SET ${db({owner, ownerAddress: addressWithProtocol})}`;\n const elapsed = (performance.now() - start).toFixed(2);\n this.#lc.info?.(\n `assumed ownership at ${addressWithProtocol} (${elapsed} ms)`,\n );\n\n if (purgeLock) {\n // Once ownership has been assumed, any initial purge-lock preventing the\n // purging of change-log records can be released, as a change-streamer\n // that was attempting to purge records will correspondingly abort on the\n // ownership check.\n void purgeLock.release();\n }\n }\n\n async getStartStreamInitializationParameters(): Promise<{\n lastWatermark: string;\n backfillRequests: BackfillRequest[];\n }> {\n const [[{lastWatermark}], result] = await runTx(\n this.#db,\n sql => [\n sql<{lastWatermark: string}[]>`\n SELECT \"lastWatermark\" FROM ${this.#cdc('replicationState')}`,\n\n // Formats a BackfillRequest using json_object_agg() to construct the\n // `columns` object. It is LEFT JOIN'ed with the `tableMetadata` table\n // to make it optional and possibly `null`.\n sql`\n SELECT \n json_build_object(\n 'schema', b.\"schema\",\n 'name', b.\"table\",\n 'metadata', t.\"metadata\"\n ) as \"table\",\n json_object_agg(b.\"column\", b.\"backfill\") \n as \"columns\"\n FROM ${this.#cdc('backfilling')} as b\n LEFT JOIN ${this.#cdc('tableMetadata')} as t\n ON (b.\"schema\" = t.\"schema\" AND b.\"table\" = t.\"table\")\n GROUP BY b.\"schema\", b.\"table\", t.\"metadata\"\n `,\n ],\n {mode: Mode.READONLY},\n );\n\n return {\n lastWatermark,\n backfillRequests: v.parse(result, backfillRequestsSchema),\n };\n }\n\n async getMinWatermarkForCatchup(): Promise<string \| null> {\n const [{minWatermark}] = await this.#db<{minWatermark: string \| null}[]>\n /sql/ `\n SELECT min(watermark) as \"minWatermark\" FROM ${this.#cdc('changeLog')}`;\n return minWatermark;\n }\n\n purgeRecordsBefore(watermark: string): Promise<number> {\n return runTx(this.#db, async sql => {\n // This NOWAIT pre-check is an optimization to abort the transaction\n // (and release associated resources) early.\n await sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR UPDATE NOWAIT\n `;\n // If the row is purge-locked by an incoming replication-manager, it\n // will assume ownership of the change-log before releasing the lock.\n // This DELETE blocks until the lock is released, allowing the change\n // in ownership to be reliably detected (and the transaction aborted)\n // in the subsequent check.\n const [{deleted}] = await sql<{deleted: bigint}[]>`\n WITH purged AS (\n DELETE FROM ${this.#cdc('changeLog')} WHERE watermark < ${watermark} \n RETURNING watermark, pos\n ) SELECT COUNT() as deleted FROM purged;`;\n\n const [{owner}] = await sql<ReplicationState[]>`\n SELECT FROM ${this.#cdc('replicationState')} FOR SHARE`;\n if (owner !== this.#taskID) {\n throw new AbortError(\n `aborting changeLog purge to ${watermark} because ownership has been taken by ${owner}`,\n );\n }\n return Number(deleted);\n });\n }\n\n /*\n @returns The size of the serialized entry, for memory / I/O estimations.\n /\n store(entry: WatermarkedChange) {\n const [watermark, [_tag, change]] = entry;\n // Eagerly stringify the JSON object so that the memory usage can be\n // more accurately measured (i.e. without an extra object traversal and\n // ad hoc memory counting heuristics).\n //\n // This essentially moves the stringify() computation out of the pg client,\n // which is instead configured to pass `string` objects directly as JSON\n // strings for JSON-valued columns (see TypeOptions.sendStringAsJson).\n const json = BigIntJSON.stringify(change);\n this.#approximateQueuedBytes += json.length;\n\n this.#queue.enqueue([\n 'change',\n watermark,\n json,\n isDataChange(change) ? null : change, // drop DataChanges to save memory\n ]);\n\n return json.length;\n }\n\n abort() {\n this.#queue.enqueue(['abort']);\n }\n\n status(s: DownstreamStatusMessage) {\n this.#queue.enqueue(s);\n }\n\n catchup(subscriber: Subscriber, mode: ReplicatorMode) {\n this.#queue.enqueue(['subscriber', {subscriber, mode}]);\n }\n\n #readyForMore: Resolver<void> \| null = null;\n\n readyForMore(): Promise<void> \| undefined {\n if (!this.#running) {\n return undefined;\n }\n if (\n this.#readyForMore === null &&\n this.#approximateQueuedBytes > this.#backPressureThresholdBytes\n ) {\n this.#lc.warn?.(\n `applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 * 2).toFixed(2)} MB)\\n` +\n `\\n` +\n `To inspect changeLog backlog in your change DB:\\n` +\n ` SELECT\\n` +\n ` (change->'relation'->>'schema') \|\| '.' \|\| (change->'relation'->>'name') AS table_name,\\n` +\n ` change->>'tag' AS operation,\\n` +\n ` COUNT() AS count\\n` +\n ` FROM \"<app_id>/cdc\".\"changeLog\"\\n` +\n ` GROUP BY 1, 2\\n` +\n ` ORDER BY 3 DESC\\n` +\n ` LIMIT 20;`,\n );\n this.#readyForMore = resolver();\n }\n return this.#readyForMore?.promise;\n }\n\n #maybeReleaseBackPressure() {\n if (\n this.#readyForMore !== null &&\n // Wait for at least 20% of the threshold to free up.\n this.#approximateQueuedBytes < this.#backPressureThresholdBytes 0.8\n ) {\n this.#lc.info?.(\n `releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 2).toFixed(2)} MB)`,\n );\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n }\n\n #stopped = promiseVoid;\n\n /\n * Runs the storer loop until {@link stop()} is called, or an error is thrown.\n * Once {@link run()} completes, it can be called again.\n /\n async run() {\n assert(!this.#running, `storer is already running`);\n\n const {promise: stopped, resolve: signalStopped} = resolver();\n this.#running = true;\n this.#stopped = stopped;\n\n this.#lc.info?.('starting storer');\n let err: unknown;\n try {\n await this.#processQueue();\n } catch (e) {\n err = e; // used in finally\n throw e;\n } finally {\n // Release any pending backpressure so the upstream can proceed\n if (this.#readyForMore !== null) {\n this.#readyForMore.resolve();\n this.#readyForMore = null;\n }\n this.#cancelQueueEntries(\n this.#queue.drain().filter(entry => entry !== undefined),\n err,\n );\n this.#running = false;\n signalStopped();\n this.#lc.info?.('storer stopped');\n }\n }\n\n #cancelQueueEntries(queue: QueueEntry[], e: unknown) {\n if (queue.length === 0) {\n return;\n }\n this.#lc.info?.(\n `canceling ${queue.length} entries from the changeLog queue`,\n );\n const err = e instanceof Error ? e : new AbortError('server shutting down');\n for (const entry of queue) {\n if (entry === 'stop') {\n continue;\n }\n const type = entry[0];\n switch (type) {\n case 'subscriber': {\n // Disconnect subscribers waiting to be caught up so that they can\n // reconnect and try again.\n const {subscriber} = entry[1];\n this.#lc.info?.(`disconnecting ${subscriber.id}`);\n subscriber.fail(err);\n break;\n }\n }\n }\n }\n\n async #processQueue() {\n let tx: PendingTransaction \| null = null;\n let msg: QueueEntry \| false;\n\n const catchupQueue: SubscriberAndMode[] = [];\n try {\n while ((msg = await this.#queue.dequeue()) !== 'stop') {\n const [msgType] = msg;\n switch (msgType) {\n case 'ready': {\n const signalReady = msg[1];\n signalReady();\n continue;\n }\n case 'subscriber': {\n const subscriber = msg[1];\n if (tx) {\n catchupQueue.push(subscriber); // Wait for the current tx to complete.\n } else {\n await this.#startCatchup([subscriber]); // Catch up immediately.\n }\n continue;\n }\n case 'status':\n this.#onConsumed(msg);\n continue;\n case 'abort': {\n if (tx) {\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n }\n continue;\n }\n }\n // msgType === 'change'\n const [_, watermark, json, change] = msg;\n const tag = change?.tag;\n this.#approximateQueuedBytes -= json.length;\n\n if (tag === 'begin') {\n assert(!tx, 'received BEGIN in the middle of a transaction');\n const {promise, resolve, reject} = resolver<ReplicationState>();\n void promise.catch(() => {}); // handle rejections before the await\n tx = {\n pool: new TransactionPool(\n this.#lc.withContext('watermark', watermark),\n {\n mode: Mode.READ_COMMITTED,\n statementResponseTimeout: this.#statementTimeoutMs,\n },\n ),\n preCommitWatermark: watermark,\n pos: 0,\n startingReplicationState: promise,\n ack: !change.skipAck,\n };\n tx.pool.run(this.#db);\n // Acquire a lock on the replicationState row to detect and/or prevent\n // a concurrent ownership change.\n void tx.pool.process(tx => {\n tx<ReplicationState[]> /sql/ `\n SELECT FROM ${this.#cdc('replicationState')} FOR UPDATE`.then(\n ([result]) => resolve(result),\n reject,\n );\n return [];\n });\n } else {\n assert(tx, () => `received change outside of transaction: ${json}`);\n tx.pos++;\n }\n\n const entry = {\n watermark: tag === 'commit' ? watermark : tx.preCommitWatermark,\n precommit: tag === 'commit' ? tx.preCommitWatermark : null,\n pos: tx.pos,\n change: json,\n };\n\n const processed = tx.pool.process(sql => [\n sql`INSERT INTO ${this.#cdc('changeLog')} ${sql(entry)}`,\n ...(change !== null && isSchemaChange(change)\n ? this.#trackBackfillMetadata(sql, change)\n : []),\n ]);\n\n if (tx.pos % 100 === 0) {\n // Backpressure is exerted on commit when awaiting tx.pool.done().\n // However, backpressure checks need to be regularly done for\n // very large transactions in order to avoid memory blowup.\n await processed;\n }\n this.#maybeReleaseBackPressure();\n\n if (tag === 'commit') {\n const {owner} = await tx.startingReplicationState;\n if (owner !== this.#taskID) {\n // Ownership change reflected in the replicationState read in 'begin'.\n tx.pool.fail(\n new AbortError(\n `changeLog ownership has been assumed by ${owner}`,\n ),\n );\n } else {\n // Update the replication state.\n const lastWatermark = watermark;\n void tx.pool.process(tx => [\n tx`\n UPDATE ${this.#cdc('replicationState')} SET ${tx({lastWatermark})}`,\n ]);\n tx.pool.setDone();\n }\n\n await tx.pool.done();\n\n // ACK the LSN to the upstream Postgres.\n if (tx.ack) {\n this.#onConsumed(['commit', change, {watermark}]);\n }\n tx = null;\n\n // Before beginning the next transaction, open a READONLY snapshot to\n // concurrently catchup any queued subscribers.\n await this.#startCatchup(catchupQueue.splice(0));\n } else if (tag === 'rollback') {\n // Aborted transactions are not stored in the changeLog. Abort the current tx\n // and process catchup of subscribers that were waiting for it to end.\n tx.pool.abort();\n await tx.pool.done();\n tx = null;\n\n await this.#startCatchup(catchupQueue.splice(0));\n }\n }\n } catch (e) {\n catchupQueue.forEach(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n }\n\n async #startCatchup(subs: SubscriberAndMode[]) {\n if (subs.length === 0) {\n return;\n }\n\n const reader = new TransactionPool(\n this.#lc.withContext('pool', 'catchup'),\n {mode: Mode.READONLY},\n );\n reader.run(this.#db);\n\n let lastWatermark: string \| undefined;\n try {\n // Ensure that the transaction has started (and is thus holding a snapshot\n // of the database) before continuing on to commit more changes. This is\n // done by performing a single read on the db, which determines the\n // snapshot for the REPEATABLE_READ transaction.\n [{lastWatermark}] = await reader.processReadTask(\n sql => sql<ReplicationState[]>`\n SELECT * FROM ${this.#cdc('replicationState')}\n `,\n );\n } catch (e) {\n subs.map(({subscriber}) => subscriber.fail(e));\n throw e;\n }\n\n // Run the actual catchup queries in the background. Errors are handled in\n // #catchup() by disconnecting the associated subscriber.\n void Promise.all(\n subs.map(sub => this.#catchup(sub, lastWatermark, reader)),\n ).finally(() => reader.setDone());\n }\n\n async #catchup(\n {subscriber: sub, mode}: SubscriberAndMode,\n lastWatermark: string,\n reader: TransactionPool,\n ) {\n try {\n await reader.processReadTask(async tx => {\n const start = Date.now();\n\n // When starting from initial-sync, there won't be a change with a watermark\n // equal to the replica version. This is the empty changeLog scenario.\n let watermarkFound = sub.watermark === this.#replicaVersion;\n let count = 0;\n let lastBatchConsumed: Promise<unknown> \| undefined;\n\n for await (const entries of tx<ChangeEntry[]> /sql/ `\n SELECT watermark, change FROM ${this.#cdc('changeLog')}\n WHERE watermark >= ${sub.watermark}\n AND watermark <= ${lastWatermark}\n ORDER BY watermark, pos`.cursor(2000)) {\n // Wait for the last batch of entries to be consumed by the\n // subscriber before sending down the current batch. This pipelining\n // allows one batch of changes to be received from the change-db\n // while the previous batch of changes are sent to the subscriber,\n // resulting in flow control that caps the number of changes\n // referenced in memory to 2 * batch-size.\n const start = performance.now();\n await lastBatchConsumed;\n const elapsed = performance.now() - start;\n if (lastBatchConsumed) {\n (elapsed > 100 ? this.#lc.info : this.#lc.debug)?.(\n `waited ${elapsed.toFixed(3)} ms for ${sub.id} to consume last batch of catchup entries`,\n );\n }\n\n for (const entry of entries) {\n if (entry.watermark === sub.watermark) {\n // This should be the first entry.\n // Catchup starts from after the watermark.\n watermarkFound = true;\n } else if (watermarkFound) {\n lastBatchConsumed = sub.catchup(toDownstream(entry));\n count++;\n } else if (mode === 'backup') {\n throw new AutoResetSignal(\n `backup replica at watermark ${sub.watermark} is behind change db: ${entry.watermark})`,\n );\n } else {\n this.#lc.warn?.(\n `rejecting subscriber at watermark ${sub.watermark} (earliest watermark: ${entry.watermark})`,\n );\n sub.close(\n ErrorType.WatermarkTooOld,\n `earliest supported watermark is ${entry.watermark} (requested ${sub.watermark})`,\n );\n return;\n }\n }\n }\n if (watermarkFound) {\n await lastBatchConsumed;\n this.#lc.info?.(\n `caught up ${sub.id} with ${count} changes (${\n Date.now() - start\n } ms)`,\n );\n } else {\n this.#lc.warn?.(\n `subscriber at watermark ${sub.watermark} is ahead of latest watermark`,\n );\n }\n // Flushes the backlog of messages buffered during catchup and\n // allows the subscription to forward subsequent messages immediately.\n sub.setCaughtUp();\n });\n } catch (err) {\n this.#lc.error?.(`error while catching up subscriber ${sub.id}`, err);\n if (err instanceof AutoResetSignal) {\n await markResetRequired(this.#db, this.#shard);\n this.#onFatal(err);\n }\n sub.fail(err);\n }\n }\n\n /*\n Returns the db statements necessary to track backfill and table metadata\n * presented in the `change`, if any.\n /\n #trackBackfillMetadata(sql: PostgresTransaction, change: SchemaChange) {\n const stmts: PendingQuery<Row[]>[] = [];\n\n switch (change.tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n stmts.push(this.#upsertTableMetadataStmt(sql, table, metadata));\n break;\n }\n\n case 'create-table': {\n const {spec, metadata, backfill} = change;\n if (metadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, spec, metadata));\n }\n if (backfill) {\n Object.entries(backfill).forEach(([col, backfill]) => {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, spec, col, backfill),\n );\n });\n }\n break;\n }\n\n case 'rename-table': {\n const {old} = change;\n const row = {schema: change.new.schema, table: change.new.name};\n stmts.push(\n sql`UPDATE ${this.#cdc('tableMetadata')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n sql`UPDATE ${this.#cdc('backfilling')} SET ${sql(row)}\n WHERE \"schema\" = ${old.schema} AND \"table\" = ${old.name}`,\n );\n break;\n }\n\n case 'drop-table': {\n const {\n id: {schema, name},\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('tableMetadata')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name}`,\n );\n break;\n }\n\n case 'add-column': {\n const {table, tableMetadata, column, backfill} = change;\n if (tableMetadata) {\n stmts.push(this.#upsertTableMetadataStmt(sql, table, tableMetadata));\n }\n if (backfill) {\n stmts.push(\n this.#upsertColumnBackfillStmt(sql, table, column.name, backfill),\n );\n }\n break;\n }\n\n case 'update-column': {\n const {\n table: {schema, name: table},\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n stmts.push(\n sql`UPDATE ${this.#cdc('backfilling')} SET \"column\" = ${newName}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" = ${oldName}`,\n );\n }\n break;\n }\n\n case 'drop-column': {\n const {\n table: {schema, name},\n column,\n } = change;\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${name} AND \"column\" = ${column}`,\n );\n break;\n }\n\n case 'backfill-completed': {\n const {\n relation: {schema, name: table, rowKey},\n columns,\n } = change;\n const cols = [...rowKey.columns, ...columns];\n stmts.push(\n sql`DELETE FROM ${this.#cdc('backfilling')}\n WHERE \"schema\" = ${schema} AND \"table\" = ${table} AND \"column\" IN ${sql(cols)}`,\n );\n }\n }\n return stmts;\n }\n\n #upsertTableMetadataStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n metadata: TableMetadata,\n ) {\n const row: TableMetadataRow = {schema, table, metadata};\n return sql`\n INSERT INTO ${this.#cdc('tableMetadata')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n #upsertColumnBackfillStmt(\n sql: PostgresTransaction,\n {schema, name: table}: Identifier,\n column: string,\n backfill: BackfillID,\n ) {\n const row: BackfillingColumn = {schema, table, column, backfill};\n return sql`\n INSERT INTO ${this.#cdc('backfilling')} ${sql(row)}\n ON CONFLICT (\"schema\", \"table\", \"column\") \n DO UPDATE SET ${sql(row)};\n `;\n }\n\n /\n Waits until all currently queued entries have been processed.\n * This is only used in tests.\n */\n async allProcessed() {\n if (this.#running) {\n const {promise, resolve} = resolver();\n this.#queue.enqueue(['ready', resolve]);\n await promise;\n }\n }\n\n stop() {\n if (this.#running) {\n this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);\n this.#queue.enqueue('stop');\n }\n return this.#stopped;\n }\n}\n\nfunction toDownstream(entry: ChangeEntry): WatermarkedChange {\n const {watermark, change} = entry;\n switch (change.tag) {\n case 'begin':\n return [watermark, ['begin', change, {commitWatermark: watermark}]];\n case 'commit':\n return [watermark, ['commit', change, {watermark}]];\n case 'rollback':\n return [watermark, ['rollback', change]];\n default:\n return [watermark, ['data', change]];\n }\n}\n\nexport class PurgeLock {\n readonly #lc: LogContext;\n readonly #tx: TransactionPool;\n readonly replicaVersion: string;\n readonly minWatermark: string;\n\n constructor(\n lc: LogContext,\n tx: TransactionPool,\n replicaVersion: string,\n watermark: string,\n ) {\n this.#lc = lc;\n this.#tx = tx;\n this.replicaVersion = replicaVersion;\n this.minWatermark = watermark;\n }\n\n #released = false;\n\n async release() {\n if (this.#released) {\n return;\n }\n this.#released = true;\n this.#tx.setDone();\n await this.#tx\n .done()\n .catch(e => this.#lc.warn?.(`error from purge-lock release`, e));\n this.#lc.info?.(`released purge lock on ${this.minWatermark}`);\n }\n}\n\nexport class PurgeLocker {\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #db: PostgresDB;\n\n constructor(lc: LogContext, shard: ShardID, db: PostgresDB) {\n this.#lc = lc.withContext('component', 'purge-locker');\n this.#shard = shard;\n this.#db = db;\n }\n\n // For readability in SQL statements.\n #cdc(table: string) {\n return this.#db(`${cdcSchema(this.#shard)}.${table}`);\n }\n\n async acquire() {\n const tx = new TransactionPool(this.#lc, {mode: Mode.READ_COMMITTED}).run(\n this.#db,\n );\n const row = await tx.processReadTask(\n sql => sql<{watermark: string}[]>`\n SELECT watermark FROM ${this.#cdc('changeLog')}\n ORDER BY watermark, pos LIMIT 1\n FOR SHARE \n `,\n );\n if (row.length === 0) {\n this.#lc.info?.(`changeLog is empty. No rows to purge-lock.`);\n tx.setDone();\n await tx.done();\n return null;\n }\n const [{watermark}] = row;\n const [{replicaVersion}] = await tx.processReadTask(\n sql => sql<{replicaVersion: string}[]>`\n SELECT \"replicaVersion\" FROM ${this.#cdc('replicationConfig')}\n `,\n );\n this.#lc.info?.(\n `locked watermark ${watermark} from being purged from replica@${replicaVersion}`,\n );\n return new PurgeLock(this.#lc, tx, replicaVersion, watermark);\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwEA,IAAM,yBAAyB,eAAE,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqC7D,IAAa,SAAb,MAAuC;CACrC,KAAc;CACd;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,OAAmB;CACzC;CACA;CAEA,0BAA0B;CAC1B,WAAW;CAEX,YACE,IACA,OACA,QACA,kBACA,mBACA,IACA,gBACA,YACA,SACA,EAAC,iCAAiC,sBAClC;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,aAAa;AACpD,QAAA,QAAc;AACd,QAAA,SAAe;AACf,QAAA,mBAAyB;AACzB,QAAA,oBAA0B;AAC1B,QAAA,KAAW;AACX,QAAA,iBAAuB;AACvB,QAAA,aAAmB;AACnB,QAAA,UAAgB;AAChB,QAAA,qBAA2B;EAE3B,MAAM,YAAY,mBAAmB;AACrC,QAAA,8BACG,UAAU,kBAAkB,UAAU,kBACvC;AAEF,QAAA,GAAS,OACP,gBAAgB,MAAA,6BAAmC,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAC5C,UAAU,kBAAkB,QAAQ,GAAG,QAAQ,EAAE,CAAC,iCAE/E,EAAC,WAAU,CACZ;;CAIH,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,gBAAgB,WAA8B;EAClD,MAAM,KAAK,MAAA;EACX,MAAM,QAAQ,MAAA;EACd,MAAM,eAAe,MAAA;EACrB,MAAM,gBAAgB,MAAA;EAEtB,MAAM,sBACJ,kBAAkB,OACd,eACA,GAAG,cAAc,KAAK;AAC5B,QAAA,GAAS,OAAO,yBAAyB,sBAAsB;EAC/D,MAAM,QAAQ,YAAY,KAAK;AAC/B,QAAM,EAAE,UAAU,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG;GAAC;GAAO,cAAc;GAAoB,CAAC;EACrG,MAAM,WAAW,YAAY,KAAK,GAAG,OAAO,QAAQ,EAAE;AACtD,QAAA,GAAS,OACP,wBAAwB,oBAAoB,IAAI,QAAQ,MACzD;AAED,MAAI,UAKG,WAAU,SAAS;;CAI5B,MAAM,yCAGH;EACD,MAAM,CAAC,CAAC,EAAC,kBAAiB,UAAU,MAAM,MACxC,MAAA,KACA,QAAO,CACL,GAA8B;sCACA,MAAA,IAAU,mBAAmB,IAK3D,GAAG;;;;;;;;;iBASM,MAAA,IAAU,cAAc,CAAC;sBACpB,MAAA,IAAU,gBAAgB,CAAC;;;UAI1C,EACD,EAAC,MAAM,UAAc,CACtB;AAED,SAAO;GACL;GACA,kBAAkB,MAAQ,QAAQ,uBAAuB;GAC1D;;CAGH,MAAM,4BAAoD;EACxD,MAAM,CAAC,EAAC,kBAAiB,MAAM,MAAA,EACvB;qDACyC,MAAA,IAAU,YAAY;AACvE,SAAO;;CAGT,mBAAmB,WAAoC;AACrD,SAAO,MAAM,MAAA,IAAU,OAAM,QAAO;AAGlC,SAAM,GAA0B;kCACJ,MAAA,IAAU,YAAY,CAAC;;;;GASnD,MAAM,CAAC,EAAC,aAAY,MAAM,GAAwB;;wBAEhC,MAAA,IAAU,YAAY,CAAC,qBAAqB,UAAU;;;GAIxE,MAAM,CAAC,EAAC,WAAU,MAAM,GAAuB;wBAC7B,MAAA,IAAU,mBAAmB,CAAC;AAChD,OAAI,UAAU,MAAA,OACZ,OAAM,IAAI,WACR,+BAA+B,UAAU,uCAAuC,QACjF;AAEH,UAAO,OAAO,QAAQ;IACtB;;;;;CAMJ,MAAM,OAA0B;EAC9B,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW;EAQpC,MAAM,OAAO,WAAW,UAAU,OAAO;AACzC,QAAA,0BAAgC,KAAK;AAErC,QAAA,MAAY,QAAQ;GAClB;GACA;GACA;GACA,aAAa,OAAO,GAAG,OAAO;GAC/B,CAAC;AAEF,SAAO,KAAK;;CAGd,QAAQ;AACN,QAAA,MAAY,QAAQ,CAAC,QAAQ,CAAC;;CAGhC,OAAO,GAA4B;AACjC,QAAA,MAAY,QAAQ,EAAE;;CAGxB,QAAQ,YAAwB,MAAsB;AACpD,QAAA,MAAY,QAAQ,CAAC,cAAc;GAAC;GAAY;GAAK,CAAC,CAAC;;CAGzD,gBAAuC;CAEvC,eAA0C;AACxC,MAAI,CAAC,MAAA,QACH;AAEF,MACE,MAAA,iBAAuB,QACvB,MAAA,yBAA+B,MAAA,4BAC/B;AACA,SAAA,GAAS,OACP,+BAA+B,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,4SAW7H;AACD,SAAA,eAAqB,UAAU;;AAEjC,SAAO,MAAA,cAAoB;;CAG7B,4BAA4B;AAC1B,MACE,MAAA,iBAAuB,QAEvB,MAAA,yBAA+B,MAAA,6BAAmC,IAClE;AACA,SAAA,GAAS,OACP,gCAAgC,MAAA,MAAY,MAAM,CAAC,qBAAqB,MAAA,yBAA+B,QAAQ,GAAG,QAAQ,EAAE,CAAC,MAC9H;AACD,SAAA,aAAmB,SAAS;AAC5B,SAAA,eAAqB;;;CAIzB,WAAW;;;;;CAMX,MAAM,MAAM;AACV,SAAO,CAAC,MAAA,SAAe,4BAA4B;EAEnD,MAAM,EAAC,SAAS,SAAS,SAAS,kBAAiB,UAAU;AAC7D,QAAA,UAAgB;AAChB,QAAA,UAAgB;AAEhB,QAAA,GAAS,OAAO,kBAAkB;EAClC,IAAI;AACJ,MAAI;AACF,SAAM,MAAA,cAAoB;WACnB,GAAG;AACV,SAAM;AACN,SAAM;YACE;AAER,OAAI,MAAA,iBAAuB,MAAM;AAC/B,UAAA,aAAmB,SAAS;AAC5B,UAAA,eAAqB;;AAEvB,SAAA,mBACE,MAAA,MAAY,OAAO,CAAC,QAAO,UAAS,UAAU,KAAA,EAAU,EACxD,IACD;AACD,SAAA,UAAgB;AAChB,kBAAe;AACf,SAAA,GAAS,OAAO,iBAAiB;;;CAIrC,oBAAoB,OAAqB,GAAY;AACnD,MAAI,MAAM,WAAW,EACnB;AAEF,QAAA,GAAS,OACP,aAAa,MAAM,OAAO,mCAC3B;EACD,MAAM,MAAM,aAAa,QAAQ,IAAI,IAAI,WAAW,uBAAuB;AAC3E,OAAK,MAAM,SAAS,OAAO;AACzB,OAAI,UAAU,OACZ;AAGF,WADa,MAAM,IACnB;IACE,KAAK,cAAc;KAGjB,MAAM,EAAC,eAAc,MAAM;AAC3B,WAAA,GAAS,OAAO,iBAAiB,WAAW,KAAK;AACjD,gBAAW,KAAK,IAAI;AACpB;;;;;CAMR,OAAA,eAAsB;EACpB,IAAI,KAAgC;EACpC,IAAI;EAEJ,MAAM,eAAoC,EAAE;AAC5C,MAAI;AACF,WAAQ,MAAM,MAAM,MAAA,MAAY,SAAS,MAAM,QAAQ;IACrD,MAAM,CAAC,WAAW;AAClB,YAAQ,SAAR;KACE,KAAK,SAAS;MACZ,MAAM,cAAc,IAAI;AACxB,mBAAa;AACb;;KAEF,KAAK,cAAc;MACjB,MAAM,aAAa,IAAI;AACvB,UAAI,GACF,cAAa,KAAK,WAAW;UAE7B,OAAM,MAAA,aAAmB,CAAC,WAAW,CAAC;AAExC;;KAEF,KAAK;AACH,YAAA,WAAiB,IAAI;AACrB;KACF,KAAK;AACH,UAAI,IAAI;AACN,UAAG,KAAK,OAAO;AACf,aAAM,GAAG,KAAK,MAAM;AACpB,YAAK;;AAEP;;IAIJ,MAAM,CAAC,GAAG,WAAW,MAAM,UAAU;IACrC,MAAM,MAAM,QAAQ;AACpB,UAAA,0BAAgC,KAAK;AAErC,QAAI,QAAQ,SAAS;AACnB,YAAO,CAAC,IAAI,gDAAgD;KAC5D,MAAM,EAAC,SAAS,SAAS,WAAU,UAA4B;AAC1D,aAAQ,YAAY,GAAG;AAC5B,UAAK;MACH,MAAM,IAAI,gBACR,MAAA,GAAS,YAAY,aAAa,UAAU,EAC5C;OACE,MAAM;OACN,0BAA0B,MAAA;OAC3B,CACF;MACD,oBAAoB;MACpB,KAAK;MACL,0BAA0B;MAC1B,KAAK,CAAC,OAAO;MACd;AACD,QAAG,KAAK,IAAI,MAAA,GAAS;AAGhB,QAAG,KAAK,SAAQ,OAAM;AACzB,QAA+B;0BACjB,MAAA,IAAU,mBAAmB,CAAC,aAAa,MACtD,CAAC,YAAY,QAAQ,OAAO,EAC7B,OACD;AACD,aAAO,EAAE;OACT;WACG;AACL,YAAO,UAAU,2CAA2C,OAAO;AACnE,QAAG;;IAGL,MAAM,QAAQ;KACZ,WAAW,QAAQ,WAAW,YAAY,GAAG;KAC7C,WAAW,QAAQ,WAAW,GAAG,qBAAqB;KACtD,KAAK,GAAG;KACR,QAAQ;KACT;IAED,MAAM,YAAY,GAAG,KAAK,SAAQ,QAAO,CACvC,GAAG,eAAe,MAAA,IAAU,YAAY,CAAC,GAAG,IAAI,MAAM,IACtD,GAAI,WAAW,QAAQ,eAAe,OAAO,GACzC,MAAA,sBAA4B,KAAK,OAAO,GACxC,EAAE,CACP,CAAC;AAEF,QAAI,GAAG,MAAM,QAAQ,EAInB,OAAM;AAER,UAAA,0BAAgC;AAEhC,QAAI,QAAQ,UAAU;KACpB,MAAM,EAAC,UAAS,MAAM,GAAG;AACzB,SAAI,UAAU,MAAA,OAEZ,IAAG,KAAK,KACN,IAAI,WACF,2CAA2C,QAC5C,CACF;UACI;MAEL,MAAM,gBAAgB;AACjB,SAAG,KAAK,SAAQ,OAAM,CACzB,EAAE;qBACK,MAAA,IAAU,mBAAmB,CAAC,OAAO,GAAG,EAAC,eAAc,CAAC,GAChE,CAAC;AACF,SAAG,KAAK,SAAS;;AAGnB,WAAM,GAAG,KAAK,MAAM;AAGpB,SAAI,GAAG,IACL,OAAA,WAAiB;MAAC;MAAU;MAAQ,EAAC,WAAU;MAAC,CAAC;AAEnD,UAAK;AAIL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;eACvC,QAAQ,YAAY;AAG7B,QAAG,KAAK,OAAO;AACf,WAAM,GAAG,KAAK,MAAM;AACpB,UAAK;AAEL,WAAM,MAAA,aAAmB,aAAa,OAAO,EAAE,CAAC;;;WAG7C,GAAG;AACV,gBAAa,SAAS,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC1D,SAAM;;;CAIV,OAAA,aAAoB,MAA2B;AAC7C,MAAI,KAAK,WAAW,EAClB;EAGF,MAAM,SAAS,IAAI,gBACjB,MAAA,GAAS,YAAY,QAAQ,UAAU,EACvC,EAAC,MAAM,UAAc,CACtB;AACD,SAAO,IAAI,MAAA,GAAS;EAEpB,IAAI;AACJ,MAAI;AAKF,IAAC,CAAC,kBAAkB,MAAM,OAAO,iBAC/B,QAAO,GAAuB;wBACd,MAAA,IAAU,mBAAmB,CAAC;QAE/C;WACM,GAAG;AACV,QAAK,KAAK,EAAC,iBAAgB,WAAW,KAAK,EAAE,CAAC;AAC9C,SAAM;;AAKH,UAAQ,IACX,KAAK,KAAI,QAAO,MAAA,QAAc,KAAK,eAAe,OAAO,CAAC,CAC3D,CAAC,cAAc,OAAO,SAAS,CAAC;;CAGnC,OAAA,QACE,EAAC,YAAY,KAAK,QAClB,eACA,QACA;AACA,MAAI;AACF,SAAM,OAAO,gBAAgB,OAAM,OAAM;IACvC,MAAM,QAAQ,KAAK,KAAK;IAIxB,IAAI,iBAAiB,IAAI,cAAc,MAAA;IACvC,IAAI,QAAQ;IACZ,IAAI;AAEJ,eAAW,MAAM,WAAW,EAA0B;0CACpB,MAAA,IAAU,YAAY,CAAC;gCACjC,IAAI,UAAU;gCACd,cAAc;oCACV,OAAO,IAAK,EAAE;KAOxC,MAAM,QAAQ,YAAY,KAAK;AAC/B,WAAM;KACN,MAAM,UAAU,YAAY,KAAK,GAAG;AACpC,SAAI,kBACF,EAAC,UAAU,MAAM,MAAA,GAAS,OAAO,MAAA,GAAS,SACxC,UAAU,QAAQ,QAAQ,EAAE,CAAC,UAAU,IAAI,GAAG,2CAC/C;AAGH,UAAK,MAAM,SAAS,QAClB,KAAI,MAAM,cAAc,IAAI,UAG1B,kBAAiB;cACR,gBAAgB;AACzB,0BAAoB,IAAI,QAAQ,aAAa,MAAM,CAAC;AACpD;gBACS,SAAS,SAClB,OAAM,IAAI,gBACR,+BAA+B,IAAI,UAAU,wBAAwB,MAAM,UAAU,GACtF;UACI;AACL,YAAA,GAAS,OACP,qCAAqC,IAAI,UAAU,wBAAwB,MAAM,UAAU,GAC5F;AACD,UAAI,MACF,GACA,mCAAmC,MAAM,UAAU,cAAc,IAAI,UAAU,GAChF;AACD;;;AAIN,QAAI,gBAAgB;AAClB,WAAM;AACN,WAAA,GAAS,OACP,aAAa,IAAI,GAAG,QAAQ,MAAM,YAChC,KAAK,KAAK,GAAG,MACd,MACF;UAED,OAAA,GAAS,OACP,2BAA2B,IAAI,UAAU,+BAC1C;AAIH,QAAI,aAAa;KACjB;WACK,KAAK;AACZ,SAAA,GAAS,QAAQ,sCAAsC,IAAI,MAAM,IAAI;AACrE,OAAI,eAAe,iBAAiB;AAClC,UAAM,kBAAkB,MAAA,IAAU,MAAA,MAAY;AAC9C,UAAA,QAAc,IAAI;;AAEpB,OAAI,KAAK,IAAI;;;;;;;CAQjB,uBAAuB,KAA0B,QAAsB;EACrE,MAAM,QAA+B,EAAE;AAEvC,UAAQ,OAAO,KAAf;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;AAC/B,UAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,SAAS,CAAC;AAC/D;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,MAAM,UAAU,aAAY;AACnC,QAAI,SACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,MAAM,SAAS,CAAC;AAEhE,QAAI,SACF,QAAO,QAAQ,SAAS,CAAC,SAAS,CAAC,KAAK,cAAc;AACpD,WAAM,KACJ,MAAA,yBAA+B,KAAK,MAAM,KAAK,SAAS,CACzD;MACD;AAEJ;;GAGF,KAAK,gBAAgB;IACnB,MAAM,EAAC,QAAO;IACd,MAAM,MAAM;KAAC,QAAQ,OAAO,IAAI;KAAQ,OAAO,OAAO,IAAI;KAAK;AAC/D,UAAM,KACJ,GAAG,UAAU,MAAA,IAAU,gBAAgB,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC/B,IAAI,OAAO,iBAAiB,IAAI,QACzD,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,OAAO,IAAI,IAAI,CAAC;mCAC7B,IAAI,OAAO,iBAAiB,IAAI,OAC1D;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EACJ,IAAI,EAAC,QAAQ,WACX;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,gBAAgB,CAAC;mCACpB,OAAO,iBAAiB,QACjD,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,OAClD;AACD;;GAGF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAO,eAAe,QAAQ,aAAY;AACjD,QAAI,cACF,OAAM,KAAK,MAAA,wBAA8B,KAAK,OAAO,cAAc,CAAC;AAEtE,QAAI,SACF,OAAM,KACJ,MAAA,yBAA+B,KAAK,OAAO,OAAO,MAAM,SAAS,CAClE;AAEH;;GAGF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OAAO,EAAC,QAAQ,MAAM,SACtB,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,QACd,OAAM,KACJ,GAAG,UAAU,MAAA,IAAU,cAAc,CAAC,kBAAkB,QAAQ;mCACzC,OAAO,iBAAiB,MAAM,kBAAkB,UACxE;AAEH;;GAGF,KAAK,eAAe;IAClB,MAAM,EACJ,OAAO,EAAC,QAAQ,QAChB,WACE;AACJ,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,KAAK,kBAAkB,SACzE;AACD;;GAGF,KAAK,sBAAsB;IACzB,MAAM,EACJ,UAAU,EAAC,QAAQ,MAAM,OAAO,UAChC,YACE;IACJ,MAAM,OAAO,CAAC,GAAG,OAAO,SAAS,GAAG,QAAQ;AAC5C,UAAM,KACJ,GAAG,eAAe,MAAA,IAAU,cAAc,CAAC;mCAClB,OAAO,iBAAiB,MAAM,mBAAmB,IAAI,KAAK,GACpF;;;AAGL,SAAO;;CAGT,yBACE,KACA,EAAC,QAAQ,MAAM,SACf,UACA;EACA,MAAM,MAAwB;GAAC;GAAQ;GAAO;GAAS;AACvD,SAAO,GAAG;sBACQ,MAAA,IAAU,gBAAgB,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEnC,IAAI,IAAI,CAAC;;;CAIjC,0BACE,KACA,EAAC,QAAQ,MAAM,SACf,QACA,UACA;EACA,MAAM,MAAyB;GAAC;GAAQ;GAAO;GAAQ;GAAS;AAChE,SAAO,GAAG;sBACQ,MAAA,IAAU,cAAc,CAAC,GAAG,IAAI,IAAI,CAAC;;0BAEjC,IAAI,IAAI,CAAC;;;;;;;CAQjC,MAAM,eAAe;AACnB,MAAI,MAAA,SAAe;GACjB,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,SAAA,MAAY,QAAQ,CAAC,SAAS,QAAQ,CAAC;AACvC,SAAM;;;CAIV,OAAO;AACL,MAAI,MAAA,SAAe;AACjB,SAAA,GAAS,OAAO,YAAY,MAAA,MAAY,MAAM,CAAC,oBAAoB;AACnE,SAAA,MAAY,QAAQ,OAAO;;AAE7B,SAAO,MAAA;;;AAIX,SAAS,aAAa,OAAuC;CAC3D,MAAM,EAAC,WAAW,WAAU;AAC5B,SAAQ,OAAO,KAAf;EACE,KAAK,QACH,QAAO,CAAC,WAAW;GAAC;GAAS;GAAQ,EAAC,iBAAiB,WAAU;GAAC,CAAC;EACrE,KAAK,SACH,QAAO,CAAC,WAAW;GAAC;GAAU;GAAQ,EAAC,WAAU;GAAC,CAAC;EACrD,KAAK,WACH,QAAO,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC;EAC1C,QACE,QAAO,CAAC,WAAW,CAAC,QAAQ,OAAO,CAAC;;;AAI1C,IAAa,YAAb,MAAuB;CACrB;CACA;CACA;CACA;CAEA,YACE,IACA,IACA,gBACA,WACA;AACA,QAAA,KAAW;AACX,QAAA,KAAW;AACX,OAAK,iBAAiB;AACtB,OAAK,eAAe;;CAGtB,YAAY;CAEZ,MAAM,UAAU;AACd,MAAI,MAAA,SACF;AAEF,QAAA,WAAiB;AACjB,QAAA,GAAS,SAAS;AAClB,QAAM,MAAA,GACH,MAAM,CACN,OAAM,MAAK,MAAA,GAAS,OAAO,iCAAiC,EAAE,CAAC;AAClE,QAAA,GAAS,OAAO,0BAA0B,KAAK,eAAe;;;AAIlE,IAAa,cAAb,MAAyB;CACvB;CACA;CACA;CAEA,YAAY,IAAgB,OAAgB,IAAgB;AAC1D,QAAA,KAAW,GAAG,YAAY,aAAa,eAAe;AACtD,QAAA,QAAc;AACd,QAAA,KAAW;;CAIb,KAAK,OAAe;AAClB,SAAO,MAAA,GAAS,GAAG,UAAU,MAAA,MAAY,CAAC,GAAG,QAAQ;;CAGvD,MAAM,UAAU;EACd,MAAM,KAAK,IAAI,gBAAgB,MAAA,IAAU,EAAC,MAAM,gBAAoB,CAAC,CAAC,IACpE,MAAA,GACD;EACD,MAAM,MAAM,MAAM,GAAG,iBACnB,QAAO,GAA0B;8BACT,MAAA,IAAU,YAAY,CAAC;;;MAIhD;AACD,MAAI,IAAI,WAAW,GAAG;AACpB,SAAA,GAAS,OAAO,6CAA6C;AAC7D,MAAG,SAAS;AACZ,SAAM,GAAG,MAAM;AACf,UAAO;;EAET,MAAM,CAAC,EAAC,eAAc;EACtB,MAAM,CAAC,EAAC,oBAAmB,MAAM,GAAG,iBAClC,QAAO,GAA+B;uCACL,MAAA,IAAU,oBAAoB,CAAC;QAEjE;AACD,QAAA,GAAS,OACP,oBAAoB,UAAU,kCAAkC,iBACjE;AACD,SAAO,IAAI,UAAU,MAAA,IAAU,IAAI,gBAAgB,UAAU"}

package/out/zero-cache/src/services/heapz.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"heapz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/heapz.ts"],"names":[],"mappings":"~~AAAA~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;~~AAG1D~~,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAGjE,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,QA0BlB"}
1	+ {"version":3,"file":"heapz.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/heapz.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,SAAS,CAAC;AAC1D,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAGjE,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,GAAG,EAAE,cAAc,EACnB,GAAG,EAAE,YAAY,QA0BlB"}

package/out/zero-cache/src/services/heapz.js CHANGED Viewed

@@ -1,7 +1,7 @@
 import { isAdminPasswordValid } from "../config/zero-config.js";
-import auth from "basic-auth";
 import fs from "fs";
 import v8 from "v8";
+import auth from "basic-auth";
 //#region ../zero-cache/src/services/heapz.ts
 function handleHeapzRequest(lc, config, req, res) {
 	if (!isAdminPasswordValid(lc, config, auth(req)?.pass)) {

package/out/zero-cache/src/services/heapz.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport ~~fs from 'fs';\nimport v8 from 'v8';\nimport~~ type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n lc: LogContext,\n config: NormalizedZeroConfig,\n req: FastifyRequest,\n res: FastifyReply,\n) {\n const credentials = auth(req);\n if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n void res\n .code(401)\n .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n .send('Unauthorized');\n return;\n }\n\n const filename = v8.writeHeapSnapshot();\n const stream = fs.createReadStream(filename);\n void res\n .header('Content-Type', 'application/octet-stream')\n .header('Content-Disposition', `attachment; filename=${filename}`)\n .send(stream);\n\n // Clean up temp file after streaming\n stream.on('end', () => {\n fs.unlink(filename, err => {\n if (err) {\n lc.error?.('Error deleting heap snapshot:', err);\n }\n });\n });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;AAEA,KAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,IAAI,EACsB,KAAK,EAAE;AACnD,MACF,KAAK,IAAI,CACT,OAAO,oBAAoB,uCAAqC,CAChE,KAAK,eAAe;AACvB;;CAGF,MAAM,WAAW,GAAG,mBAAmB;CACvC,MAAM,SAAS,GAAG,iBAAiB,SAAS;AACvC,KACF,OAAO,gBAAgB,2BAA2B,CAClD,OAAO,uBAAuB,wBAAwB,WAAW,CACjE,KAAK,OAAO;AAGf,QAAO,GAAG,aAAa;AACrB,KAAG,OAAO,WAAU,QAAO;AACzB,OAAI,IACF,IAAG,QAAQ,iCAAiC,IAAI;IAElD;GACF"}
1	+ {"version":3,"file":"heapz.js","names":[],"sources":["../../../../../zero-cache/src/services/heapz.ts"],"sourcesContent":["import fs from 'fs';\nimport v8 from 'v8';\nimport type {LogContext} from '@rocicorp/logger';\nimport auth from 'basic-auth';\nimport type {FastifyReply, FastifyRequest} from 'fastify';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {isAdminPasswordValid} from '../config/zero-config.ts';\n\nexport function handleHeapzRequest(\n lc: LogContext,\n config: NormalizedZeroConfig,\n req: FastifyRequest,\n res: FastifyReply,\n) {\n const credentials = auth(req);\n if (!isAdminPasswordValid(lc, config, credentials?.pass)) {\n void res\n .code(401)\n .header('WWW-Authenticate', 'Basic realm=\"Heapz Protected Area\"')\n .send('Unauthorized');\n return;\n }\n\n const filename = v8.writeHeapSnapshot();\n const stream = fs.createReadStream(filename);\n void res\n .header('Content-Type', 'application/octet-stream')\n .header('Content-Disposition', `attachment; filename=${filename}`)\n .send(stream);\n\n // Clean up temp file after streaming\n stream.on('end', () => {\n fs.unlink(filename, err => {\n if (err) {\n lc.error?.('Error deleting heap snapshot:', err);\n }\n });\n });\n}\n"],"mappings":";;;;;AAQA,SAAgB,mBACd,IACA,QACA,KACA,KACA;AAEA,KAAI,CAAC,qBAAqB,IAAI,QADV,KAAK,IAAI,EACsB,KAAK,EAAE;AACnD,MACF,KAAK,IAAI,CACT,OAAO,oBAAoB,uCAAqC,CAChE,KAAK,eAAe;AACvB;;CAGF,MAAM,WAAW,GAAG,mBAAmB;CACvC,MAAM,SAAS,GAAG,iBAAiB,SAAS;AACvC,KACF,OAAO,gBAAgB,2BAA2B,CAClD,OAAO,uBAAuB,wBAAwB,WAAW,CACjE,KAAK,OAAO;AAGf,QAAO,GAAG,aAAa;AACrB,KAAG,OAAO,WAAU,QAAO;AACzB,OAAI,IACF,IAAG,QAAQ,iCAAiC,IAAI;IAElD;GACF"}

package/out/zero-cache/src/services/life-cycle.d.ts CHANGED Viewed

@@ -1,6 +1,6 @@
-import type { LogContext } from '@rocicorp/logger';
 import type { IncomingHttpHeaders } from 'node:http';
 import type { EventEmitter } from 'stream';
+import type { LogContext } from '@rocicorp/logger';
 import { type Subprocess, type Worker } from '../types/processes.ts';
 import type { SingletonService } from './service.ts';
 /**
@@ -20,6 +20,7 @@ export type WorkerType = 'user-facing' | 'supporting';
 export declare const GRACEFUL_SHUTDOWN: readonly ["SIGTERM", "SIGINT"];
 export declare const FORCEFUL_SHUTDOWN: readonly ["SIGQUIT", "SIGABRT"];
 export declare const UNHANDLED_EXCEPTION_ERROR_CODE = 13;
+export declare const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;
 /**
  * Handles readiness, termination signals, and coordination of graceful
  * shutdown.

package/out/zero-cache/src/services/life-cycle.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,~~UAAU~~,EAAC,MAAM,~~kBAAkB~~,CAAC;~~AAEjD~~,OAAO,KAAK,EAAC,~~mBAAmB~~,EAAC,MAAM,~~WAAW~~,CAAC;~~AAEnD~~,OAAO,KAAK,EAAC,~~YAAY~~,EAAC,MAAM,~~QAAQ~~,CAAC;~~AACzC~~,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AAOjE,eAAO,MAAM,8BAA8B,KAAK,CAAC;~~AAOjD~~;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;~~IAoBJ~~,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}
1	+ {"version":3,"file":"life-cycle.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,WAAW,CAAC;AAEnD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,YAAY,CAAC;AAEtD,eAAO,MAAM,iBAAiB,gCAAiC,CAAC;AAChE,eAAO,MAAM,iBAAiB,iCAAkC,CAAC;AAOjE,eAAO,MAAM,8BAA8B,KAAK,CAAC;AAKjD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD;;;GAGG;AACH,qBAAa,cAAc;;gBAWb,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY;IA2C9C,IAAI;IAyBJ,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM;IA2B9D,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAiBjE,YAAY,IAAI,MAAM,EAAE;IAIlB,eAAe;IAIrB,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;CAiF3C;AAED;;;;;;GAMG;AAEH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,YAAY,EACpB,GAAG,QAAQ,EAAE,gBAAgB,EAAE,GAC9B,OAAO,CAAC,IAAI,CAAC,CA6Bf;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,iBAWvD;AAID;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;;gBAQf,EAAE,EAAE,UAAU,EAAE,YAAY,SAA2B;IAKnE,WAAW,CAAC,UAAU,EAAE,mBAAmB;IAsD3C,IAAI;CAML"}

package/out/zero-cache/src/services/life-cycle.js CHANGED Viewed

@@ -5,7 +5,6 @@ import { pid } from "node:process";
 //#region ../zero-cache/src/services/life-cycle.ts
 var GRACEFUL_SHUTDOWN = ["SIGTERM", "SIGINT"];
 var FORCEFUL_SHUTDOWN = ["SIGQUIT", "SIGABRT"];
-var INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;
 /**
 * Handles readiness, termination signals, and coordination of graceful
 * shutdown.
@@ -22,8 +21,8 @@ var ProcessManager = class {
 	constructor(lc, proc) {
 		this.#lc = lc.withContext("component", "process-manager");
 		for (const signal of GRACEFUL_SHUTDOWN) proc.on(signal, () => this.#startDrain(signal));
-		proc.on("exit", (code) => this.#kill(this.#all, code === 0 ? "SIGTERM" : code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "SIGQUIT" : "SIGABRT"));
-		for (const signal of FORCEFUL_SHUTDOWN) proc.on(signal, () => this.#exit(signal === "SIGQUIT" ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1));
+		proc.on("exit", (code) => this.#kill(this.#all, code === 0 ? "SIGTERM" : code === 14 ? "SIGQUIT" : "SIGABRT"));
+		for (const signal of FORCEFUL_SHUTDOWN) proc.on(signal, () => this.#exit(signal === "SIGQUIT" ? 14 : -1));
 		this.#exitImpl = (code) => {
 			if (singleProcessMode()) return proc.emit("exit", code);
 			process.exit(code);
@@ -38,6 +37,10 @@ var ProcessManager = class {
 		this.#lc.flush().finally(() => this.#exitImpl(code));
 	}
 	#startDrain(signal) {
+		if (this.#all.size === 0) {
+			this.#lc.info?.(`exiting on ${signal}`);
+			this.#exit(0);
+		}
 		this.#lc.info?.(`initiating drain (${signal})`);
 		this.#drainStart = Date.now();
 		if (this.#userFacing.size) this.#kill(this.#userFacing, signal);
@@ -51,7 +54,7 @@ var ProcessManager = class {
 			isOpen = false;
 			this.#onExit(code, signal, null, type, name, proc);
 		});
-		proc.on("error", (err) => this.#lc[isOpen ? "error" : "warn"]?.(`error from ${name} ${proc.pid}`, err));
+		proc.on("error", (err) => this.#lc[!isOpen || this.#drainStart > 0 ? "warn" : "error"]?.(`error from ${name} ${proc.pid}`, err));
 	}
 	#initializing = /* @__PURE__ */ new Map();
 	#nextID = 0;
@@ -84,12 +87,12 @@ var ProcessManager = class {
 		}
 		const pid = worker?.pid ?? process.pid;
 		if (type === "supporting") {
-			if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) code = INTENTIONAL_SHUTDOWN_ERROR_CODE;
-			const log = code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "info" : "warn";
+			if (code === 0 && (this.#drainStart === 0 || this.#userFacing.size > 0)) code = 14;
+			const log = code === 0 || code === 14 ? "info" : "warn";
 			this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? "");
 			return this.#exit(code);
 		}
-		const log = code === 0 || code === INTENTIONAL_SHUTDOWN_ERROR_CODE ? "info" : this.#drainStart > 0 || code === 13 ? "warn" : "error";
+		const log = code === 0 || code === 14 ? "info" : this.#drainStart > 0 || code === 13 ? "warn" : "error";
 		this.#lc[log]?.(sig ? `${name} (${pid}) killed with (${sig})` : `${name} (${pid}) exited with code (${code})`, err ?? "");
 		if (this.#userFacing.size === 0) {
 			this.#lc.info?.(this.#drainStart ? `all user-facing workers drained (${Date.now() - this.#drainStart} ms)` : `all user-facing workers exited`);

package/out/zero-cache/src/services/life-cycle.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/*\n * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n \n * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n \n For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n /\nexport type WorkerType = 'user-facing' \| 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nconst INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/\n Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n /\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0\n ? 'SIGTERM' // graceful, drained shutdown\n : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'SIGQUIT' // intentional abort without drain\n : 'SIGABRT', // unintentional shutdown, alertable error\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n // shutdown) to all workers. For these signals, workers are stopped\n // immediately without draining, since there is no merit to slowly draining\n // when supporting workers have stopped.\n //\n // The logic for handling these signals is in `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () =>\n this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n );\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n done() {\n return this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: GracefulShutdownSignal) {\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[isOpen ? 'error' : 'warn']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals \| null,\n err: unknown,\n type: WorkerType,\n name: string,\n worker: Subprocess \| undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // Supporting workers like the replication-manager shut down without a\n // drain signal when receiving protocol-specific instructions (like auto\n // reset). In this case, a special error code is used to signal that the\n // server should be shut down without draining, but it is otherwise not\n // considered an unexpected/alertable error.\n if (code === 0 && (this.#drainStart === 0 \|\| this.#userFacing.size > 0)) {\n code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n }\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(code);\n }\n\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : this.#drainStart > 0 \|\| code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n this.#lc[log]?.(\n sig\n ? `${name} (${pid}) killed with (${sig})`\n : `${name} (${pid}) exited with code (${code})`,\n err ?? '',\n );\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(0);\n }\n\n if (this.#drainStart === 0) {\n // If a user-facing worker exits without receiving a drain signal,\n // shutdown the server.\n return this.#exit(code \|\| -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/\n Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n \n @returns a Promise that resolves/rejects when any of the services stops/throws.\n /\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n try {\n await run();\n // oxlint-disable-next-line no-console\n console.info(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n // oxlint-disable-next-line no-console\n console.error(`pid ${pid} exiting with error`, e);\n process.exit(-1);\n }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/\n The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n \n The HeartbeatMonitor is opt-in in that it only kicks in after it\n * starts receiving keepalives.\n /\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout \| undefined;\n #checkImmediateTimer: NodeJS.Immediate \| undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run before the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop after\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;AAYvD,IAAM,kCAAkC;;;;;AAMxC,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAS,kCACP,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAY,kCAAkC,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,SAAS,UAAU,UAC1B,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAO;GAET,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAS,kCACnB,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}
1	+ {"version":3,"file":"life-cycle.js","names":["#lc","#userFacing","#all","#exitImpl","#start","#ready","#startDrain","#kill","#exit","#runningState","#drainStart","#onExit","#initializing","#nextID","#stopInterval","#lastHeartbeat","#checkIntervalTimer","#checkStopInterval","#checkImmediateTimer"],"sources":["../../../../../zero-cache/src/services/life-cycle.ts"],"sourcesContent":["import type {IncomingHttpHeaders} from 'node:http';\nimport {pid} from 'node:process';\nimport type {EventEmitter} from 'stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n singleProcessMode,\n type Subprocess,\n type Worker,\n} from '../types/processes.ts';\nimport {RunningState} from './running-state.ts';\nimport type {SingletonService} from './service.ts';\n\n/*\n * `user-facing` workers serve external requests and are the first to\n * receive a `SIGTERM` or `SIGINT` signal for graceful shutdown.\n \n * `supporting` workers support `user-facing` workers and are sent\n * the `SIGTERM` signal only after all `user-facing` workers have\n * exited.\n \n For other kill signals, such as `SIGQUIT` and `SIGABRT`, all workers\n * are stopped without draining. `SIGQUIT` is used to represent an\n * intentional shutdown (for which draining is not beneficial), whereas\n * `SIGABRT` is used for unexpected process exits.\n /\nexport type WorkerType = 'user-facing' \| 'supporting';\n\nexport const GRACEFUL_SHUTDOWN = ['SIGTERM', 'SIGINT'] as const;\nexport const FORCEFUL_SHUTDOWN = ['SIGQUIT', 'SIGABRT'] as const;\n\ntype GracefulShutdownSignal = (typeof GRACEFUL_SHUTDOWN)[number];\n\n// An internal error code used to indicate that a message has already been\n// logged at level ERRROR. When a process exits with this error code, the\n// parent process logs the exit at level WARN instead of ERROR.\nexport const UNHANDLED_EXCEPTION_ERROR_CODE = 13;\n\n// An internal error code used to indicate that the server should exit\n// without draining (e.g. due to a supporting worker get a signal to shut\n// down), but the exit is otherwise intentional.\nexport const INTENTIONAL_SHUTDOWN_ERROR_CODE = 14;\n\n/\n Handles readiness, termination signals, and coordination of graceful\n * shutdown.\n /\nexport class ProcessManager {\n readonly #lc: LogContext;\n readonly #userFacing = new Set<Subprocess>();\n readonly #all = new Set<Subprocess>();\n readonly #exitImpl: (code: number) => never;\n readonly #start = Date.now();\n readonly #ready: Promise<void>[] = [];\n\n #runningState = new RunningState('process-manager');\n #drainStart = 0;\n\n constructor(lc: LogContext, proc: EventEmitter) {\n this.#lc = lc.withContext('component', 'process-manager');\n\n // Propagate `SIGTERM` and `SIGINT` to all user-facing workers,\n // initiating a graceful shutdown. The parent process will\n // exit once all user-facing workers have exited ...\n for (const signal of GRACEFUL_SHUTDOWN) {\n proc.on(signal, () => this.#startDrain(signal));\n }\n\n // ... which will result in sending `SIGTERM` to the remaining workers.\n proc.on('exit', code =>\n this.#kill(\n this.#all,\n code === 0\n ? 'SIGTERM' // graceful, drained shutdown\n : code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'SIGQUIT' // intentional abort without drain\n : 'SIGABRT', // unintentional shutdown, alertable error\n ),\n );\n\n // For other (catchable) kill signals, exit with a non-zero error code\n // to send a `SIGQUIT` (intentional shutdown) or `SIGABRT` (unexpected\n // shutdown) to all workers. For these signals, workers are stopped\n // immediately without draining, since there is no merit to slowly draining\n // when supporting workers have stopped.\n //\n // The logic for handling these signals is in `runUntilKilled()`.\n for (const signal of FORCEFUL_SHUTDOWN) {\n proc.on(signal, () =>\n this.#exit(signal === 'SIGQUIT' ? INTENTIONAL_SHUTDOWN_ERROR_CODE : -1),\n );\n }\n\n this.#exitImpl = (code: number) => {\n if (singleProcessMode()) {\n return proc.emit('exit', code) as never; // For unit / integration tests.\n }\n process.exit(code);\n };\n }\n\n done() {\n return this.#runningState.stopped();\n }\n\n #exit(code: number) {\n this.#lc.info?.('exiting with code', code);\n this.#runningState.stop(this.#lc);\n void this.#lc.flush().finally(() => this.#exitImpl(code));\n }\n\n #startDrain(signal: GracefulShutdownSignal) {\n if (this.#all.size === 0) {\n // Shutdown if a signal is received before any subprocesses are added.\n this.#lc.info?.(`exiting on ${signal}`);\n this.#exit(0);\n }\n this.#lc.info?.(`initiating drain (${signal})`);\n this.#drainStart = Date.now();\n if (this.#userFacing.size) {\n this.#kill(this.#userFacing, signal);\n } else {\n this.#kill(this.#all, signal);\n }\n }\n\n addSubprocess(proc: Subprocess, type: WorkerType, name: string) {\n if (type === 'user-facing') {\n this.#userFacing.add(proc);\n }\n this.#all.add(proc);\n\n let isOpen = true;\n proc.on('close', (code, signal) => {\n isOpen = false;\n this.#onExit(code, signal, null, type, name, proc);\n });\n\n // As per https://nodejs.org/api/child_process.html#event-error\n // 'error' events can happen when sending a message to a child process\n // fails. This is not really an error when the server is shutting down,\n // so log any post-close errors at 'warn'.\n proc.on('error', err =>\n this.#lc[!isOpen \|\| this.#drainStart > 0 ? 'warn' : 'error']?.(\n `error from ${name} ${proc.pid}`,\n err,\n ),\n );\n }\n\n readonly #initializing = new Map<number, string>();\n #nextID = 0;\n\n addWorker(worker: Worker, type: WorkerType, name: string): Worker {\n this.addSubprocess(worker, type, name);\n\n const id = ++this.#nextID;\n this.#initializing.set(id, name);\n const {promise, resolve} = resolver();\n this.#ready.push(promise);\n\n worker.onceMessageType('ready', () => {\n this.#lc.debug?.(`${name} ready (${Date.now() - this.#start} ms)`);\n this.#initializing.delete(id);\n resolve();\n });\n\n return worker;\n }\n\n initializing(): string[] {\n return [...this.#initializing.values()];\n }\n\n async allWorkersReady() {\n await Promise.all(this.#ready);\n }\n\n logErrorAndExit(err: unknown, name: string) {\n // only accessible by the main (i.e. user-facing) process.\n this.#onExit(-1, null, err, 'user-facing', name, undefined);\n }\n\n #onExit(\n code: number,\n sig: NodeJS.Signals \| null,\n err: unknown,\n type: WorkerType,\n name: string,\n worker: Subprocess \| undefined,\n ) {\n // Remove the worker from maps to avoid attempting to send more signals to it.\n if (worker) {\n this.#userFacing.delete(worker);\n this.#all.delete(worker);\n }\n\n const pid = worker?.pid ?? process.pid;\n\n if (type === 'supporting') {\n // Supporting workers like the replication-manager shut down without a\n // drain signal when receiving protocol-specific instructions (like auto\n // reset). In this case, a special error code is used to signal that the\n // server should be shut down without draining, but it is otherwise not\n // considered an unexpected/alertable error.\n if (code === 0 && (this.#drainStart === 0 \|\| this.#userFacing.size > 0)) {\n code = INTENTIONAL_SHUTDOWN_ERROR_CODE;\n }\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : 'warn';\n this.#lc[log]?.(`${name} (${pid}) exited with code (${code})`, err ?? '');\n return this.#exit(code);\n }\n\n const log =\n code === 0 \|\| code === INTENTIONAL_SHUTDOWN_ERROR_CODE\n ? 'info'\n : this.#drainStart > 0 \|\| code === UNHANDLED_EXCEPTION_ERROR_CODE\n ? 'warn'\n : 'error';\n this.#lc[log]?.(\n sig\n ? `${name} (${pid}) killed with (${sig})`\n : `${name} (${pid}) exited with code (${code})`,\n err ?? '',\n );\n\n // user-facing workers exited or finished draining.\n if (this.#userFacing.size === 0) {\n this.#lc.info?.(\n this.#drainStart\n ? `all user-facing workers drained (${\n Date.now() - this.#drainStart\n } ms)`\n : `all user-facing workers exited`,\n );\n return this.#exit(0);\n }\n\n if (this.#drainStart === 0) {\n // If a user-facing worker exits without receiving a drain signal,\n // shutdown the server.\n return this.#exit(code \|\| -1);\n }\n\n return undefined;\n }\n\n #kill(workers: Iterable<Subprocess>, signal: NodeJS.Signals) {\n for (const worker of workers) {\n try {\n worker.kill(signal);\n } catch (e) {\n this.#lc.error?.(e);\n }\n }\n }\n}\n\n/\n Runs the specified services, stopping them on `SIGTERM` or `SIGINT` with\n * an optional {@link SingletonService.drain drain()}, or stopping them\n * without draining for `SIGQUIT`.\n \n @returns a Promise that resolves/rejects when any of the services stops/throws.\n /\n\nexport async function runUntilKilled(\n lc: LogContext,\n parent: EventEmitter,\n ...services: SingletonService[]\n): Promise<void> {\n if (services.length === 0) {\n return;\n }\n for (const signal of [...GRACEFUL_SHUTDOWN, ...FORCEFUL_SHUTDOWN]) {\n parent.once(signal, () => {\n const GRACEFUL_SIGNALS = GRACEFUL_SHUTDOWN as readonly NodeJS.Signals[];\n\n services.forEach(async svc => {\n if (GRACEFUL_SIGNALS.includes(signal) && svc.drain) {\n lc.info?.(`draining ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.drain();\n }\n lc.info?.(`stopping ${svc.constructor.name} ${svc.id} (${signal})`);\n await svc.stop();\n });\n });\n }\n\n try {\n // Run all services and resolve when any of them stops.\n const svc = await Promise.race(\n services.map(svc => svc.run().then(() => svc)),\n );\n lc.info?.(`${svc.constructor.name} (${svc.id}) stopped`);\n } catch (e) {\n lc.error?.(`exiting on error`, e);\n throw e;\n }\n}\n\nexport async function exitAfter(run: () => Promise<void>) {\n try {\n await run();\n // oxlint-disable-next-line no-console\n console.info(`pid ${pid} exiting normally`);\n process.exit(0);\n } catch (e) {\n // oxlint-disable-next-line no-console\n console.error(`pid ${pid} exiting with error`, e);\n process.exit(-1);\n }\n}\n\nconst DEFAULT_STOP_INTERVAL_MS = 20_000;\n\n/\n The HeartbeatMonitor monitors the cadence heartbeats (e.g. \"/keepalive\"\n * health checks made to HttpServices) that signal that the server\n * should continue processing requests. When a configurable `stopInterval`\n * elapses without receiving these heartbeats, the monitor initiates a\n * graceful shutdown of the server. This works with common load balancing\n * frameworks such as AWS Elastic Load Balancing.\n \n The HeartbeatMonitor is opt-in in that it only kicks in after it\n * starts receiving keepalives.\n /\nexport class HeartbeatMonitor {\n readonly #stopInterval: number;\n\n #lc: LogContext;\n #checkIntervalTimer: NodeJS.Timeout \| undefined;\n #checkImmediateTimer: NodeJS.Immediate \| undefined;\n #lastHeartbeat = 0;\n\n constructor(lc: LogContext, stopInterval = DEFAULT_STOP_INTERVAL_MS) {\n this.#lc = lc;\n this.#stopInterval = stopInterval;\n }\n\n onHeartbeat(reqHeaders: IncomingHttpHeaders) {\n this.#lastHeartbeat = Date.now();\n if (this.#checkIntervalTimer === undefined) {\n this.#lc.info?.(\n `starting heartbeat monitor at ${\n this.#stopInterval / 1000\n } second interval`,\n reqHeaders,\n );\n // e.g. check every 5 seconds to see if it's been over 20 seconds\n // since the last heartbeat.\n this.#checkIntervalTimer = setInterval(\n this.#checkStopInterval,\n this.#stopInterval / 4,\n );\n }\n }\n\n #checkStopInterval = () => {\n // In the Node.js event loop, timers like setInterval and setTimeout\n // run before* I/O events coming from network sockets or file reads/writes.\n // When this process gets starved of CPU resources for long periods of time,\n // for example when other processes are monopolizing all available cores,\n // pathological behavior can emerge:\n // - keepalive network request comes in, but is queued in Node internals waiting\n // for time on the event loop\n // - CPU is starved/monopolized by other processes for longer than the time\n // configured via this.#stopInterval\n // - When CPU becomes available and the event loop wakes up, this stop interval\n // check is run before the keepalive request is processed. The value of\n // this.#lastHeartbeat is now very stale, and erroneously triggers a shutdown\n // even though keepalive requests were about to be processed and update\n // this.#lastHeartbeat. Downtime ensues.\n //\n // To avoid this, we push the check out to a phase of the event loop after\n // I/O events are processed, using setImmediate():\n // https://nodejs.org/en/learn/asynchronous-work/event-loop-timers-and-nexttick#setimmediate-vs-settimeout\n //\n // This ensures we see a value for this.#lastHeartbeat that reflects\n // any keepalive requests that came in during the current event loop turn.\n this.#checkImmediateTimer = setImmediate(() => {\n this.#checkImmediateTimer = undefined;\n const timeSinceLastHeartbeat = Date.now() - this.#lastHeartbeat;\n if (timeSinceLastHeartbeat >= this.#stopInterval) {\n this.#lc.info?.(\n `last heartbeat received ${\n timeSinceLastHeartbeat / 1000\n } seconds ago. draining.`,\n );\n process.kill(process.pid, GRACEFUL_SHUTDOWN[0]);\n }\n });\n };\n\n stop() {\n clearTimeout(this.#checkIntervalTimer);\n if (this.#checkImmediateTimer) {\n clearImmediate(this.#checkImmediateTimer);\n }\n }\n}\n"],"mappings":";;;;;AA4BA,IAAa,oBAAoB,CAAC,WAAW,SAAS;AACtD,IAAa,oBAAoB,CAAC,WAAW,UAAU;;;;;AAkBvD,IAAa,iBAAb,MAA4B;CAC1B;CACA,8BAAuB,IAAI,KAAiB;CAC5C,uBAAgB,IAAI,KAAiB;CACrC;CACA,SAAkB,KAAK,KAAK;CAC5B,SAAmC,EAAE;CAErC,gBAAgB,IAAI,aAAa,kBAAkB;CACnD,cAAc;CAEd,YAAY,IAAgB,MAAoB;AAC9C,QAAA,KAAW,GAAG,YAAY,aAAa,kBAAkB;AAKzD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cAAc,MAAA,WAAiB,OAAO,CAAC;AAIjD,OAAK,GAAG,SAAQ,SACd,MAAA,KACE,MAAA,KACA,SAAS,IACL,YACA,SAAA,KACE,YACA,UACP,CACF;AASD,OAAK,MAAM,UAAU,kBACnB,MAAK,GAAG,cACN,MAAA,KAAW,WAAW,YAAA,KAA8C,GAAG,CACxE;AAGH,QAAA,YAAkB,SAAiB;AACjC,OAAI,mBAAmB,CACrB,QAAO,KAAK,KAAK,QAAQ,KAAK;AAEhC,WAAQ,KAAK,KAAK;;;CAItB,OAAO;AACL,SAAO,MAAA,aAAmB,SAAS;;CAGrC,MAAM,MAAc;AAClB,QAAA,GAAS,OAAO,qBAAqB,KAAK;AAC1C,QAAA,aAAmB,KAAK,MAAA,GAAS;AAC5B,QAAA,GAAS,OAAO,CAAC,cAAc,MAAA,SAAe,KAAK,CAAC;;CAG3D,YAAY,QAAgC;AAC1C,MAAI,MAAA,IAAU,SAAS,GAAG;AAExB,SAAA,GAAS,OAAO,cAAc,SAAS;AACvC,SAAA,KAAW,EAAE;;AAEf,QAAA,GAAS,OAAO,qBAAqB,OAAO,GAAG;AAC/C,QAAA,aAAmB,KAAK,KAAK;AAC7B,MAAI,MAAA,WAAiB,KACnB,OAAA,KAAW,MAAA,YAAkB,OAAO;MAEpC,OAAA,KAAW,MAAA,KAAW,OAAO;;CAIjC,cAAc,MAAkB,MAAkB,MAAc;AAC9D,MAAI,SAAS,cACX,OAAA,WAAiB,IAAI,KAAK;AAE5B,QAAA,IAAU,IAAI,KAAK;EAEnB,IAAI,SAAS;AACb,OAAK,GAAG,UAAU,MAAM,WAAW;AACjC,YAAS;AACT,SAAA,OAAa,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;IAClD;AAMF,OAAK,GAAG,UAAS,QACf,MAAA,GAAS,CAAC,UAAU,MAAA,aAAmB,IAAI,SAAS,WAClD,cAAc,KAAK,GAAG,KAAK,OAC3B,IACD,CACF;;CAGH,gCAAyB,IAAI,KAAqB;CAClD,UAAU;CAEV,UAAU,QAAgB,MAAkB,MAAsB;AAChE,OAAK,cAAc,QAAQ,MAAM,KAAK;EAEtC,MAAM,KAAK,EAAE,MAAA;AACb,QAAA,aAAmB,IAAI,IAAI,KAAK;EAChC,MAAM,EAAC,SAAS,YAAW,UAAU;AACrC,QAAA,MAAY,KAAK,QAAQ;AAEzB,SAAO,gBAAgB,eAAe;AACpC,SAAA,GAAS,QAAQ,GAAG,KAAK,UAAU,KAAK,KAAK,GAAG,MAAA,MAAY,MAAM;AAClE,SAAA,aAAmB,OAAO,GAAG;AAC7B,YAAS;IACT;AAEF,SAAO;;CAGT,eAAyB;AACvB,SAAO,CAAC,GAAG,MAAA,aAAmB,QAAQ,CAAC;;CAGzC,MAAM,kBAAkB;AACtB,QAAM,QAAQ,IAAI,MAAA,MAAY;;CAGhC,gBAAgB,KAAc,MAAc;AAE1C,QAAA,OAAa,IAAI,MAAM,KAAK,eAAe,MAAM,KAAA,EAAU;;CAG7D,QACE,MACA,KACA,KACA,MACA,MACA,QACA;AAEA,MAAI,QAAQ;AACV,SAAA,WAAiB,OAAO,OAAO;AAC/B,SAAA,IAAU,OAAO,OAAO;;EAG1B,MAAM,MAAM,QAAQ,OAAO,QAAQ;AAEnC,MAAI,SAAS,cAAc;AAMzB,OAAI,SAAS,MAAM,MAAA,eAAqB,KAAK,MAAA,WAAiB,OAAO,GACnE,QAAA;GAEF,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA;AACN,SAAA,GAAS,OAAO,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAAI,OAAO,GAAG;AACzE,UAAO,MAAA,KAAW,KAAK;;EAGzB,MAAM,MACJ,SAAS,KAAK,SAAA,KACV,SACA,MAAA,aAAmB,KAAK,SAAA,KACtB,SACA;AACR,QAAA,GAAS,OACP,MACI,GAAG,KAAK,IAAI,IAAI,iBAAiB,IAAI,KACrC,GAAG,KAAK,IAAI,IAAI,sBAAsB,KAAK,IAC/C,OAAO,GACR;AAGD,MAAI,MAAA,WAAiB,SAAS,GAAG;AAC/B,SAAA,GAAS,OACP,MAAA,aACI,oCACE,KAAK,KAAK,GAAG,MAAA,WACd,QACD,iCACL;AACD,UAAO,MAAA,KAAW,EAAE;;AAGtB,MAAI,MAAA,eAAqB,EAGvB,QAAO,MAAA,KAAW,QAAQ,GAAG;;CAMjC,MAAM,SAA+B,QAAwB;AAC3D,OAAK,MAAM,UAAU,QACnB,KAAI;AACF,UAAO,KAAK,OAAO;WACZ,GAAG;AACV,SAAA,GAAS,QAAQ,EAAE;;;;;;;;;;;AAc3B,eAAsB,eACpB,IACA,QACA,GAAG,UACY;AACf,KAAI,SAAS,WAAW,EACtB;AAEF,MAAK,MAAM,UAAU,CAAC,GAAG,mBAAmB,GAAG,kBAAkB,CAC/D,QAAO,KAAK,cAAc;EACxB,MAAM,mBAAmB;AAEzB,WAAS,QAAQ,OAAM,QAAO;AAC5B,OAAI,iBAAiB,SAAS,OAAO,IAAI,IAAI,OAAO;AAClD,OAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,UAAM,IAAI,OAAO;;AAEnB,MAAG,OAAO,YAAY,IAAI,YAAY,KAAK,GAAG,IAAI,GAAG,IAAI,OAAO,GAAG;AACnE,SAAM,IAAI,MAAM;IAChB;GACF;AAGJ,KAAI;EAEF,MAAM,MAAM,MAAM,QAAQ,KACxB,SAAS,KAAI,QAAO,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAC/C;AACD,KAAG,OAAO,GAAG,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG,WAAW;UACjD,GAAG;AACV,KAAG,QAAQ,oBAAoB,EAAE;AACjC,QAAM;;;AAIV,eAAsB,UAAU,KAA0B;AACxD,KAAI;AACF,QAAM,KAAK;AAEX,UAAQ,KAAK,OAAO,IAAI,mBAAmB;AAC3C,UAAQ,KAAK,EAAE;UACR,GAAG;AAEV,UAAQ,MAAM,OAAO,IAAI,sBAAsB,EAAE;AACjD,UAAQ,KAAK,GAAG;;;AAIpB,IAAM,2BAA2B;;;;;;;;;;;;AAajC,IAAa,mBAAb,MAA8B;CAC5B;CAEA;CACA;CACA;CACA,iBAAiB;CAEjB,YAAY,IAAgB,eAAe,0BAA0B;AACnE,QAAA,KAAW;AACX,QAAA,eAAqB;;CAGvB,YAAY,YAAiC;AAC3C,QAAA,gBAAsB,KAAK,KAAK;AAChC,MAAI,MAAA,uBAA6B,KAAA,GAAW;AAC1C,SAAA,GAAS,OACP,iCACE,MAAA,eAAqB,IACtB,mBACD,WACD;AAGD,SAAA,qBAA2B,YACzB,MAAA,mBACA,MAAA,eAAqB,EACtB;;;CAIL,2BAA2B;AAsBzB,QAAA,sBAA4B,mBAAmB;AAC7C,SAAA,sBAA4B,KAAA;GAC5B,MAAM,yBAAyB,KAAK,KAAK,GAAG,MAAA;AAC5C,OAAI,0BAA0B,MAAA,cAAoB;AAChD,UAAA,GAAS,OACP,2BACE,yBAAyB,IAC1B,yBACF;AACD,YAAQ,KAAK,QAAQ,KAAK,kBAAkB,GAAG;;IAEjD;;CAGJ,OAAO;AACL,eAAa,MAAA,mBAAyB;AACtC,MAAI,MAAA,oBACF,gBAAe,MAAA,oBAA0B"}

package/out/zero-cache/src/services/litestream/commands.d.ts CHANGED Viewed

@@ -1,10 +1,21 @@
-import type { LogContext } from '@rocicorp/logger';
 import type { ChildProcess } from 'node:child_process';
+import type { LogContext } from '@rocicorp/logger';
 import type { ZeroConfig } from '../../config/zero-config.ts';
+type ReplicaConstraints = {
+    replicaVersion: string;
+    minWatermark: string;
+};
+export declare class BackupNotFoundException extends Error {
+    static readonly name = "BackupNotFoundException";
+    constructor(backupURL: string | undefined);
+}
 /**
- * @returns The time at which the last restore started
- *          (i.e. not counting failed attempts).
+ * @param replicaConstraints The constraints of the restored backup when
+ *        restoring for the change-streamer (replication-manager). For the
+ *        view-syncer, this should be unspecified so that the constraints are
+ *        retrieved from the replication-manager via the snapshot protocol.
  */
-export declare function restoreReplica(lc: LogContext, config: ZeroConfig): Promise<Date>;
+export declare function restoreReplica(lc: LogContext, config: ZeroConfig, replicaConstraints: ReplicaConstraints | null): Promise<void>;
 export declare function startReplicaBackupProcess(lc: LogContext, config: ZeroConfig): ChildProcess;
+export {};
 //# sourceMappingURL=commands.d.ts.map

package/out/zero-cache/src/services/litestream/commands.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,~~UAAU~~,~~EAAW~~,MAAM,~~kBAAkB~~,CAAC;~~AAE3D~~,OAAO,KAAK,EAAC,~~YAAY~~,~~EAAC~~,MAAM,~~oBAAoB~~,CAAC;~~AAOrD~~,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D~~;;;GAGG~~;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,~~GACjB~~,~~OAAO~~,~~CAAC~~,IAAI,~~CAAC,CAwBf~~;~~AA2KD~~,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}
1	+ {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAM3D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D,KAAK,kBAAkB,GAAG;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;gBAErC,SAAS,EAAE,MAAM,GAAG,SAAS;CAG1C;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,EAClB,kBAAkB,EAAE,kBAAkB,GAAG,IAAI,iBA8B9C;AAgKD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}