@rocicorp/zero 1.11.0-canary.40 → 1.11.0-canary.41

package/out/zero-cache/src/services/change-streamer/schema/tables.js

@@ -77,8 +77,58 @@ async function markResetRequired(sql, shard) {
   UPDATE ${sql(cdcSchema(shard))}."replicationConfig"
     SET "resetRequired" = true`;
 }
-async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout, pgChangeLogEnabled = true) {
-	const { publications, replicaVersion, watermark } = subscriptionState;
+/**
+* Validates the `replicationConfig` of the change db against the replica,
+* throwing an {@link AutoResetSignal} if the replica cannot be served from it.
+*
+* @param behindSlot Whether the change log is behind the replication slot,
+*     in which case it is re-initialized from the replica (see
+*     {@link reinitializeChangeLog}) and can thus be reset for a replica
+*     at any watermark.
+* @returns `'init'` if the tables are uninitialized, `'reset'` if they are
+*     for a different replica, or `'compatible'`.
+*/
+function validateReplicationConfig(lc, current, { publications, replicaVersion, watermark }, autoReset, behindSlot) {
+	if (!current) return "init";
+	if (current.replicaVersion !== replicaVersion || !equals(new Set(current.publications), new Set(publications))) {
+		if (replicaVersion !== watermark && !behindSlot) throw new AutoResetSignal(`Cannot reset change db@${current.replicaVersion} to service replica@${replicaVersion} from watermark ${watermark}`);
+		return "reset";
+	}
+	if (current.resetRequired) {
+		if (autoReset) throw new AutoResetSignal("reset required by replication stream");
+		lc.error?.("\n\n\nReset required but --auto-reset is not enabled.\nThis can happen for upstream databases that do not support event triggers.\nTo correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes\n\n\n");
+	}
+	return "compatible";
+}
+function selectReplicationConfig(sql, shard) {
+	return sql`
+    SELECT "replicaVersion", "publications", "resetRequired" 
+      FROM ${sql(cdcSchema(shard))}."replicationConfig"`;
+}
+/**
+* Checks, without modifying the change db, that the replica can be served
+* by the change log once {@link ensureReplicationConfig} is called upon
+* taking over the change log. Throws an {@link AutoResetSignal} otherwise.
+*
+* This allows a replication-manager to detect an incompatible replica at
+* startup while leaving the change log to its current owner until then.
+*/
+async function checkReplicationConfig(lc, db, subscriptionState, shard, autoReset, behindSlot = false) {
+	const [current] = await runTx(db, (sql) => selectReplicationConfig(sql, shard));
+	if (validateReplicationConfig(lc, current, subscriptionState, autoReset, behindSlot) === "reset") lc.info?.(`Data in cdc tables @${current?.replicaVersion} is incompatible with replica @${subscriptionState.replicaVersion}. Tables will be reset when taking over the change log.`);
+}
+/**
+* Initializes the cdc tables for the replica, resetting them if they were
+* written for a different replica (version or publications). This disrupts
+* a replication-manager that is serving the change log, and is thus only
+* done when taking over the change log (see `Storer.assumeOwnershipAtHead`).
+*
+* @param purgeLock A purge lock held by the caller, if any, which is released
+*     before the tables are reset.
+* @param behindSlot See {@link checkReplicationConfig}.
+*/
+async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoReset, purgeLock, setTimeoutFn = setTimeout, behindSlot = false) {
+	const { publications, replicaVersion } = subscriptionState;
 	const replicaConfig = {
 		publications,
 		replicaVersion
@@ -90,43 +140,28 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
 	};
 	const schema = cdcSchema(shard);
 	await runTx(db, async (sql) => {
+		const [current] = await selectReplicationConfig(sql, shard);
+		const status = validateReplicationConfig(lc, current, subscriptionState, autoReset, behindSlot);
+		if (status === "compatible") return [];
 		const stmts = [];
-		let needsTruncate = false;
-		const results = await sql`
-    SELECT "replicaVersion", "publications", "resetRequired" 
-      FROM ${sql(schema)}."replicationConfig"`;
-		if (results.length) {
-			const { replicaVersion, publications } = results[0];
-			if (replicaVersion !== replicaConfig.replicaVersion || !equals(new Set(publications), new Set(replicaConfig.publications))) {
-				if (replicaConfig.replicaVersion !== watermark) throw new AutoResetSignal(`Cannot reset change db@${replicaVersion} to service replica@${replicaConfig.replicaVersion} from watermark ${watermark}`);
-				lc.info?.(`Data in cdc tables @${replicaVersion} is incompatible with replica @${replicaConfig.replicaVersion}. Clearing tables.`);
-				purgeLock?.release();
-				needsTruncate = true;
-				stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, ...pgChangeLogEnabled ? [sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`] : [], sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
-			}
+		if (status === "reset") {
+			lc.info?.(`Data in cdc tables @${current?.replicaVersion} is incompatible with replica @${replicaVersion}. Clearing tables.`);
+			purgeLock?.release();
+			stmts.push(sql`TRUNCATE TABLE ${sql(schema)}."replicationState"`, sql`TRUNCATE TABLE ${sql(schema)}."changeLog"`, sql`TRUNCATE TABLE ${sql(schema)}."replicationConfig"`, sql`TRUNCATE TABLE ${sql(schema)}."tableMetadata"`, sql`TRUNCATE TABLE ${sql(schema)}."backfilling"`);
 		}
-		if (results.length === 0 || needsTruncate) {
-			const watermark = replicaConfig.replicaVersion;
-			const initialTx = pgChangeLogEnabled ? [{
-				watermark,
-				pos: 0,
-				change: { tag: "begin" }
-			}, {
-				watermark,
-				pos: 1,
-				change: { tag: "commit" }
-			}] : [];
-			stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
-              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
-			if (needsTruncate) return runStatementsAndTerminateLockHolders(lc, db, shard, "ensureReplicationConfig", stmts, setTimeoutFn);
-			return Promise.all(stmts);
-		}
-		const { resetRequired } = results[0];
-		if (resetRequired) {
-			if (autoReset) throw new AutoResetSignal("reset required by replication stream");
-			lc.error?.("\n\n\nReset required but --auto-reset is not enabled.\nThis can happen for upstream databases that do not support event triggers.\nTo correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes\n\n\n");
-		}
-		return [];
+		const initialTx = [{
+			watermark: replicaVersion,
+			pos: 0,
+			change: { tag: "begin" }
+		}, {
+			watermark: replicaVersion,
+			pos: 1,
+			change: { tag: "commit" }
+		}];
+		stmts.push(sql`INSERT INTO ${sql(schema)}."replicationConfig" ${sql(replicaConfig)}`, sql`INSERT INTO ${sql(schema)}."replicationState"  ${sql(replicationState)} 
+            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`, ...initialTx.map((change) => sql`INSERT INTO ${sql(schema)}."changeLog" ${sql(change)}`));
+		if (status === "reset") return runStatementsAndTerminateLockHolders(lc, db, shard, "ensureReplicationConfig", stmts, setTimeoutFn);
+		return Promise.all(stmts);
 	});
 }
 var LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5e3;
@@ -278,6 +313,6 @@ async function terminateChangeDBLockHolders(lc, db, shard) {
 	else for (const { pid, applicationName, query, terminated: ok } of terminated) lc.info?.(`terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`);
 }
 //#endregion
-export { AutoResetSignal, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, reinitializeChangeLog, setupCDCTables };
+export { AutoResetSignal, checkReplicationConfig, createBackfillTables, createReplicationStateTable, discoverChangeStreamerAddress, ensureReplicationConfig, markResetRequired, reinitializeChangeLog, setupCDCTables };
 
 //# sourceMappingURL=tables.js.map

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: Pick<\n    SubscriptionState,\n    'publications' | 'replicaVersion' | 'watermark'\n  >,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  pgChangeLogEnabled = true,\n) {\n  const {publications, replicaVersion, watermark} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const stmts: PendingQuery<Row[]>[] = [];\n    let needsTruncate = false;\n    const results = await sql<\n      {\n        replicaVersion: string;\n        publications: string[];\n        resetRequired: boolean | null;\n      }[]\n    > /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(schema)}.\"replicationConfig\"`;\n\n    if (results.length) {\n      const {replicaVersion, publications} = results[0];\n      if (\n        replicaVersion !== replicaConfig.replicaVersion ||\n        !equals(new Set(publications), new Set(replicaConfig.publications))\n      ) {\n        if (replicaConfig.replicaVersion !== watermark) {\n          throw new AutoResetSignal(\n            `Cannot reset change db@${replicaVersion} to ` +\n              `service replica@${replicaConfig.replicaVersion} ` +\n              `from watermark ${watermark}`,\n          );\n        }\n        lc.info?.(\n          `Data in cdc tables @${replicaVersion} is incompatible ` +\n            `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n        );\n        // Release any purge lock held by the caller; the changeLog is\n        // incompatible with the replica and needs to be truncated.\n        void purgeLock?.release();\n\n        // Note: The order of TRUNCATE matters. The replicationState table is\n        //       truncated before the changeLog table, in order to acquire\n        //       (exclusive) locks on the tables in the same order that the\n        //       storer.ts acquires locks when writing the changeLog, namely:\n        //\n        // 1. SELECT ... FROM replicationState FOR UPDATE;\n        // 2. INSERT INTO changeLog ...;\n        //    ...\n        // n. UPDATE replicationState ...;\n        needsTruncate = true;\n        stmts.push(\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n          ...(pgChangeLogEnabled\n            ? [sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`]\n            : []),\n          sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n          sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n        );\n      }\n    }\n    // Initialize (or re-initialize TRUNCATED) tables\n    if (results.length === 0 || needsTruncate) {\n      // When enabled, the PG storer uses the earliest changeLog entry as the\n      // safe watermark from which subscribers can be resumed. These initial\n      // entries ensure that subscribers can start from a freshly synced\n      // replica, even if new changes have been replicated and not purged.\n      //\n      // TODO: Replace this with an explicit `firstWatermark` column in the\n      //       change db.\n      const watermark = replicaConfig.replicaVersion;\n      const initialTx: FullChangeLogEntry[] = pgChangeLogEnabled\n        ? [\n            {watermark, pos: 0, change: {tag: 'begin'}},\n            {watermark, pos: 1, change: {tag: 'commit'}},\n          ]\n        : [];\n\n      stmts.push(\n        sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n        sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n        ...initialTx.map(\n          change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n        ),\n      );\n\n      if (needsTruncate) {\n        return runStatementsAndTerminateLockHolders(\n          lc,\n          db,\n          shard,\n          'ensureReplicationConfig',\n          stmts,\n          setTimeoutFn,\n        );\n      }\n      return Promise.all(stmts);\n    }\n\n    const {resetRequired} = results[0];\n    if (resetRequired) {\n      if (autoReset) {\n        throw new AutoResetSignal('reset required by replication stream');\n      }\n      lc.error?.(\n        '\\n\\n\\n' +\n          'Reset required but --auto-reset is not enabled.\\n' +\n          'This can happen for upstream databases that do not support event triggers.\\n' +\n          'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n          '\\n\\n\\n',\n      );\n    }\n\n    return [];\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;AAEA,eAAsB,wBACpB,IACA,IACA,mBAIA,OACA,WACA,WACA,eAAkC,YAClC,qBAAqB,MACrB;CACA,MAAM,EAAC,cAAc,gBAAgB,cAAa;CAClD,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,QAA+B,CAAC;EACtC,IAAI,gBAAgB;EACpB,MAAM,UAAU,MAAM,GAMZ;;aAED,IAAI,MAAM,EAAE;EAErB,IAAI,QAAQ,QAAQ;GAClB,MAAM,EAAC,gBAAgB,iBAAgB,QAAQ;GAC/C,IACE,mBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAI,YAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;IACA,IAAI,cAAc,mBAAmB,WACnC,MAAM,IAAI,gBACR,0BAA0B,eAAe,sBACpB,cAAc,eAAe,kBAC9B,WACtB;IAEF,GAAG,OACD,uBAAuB,eAAe,iCACnB,cAAc,eAAe,mBAClD;IAGA,WAAgB,QAAQ;IAWxB,gBAAgB;IAChB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAI,qBACA,CAAC,GAAG,kBAAkB,IAAI,MAAM,EAAE,aAAa,IAC/C,CAAC,GACL,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;GACF;EACF;EAEA,IAAI,QAAQ,WAAW,KAAK,eAAe;GAQzC,MAAM,YAAY,cAAc;GAChC,MAAM,YAAkC,qBACpC,CACE;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,QAAO;GAAC,GAC1C;IAAC;IAAW,KAAK;IAAG,QAAQ,EAAC,KAAK,SAAQ;GAAC,CAC7C,IACA,CAAC;GAEL,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;iDAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;GAEA,IAAI,eACF,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;GAEF,OAAO,QAAQ,IAAI,KAAK;EAC1B;EAEA,MAAM,EAAC,kBAAiB,QAAQ;EAChC,IAAI,eAAe;GACjB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;GAElE,GAAG,QACD,oOAKF;EACF;EAEA,OAAO,CAAC;CACV,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,iCAChB,IAAI;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\ntype ReplicationConfigRow = ReplicationConfig & {\n  resetRequired: boolean | null;\n};\n\nexport type ReplicaSubscriptionState = Pick<\n  SubscriptionState,\n  'publications' | 'replicaVersion' | 'watermark'\n>;\n\n/**\n * Validates the `replicationConfig` of the change db against the replica,\n * throwing an {@link AutoResetSignal} if the replica cannot be served from it.\n *\n * @param behindSlot Whether the change log is behind the replication slot,\n *     in which case it is re-initialized from the replica (see\n *     {@link reinitializeChangeLog}) and can thus be reset for a replica\n *     at any watermark.\n * @returns `'init'` if the tables are uninitialized, `'reset'` if they are\n *     for a different replica, or `'compatible'`.\n */\nfunction validateReplicationConfig(\n  lc: LogContext,\n  current: ReplicationConfigRow | undefined,\n  {publications, replicaVersion, watermark}: ReplicaSubscriptionState,\n  autoReset: boolean,\n  behindSlot: boolean,\n): 'init' | 'reset' | 'compatible' {\n  if (!current) {\n    return 'init';\n  }\n  if (\n    current.replicaVersion !== replicaVersion ||\n    !equals(new Set(current.publications), new Set(publications))\n  ) {\n    if (replicaVersion !== watermark && !behindSlot) {\n      throw new AutoResetSignal(\n        `Cannot reset change db@${current.replicaVersion} to ` +\n          `service replica@${replicaVersion} from watermark ${watermark}`,\n      );\n    }\n    return 'reset';\n  }\n  if (current.resetRequired) {\n    if (autoReset) {\n      throw new AutoResetSignal('reset required by replication stream');\n    }\n    lc.error?.(\n      '\\n\\n\\n' +\n        'Reset required but --auto-reset is not enabled.\\n' +\n        'This can happen for upstream databases that do not support event triggers.\\n' +\n        'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n        '\\n\\n\\n',\n    );\n  }\n  return 'compatible';\n}\n\nfunction selectReplicationConfig(sql: PostgresTransaction, shard: ShardID) {\n  return sql<ReplicationConfigRow[]> /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(cdcSchema(shard))}.\"replicationConfig\"`;\n}\n\n/**\n * Checks, without modifying the change db, that the replica can be served\n * by the change log once {@link ensureReplicationConfig} is called upon\n * taking over the change log. Throws an {@link AutoResetSignal} otherwise.\n *\n * This allows a replication-manager to detect an incompatible replica at\n * startup while leaving the change log to its current owner until then.\n */\nexport async function checkReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  behindSlot = false,\n): Promise<void> {\n  const [current] = await runTx(db, sql => selectReplicationConfig(sql, shard));\n  const status = validateReplicationConfig(\n    lc,\n    current,\n    subscriptionState,\n    autoReset,\n    behindSlot,\n  );\n  if (status === 'reset') {\n    lc.info?.(\n      `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n        `with replica @${subscriptionState.replicaVersion}. ` +\n        `Tables will be reset when taking over the change log.`,\n    );\n  }\n}\n\n/**\n * Initializes the cdc tables for the replica, resetting them if they were\n * written for a different replica (version or publications). This disrupts\n * a replication-manager that is serving the change log, and is thus only\n * done when taking over the change log (see `Storer.assumeOwnershipAtHead`).\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are reset.\n * @param behindSlot See {@link checkReplicationConfig}.\n */\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  behindSlot = false,\n) {\n  const {publications, replicaVersion} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const [current] = await selectReplicationConfig(sql, shard);\n    const status = validateReplicationConfig(\n      lc,\n      current,\n      subscriptionState,\n      autoReset,\n      behindSlot,\n    );\n    if (status === 'compatible') {\n      return [];\n    }\n    const stmts: PendingQuery<Row[]>[] = [];\n    if (status === 'reset') {\n      lc.info?.(\n        `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n          `with replica @${replicaVersion}. Clearing tables.`,\n      );\n      // Release any purge lock held by the caller; the changeLog is\n      // incompatible with the replica and needs to be truncated.\n      void purgeLock?.release();\n\n      // Note: The order of TRUNCATE matters. The replicationState table is\n      //       truncated before the changeLog table, in order to acquire\n      //       (exclusive) locks on the tables in the same order that the\n      //       storer.ts acquires locks when writing the changeLog, namely:\n      //\n      // 1. SELECT ... FROM replicationState FOR UPDATE;\n      // 2. INSERT INTO changeLog ...;\n      //    ...\n      // n. UPDATE replicationState ...;\n      stmts.push(\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      );\n    }\n    // Initialize (or re-initialize TRUNCATED) tables.\n    //\n    // The PG storer uses the earliest changeLog entry as the safe watermark\n    // from which subscribers can be resumed. These initial entries ensure\n    // that subscribers can start from a freshly synced replica, even if new\n    // changes have been replicated and not purged.\n    //\n    // TODO: Replace this with an explicit `firstWatermark` column in the\n    //       change db.\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark: replicaVersion, pos: 0, change: {tag: 'begin'}},\n      {watermark: replicaVersion, pos: 1, change: {tag: 'commit'}},\n    ];\n    stmts.push(\n      sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n      sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n      ...initialTx.map(\n        change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n      ),\n    );\n\n    if (status === 'reset') {\n      return runStatementsAndTerminateLockHolders(\n        lc,\n        db,\n        shard,\n        'ensureReplicationConfig',\n        stmts,\n        setTimeoutFn,\n      );\n    }\n    return Promise.all(stmts);\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;;;;;;;;;;;;AAsBA,SAAS,0BACP,IACA,SACA,EAAC,cAAc,gBAAgB,aAC/B,WACA,YACiC;CACjC,IAAI,CAAC,SACH,OAAO;CAET,IACE,QAAQ,mBAAmB,kBAC3B,CAAC,OAAO,IAAI,IAAI,QAAQ,YAAY,GAAG,IAAI,IAAI,YAAY,CAAC,GAC5D;EACA,IAAI,mBAAmB,aAAa,CAAC,YACnC,MAAM,IAAI,gBACR,0BAA0B,QAAQ,eAAe,sBAC5B,eAAe,kBAAkB,WACxD;EAEF,OAAO;CACT;CACA,IAAI,QAAQ,eAAe;EACzB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;EAElE,GAAG,QACD,oOAKF;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBAAwB,KAA0B,OAAgB;CACzE,OAAO,GAAoC;;aAEhC,IAAI,UAAU,KAAK,CAAC,EAAE;AACnC;;;;;;;;;AAUA,eAAsB,uBACpB,IACA,IACA,mBACA,OACA,WACA,aAAa,OACE;CACf,MAAM,CAAC,WAAW,MAAM,MAAM,KAAI,QAAO,wBAAwB,KAAK,KAAK,CAAC;CAQ5E,IAPe,0BACb,IACA,SACA,mBACA,WACA,UAEE,MAAW,SACb,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,kBAAkB,eAAe,wDAEtD;AAEJ;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA,WACA,eAAkC,YAClC,aAAa,OACb;CACA,MAAM,EAAC,cAAc,mBAAkB;CACvC,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,CAAC,WAAW,MAAM,wBAAwB,KAAK,KAAK;EAC1D,MAAM,SAAS,0BACb,IACA,SACA,mBACA,WACA,UACF;EACA,IAAI,WAAW,cACb,OAAO,CAAC;EAEV,MAAM,QAA+B,CAAC;EACtC,IAAI,WAAW,SAAS;GACtB,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,eAAe,mBACpC;GAGA,WAAgB,QAAQ;GAWxB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;EACF;EAUA,MAAM,YAAkC,CACtC;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1D;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7D;EACA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;+CAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;EAEA,IAAI,WAAW,SACb,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;EAEF,OAAO,QAAQ,IAAI,KAAK;CAC1B,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,iCAChB,IAAI;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -10,7 +10,7 @@ import { type CookieSet } from '../replicator/change-log-cookies.ts';
 import type { ReplicatorMode } from '../replicator/replicator.ts';
 import type { Service } from '../service.ts';
 import { type ChangeLogEntry, type SerializedChangeStreamData } from './change-log-codec.ts';
-import { type ChangeLogReinitialization } from './schema/tables.ts';
+import { type ReplicaSubscriptionState, type ChangeLogReinitialization } from './schema/tables.ts';
 import type { Subscriber } from './subscriber.ts';
 /**
  * Factory for creating dynamic connection pools used by the Storer to
@@ -29,6 +29,16 @@ export type OwnershipTakeover = {
      * which is used to start pending backfills once ownership is assumed.
      */
     requestRestart: (reason: string) => void;
+    /**
+     * Whether to throw an AutoResetSignal if the change db requires a reset.
+     */
+    autoReset: boolean;
+    /**
+     * Whether the change log was behind the replication slot, in which case it
+     * is re-initialized from the replica (see `reinitializeFromReplica()`), and
+     * can thus be reset for a replica at any watermark.
+     */
+    pgChangeLogBehindSlot: boolean;
 };
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
@@ -69,7 +79,7 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, replicaVersion: string, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, dbProvider: PostgresDBProvider, subscriptionState: ReplicaSubscriptionState, onCommitted: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     /**
      * Re-initializes a change log that is no longer maintained from the
      * replica's state, taking ownership of it in the process (which is then
@@ -109,7 +119,7 @@ export declare class Storer implements Service {
      * assumed, a restart of the change stream is requested if the change log has
      * pending backfills or if a backfill transaction was skipped.
      */
-    assumeOwnershipAtHead(takeover: OwnershipTakeover): void;
+    assumeOwnershipAtHead(takeover: OwnershipTakeover): Promise<void>;
     /**
      * Whether ownership of the change log is pending, i.e. after
      * {@link assumeOwnershipAtHead} and before ownership is assumed.

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,0BAA0B,EAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,yBAAyB,EAE/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsDhB,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;OAIG;IACH,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAC1C,CAAC;AAQF,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA0CvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAkCjB;IAyCD;;;;;OAKG;IACH,uBAAuB,CACrB,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,GAAG,cAAc,CAAC,GAChE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAUzC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,QAOhD;IAED;;;OAGG;IACH,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,QAIvB;IA6DD;;;;;;OAMG;IACG,eAAe,kBAcpB;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CA4DD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,EACtB,UAAU,GAAE,0BAEX,UAkBF;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IA6kBD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAE/B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOD;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,CACL,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;CA6D9C;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,uCAAuC,CAAC;AAO/C,OAAO,EAAiB,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EAMrB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,6CAA6C,CAAC;AACvF,OAAO,EAIL,KAAK,SAAS,EAEf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,EAGL,KAAK,cAAc,EACnB,KAAK,0BAA0B,EAChC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAKL,KAAK,wBAAwB,EAE7B,KAAK,yBAAyB,EAE/B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAEhD;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,eAAe,EAAE,MAAM,EACvB,QAAQ,EAAE,MAAM,KACb,UAAU,CAAC;AAsDhB,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;OAIG;IACH,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC;;OAEG;IACH,SAAS,EAAE,OAAO,CAAC;IACnB;;;;OAIG;IACH,qBAAqB,EAAE,OAAO,CAAC;CAChC,CAAC;AAQF,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;IA2CvB,YACE,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,UAAU,EAAE,kBAAkB,EAC9B,iBAAiB,EAAE,wBAAwB,EAC3C,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACxD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa,EAmCjB;IAyCD;;;;;OAKG;IACH,uBAAuB,CACrB,MAAM,EAAE,IAAI,CAAC,yBAAyB,EAAE,OAAO,GAAG,cAAc,CAAC,GAChE,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAUzC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,iBAqBtD;IAED;;;OAGG;IACH,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,QAIvB;IA6DD;;;;;;OAMG;IACG,eAAe,kBAcpB;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;QACpC,OAAO,EAAE,SAAS,CAAC;KACpB,CAAC,CA4DD;IAEK,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAOxD;IAED;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC;QAChC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CASD;IAED;;;OAGG;IACG,oBAAoB,CACxB,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAED;;;;;;OAMG;IACH,gBAAgB,CACd,cAAc,EAAE,MAAM,EACtB,gBAAgB,EAAE,MAAM,EACxB,SAAS,SAAO,GACf,aAAa,CAAC,cAAc,EAAE,CAAC,CAYjC;IAMD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuCrD;IAED;;OAEG;IACH,KAAK,CACH,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gBAAgB,EACtB,UAAU,GAAE,0BAEX,UAkBF;IAED,KAAK,SAEJ;IAED,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc,QAEnD;IAID,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAwBxC;IAmDD;;;OAGG;IACG,GAAG,kBA6BR;IA6kBD;;;OAGG;IACG,YAAY,kBAMjB;IAED;;;;OAIG;IACG,IAAI,kBAcT;CACF;AAED,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAE/B,YACE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAOtB;IAIK,OAAO,kBAUZ;CACF;AAED,qBAAa,WAAW;;IAKtB,YAAY,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAIzD;IAOD;;;;;;;;;;;OAWG;IACH,OAAO,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,CACL,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,GACjC,OAAO,CAAC,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,CAAC;CA6D9C;AAKD,KAAK,OAAO,GAAG,UAAU,CAAC;AAE1B,KAAK,UAAU,GAAG,MAAM,IAAI,CAAC;AAE7B;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,eAAe;;IAQ1B,YACE,EAAE,EAAE,UAAU,EACd,kBAAkB,EAAE,MAAM,EAC1B,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAKhC;IAED,KAAK,SAMJ;IAGD,iBAAiB,CAAC,GAAG,EAAE,IAAI,QAgB1B;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,OAAa,GAAG,UAAU,CAKrD;IAED,IAAI,SAGH;CACF"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -15,7 +15,7 @@ import { orTimeout } from "../../types/timeout.js";
 import { READONLY, READ_COMMITTED } from "../../db/mode-enum.js";
 import { runTx } from "../../db/run-transaction.js";
 import { Queue } from "../../../../shared/src/queue.js";
-import { AutoResetSignal, markResetRequired, reinitializeChangeLog } from "./schema/tables.js";
+import { AutoResetSignal, ensureReplicationConfig, markResetRequired, reinitializeChangeLog } from "./schema/tables.js";
 import { TransactionPool, sharedSnapshot } from "../../db/transaction-pool.js";
 import { reconstructWatermarkedChange, serializeChangeStreamDataWithChange } from "./change-log-codec.js";
 import { resolver } from "@rocicorp/resolver";
@@ -64,6 +64,7 @@ var Storer = class {
 	#db;
 	#inserter;
 	#purger;
+	#subscriptionState;
 	#replicaVersion;
 	#onCommitted;
 	#onFatal;
@@ -80,7 +81,7 @@ var Storer = class {
 	#progressMonitor;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, dbProvider, replicaVersion, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, dbProvider, subscriptionState, onCommitted, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
@@ -90,7 +91,8 @@ var Storer = class {
 		this.#db = dbProvider("change-stream-init", 3);
 		this.#inserter = dbProvider("change-log-inserter", 1);
 		this.#purger = dbProvider("change-log-purger", 1);
-		this.#replicaVersion = replicaVersion;
+		this.#subscriptionState = subscriptionState;
+		this.#replicaVersion = subscriptionState.replicaVersion;
 		this.#onCommitted = onCommitted;
 		this.#onFatal = onFatal;
 		this.#statementTimeoutMs = statementTimeoutMs;
@@ -176,7 +178,8 @@ var Storer = class {
 	* assumed, a restart of the change stream is requested if the change log has
 	* pending backfills or if a backfill transaction was skipped.
 	*/
-	assumeOwnershipAtHead(takeover) {
+	async assumeOwnershipAtHead(takeover) {
+		await ensureReplicationConfig(this.#lc, this.#db, this.#subscriptionState, this.#shard, takeover.autoReset, takeover.purgeLock ?? void 0, void 0, takeover.pgChangeLogBehindSlot);
 		this.#pendingOwnership = {
 			...takeover,
 			head: "",