@rocicorp/zero 1.12.0-canary.2 → 1.12.0-canary.4

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\ntype ReplicationConfigRow = ReplicationConfig & {\n  resetRequired: boolean | null;\n};\n\nexport type ReplicaSubscriptionState = Pick<\n  SubscriptionState,\n  'publications' | 'replicaVersion' | 'watermark'\n>;\n\n/**\n * Validates the `replicationConfig` of the change db against the replica,\n * throwing an {@link AutoResetSignal} if the replica cannot be served from it.\n *\n * @param behindSlot Whether the change log is behind the replication slot,\n *     in which case it is re-initialized from the replica (see\n *     {@link reinitializeChangeLog}) and can thus be reset for a replica\n *     at any watermark.\n * @returns `'init'` if the tables are uninitialized, `'reset'` if they are\n *     for a different replica, or `'compatible'`.\n */\nfunction validateReplicationConfig(\n  lc: LogContext,\n  current: ReplicationConfigRow | undefined,\n  {publications, replicaVersion, watermark}: ReplicaSubscriptionState,\n  autoReset: boolean,\n  behindSlot: boolean,\n): 'init' | 'reset' | 'compatible' {\n  if (!current) {\n    return 'init';\n  }\n  if (\n    current.replicaVersion !== replicaVersion ||\n    !equals(new Set(current.publications), new Set(publications))\n  ) {\n    if (replicaVersion !== watermark && !behindSlot) {\n      throw new AutoResetSignal(\n        `Cannot reset change db@${current.replicaVersion} to ` +\n          `service replica@${replicaVersion} from watermark ${watermark}`,\n      );\n    }\n    return 'reset';\n  }\n  if (current.resetRequired) {\n    if (autoReset) {\n      throw new AutoResetSignal('reset required by replication stream');\n    }\n    lc.error?.(\n      '\\n\\n\\n' +\n        'Reset required but --auto-reset is not enabled.\\n' +\n        'This can happen for upstream databases that do not support event triggers.\\n' +\n        'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n        '\\n\\n\\n',\n    );\n  }\n  return 'compatible';\n}\n\nfunction selectReplicationConfig(sql: PostgresTransaction, shard: ShardID) {\n  return sql<ReplicationConfigRow[]> /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(cdcSchema(shard))}.\"replicationConfig\"`;\n}\n\n/**\n * Checks, without modifying the change db, that the replica can be served\n * by the change log once {@link ensureReplicationConfig} is called upon\n * taking over the change log. Throws an {@link AutoResetSignal} otherwise.\n *\n * This allows a replication-manager to detect an incompatible replica at\n * startup while leaving the change log to its current owner until then.\n */\nexport async function checkReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  behindSlot = false,\n): Promise<void> {\n  const [current] = await runTx(db, sql => selectReplicationConfig(sql, shard));\n  const status = validateReplicationConfig(\n    lc,\n    current,\n    subscriptionState,\n    autoReset,\n    behindSlot,\n  );\n  if (status === 'reset') {\n    lc.info?.(\n      `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n        `with replica @${subscriptionState.replicaVersion}. ` +\n        `Tables will be reset when taking over the change log.`,\n    );\n  }\n}\n\n/**\n * Initializes the cdc tables for the replica, resetting them if they were\n * written for a different replica (version or publications). This disrupts\n * a replication-manager that is serving the change log, and is thus only\n * done when taking over the change log (see `Storer.assumeOwnershipAtHead`).\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are reset.\n * @param behindSlot See {@link checkReplicationConfig}.\n */\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  behindSlot = false,\n) {\n  const {publications, replicaVersion} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const [current] = await selectReplicationConfig(sql, shard);\n    const status = validateReplicationConfig(\n      lc,\n      current,\n      subscriptionState,\n      autoReset,\n      behindSlot,\n    );\n    if (status === 'compatible') {\n      return [];\n    }\n    const stmts: PendingQuery<Row[]>[] = [];\n    if (status === 'reset') {\n      lc.info?.(\n        `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n          `with replica @${replicaVersion}. Clearing tables.`,\n      );\n      // Release any purge lock held by the caller; the changeLog is\n      // incompatible with the replica and needs to be truncated.\n      void purgeLock?.release();\n\n      // Note: The order of TRUNCATE matters. The replicationState table is\n      //       truncated before the changeLog table, in order to acquire\n      //       (exclusive) locks on the tables in the same order that the\n      //       storer.ts acquires locks when writing the changeLog, namely:\n      //\n      // 1. SELECT ... FROM replicationState FOR UPDATE;\n      // 2. INSERT INTO changeLog ...;\n      //    ...\n      // n. UPDATE replicationState ...;\n      stmts.push(\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      );\n    }\n    // Initialize (or re-initialize TRUNCATED) tables.\n    //\n    // The PG storer uses the earliest changeLog entry as the safe watermark\n    // from which subscribers can be resumed. These initial entries ensure\n    // that subscribers can start from a freshly synced replica, even if new\n    // changes have been replicated and not purged.\n    //\n    // TODO: Replace this with an explicit `firstWatermark` column in the\n    //       change db.\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark: replicaVersion, pos: 0, change: {tag: 'begin'}},\n      {watermark: replicaVersion, pos: 1, change: {tag: 'commit'}},\n    ];\n    stmts.push(\n      sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n      sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n      ...initialTx.map(\n        change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n      ),\n    );\n\n    if (status === 'reset') {\n      return runStatementsAndTerminateLockHolders(\n        lc,\n        db,\n        shard,\n        'ensureReplicationConfig',\n        stmts,\n        setTimeoutFn,\n      );\n    }\n    return Promise.all(stmts);\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,sBACvC,GAAG;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;;;;;;;;;;;;AAsBA,SAAS,0BACP,IACA,SACA,EAAC,cAAc,gBAAgB,aAC/B,WACA,YACiC;CACjC,IAAI,CAAC,SACH,OAAO;CAET,IACE,QAAQ,mBAAmB,kBAC3B,CAAC,OAAO,IAAI,IAAI,QAAQ,YAAY,GAAG,IAAI,IAAI,YAAY,CAAC,GAC5D;EACA,IAAI,mBAAmB,aAAa,CAAC,YACnC,MAAM,IAAI,gBACR,0BAA0B,QAAQ,eAAe,sBAC5B,eAAe,kBAAkB,WACxD;EAEF,OAAO;CACT;CACA,IAAI,QAAQ,eAAe;EACzB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;EAElE,GAAG,QACD,oOAKF;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBAAwB,KAA0B,OAAgB;CACzE,OAAO,GAAoC;;aAEhC,IAAI,UAAU,KAAK,CAAC,EAAE;AACnC;;;;;;;;;AAUA,eAAsB,uBACpB,IACA,IACA,mBACA,OACA,WACA,aAAa,OACE;CACf,MAAM,CAAC,WAAW,MAAM,MAAM,KAAI,QAAO,wBAAwB,KAAK,KAAK,CAAC;CAQ5E,IAPe,0BACb,IACA,SACA,mBACA,WACA,UAEE,MAAW,SACb,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,kBAAkB,eAAe,wDAEtD;AAEJ;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA,WACA,eAAkC,YAClC,aAAa,OACb;CACA,MAAM,EAAC,cAAc,mBAAkB;CACvC,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,CAAC,WAAW,MAAM,wBAAwB,KAAK,KAAK;EAC1D,MAAM,SAAS,0BACb,IACA,SACA,mBACA,WACA,UACF;EACA,IAAI,WAAW,cACb,OAAO,CAAC;EAEV,MAAM,QAA+B,CAAC;EACtC,IAAI,WAAW,SAAS;GACtB,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,eAAe,mBACpC;GAGA,WAAgB,QAAQ;GAWxB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;EACF;EAUA,MAAM,YAAkC,CACtC;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1D;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7D;EACA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;+CAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;EAEA,IAAI,WAAW,SACb,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;EAEF,OAAO,QAAQ,IAAI,KAAK;CAC1B,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,iCAChB,IAAI;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}
+{"version":3,"file":"tables.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {type PostgresDB, type PostgresTransaction} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {\n  BackfillID,\n  Change,\n  TableMetadata,\n} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n  return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n  return /*sql*/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n  // A strictly monotonically increasing, lexicographically sortable\n  // value that uniquely identifies a position in the change stream.\n  watermark: string;\n  change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n  // Note: The \"change\" column used to be JSONB, but that was problematic in that\n  // it does not handle the NULL unicode character.\n  // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"changeLog\" (\n    watermark  TEXT,\n    pos        INT8,\n    change     JSON NOT NULL,\n    precommit  TEXT,  -- Only exists on commit entries. Purely for debugging.\n    PRIMARY KEY (watermark, pos)\n  );\n`;\n}\n\n/**\n * Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n */\nexport type ReplicationState = {\n  lastWatermark: string;\n  owner: string | null;\n  ownerAddress: string | null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationState\" (\n    \"lastWatermark\" TEXT NOT NULL,\n    \"owner\" TEXT,\n    \"ownerAddress\" TEXT,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n  shard: ShardID,\n  sql: PostgresDB,\n): Promise<string | null> {\n  const result = await sql<{ownerAddress: string | null}[]> /*sql*/ `\n    SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n  return result[0].ownerAddress;\n}\n\n/**\n * This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n */\nexport type ReplicationConfig = {\n  replicaVersion: string;\n  publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n    \"replicaVersion\" TEXT NOT NULL,\n    \"publications\" TEXT[] NOT NULL,\n    \"resetRequired\" BOOL,\n    \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n  );\n`;\n}\n\nexport function createBackfillTables(shard: ShardID) {\n  return /*sql*/ `\n  CREATE TABLE ${schema(shard)}.\"tableMetadata\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"metadata\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\")\n  );\n\n  CREATE TABLE ${schema(shard)}.\"backfilling\" (\n    \"schema\" TEXT NOT NULL,\n    \"table\" TEXT NOT NULL,\n    \"column\" TEXT NOT NULL,\n    \"backfill\" JSONB NOT NULL,\n    PRIMARY KEY(\"schema\", \"table\", \"column\")\n  );\n  `;\n}\n\nexport type TableMetadataRow = {\n  schema: string;\n  table: string;\n  metadata: TableMetadata;\n};\n\nexport type BackfillingColumn = {\n  schema: string;\n  table: string;\n  column: string;\n  backfill: BackfillID;\n};\n\nfunction createTables(shard: ShardID) {\n  return (\n    createSchema(shard) +\n    createChangeLogTable(shard) +\n    createReplicationStateTable(shard) +\n    createReplicationConfigTable(shard) +\n    createBackfillTables(shard)\n  );\n}\n\ninterface PurgeLock {\n  release(): Promise<void>;\n}\n\nexport async function setupCDCTables(\n  lc: LogContext,\n  db: postgres.TransactionSql,\n  shard: ShardID,\n) {\n  lc.info?.(`Setting up CDC tables`);\n  await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n  const schema = cdcSchema(shard);\n  await sql`\n  UPDATE ${sql(schema)}.\"replicationConfig\"\n    SET \"resetRequired\" = true`;\n}\n\ntype ReplicationConfigRow = ReplicationConfig & {\n  resetRequired: boolean | null;\n};\n\nexport type ReplicaSubscriptionState = Pick<\n  SubscriptionState,\n  'publications' | 'replicaVersion' | 'watermark'\n>;\n\n/**\n * Validates the `replicationConfig` of the change db against the replica,\n * throwing an {@link AutoResetSignal} if the replica cannot be served from it.\n *\n * @param behindSlot Whether the change log is behind the replication slot,\n *     in which case it is re-initialized from the replica (see\n *     {@link reinitializeChangeLog}) and can thus be reset for a replica\n *     at any watermark.\n * @returns `'init'` if the tables are uninitialized, `'reset'` if they are\n *     for a different replica, or `'compatible'`.\n */\nfunction validateReplicationConfig(\n  lc: LogContext,\n  current: ReplicationConfigRow | undefined,\n  {publications, replicaVersion, watermark}: ReplicaSubscriptionState,\n  autoReset: boolean,\n  behindSlot: boolean,\n): 'init' | 'reset' | 'compatible' {\n  if (!current) {\n    return 'init';\n  }\n  if (\n    current.replicaVersion !== replicaVersion ||\n    !equals(new Set(current.publications), new Set(publications))\n  ) {\n    if (replicaVersion !== watermark && !behindSlot) {\n      throw new AutoResetSignal(\n        `Cannot reset change db@${current.replicaVersion} to ` +\n          `service replica@${replicaVersion} from watermark ${watermark}`,\n      );\n    }\n    return 'reset';\n  }\n  if (current.resetRequired) {\n    if (autoReset) {\n      throw new AutoResetSignal('reset required by replication stream');\n    }\n    lc.error?.(\n      '\\n\\n\\n' +\n        'Reset required but --auto-reset is not enabled.\\n' +\n        'This can happen for upstream databases that do not support event triggers.\\n' +\n        'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n        '\\n\\n\\n',\n    );\n  }\n  return 'compatible';\n}\n\nfunction selectReplicationConfig(sql: PostgresTransaction, shard: ShardID) {\n  return sql<ReplicationConfigRow[]> /*sql*/ `\n    SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n      FROM ${sql(cdcSchema(shard))}.\"replicationConfig\"`;\n}\n\n/**\n * Checks, without modifying the change db, that the replica can be served\n * by the change log once {@link ensureReplicationConfig} is called upon\n * taking over the change log. Throws an {@link AutoResetSignal} otherwise.\n *\n * This allows a replication-manager to detect an incompatible replica at\n * startup while leaving the change log to its current owner until then.\n */\nexport async function checkReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  behindSlot = false,\n): Promise<void> {\n  const [current] = await runTx(db, sql => selectReplicationConfig(sql, shard));\n  const status = validateReplicationConfig(\n    lc,\n    current,\n    subscriptionState,\n    autoReset,\n    behindSlot,\n  );\n  if (status === 'reset') {\n    lc.info?.(\n      `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n        `with replica @${subscriptionState.replicaVersion}. ` +\n        `Tables will be reset when taking over the change log.`,\n    );\n  }\n}\n\n/**\n * Initializes the cdc tables for the replica, resetting them if they were\n * written for a different replica (version or publications). This disrupts\n * a replication-manager that is serving the change log, and is thus only\n * done when taking over the change log (see `Storer.assumeOwnershipAtHead`).\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are reset.\n * @param behindSlot See {@link checkReplicationConfig}.\n */\nexport async function ensureReplicationConfig(\n  lc: LogContext,\n  db: PostgresDB,\n  subscriptionState: ReplicaSubscriptionState,\n  shard: ShardID,\n  autoReset: boolean,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n  behindSlot = false,\n) {\n  const {publications, replicaVersion} = subscriptionState;\n  const replicaConfig = {publications, replicaVersion};\n  const replicationState: ReplicationState = {\n    lastWatermark: replicaVersion,\n    owner: null,\n    ownerAddress: null,\n  };\n  const schema = cdcSchema(shard);\n\n  await runTx(db, async sql => {\n    const [current] = await selectReplicationConfig(sql, shard);\n    const status = validateReplicationConfig(\n      lc,\n      current,\n      subscriptionState,\n      autoReset,\n      behindSlot,\n    );\n    if (status === 'compatible') {\n      return [];\n    }\n    const stmts: PendingQuery<Row[]>[] = [];\n    if (status === 'reset') {\n      lc.info?.(\n        `Data in cdc tables @${current?.replicaVersion} is incompatible ` +\n          `with replica @${replicaVersion}. Clearing tables.`,\n      );\n      // Release any purge lock held by the caller; the changeLog is\n      // incompatible with the replica and needs to be truncated.\n      void purgeLock?.release();\n\n      // Note: The order of TRUNCATE matters. The replicationState table is\n      //       truncated before the changeLog table, in order to acquire\n      //       (exclusive) locks on the tables in the same order that the\n      //       storer.ts acquires locks when writing the changeLog, namely:\n      //\n      // 1. SELECT ... FROM replicationState FOR UPDATE;\n      // 2. INSERT INTO changeLog ...;\n      //    ...\n      // n. UPDATE replicationState ...;\n      stmts.push(\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      );\n    }\n    // Initialize (or re-initialize TRUNCATED) tables.\n    //\n    // The PG storer uses the earliest changeLog entry as the safe watermark\n    // from which subscribers can be resumed. These initial entries ensure\n    // that subscribers can start from a freshly synced replica, even if new\n    // changes have been replicated and not purged.\n    //\n    // TODO: Replace this with an explicit `firstWatermark` column in the\n    //       change db.\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark: replicaVersion, pos: 0, change: {tag: 'begin'}},\n      {watermark: replicaVersion, pos: 1, change: {tag: 'commit'}},\n    ];\n    stmts.push(\n      sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n      sql`INSERT INTO ${sql(schema)}.\"replicationState\"  ${sql(replicationState)} \n            ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n      ...initialTx.map(\n        change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n      ),\n    );\n\n    if (status === 'reset') {\n      return runStatementsAndTerminateLockHolders(\n        lc,\n        db,\n        shard,\n        'ensureReplicationConfig',\n        stmts,\n        setTimeoutFn,\n      );\n    }\n    return Promise.all(stmts);\n  });\n}\n\n// The time to wait for a TRUNCATE in ensureReplicationConfig before\n// terminating blocking backends via terminateChangeDBLockHolders.\nconst LOCK_HOLDER_TERMINATE_TIMEOUT_MS = 5_000;\n\n/**\n * Executes `stmts` that require ACCESS EXCLUSIVE locks (e.g. `TRUNCATE`),\n * which may be blocked by old storer catchup reads. Races against a\n * timeout that terminates the blocking backends if the `stmts` take too\n * long. The check is repeated until the statements complete, since the\n * `stmts` may not yet be waiting on a lock when the timer fires.\n */\nasync function runStatementsAndTerminateLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  name: string,\n  stmts: PendingQuery<Row[]>[],\n  setTimeoutFn: typeof setTimeout,\n) {\n  let done = false;\n  let timer: ReturnType<typeof setTimeout> | undefined;\n  const scheduleTerminate = () => {\n    timer = setTimeoutFn(async () => {\n      lc.info?.(`${name} blocked, terminating lock holders`);\n      try {\n        await terminateChangeDBLockHolders(lc, db, shard);\n      } catch (e) {\n        lc.warn?.('error terminating lock holders', e);\n      }\n      if (!done) {\n        scheduleTerminate();\n      }\n    }, LOCK_HOLDER_TERMINATE_TIMEOUT_MS);\n  };\n  scheduleTerminate();\n  try {\n    return await Promise.all(stmts);\n  } finally {\n    done = true;\n    clearTimeout(timer);\n  }\n}\n\nexport type ChangeLogReinitialization = {\n  /**\n   * The watermark of the replica from which the change log is re-initialized,\n   * i.e. the watermark from which the change stream will resume.\n   */\n  watermark: string;\n  cookies: {\n    tableMetadata: readonly {\n      schema: string;\n      table: string;\n      metadata: TableMetadata;\n    }[];\n    backfilling: readonly {\n      schema: string;\n      table: string;\n      column: string;\n      backfill: BackfillID;\n    }[];\n  };\n  owner: string;\n  ownerAddress: string;\n};\n\n/**\n * Re-initializes the change log from the state of the replica if it does not\n * contain all of the replica's changes, i.e. if its `lastWatermark` is before\n * the replica's watermark. This is the case if the change log is no longer\n * maintained (e.g. after running in high-availability mode), or if its last\n * owner stopped before persisting changes that it had already forwarded\n * (and backed up). Ownership of the change log is taken in the same\n * transaction, in which:\n *\n * * The `replicationState`, `changeLog`, `tableMetadata`, and `backfilling`\n *   tables are truncated. (The `replicationConfig` is kept;\n *   {@link ensureReplicationConfig} handles changes of the replica version\n *   or publications.)\n * * The change log is seeded with an empty transaction at the replica's\n *   watermark, so that subscribers at that watermark can be caught up, while\n *   subscribers at earlier watermarks are rejected as being too old.\n * * The `tableMetadata` and `backfilling` cookies are set from the replica.\n *\n * Returns `'maintained'` without modifying anything if the change log's\n * `lastWatermark` is at or past the replica's watermark, i.e. the change log\n * contains all of the replica's changes. Because a replication slot is only\n * acked past a transaction once it has been backed up, this means that the\n * change log contains every transaction up to the position of the replica's\n * replication slot, and it can be taken over (and resumed from its head).\n *\n * This is called when the change log's head is behind the position of the\n * replication slot (see `PgChangeLogPurgeLocker`). That position cannot be\n * compared with the watermarks of transactions to determine whether the\n * change log is maintained, as it is also advanced past positions of changes\n * outside of the publication.\n *\n * @param purgeLock A purge lock held by the caller, if any, which is released\n *     before the tables are truncated.\n */\nexport function reinitializeChangeLog(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n  {watermark, cookies, owner, ownerAddress}: ChangeLogReinitialization,\n  purgeLock?: PurgeLock,\n  setTimeoutFn: typeof setTimeout = setTimeout,\n): Promise<'reinitialized' | 'maintained'> {\n  const schema = cdcSchema(shard);\n  return runTx(db, async sql => {\n    // Lock the replicationState first, which is the order in which the\n    // storer acquires locks when writing the changeLog.\n    const state = await sql<{lastWatermark: string}[]>`\n      SELECT \"lastWatermark\" FROM ${sql(schema)}.\"replicationState\" FOR UPDATE`;\n    const lastWatermark = state[0]?.lastWatermark;\n    if (lastWatermark !== undefined && lastWatermark >= watermark) {\n      lc.info?.(\n        `changeLog@${lastWatermark} contains the replica's changes (@${watermark})`,\n      );\n      return 'maintained';\n    }\n    lc.info?.(\n      `re-initializing changeLog@${lastWatermark} from the replica @${watermark}`,\n    );\n    const replicationState: ReplicationState = {\n      lastWatermark: watermark,\n      owner,\n      ownerAddress,\n    };\n    const initialTx: FullChangeLogEntry[] = [\n      {watermark, pos: 0, change: {tag: 'begin'}},\n      {watermark, pos: 1, change: {tag: 'commit'}},\n    ];\n    // The caller's purge lock would block the TRUNCATE. Releasing it is safe:\n    // a concurrent purge checks the ownership after deleting, which blocks\n    // on the replicationState lock held by this transaction.\n    await purgeLock?.release();\n    await runStatementsAndTerminateLockHolders(\n      lc,\n      db,\n      shard,\n      'reinitializeChangeLog',\n      [\n        sql`TRUNCATE TABLE ${sql(schema)}.\"replicationState\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"tableMetadata\"`,\n        sql`TRUNCATE TABLE ${sql(schema)}.\"backfilling\"`,\n      ],\n      setTimeoutFn,\n    );\n    await sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)}`;\n    for (const entry of initialTx) {\n      await sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(entry)}`;\n    }\n    for (const row of cookies.tableMetadata) {\n      await sql`INSERT INTO ${sql(schema)}.\"tableMetadata\" ${sql({...row})}`;\n    }\n    for (const row of cookies.backfilling) {\n      await sql`INSERT INTO ${sql(schema)}.\"backfilling\" ${sql({...row})}`;\n    }\n    return 'reinitialized';\n  });\n}\n\nexport const CHANGE_STREAMER_APP_NAME = 'zero-change-streamer';\n\nexport class AutoResetSignal extends AbortError {\n  readonly name = 'AutoResetSignal';\n}\n\n/**\n * Terminates zero-cache backends that are blocking the current backend\n * from acquiring locks on CDC tables (e.g., during TRUNCATE).\n *\n * This is used during change-DB takeover when the new replication-manager's\n * `ensureReplicationConfig` needs to TRUNCATE tables, but the old\n * replication-manager's storer is still reading from them (e.g., large\n * catchup cursors).\n *\n * The function:\n * 1. Finds backends waiting for a lock on a TRUNCATE in {schema}\n * 2. Uses `pg_blocking_pids()` to identify which backends are blocking them\n * 3. Terminates blocking backends that have `application_name = 'zero-change-streamer'`\n *\n * Must be called on a **separate connection** from the one that is blocked,\n * since the blocked connection is inside a pending transaction.\n */\nexport async function terminateChangeDBLockHolders(\n  lc: LogContext,\n  db: PostgresDB,\n  shard: ShardID,\n) {\n  const schema = cdcSchema(shard);\n\n  // Step 1: Find backends that are blocked waiting for a lock,\n  // whose query involves a TRUNCATE on this shard's CDC schema.\n  const blocked = await db<{pid: number}[]>`\n    SELECT pid FROM pg_stat_activity\n      WHERE wait_event_type = 'Lock'\n        AND application_name = ${CHANGE_STREAMER_APP_NAME}\n        AND query LIKE ${'%TRUNCATE%' + schema + '%'}`;\n\n  if (blocked.length === 0) {\n    lc.info?.('no blocked TRUNCATE backends found');\n    return;\n  }\n\n  const blockedPids = blocked.map(r => r.pid);\n  lc.info?.(`found blocked TRUNCATE backends: ${JSON.stringify(blockedPids)}`);\n\n  // Step 2: For each blocked backend, find and terminate its blockers\n  // that are zero-change-streamer connections.\n  const terminated = await db<\n    {pid: number; applicationName: string; query: string; terminated: boolean}[]\n  >`\n    SELECT pid, application_name as \"applicationName\", query,\n           pg_terminate_backend(pid) as terminated\n      FROM pg_stat_activity\n      WHERE pid = ANY(\n        SELECT unnest(pg_blocking_pids(blocked.pid))\n          FROM unnest(${blockedPids}::int[]) AS blocked(pid)\n      )\n      AND application_name = ${CHANGE_STREAMER_APP_NAME}\n      AND pid != ALL(${blockedPids}::int[])`;\n\n  if (terminated.length === 0) {\n    lc.info?.(`no ${CHANGE_STREAMER_APP_NAME} blockers found to terminate`);\n  } else {\n    for (const {pid, applicationName, query, terminated: ok} of terminated) {\n      lc.info?.(\n        `terminated blocking backend pid=${pid} app=${applicationName} ok=${ok} query=${query.slice(0, 200)}`,\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAiBA,SAAS,OAAO,OAAgB;CAC9B,OAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;CACpC,OAAe,+BAA+B,OAAO,KAAK,EAAE;AAC9D;AAWA,SAAS,qBAAqB,OAAgB;CAI5C,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAYA,SAAgB,4BAA4B,OAAgB;CAC1D,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,eAAsB,8BACpB,OACA,KACwB;CAGxB,QAAO,MAFc,GAA6C;iCACnC,IAAI,UAAU,KAAK,CAAC,EAAE,qBAAA,CACvC,EAAE,CAAC;AACnB;AAaA,SAAS,6BAA6B,OAAgB;CACpD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;AAO/B;AAEA,SAAgB,qBAAqB,OAAgB;CACnD,OAAe;iBACA,OAAO,KAAK,EAAE;;;;;;;iBAOd,OAAO,KAAK,EAAE;;;;;;;;AAQ/B;AAeA,SAAS,aAAa,OAAgB;CACpC,OACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK,IAClC,qBAAqB,KAAK;AAE9B;AAMA,eAAsB,eACpB,IACA,IACA,OACA;CACA,GAAG,OAAO,uBAAuB;CACjC,MAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;CAEvE,MAAM,GAAG;WACA,IAFM,UAAU,KAEZ,CAAM,EAAE;;AAEvB;;;;;;;;;;;;AAsBA,SAAS,0BACP,IACA,SACA,EAAC,cAAc,gBAAgB,aAC/B,WACA,YACiC;CACjC,IAAI,CAAC,SACH,OAAO;CAET,IACE,QAAQ,mBAAmB,kBAC3B,CAAC,OAAO,IAAI,IAAI,QAAQ,YAAY,GAAG,IAAI,IAAI,YAAY,CAAC,GAC5D;EACA,IAAI,mBAAmB,aAAa,CAAC,YACnC,MAAM,IAAI,gBACR,0BAA0B,QAAQ,eAAe,sBAC5B,eAAe,kBAAkB,WACxD;EAEF,OAAO;CACT;CACA,IAAI,QAAQ,eAAe;EACzB,IAAI,WACF,MAAM,IAAI,gBAAgB,sCAAsC;EAElE,GAAG,QACD,oOAKF;CACF;CACA,OAAO;AACT;AAEA,SAAS,wBAAwB,KAA0B,OAAgB;CACzE,OAAO,GAAoC;;aAEhC,IAAI,UAAU,KAAK,CAAC,EAAE;AACnC;;;;;;;;;AAUA,eAAsB,uBACpB,IACA,IACA,mBACA,OACA,WACA,aAAa,OACE;CACf,MAAM,CAAC,WAAW,MAAM,MAAM,KAAI,QAAO,wBAAwB,KAAK,KAAK,CAAC;CAQ5E,IAPe,0BACb,IACA,SACA,mBACA,WACA,UAEE,MAAW,SACb,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,kBAAkB,eAAe,wDAEtD;AAEJ;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA,WACA,eAAkC,YAClC,aAAa,OACb;CACA,MAAM,EAAC,cAAc,mBAAkB;CACvC,MAAM,gBAAgB;EAAC;EAAc;CAAc;CACnD,MAAM,mBAAqC;EACzC,eAAe;EACf,OAAO;EACP,cAAc;CAChB;CACA,MAAM,SAAS,UAAU,KAAK;CAE9B,MAAM,MAAM,IAAI,OAAM,QAAO;EAC3B,MAAM,CAAC,WAAW,MAAM,wBAAwB,KAAK,KAAK;EAC1D,MAAM,SAAS,0BACb,IACA,SACA,mBACA,WACA,UACF;EACA,IAAI,WAAW,cACb,OAAO,CAAC;EAEV,MAAM,QAA+B,CAAC;EACtC,IAAI,WAAW,SAAS;GACtB,GAAG,OACD,uBAAuB,SAAS,eAAe,iCAC5B,eAAe,mBACpC;GAGA,WAAgB,QAAQ;GAWxB,MAAM,KACJ,GAAG,kBAAkB,IAAI,MAAM,EAAE,sBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,uBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,mBACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE,eACnC;EACF;EAUA,MAAM,YAAkC,CACtC;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1D;GAAC,WAAW;GAAgB,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7D;EACA,MAAM,KACJ,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,aAAa,KACtE,GAAG,eAAe,IAAI,MAAM,EAAE,uBAAuB,IAAI,gBAAgB,EAAE;+CAClC,IAAI,gBAAgB,KAC7D,GAAG,UAAU,KACX,WAAU,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,MAAM,GACnE,CACF;EAEA,IAAI,WAAW,SACb,OAAO,qCACL,IACA,IACA,OACA,2BACA,OACA,YACF;EAEF,OAAO,QAAQ,IAAI,KAAK;CAC1B,CAAC;AACH;AAIA,IAAM,mCAAmC;;;;;;;;AASzC,eAAe,qCACb,IACA,IACA,OACA,MACA,OACA,cACA;CACA,IAAI,OAAO;CACX,IAAI;CACJ,MAAM,0BAA0B;EAC9B,QAAQ,aAAa,YAAY;GAC/B,GAAG,OAAO,GAAG,KAAK,mCAAmC;GACrD,IAAI;IACF,MAAM,6BAA6B,IAAI,IAAI,KAAK;GAClD,SAAS,GAAG;IACV,GAAG,OAAO,kCAAkC,CAAC;GAC/C;GACA,IAAI,CAAC,MACH,kBAAkB;EAEtB,GAAG,gCAAgC;CACrC;CACA,kBAAkB;CAClB,IAAI;EACF,OAAO,MAAM,QAAQ,IAAI,KAAK;CAChC,UAAU;EACR,OAAO;EACP,aAAa,KAAK;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,sBACd,IACA,IACA,OACA,EAAC,WAAW,SAAS,OAAO,gBAC5B,WACA,eAAkC,YACO;CACzC,MAAM,SAAS,UAAU,KAAK;CAC9B,OAAO,MAAM,IAAI,OAAM,QAAO;EAK5B,MAAM,iBAAgB,MAFF,GAA8B;oCAClB,IAAI,MAAM,EAAE,gCAAA,CAChB,EAAE,EAAE;EAChC,IAAI,kBAAkB,KAAA,KAAa,iBAAiB,WAAW;GAC7D,GAAG,OACD,aAAa,cAAc,oCAAoC,UAAU,EAC3E;GACA,OAAO;EACT;EACA,GAAG,OACD,6BAA6B,cAAc,qBAAqB,WAClE;EACA,MAAM,mBAAqC;GACzC,eAAe;GACf;GACA;EACF;EACA,MAAM,YAAkC,CACtC;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,QAAO;EAAC,GAC1C;GAAC;GAAW,KAAK;GAAG,QAAQ,EAAC,KAAK,SAAQ;EAAC,CAC7C;EAIA,MAAM,WAAW,QAAQ;EACzB,MAAM,qCACJ,IACA,IACA,OACA,yBACA;GACE,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;GACjC,GAAG,kBAAkB,IAAI,MAAM,EAAE;EACnC,GACA,YACF;EACA,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,sBAAsB,IAAI,gBAAgB;EAC9E,KAAK,MAAM,SAAS,WAClB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,eAAe,IAAI,KAAK;EAE9D,KAAK,MAAM,OAAO,QAAQ,eACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,mBAAmB,IAAI,EAAC,GAAG,IAAG,CAAC;EAErE,KAAK,MAAM,OAAO,QAAQ,aACxB,MAAM,GAAG,eAAe,IAAI,MAAM,EAAE,iBAAiB,IAAI,EAAC,GAAG,IAAG,CAAC;EAEnE,OAAO;CACT,CAAC;AACH;AAEA,IAAa,2BAA2B;AAExC,IAAa,kBAAb,cAAqC,WAAW;CAC9C,OAAgB;AAClB;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,IACA,OACA;CAKA,MAAM,UAAU,MAAM,EAAmB;;;iCAGV,yBAAyB;yBACjC,eARR,UAAU,KAQa,IAAS;CAE/C,IAAI,QAAQ,WAAW,GAAG;EACxB,GAAG,OAAO,oCAAoC;EAC9C;CACF;CAEA,MAAM,cAAc,QAAQ,KAAI,MAAK,EAAE,GAAG;CAC1C,GAAG,OAAO,oCAAoC,KAAK,UAAU,WAAW,GAAG;CAI3E,MAAM,aAAa,MAAM,EAExB;;;;;;wBAMqB,YAAY;;+BAEL,yBAAyB;uBACjC,YAAY;CAEjC,IAAI,WAAW,WAAW,GACxB,GAAG,OAAO,MAAM,yBAAyB,6BAA6B;MAEtE,KAAK,MAAM,EAAC,KAAK,iBAAiB,OAAO,YAAY,QAAO,YAC1D,GAAG,OACD,mCAAmC,IAAI,OAAO,gBAAgB,MAAM,GAAG,SAAS,MAAM,MAAM,GAAG,GAAG,GACpG;AAGN"}

package/out/zero-cache/src/services/change-streamer/snapshot-message.js

@@ -37,6 +37,6 @@ var statusSchema = valita_exports.object({
 var statusMessageSchema = valita_exports.tuple([valita_exports.literal("status"), statusSchema]);
 valita_exports.union(statusMessageSchema);
 //#endregion
-export { statusSchema };
+export { statusMessageSchema, statusSchema };
 
 //# sourceMappingURL=snapshot-message.js.map

package/out/zero-cache/src/services/change-streamer/snapshot-message.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-message.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"sourcesContent":["/**\n * Pure schema definitions for the `/snapshot` reservation status message.\n *\n * These live in their own module (depending only on valita) so that both\n * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and\n * {@link subscribe.ts} can import them without forming an import cycle.\n */\n\nimport * as v from '../../../../shared/src/valita.ts';\n\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n\n  /**\n   * The size in bytes of the replication-manager's replica when the snapshot\n   * was reserved, used as an estimate of the size of the restored replica\n   * when reporting restore progress. Absent from older replication-managers.\n   */\n  replicaSize: v.number().optional(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n"],"mappings":";;;;;;;;;AAUA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAW,eAAE,OAAO;;;;;;;CAQpB,gBAAgB,eAAE,OAAO;;;;;;;CAQzB,cAAc,eAAE,OAAO;;;;;;CAOvB,aAAa,eAAE,OAAO,EAAE,SAAS;AACnC,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEzC,eAAE,MAAM,mBAAmB"}
+{"version":3,"file":"snapshot-message.js","names":["v"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-message.ts"],"sourcesContent":["/**\n * Pure schema definitions for the `/snapshot` reservation status message.\n *\n * These live in their own module (depending only on valita) so that both\n * {@link snapshot.ts} (which also pulls in the change-streamer HTTP client) and\n * {@link subscribe.ts} can import them without forming an import cycle.\n */\n\nimport * as v from '../../../../shared/src/valita.ts';\n\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  /**\n   * The location from which litestream should perform the restore.\n   */\n  backupURL: v.string(),\n\n  /**\n   * The `replicaVersion` of the backup. If a subscriber's restored or\n   * existing replica is of a different version, it should delete it and\n   * retry the restore from litestream (i.e. equivalent to a\n   * `WrongReplicaVersion` response from a `/changes` subscription).\n   */\n  replicaVersion: v.string(),\n\n  /**\n   * The earliest watermark from which catchup is possible. If the\n   * subscriber's replica is older that this watermark, it should delete it\n   * and (retry the) restore from litestream (i.e. equivalent to a\n   * `WatermarkTooOld` response from a `/changes` subscription).\n   */\n  minWatermark: v.string(),\n\n  /**\n   * The size in bytes of the replication-manager's replica when the snapshot\n   * was reserved, used as an estimate of the size of the restored replica\n   * when reporting restore progress. Absent from older replication-managers.\n   */\n  replicaSize: v.number().optional(),\n});\n\nexport type SnapshotStatus = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\nexport const snapshotMessageSchema = v.union(statusMessageSchema);\n\nexport type SnapshotMessage = v.Infer<typeof statusMessageSchema>;\n"],"mappings":";;;;;;;;;AAUA,IAAa,eAAeA,eAAE,OAAO;CACnC,KAAKA,eAAE,QAAQ,QAAQ;;;;CAKvB,WAAWA,eAAE,OAAO;;;;;;;CAQpB,gBAAgBA,eAAE,OAAO;;;;;;;CAQzB,cAAcA,eAAE,OAAO;;;;;;CAOvB,aAAaA,eAAE,OAAO,CAAC,CAAC,SAAS;AACnC,CAAC;AAID,IAAa,sBAAsBA,eAAE,MAAM,CAACA,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAEzCA,eAAE,MAAM,mBAAmB"}

package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-reservations.js","names":["#lc","#backupConfig","#callbacks","#reservations","#close","#metricAttrs","#replicaSize","#downstream","#watermark","#delayNoted"],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport type ReservationCallbacks = {\n  /**\n   * Called whenever a reservation is closed, including when it is replaced\n   * by a new reservation (e.g. a retry) for the same task.\n   */\n  onClose?: ((taskID: string) => void) | undefined;\n\n  /**\n   * Called when a task's reservation is closed without being replaced,\n   * i.e. when the task no longer holds a reservation.\n   */\n  onEnd?: ((taskID: string) => void) | undefined;\n};\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #callbacks: ReservationCallbacks;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    callbacks: ReservationCallbacks = {},\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#callbacks = callbacks;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.#close(taskID, undefined, true);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(\n    taskID: string,\n    cancelledInstanceID: InstanceID | undefined,\n    replaced = false,\n  ) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#callbacks.onClose?.(taskID);\n      if (!replaced) {\n        this.#callbacks.onEnd?.(taskID);\n      }\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AA2BA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,YAAkC,CAAC,GACnC;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAKC,gBAAgB;EACrB,KAAKC,aAAa;CACpB;CAEA,KAAK,QAAyC;EAC5C,KAAKE,OAAO,QAAQ,KAAA,GAAW,IAAI;EAEnC,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAKA,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAKD,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAKH,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAKI,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAKD,cAAc,IAAI,MAAM,GAAG,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAKF,cAAc,WACnB,KAAKA,cAAc,mBACnB,aACF;CACF;CAEA,OACE,QACA,qBACA,WAAW,OACX;EACA,MAAM,MAAM,KAAKE,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAKA,cAAc,OAAO,MAAM;GAChC,KAAKD,WAAW,UAAU,MAAM;GAChC,IAAI,CAAC,UACH,KAAKA,WAAW,QAAQ,MAAM;GAEhC,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAKF,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,EAAE,SAAS,UAAU;IACzD,GAAG,KAAKK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAKF,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAKA,cAAc,QAAQ,CAAC,EACpC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,EACpD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAKA,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAKH,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAKC,cAAc,WACnB,gBACA,cACA,KAAKK,aAAa,CACpB;GAGA,6CAA6C,EAAE,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAKD,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAKJ;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,EAAE;EAC/B,SAAS,GAAG;GACV,KAAKD,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAKG,cAAc,IAAI,MAAM,GAAG,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAKA,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,EAAE,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAKI,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAKC;CACd;CAEA,YAAY;EACV,OAAO,KAAKA,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAKD,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAKE,aACP,OAAO;EAET,KAAKA,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAKD,eAAe,MAAM;GAC5B,IAAI,KAAKD,YAAY,QACnB,KAAKA,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAKC,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAKD,YAAY,OAAO;CAC1B;AACF"}
+{"version":3,"file":"snapshot-reservations.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/snapshot-reservations.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n  litestreamMonitorMetricAttrs,\n  litestreamSnapshotReservationConfirmDuration,\n  litestreamSnapshotReservationDuration,\n} from '../litestream/metrics.ts';\nimport type {BackupConfig} from './change-streamer-service.ts';\nimport type {SnapshotMessage} from './snapshot-message.ts';\nimport type {ChangeLogReadSource} from './sqlite-change-log-read-router.ts';\n\nexport type ReservationCallbacks = {\n  /**\n   * Called whenever a reservation is closed, including when it is replaced\n   * by a new reservation (e.g. a retry) for the same task.\n   */\n  onClose?: ((taskID: string) => void) | undefined;\n\n  /**\n   * Called when a task's reservation is closed without being replaced,\n   * i.e. when the task no longer holds a reservation.\n   */\n  onEnd?: ((taskID: string) => void) | undefined;\n};\n\nexport class SnapshotReservations {\n  readonly #lc: LogContext;\n  readonly #backupConfig: BackupConfig;\n  readonly #callbacks: ReservationCallbacks;\n  readonly #reservations = new Map<string, Reservation>();\n\n  constructor(\n    lc: LogContext,\n    backupConfig: BackupConfig,\n    callbacks: ReservationCallbacks = {},\n  ) {\n    this.#lc = lc.withContext('component', 'snapshot-reserver');\n    this.#backupConfig = backupConfig;\n    this.#callbacks = callbacks;\n  }\n\n  open(taskID: string): Source<SnapshotMessage> {\n    this.#close(taskID, undefined, true);\n\n    const instanceID = {};\n    const downstream = Subscription.create<SnapshotMessage>({\n      cleanup: () => this.#close(taskID, instanceID),\n    });\n    this.#reservations.set(taskID, new Reservation(instanceID, downstream));\n    this.#lc.info?.(`created snapshot reservation for ${taskID}`);\n    return downstream;\n  }\n\n  close(taskID: string) {\n    this.#close(taskID, undefined);\n  }\n\n  isCurrent(taskID: string, source: Source<SnapshotMessage>): boolean {\n    return this.#reservations.get(taskID)?.owns(source) ?? false;\n  }\n\n  #metricAttrs() {\n    return litestreamMonitorMetricAttrs(\n      this.#backupConfig.backupURL,\n      this.#backupConfig.litestreamVersion,\n      'view_syncer',\n    );\n  }\n\n  #close(\n    taskID: string,\n    cancelledInstanceID: InstanceID | undefined,\n    replaced = false,\n  ) {\n    const res = this.#reservations.get(taskID);\n    if (\n      res &&\n      (!cancelledInstanceID || res.instanceID === cancelledInstanceID)\n    ) {\n      // Note: delete first, so that the reservation is gone when close() is called.\n      this.#reservations.delete(taskID);\n      this.#callbacks.onClose?.(taskID);\n      if (!replaced) {\n        this.#callbacks.onEnd?.(taskID);\n      }\n      res.close();\n\n      const duration = Date.now() - res.startTime.getTime();\n      this.#lc.info?.(\n        `ended snapshot reservation for ${taskID} (${duration} ms)`,\n      );\n      // `result` reports how the reservation ended and `confirmed` whether it\n      // ever received its bounds. They are separate dimensions: a follower\n      // that gives up while waiting for a confirmation is\n      // `result=cancelled, confirmed=false`, which a single collapsed\n      // attribute cannot distinguish from a client that simply went away.\n      litestreamSnapshotReservationDuration().recordMs(duration, {\n        ...this.#metricAttrs(),\n        result: cancelledInstanceID ? 'cancelled' : 'closed',\n        confirmed: res.confirmed(),\n      });\n    }\n  }\n\n  confirmationsRequired() {\n    for (const res of this.#reservations.values()) {\n      if (!res.confirmed()) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  unconfirmedTaskIDs(): string[] {\n    return [...this.#reservations.entries()]\n      .filter(([, reservation]) => !reservation.confirmed())\n      .map(([taskID]) => taskID);\n  }\n\n  /** Confirms one reservation with the bounds of its pinned read source. */\n  confirmFor(\n    taskID: string,\n    replicaVersion: string,\n    minWatermark: string,\n    source: ChangeLogReadSource,\n  ): void {\n    const res = this.#reservations.get(taskID);\n    if (res && !res.confirmed()) {\n      this.#lc.info?.(\n        `reserving change-log entries since ${minWatermark} for ${taskID}`,\n      );\n      res.confirm(\n        this.#backupConfig.backupURL,\n        replicaVersion,\n        minWatermark,\n        this.#replicaSize(),\n      );\n      // Measured from `open()`, not from the first confirmation attempt: what\n      // matters is how long the follower waited before it could restore.\n      litestreamSnapshotReservationConfirmDuration().recordMs(\n        Date.now() - res.startTime.getTime(),\n        {...this.#metricAttrs(), source},\n      );\n    }\n  }\n\n  #replicaSize(): number | undefined {\n    const {replicaFile} = this.#backupConfig;\n    if (!replicaFile) {\n      return undefined;\n    }\n    try {\n      return statSync(replicaFile).size;\n    } catch (e) {\n      this.#lc.warn?.(`Unable to get the size of ${replicaFile}`, e);\n      return undefined;\n    }\n  }\n\n  /**\n   * Notes that a confirmation was deferred, returning true only the first time\n   * for this reservation. Confirmation is retried on every backup, so an\n   * undeduplicated count would measure backups rather than delayed followers.\n   */\n  noteConfirmationDelayed(taskID: string): boolean {\n    return this.#reservations.get(taskID)?.noteDelayed() ?? false;\n  }\n\n  getReservedWatermarks() {\n    return Array.from(\n      this.#reservations.values(),\n      ({reservedWatermark}) => reservedWatermark,\n    ).filter(watermark => watermark !== null);\n  }\n}\n\ntype InstanceID = {};\n\nclass Reservation {\n  readonly instanceID: InstanceID;\n  readonly startTime: Date = new Date();\n  readonly #downstream: Subscription<SnapshotMessage>;\n  #watermark: string | null = null;\n  #delayNoted = false;\n\n  constructor(\n    instanceID: InstanceID,\n    downstream: Subscription<SnapshotMessage>,\n  ) {\n    this.instanceID = instanceID;\n    this.#downstream = downstream;\n  }\n\n  get reservedWatermark() {\n    return this.#watermark;\n  }\n\n  confirmed() {\n    return this.#watermark !== null;\n  }\n\n  owns(source: Source<SnapshotMessage>): boolean {\n    return this.#downstream === source;\n  }\n\n  noteDelayed(): boolean {\n    if (this.#delayNoted) {\n      return false;\n    }\n    this.#delayNoted = true;\n    return true;\n  }\n\n  confirm(\n    backupURL: string,\n    replicaVersion: string,\n    minWatermark: string,\n    replicaSize: number | undefined,\n  ) {\n    if (this.#watermark === null) {\n      if (this.#downstream.active) {\n        this.#downstream.push([\n          'status',\n          {tag: 'status', backupURL, replicaVersion, minWatermark, replicaSize},\n        ]);\n      }\n      this.#watermark = minWatermark;\n    }\n  }\n\n  close() {\n    this.#downstream.cancel();\n  }\n}\n"],"mappings":";;;;AA2BA,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA,gCAAyB,IAAI,IAAyB;CAEtD,YACE,IACA,cACA,YAAkC,CAAC,GACnC;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,mBAAmB;EAC1D,KAAK,gBAAgB;EACrB,KAAK,aAAa;CACpB;CAEA,KAAK,QAAyC;EAC5C,KAAK,OAAO,QAAQ,KAAA,GAAW,IAAI;EAEnC,MAAM,aAAa,CAAC;EACpB,MAAM,aAAa,aAAa,OAAwB,EACtD,eAAe,KAAK,OAAO,QAAQ,UAAU,EAC/C,CAAC;EACD,KAAK,cAAc,IAAI,QAAQ,IAAI,YAAY,YAAY,UAAU,CAAC;EACtE,KAAK,IAAI,OAAO,oCAAoC,QAAQ;EAC5D,OAAO;CACT;CAEA,MAAM,QAAgB;EACpB,KAAK,OAAO,QAAQ,KAAA,CAAS;CAC/B;CAEA,UAAU,QAAgB,QAA0C;EAClE,OAAO,KAAK,cAAc,IAAI,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK;CACzD;CAEA,eAAe;EACb,OAAO,6BACL,KAAK,cAAc,WACnB,KAAK,cAAc,mBACnB,aACF;CACF;CAEA,OACE,QACA,qBACA,WAAW,OACX;EACA,MAAM,MAAM,KAAK,cAAc,IAAI,MAAM;EACzC,IACE,QACC,CAAC,uBAAuB,IAAI,eAAe,sBAC5C;GAEA,KAAK,cAAc,OAAO,MAAM;GAChC,KAAK,WAAW,UAAU,MAAM;GAChC,IAAI,CAAC,UACH,KAAK,WAAW,QAAQ,MAAM;GAEhC,IAAI,MAAM;GAEV,MAAM,WAAW,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ;GACpD,KAAK,IAAI,OACP,kCAAkC,OAAO,IAAI,SAAS,KACxD;GAMA,sCAAsC,CAAC,CAAC,SAAS,UAAU;IACzD,GAAG,KAAK,aAAa;IACrB,QAAQ,sBAAsB,cAAc;IAC5C,WAAW,IAAI,UAAU;GAC3B,CAAC;EACH;CACF;CAEA,wBAAwB;EACtB,KAAK,MAAM,OAAO,KAAK,cAAc,OAAO,GAC1C,IAAI,CAAC,IAAI,UAAU,GACjB,OAAO;EAGX,OAAO;CACT;CAEA,qBAA+B;EAC7B,OAAO,CAAC,GAAG,KAAK,cAAc,QAAQ,CAAC,CAAC,CACrC,QAAQ,GAAG,iBAAiB,CAAC,YAAY,UAAU,CAAC,CAAC,CACrD,KAAK,CAAC,YAAY,MAAM;CAC7B;;CAGA,WACE,QACA,gBACA,cACA,QACM;EACN,MAAM,MAAM,KAAK,cAAc,IAAI,MAAM;EACzC,IAAI,OAAO,CAAC,IAAI,UAAU,GAAG;GAC3B,KAAK,IAAI,OACP,sCAAsC,aAAa,OAAO,QAC5D;GACA,IAAI,QACF,KAAK,cAAc,WACnB,gBACA,cACA,KAAK,aAAa,CACpB;GAGA,6CAA6C,CAAC,CAAC,SAC7C,KAAK,IAAI,IAAI,IAAI,UAAU,QAAQ,GACnC;IAAC,GAAG,KAAK,aAAa;IAAG;GAAM,CACjC;EACF;CACF;CAEA,eAAmC;EACjC,MAAM,EAAC,gBAAe,KAAK;EAC3B,IAAI,CAAC,aACH;EAEF,IAAI;GACF,OAAO,SAAS,WAAW,CAAC,CAAC;EAC/B,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,6BAA6B,eAAe,CAAC;GAC7D;EACF;CACF;;;;;;CAOA,wBAAwB,QAAyB;EAC/C,OAAO,KAAK,cAAc,IAAI,MAAM,CAAC,EAAE,YAAY,KAAK;CAC1D;CAEA,wBAAwB;EACtB,OAAO,MAAM,KACX,KAAK,cAAc,OAAO,IACzB,EAAC,wBAAuB,iBAC3B,CAAC,CAAC,QAAO,cAAa,cAAc,IAAI;CAC1C;AACF;AAIA,IAAM,cAAN,MAAkB;CAChB;CACA,4BAA2B,IAAI,KAAK;CACpC;CACA,aAA4B;CAC5B,cAAc;CAEd,YACE,YACA,YACA;EACA,KAAK,aAAa;EAClB,KAAK,cAAc;CACrB;CAEA,IAAI,oBAAoB;EACtB,OAAO,KAAK;CACd;CAEA,YAAY;EACV,OAAO,KAAK,eAAe;CAC7B;CAEA,KAAK,QAA0C;EAC7C,OAAO,KAAK,gBAAgB;CAC9B;CAEA,cAAuB;EACrB,IAAI,KAAK,aACP,OAAO;EAET,KAAK,cAAc;EACnB,OAAO;CACT;CAEA,QACE,WACA,gBACA,cACA,aACA;EACA,IAAI,KAAK,eAAe,MAAM;GAC5B,IAAI,KAAK,YAAY,QACnB,KAAK,YAAY,KAAK,CACpB,UACA;IAAC,KAAK;IAAU;IAAW;IAAgB;IAAc;GAAW,CACtE,CAAC;GAEH,KAAK,aAAa;EACpB;CACF;CAEA,QAAQ;EACN,KAAK,YAAY,OAAO;CAC1B;AACF"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js

@@ -339,6 +339,6 @@ var SQLiteChangeLogBarrierBacklogError = class extends SQLiteChangeLogBarrierErr
 	name = "SQLiteChangeLogBarrierBacklogError";
 };
 //#endregion
-export { SQLiteChangeLogCatchup };
+export { SQLiteChangeLogBarrierBacklogError, SQLiteChangeLogBarrierError, SQLiteChangeLogBarrierTimeoutError, SQLiteChangeLogCatchup };
 
 //# sourceMappingURL=sqlite-change-log-catchup.js.map

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-catchup.js","names":["#lc","#forwarder","#reader","#batchSize","#barrierTimeoutMs","#barrierPollIntervalMs","#cleanupGuard","#sleep","#now","#onFailure","#barrierTimeouts","#barrierBacklogOverflows","#barrierWakeups","#catchupResults","#catchupRows","#catchupBytes","#catchupDuration","#barrierWaitDuration","#catchupBacklogPeak","#catchups","#commitWaiters","#closed","#throwIfAborted","#run","#awaitRequiredHead","#waitForPlan","#awaitChangeLogProgress"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n  /** Opens slice 11's process-local breaker after registration has committed. */\n  onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;\n};\n\nexport type SQLiteChangeLogCatchupRequest = {\n  /**\n   * Whether the routing warm-up gate classified the log as warm *for this\n   * subscriber*. The coordinator is opened once and reused, so this cannot be\n   * a property of it: `coldReadPercent` serves some subscribers from a log\n   * that is still inside its warm-up window, and the catchup metrics have to\n   * separate those from steady-state serving.\n   */\n  readonly logWarm?: boolean | undefined;\n};\n\nexport type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';\n\nexport type SQLiteChangeLogCatchupRegistration =\n  | {readonly kind: 'registered'}\n  | {readonly kind: 'uncovered'; readonly minWatermark: string}\n  // Registration failed before the subscriber was committed to SQLite, so PG\n  // catchup is still available and the caller falls back to it.\n  | {readonly kind: 'declined'; readonly error: unknown}\n  // The coordinator closed or failed the subscriber itself.\n  | {readonly kind: 'handled'};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #onFailure:\n    | ((failure: SQLiteChangeLogCatchupFailure) => void)\n    | undefined;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchupResults = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_results',\n    'SQLite catchup subscriptions by outcome.',\n  );\n  readonly #catchupRows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_rows',\n    'Rows served from the SQLite change log during catchup.',\n  );\n  readonly #catchupBytes = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_bytes',\n    {\n      description: 'Bytes served from the SQLite change log during catchup.',\n      unit: 'By',\n    },\n  );\n  readonly #catchupDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.catchup_duration',\n    'Duration of a SQLite change-log catchup.',\n  );\n  readonly #barrierWaitDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.barrier_wait_duration',\n    'Time SQLite catchup waits for its pinned required head.',\n  );\n  readonly #catchupBacklogPeak = getOrCreateValueHistogram(\n    'replication',\n    'sqlite_change_log.catchup_backlog_peak_bytes',\n    {\n      description:\n        'Peak live backlog bytes buffered while a subscriber catches up from SQLite.',\n      unit: 'By',\n      bucketBoundaries: [\n        0, 1024, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216,\n      ],\n    },\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n    this.#onFailure = opts.onFailure;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n    request: SQLiteChangeLogCatchupRequest = {},\n  ): Promise<SQLiteChangeLogCatchupRegistration> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return {kind: 'handled'};\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    let uncoveredMinWatermark: string | undefined;\n    let committed = false;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        // The router's eligibility inspection happens before this async guard\n        // is acquired. Recheck the exact boundary here: an in-flight purge can\n        // have advanced the minimum while registration waited. Once added to\n        // the Forwarder, the subscriber's ACK prevents any later purge from\n        // crossing this boundary.\n        const plan = this.#reader.plan(subscriber.watermark);\n        if (plan.kind === 'too-old') {\n          uncoveredMinWatermark = plan.minWatermark;\n          return;\n        }\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n        committed = true;\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (abort.signal.aborted) {\n        return {kind: 'handled'};\n      }\n      if (!committed) {\n        // Nothing was committed to SQLite -- the subscriber never reached\n        // Forwarder.add() -- so PG catchup is still available. Failing here\n        // would send the client a terminal ['error', ...], which\n        // IncrementalSyncer answers with a full litestream replica restore,\n        // for a fallback that was still open. Trip the breaker as well: this\n        // is a reader failure like any other, and without it the retry picks\n        // SQLite again and the reset repeats.\n        this.#onFailure?.('reader-error');\n        return {kind: 'declined', error};\n      }\n      this.#lc.error?.(\n        `error while registering SQLite catchup subscriber ${subscriber.id}`,\n        error,\n      );\n      subscriber.fail(error);\n      return {kind: 'handled'};\n    }\n    if (uncoveredMinWatermark !== undefined) {\n      this.#catchups.delete(subscriber);\n      return {kind: 'uncovered', minWatermark: uncoveredMinWatermark};\n    }\n\n    void this.#run(\n      subscriber,\n      requiredHead as string | Promise<string>,\n      abort,\n      request.logWarm,\n    );\n    return {kind: 'registered'};\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n    logWarm: boolean | undefined,\n  ): Promise<void> {\n    const {signal} = abort;\n    const catchupStart = this.#now();\n    let outcome = 'aborted';\n    let backlogPeakBytes = 0;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const barrierStart = this.#now();\n      let plan: CatchupPlan;\n      try {\n        plan = await this.#waitForPlan(subscriber, required, deadline, signal);\n      } finally {\n        // Record failed and aborted barriers too; their wait distribution is\n        // at least as operationally important as the successful path.\n        this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);\n      }\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        outcome = 'too-old';\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      let bytes = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n            bytes += Buffer.byteLength(change[2]);\n          }\n          backlogPeakBytes = Math.max(\n            backlogPeakBytes,\n            subscriber.getStats().backlogBytes,\n          );\n        }\n        await lastBatchConsumed;\n        outcome = 'range';\n      } else {\n        outcome = 'ahead';\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupRows.add(count, attributes);\n      this.#catchupBytes.add(bytes, attributes);\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n          outcome = 'barrier-timeout';\n          this.#onFailure?.('barrier-timeout');\n        } else {\n          // No #onFailure: a backlog high water mark is a property of this\n          // one subscriber's consumption, not of the log. Tripping the\n          // process-wide breaker for it would take every task in the shard\n          // off SQLite, which a client that reconnects and stalls again\n          // could hold open indefinitely.\n          outcome = 'barrier-backlog';\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      outcome = 'reader-error';\n      this.#onFailure?.('reader-error');\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      backlogPeakBytes = Math.max(\n        backlogPeakBytes,\n        subscriber.getStats().backlogBytes,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupResults.add(1, attributes);\n      this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);\n      this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAuCA,IAAM,mCAAmC;AAmCzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,kBAA2B,mBACzB,eACA,qCACA,0CACF;CACA,eAAwB,mBACtB,eACA,kCACA,wDACF;CACA,gBAAyB,mBACvB,eACA,mCACA;EACE,aAAa;EACb,MAAM;CACR,CACF;CACA,mBAA4B,4BAC1B,eACA,sCACA,0CACF;CACA,uBAAgC,4BAC9B,eACA,2CACA,yDACF;CACA,sBAA+B,0BAC7B,eACA,gDACA;EACE,aACE;EACF,MAAM;EACN,kBAAkB;GAChB;GAAG;GAAM;GAAQ;GAAQ;GAAS;GAAW;GAAW;EAC1D;CACF,CACF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,aAAa;EAClB,KAAKC,UAAU;EACf,KAAKC,aAAa,KAAK;EACvB,KAAKC,oBAAoB,KAAK;EAC9B,KAAKC,yBACH,KAAK,yBAAyB;EAChC,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKC,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAKC,aAAa,KAAK;CACzB;;;;;;CAOA,MAAM,QACJ,YACA,qBACA,UAAyC,CAAC,GACG;EAC7C,IAAI,KAAKY,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE,OAAO,EAAC,MAAM,UAAS;EACzB;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAKF,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI;GACF,MAAM,KAAKb,cAAc,2BAA2B;IAClD,KAAKgB,gBAAgB,MAAM,MAAM;IAMjC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;IACnD,IAAI,KAAK,SAAS,WAAW;KAC3B,wBAAwB,KAAK;KAC7B;IACF;IACA,eAAe,oBAAoB;IACnC,KAAKD,WAAW,IAAI,UAAU;IAC9B,YAAY;GACd,CAAC;EACH,SAAS,OAAO;GACd,KAAKkB,UAAU,OAAO,UAAU;GAChC,IAAI,MAAM,OAAO,SACf,OAAO,EAAC,MAAM,UAAS;GAEzB,IAAI,CAAC,WAAW;IAQd,KAAKV,aAAa,cAAc;IAChC,OAAO;KAAC,MAAM;KAAY;IAAK;GACjC;GACA,KAAKT,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;GACA,WAAW,KAAK,KAAK;GACrB,OAAO,EAAC,MAAM,UAAS;EACzB;EACA,IAAI,0BAA0B,KAAA,GAAW;GACvC,KAAKmB,UAAU,OAAO,UAAU;GAChC,OAAO;IAAC,MAAM;IAAa,cAAc;GAAqB;EAChE;EAEA,KAAUI,KACR,YACA,cACA,OACA,QAAQ,OACV;EACA,OAAO,EAAC,MAAM,aAAY;CAC5B;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAKH,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAKA,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAKD,UAAU,IAAI,UAAU,GAAG,MAAM;EACtC,KAAKA,UAAU,OAAO,UAAU;EAChC,KAAKlB,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAKoB,SACP;EAEF,KAAKA,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAKF,WAAW;GAChD,MAAM,MAAM;GACZ,KAAKlB,WAAW,OAAO,UAAU;EACnC;EACA,KAAKkB,UAAU,MAAM;EACrB,KAAKjB,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAMqB,KACJ,YACA,cACA,OACA,SACe;EACf,MAAM,EAAC,WAAU;EACjB,MAAM,eAAe,KAAKf,KAAK;EAC/B,IAAI,UAAU;EACd,IAAI,mBAAmB;EACvB,IAAI;GACF,MAAM,oBAAoB,KAAKA,KAAK;GACpC,MAAM,WAAW,MAAM,KAAKgB,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAKhB,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAKJ,mBAC5B,KAAKJ,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAKQ,KAAK,IAAI,KAAKJ;GACpC,MAAM,eAAe,KAAKI,KAAK;GAC/B,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,KAAKiB,aAAa,YAAY,UAAU,UAAU,MAAM;GACvE,UAAU;IAGR,KAAKR,qBAAqB,SAAS,KAAKT,KAAK,IAAI,YAAY;GAC/D;GACA,KAAKc,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,UAAU;IACV,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAKtB,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAKQ,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAKN,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAKC,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAKK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAKA,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAKR,IAAI,UAAU,MAAM,SAAS,WAChC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAKsB,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;MACA,SAAS,OAAO,WAAW,OAAO,EAAE;KACtC;KACA,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;IACF;IACA,MAAM;IACN,UAAU;GACZ,OAAO;IACL,UAAU;IACV,KAAKtB,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GACF;GAEA,KAAKsB,gBAAgB,MAAM;GAC3B,KAAKtB,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAKQ,KAAK,IAAI,MAAM,KAC5B;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKM,aAAa,IAAI,OAAO,UAAU;GACvC,KAAKC,cAAc,IAAI,OAAO,UAAU;GAGxC,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCAAoC;KACvD,KAAKL,iBAAiB,IAAI,CAAC;KAC3B,UAAU;KACV,KAAKD,aAAa,iBAAiB;IACrC,OAME,UAAU;IAUZ,KAAKT,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,UAAU;GACV,KAAKS,aAAa,cAAc;GAChC,KAAKT,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,EAAE,YACxB;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAKa,gBAAgB,IAAI,GAAG,UAAU;GACtC,KAAKG,iBAAiB,SAAS,KAAKR,KAAK,IAAI,cAAc,UAAU;GACrE,KAAKU,oBAAoB,OAAO,kBAAkB,UAAU;GAC5D,IAAI,KAAKC,UAAU,IAAI,UAAU,MAAM,OACrC,KAAKA,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAMK,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAKF,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAMG,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAKH,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAKpB,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAKM,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAKkB,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,GAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAKJ,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAKV,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAKD,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAMe,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAKN,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAKb,OACH,KAAK,IAAI,KAAKF,wBAAwB,WAAW,GACjD,KAAK,MACP,EAAE,WAAW,MAAe,CAC9B,CAAC;GACD,KAAKO,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAKQ,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAKC,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}
+{"version":3,"file":"sqlite-change-log-catchup.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-catchup.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n  getOrCreateValueHistogram,\n} from '../../observability/metrics.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport type {Forwarder} from './forwarder.ts';\nimport type {CatchupPlan} from './sqlite-change-log-reader.ts';\nimport type {Subscriber} from './subscriber.ts';\n\nexport interface SQLiteChangeLogCatchupReader {\n  plan(fromWatermark: string): CatchupPlan;\n  read(\n    fromWatermark: string,\n    throughWatermark: string,\n    batchSize: number,\n    signal?: AbortSignal,\n  ): AsyncIterable<readonly WatermarkedChange[]>;\n  close(): void;\n}\n\n/**\n * Slice 9 replaces this no-op implementation with a guard that waits for an\n * in-flight purge and blocks new purge dispatch while the subscriber's ACK is\n * made visible to the Forwarder.\n */\nexport interface SQLiteChangeLogCleanupGuard {\n  runWhilePurgeBlocked<T>(register: () => T): Promise<T>;\n}\n\n// How long the barrier waits for the change-log writer's commit notification\n// before re-reading the log itself. The notification normally arrives first, so\n// this is a backstop for the windows in which none is coming: the writer is\n// disabled, or it is between stream connections.\nconst DEFAULT_BARRIER_POLL_INTERVAL_MS = 1000;\n\nexport type SQLiteChangeLogCatchupOptions = {\n  batchSize: number;\n  barrierTimeoutMs: number;\n  barrierPollIntervalMs?: number | undefined;\n  cleanupGuard?: SQLiteChangeLogCleanupGuard | undefined;\n  sleep?: typeof sleep | undefined;\n  now?: (() => number) | undefined;\n  /** Opens slice 11's process-local breaker after registration has committed. */\n  onFailure?: ((failure: SQLiteChangeLogCatchupFailure) => void) | undefined;\n};\n\nexport type SQLiteChangeLogCatchupRequest = {\n  /**\n   * Whether the routing warm-up gate classified the log as warm *for this\n   * subscriber*. The coordinator is opened once and reused, so this cannot be\n   * a property of it: `coldReadPercent` serves some subscribers from a log\n   * that is still inside its warm-up window, and the catchup metrics have to\n   * separate those from steady-state serving.\n   */\n  readonly logWarm?: boolean | undefined;\n};\n\nexport type SQLiteChangeLogCatchupFailure = 'barrier-timeout' | 'reader-error';\n\nexport type SQLiteChangeLogCatchupRegistration =\n  | {readonly kind: 'registered'}\n  | {readonly kind: 'uncovered'; readonly minWatermark: string}\n  // Registration failed before the subscriber was committed to SQLite, so PG\n  // catchup is still available and the caller falls back to it.\n  | {readonly kind: 'declined'; readonly error: unknown}\n  // The coordinator closed or failed the subscriber itself.\n  | {readonly kind: 'handled'};\n\nconst NOOP_CLEANUP_GUARD: SQLiteChangeLogCleanupGuard = {\n  runWhilePurgeBlocked: register => Promise.resolve(register()),\n};\n\n/**\n * Coordinates a serving subscriber's gap-free transition from replica-local\n * SQLite catchup to the Forwarder's live stream.\n *\n * Registration and required-head capture happen in one synchronous callback.\n * The subscriber therefore either sees a transaction in SQLite catchup or in\n * its live backlog (duplicates across that boundary are filtered by\n * Subscriber), never in neither place.\n */\nexport class SQLiteChangeLogCatchup implements Disposable {\n  readonly #lc: LogContext;\n  readonly #forwarder: Forwarder;\n  readonly #reader: SQLiteChangeLogCatchupReader;\n  readonly #batchSize: number;\n  readonly #barrierTimeoutMs: number;\n  readonly #barrierPollIntervalMs: number;\n  readonly #cleanupGuard: SQLiteChangeLogCleanupGuard;\n  readonly #sleep: typeof sleep;\n  readonly #now: () => number;\n  readonly #onFailure:\n    | ((failure: SQLiteChangeLogCatchupFailure) => void)\n    | undefined;\n  readonly #barrierTimeouts = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_timeouts',\n    'SQLite change-log catchups that timed out waiting for the required head.',\n  );\n  readonly #barrierBacklogOverflows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_backlog_overflows',\n    'SQLite change-log catchups abandoned because the subscriber backlog ' +\n      'reached its high water mark while waiting for the required head.',\n  );\n  readonly #barrierWakeups = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.barrier_wakeups',\n    'SQLite catchup barrier waits, labeled by what ended the wait. A ' +\n      'population dominated by \"poll\" means the change-log writer\\'s commit ' +\n      'notification is not reaching the barrier.',\n  );\n  readonly #catchupResults = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_results',\n    'SQLite catchup subscriptions by outcome.',\n  );\n  readonly #catchupRows = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_rows',\n    'Rows served from the SQLite change log during catchup.',\n  );\n  readonly #catchupBytes = getOrCreateCounter(\n    'replication',\n    'sqlite_change_log.catchup_bytes',\n    {\n      description: 'Bytes served from the SQLite change log during catchup.',\n      unit: 'By',\n    },\n  );\n  readonly #catchupDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.catchup_duration',\n    'Duration of a SQLite change-log catchup.',\n  );\n  readonly #barrierWaitDuration = getOrCreateLatencyHistogram(\n    'replication',\n    'sqlite_change_log.barrier_wait_duration',\n    'Time SQLite catchup waits for its pinned required head.',\n  );\n  readonly #catchupBacklogPeak = getOrCreateValueHistogram(\n    'replication',\n    'sqlite_change_log.catchup_backlog_peak_bytes',\n    {\n      description:\n        'Peak live backlog bytes buffered while a subscriber catches up from SQLite.',\n      unit: 'By',\n      bucketBoundaries: [\n        0, 1024, 16_384, 65_536, 262_144, 1_048_576, 4_194_304, 16_777_216,\n      ],\n    },\n  );\n  readonly #catchups = new Map<Subscriber, AbortController>();\n  readonly #commitWaiters = new Set<CommitWaiter>();\n  #closed = false;\n\n  constructor(\n    lc: LogContext,\n    forwarder: Forwarder,\n    reader: SQLiteChangeLogCatchupReader,\n    opts: SQLiteChangeLogCatchupOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-catchup');\n    this.#forwarder = forwarder;\n    this.#reader = reader;\n    this.#batchSize = opts.batchSize;\n    this.#barrierTimeoutMs = opts.barrierTimeoutMs;\n    this.#barrierPollIntervalMs =\n      opts.barrierPollIntervalMs ?? DEFAULT_BARRIER_POLL_INTERVAL_MS;\n    this.#cleanupGuard = opts.cleanupGuard ?? NOOP_CLEANUP_GUARD;\n    this.#sleep = opts.sleep ?? sleep;\n    this.#now = opts.now ?? Date.now;\n    this.#onFailure = opts.onFailure;\n  }\n\n  /**\n   * Registers the subscriber before resolving. Catchup itself continues in\n   * the background so subscribe() does not wait for SQLite to reach the\n   * required head.\n   */\n  async catchup(\n    subscriber: Subscriber,\n    captureRequiredHead: () => string | Promise<string>,\n    request: SQLiteChangeLogCatchupRequest = {},\n  ): Promise<SQLiteChangeLogCatchupRegistration> {\n    if (this.#closed) {\n      subscriber.fail(new AbortError('SQLite change-log catchup is closed'));\n      return {kind: 'handled'};\n    }\n\n    const abort = new AbortController();\n    this.#catchups.set(subscriber, abort);\n    let requiredHead: string | Promise<string> | undefined;\n    let uncoveredMinWatermark: string | undefined;\n    let committed = false;\n    try {\n      await this.#cleanupGuard.runWhilePurgeBlocked(() => {\n        this.#throwIfAborted(abort.signal);\n        // The router's eligibility inspection happens before this async guard\n        // is acquired. Recheck the exact boundary here: an in-flight purge can\n        // have advanced the minimum while registration waited. Once added to\n        // the Forwarder, the subscriber's ACK prevents any later purge from\n        // crossing this boundary.\n        const plan = this.#reader.plan(subscriber.watermark);\n        if (plan.kind === 'too-old') {\n          uncoveredMinWatermark = plan.minWatermark;\n          return;\n        }\n        requiredHead = captureRequiredHead();\n        this.#forwarder.add(subscriber);\n        committed = true;\n      });\n    } catch (error) {\n      this.#catchups.delete(subscriber);\n      if (abort.signal.aborted) {\n        return {kind: 'handled'};\n      }\n      if (!committed) {\n        // Nothing was committed to SQLite -- the subscriber never reached\n        // Forwarder.add() -- so PG catchup is still available. Failing here\n        // would send the client a terminal ['error', ...], which\n        // IncrementalSyncer answers with a full litestream replica restore,\n        // for a fallback that was still open. Trip the breaker as well: this\n        // is a reader failure like any other, and without it the retry picks\n        // SQLite again and the reset repeats.\n        this.#onFailure?.('reader-error');\n        return {kind: 'declined', error};\n      }\n      this.#lc.error?.(\n        `error while registering SQLite catchup subscriber ${subscriber.id}`,\n        error,\n      );\n      subscriber.fail(error);\n      return {kind: 'handled'};\n    }\n    if (uncoveredMinWatermark !== undefined) {\n      this.#catchups.delete(subscriber);\n      return {kind: 'uncovered', minWatermark: uncoveredMinWatermark};\n    }\n\n    void this.#run(\n      subscriber,\n      requiredHead as string | Promise<string>,\n      abort,\n      request.logWarm,\n    );\n    return {kind: 'registered'};\n  }\n\n  /**\n   * Notes that the change-log writer has committed `watermark`, i.e. that the\n   * log's head has advanced to it. The writer runs in this process, so this is\n   * the event itself rather than a subscriber's ACK of it.\n   *\n   * It is a wakeup, not an authority: `plan()` still decides what can be read.\n   * A notification that never arrives costs the barrier a poll interval, and one\n   * that arrives early costs an extra `plan()` call.\n   */\n  onChangeLogCommit(watermark: string): void {\n    for (const waiter of this.#commitWaiters) {\n      if (watermark >= waiter.watermark) {\n        this.#commitWaiters.delete(waiter);\n        waiter.resolve();\n      }\n    }\n  }\n\n  remove(subscriber: Subscriber): void {\n    this.#catchups.get(subscriber)?.abort();\n    this.#catchups.delete(subscriber);\n    this.#forwarder.remove(subscriber);\n  }\n\n  close(): void {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    for (const [subscriber, abort] of this.#catchups) {\n      abort.abort();\n      this.#forwarder.remove(subscriber);\n    }\n    this.#catchups.clear();\n    this.#reader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n\n  async #run(\n    subscriber: Subscriber,\n    requiredHead: string | Promise<string>,\n    abort: AbortController,\n    logWarm: boolean | undefined,\n  ): Promise<void> {\n    const {signal} = abort;\n    const catchupStart = this.#now();\n    let outcome = 'aborted';\n    let backlogPeakBytes = 0;\n    try {\n      const requiredHeadStart = this.#now();\n      const required = await this.#awaitRequiredHead(requiredHead, signal);\n      const requiredHeadWaitMs = this.#now() - requiredHeadStart;\n      if (requiredHeadWaitMs > this.#barrierTimeoutMs) {\n        this.#lc.info?.(\n          `waited ${requiredHeadWaitMs} ms for the forwarded transaction to ` +\n            `commit before starting the SQLite barrier for ${subscriber.id}`,\n        );\n      }\n      // The barrier bounds replica lag, not upstream transaction duration, so\n      // its deadline starts once the required head is known. Sharing one\n      // deadline with the wait above would fail subscribers that registered\n      // during a transaction longer than barrierTimeoutMs, however current\n      // the replica is.\n      const deadline = this.#now() + this.#barrierTimeoutMs;\n      const barrierStart = this.#now();\n      let plan: CatchupPlan;\n      try {\n        plan = await this.#waitForPlan(subscriber, required, deadline, signal);\n      } finally {\n        // Record failed and aborted barriers too; their wait distribution is\n        // at least as operationally important as the successful path.\n        this.#barrierWaitDuration.recordMs(this.#now() - barrierStart);\n      }\n      this.#throwIfAborted(signal);\n\n      if (plan.kind === 'too-old') {\n        outcome = 'too-old';\n        const message =\n          `earliest supported watermark is ${plan.minWatermark} ` +\n          `(requested ${subscriber.watermark})`;\n        this.#lc.warn?.(\n          `rejecting subscriber at watermark ${subscriber.watermark} ` +\n            `(earliest watermark: ${plan.minWatermark})`,\n        );\n        subscriber.close(ErrorType.WatermarkTooOld, message);\n        return;\n      }\n\n      let count = 0;\n      let bytes = 0;\n      const start = this.#now();\n      if (plan.kind === 'range') {\n        let lastBatchConsumed: Promise<unknown> | undefined;\n        for await (const changes of this.#reader.read(\n          subscriber.watermark,\n          plan.headWatermark,\n          this.#batchSize,\n          signal,\n        )) {\n          const waitStart = this.#now();\n          await lastBatchConsumed;\n          const elapsed = this.#now() - waitStart;\n          if (lastBatchConsumed) {\n            this.#lc[elapsed > 100 ? 'info' : 'debug']?.(\n              `waited ${elapsed.toFixed(3)} ms for ${subscriber.id} to consume ` +\n                `the previous SQLite catchup batch`,\n            );\n          }\n          this.#throwIfAborted(signal);\n          for (const change of changes) {\n            lastBatchConsumed = subscriber.catchup(change);\n            count++;\n            bytes += Buffer.byteLength(change[2]);\n          }\n          backlogPeakBytes = Math.max(\n            backlogPeakBytes,\n            subscriber.getStats().backlogBytes,\n          );\n        }\n        await lastBatchConsumed;\n        outcome = 'range';\n      } else {\n        outcome = 'ahead';\n        this.#lc.warn?.(\n          `subscriber ${subscriber.id} at watermark ${subscriber.watermark} ` +\n            `is ahead of the SQLite change-log head ${plan.headWatermark}; ` +\n            `waiting for the replica to catch up`,\n        );\n      }\n\n      this.#throwIfAborted(signal);\n      this.#lc.info?.(\n        `caught up ${subscriber.id} from SQLite with ${count} changes ` +\n          `(${this.#now() - start} ms)`,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupRows.add(count, attributes);\n      this.#catchupBytes.add(bytes, attributes);\n      // Keep buffering live sends until the asynchronous backlog drain has\n      // established its ordering boundary.\n      void subscriber.setCaughtUp();\n    } catch (error) {\n      if (signal.aborted) {\n        return;\n      }\n      if (error instanceof SQLiteChangeLogBarrierError) {\n        if (error instanceof SQLiteChangeLogBarrierTimeoutError) {\n          this.#barrierTimeouts.add(1);\n          outcome = 'barrier-timeout';\n          this.#onFailure?.('barrier-timeout');\n        } else {\n          // No #onFailure: a backlog high water mark is a property of this\n          // one subscriber's consumption, not of the log. Tripping the\n          // process-wide breaker for it would take every task in the shard\n          // off SQLite, which a client that reconnects and stalls again\n          // could hold open indefinitely.\n          outcome = 'barrier-backlog';\n        }\n        // Giving up on the barrier means the replica has not caught up *yet*,\n        // not that it cannot serve this subscriber. End the subscription\n        // cleanly so the client reconnects and retries: IncrementalSyncer\n        // treats any ['error', ...] message as terminal and restores a fresh\n        // replica from litestream, whereas a clean end backs off and\n        // re-subscribes. If retries keep failing until the change log is purged\n        // past the subscriber's watermark, plan() returns 'too-old', which is\n        // terminal by design.\n        this.#lc.warn?.(\n          `ending subscription for ${subscriber.id} to retry SQLite catchup`,\n          error,\n        );\n        subscriber.close();\n        return;\n      }\n      outcome = 'reader-error';\n      this.#onFailure?.('reader-error');\n      this.#lc.error?.(\n        `error while catching up subscriber ${subscriber.id} from SQLite`,\n        error,\n      );\n      subscriber.fail(error);\n    } finally {\n      backlogPeakBytes = Math.max(\n        backlogPeakBytes,\n        subscriber.getStats().backlogBytes,\n      );\n      const attributes = {\n        classification: outcome,\n        log_warm: logWarm ?? 'unknown',\n      };\n      this.#catchupResults.add(1, attributes);\n      this.#catchupDuration.recordMs(this.#now() - catchupStart, attributes);\n      this.#catchupBacklogPeak.record(backlogPeakBytes, attributes);\n      if (this.#catchups.get(subscriber) === abort) {\n        this.#catchups.delete(subscriber);\n      }\n    }\n  }\n\n  /**\n   * Waits for the transaction that was in flight at registration to finish.\n   *\n   * This wait is bounded by the upstream transaction rather than by\n   * `barrierTimeoutMs`: the completion settles on both commit and rollback,\n   * and an interrupted change stream forwards a synthetic rollback, so it\n   * always settles. Aborting -- the subscriber disconnecting or the service\n   * shutting down -- is what releases it early.\n   */\n  async #awaitRequiredHead(\n    requiredHead: string | Promise<string>,\n    signal: AbortSignal,\n  ): Promise<string> {\n    if (typeof requiredHead === 'string') {\n      return requiredHead;\n    }\n    this.#throwIfAborted(signal);\n    let onAbort: (() => void) | undefined;\n    try {\n      return await Promise.race([\n        requiredHead,\n        new Promise<never>((_, reject) => {\n          onAbort = () =>\n            reject(new AbortError('SQLite change-log catchup aborted'));\n          signal.addEventListener('abort', onAbort, {once: true});\n        }),\n      ]);\n    } finally {\n      if (onAbort) {\n        signal.removeEventListener('abort', onAbort);\n      }\n    }\n  }\n\n  async #waitForPlan(\n    subscriber: Subscriber,\n    requiredHead: string,\n    deadline: number,\n    signal: AbortSignal,\n  ): Promise<Exclude<CatchupPlan, {kind: 'not-ready'}>> {\n    // Race backlog growth once around the whole polling loop. Attaching the\n    // same unresolved promise to every per-poll race would retain one reaction\n    // per iteration until the subscriber's backlog eventually filled.\n    const backlogFull = subscriber.whenBacklogFull();\n    const barrier = new AbortController();\n    const abortBarrier = () => barrier.abort();\n    signal.addEventListener('abort', abortBarrier, {once: true});\n    if (signal.aborted) {\n      barrier.abort();\n    }\n\n    let observedHead = subscriber.watermark;\n    try {\n      const planReady = (async () => {\n        while (true) {\n          this.#throwIfAborted(barrier.signal);\n          const plan = this.#reader.plan(subscriber.watermark);\n          // A not-ready log has no head to compare, so it can never satisfy\n          // the required head: keep waiting for the writer to create and\n          // reconcile it, and let the barrier deadline end the subscription\n          // cleanly for a retry. Selection declines a not-ready log before a\n          // subscriber gets here, and neither purging nor reconciliation can\n          // empty a log that had content, so this is a belt-and-braces path.\n          if (plan.kind !== 'not-ready') {\n            observedHead = plan.headWatermark;\n            if (plan.headWatermark >= requiredHead) {\n              return plan;\n            }\n          }\n          const remaining = deadline - this.#now();\n          if (remaining <= 0) {\n            throw new SQLiteChangeLogBarrierTimeoutError(\n              `timed out waiting for SQLite head ${observedHead} to ` +\n                `reach required head ${requiredHead}`,\n            );\n          }\n          await this.#awaitChangeLogProgress(\n            requiredHead,\n            remaining,\n            barrier.signal,\n          );\n        }\n      })();\n\n      const backlogOverflow = backlogFull.promise.then(() => {\n        this.#throwIfAborted(signal);\n        // close() also resolves backlog waiters so none are stranded. The\n        // backlog is cleared first, which distinguishes close from overflow.\n        if (!subscriber.backlogFull) {\n          return new Promise<never>(() => {});\n        }\n        this.#barrierWakeups.add(1, {'wakeup.source': 'backlog'});\n        // Bytes, not time, are what the barrier actually costs: past the high\n        // water mark this subscriber's send() no longer resolves, so it holds\n        // up every flush and can stall the replica this barrier waits on.\n        this.#barrierBacklogOverflows.add(1);\n        throw new SQLiteChangeLogBarrierBacklogError(\n          `backlog for subscriber ${subscriber.id} reached its high water ` +\n            `mark while waiting for SQLite head ${observedHead} to ` +\n            `reach required head ${requiredHead}`,\n        );\n      });\n\n      return await Promise.race([planReady, backlogOverflow]);\n    } finally {\n      signal.removeEventListener('abort', abortBarrier);\n      barrier.abort();\n      backlogFull.cancel();\n    }\n  }\n\n  /**\n   * Waits for the change-log writer to commit `requiredHead`, or the poll\n   * interval as a backstop.\n   *\n   * No notification can be missed by registering after `plan()` was read: the\n   * notification follows the commit it reports, so if the writer had already\n   * committed `requiredHead`, `plan()` would have seen it.\n   */\n  async #awaitChangeLogProgress(\n    requiredHead: string,\n    remainingMs: number,\n    signal: AbortSignal,\n  ): Promise<void> {\n    const committed = resolver<void>();\n    const waiter = {watermark: requiredHead, resolve: committed.resolve};\n    this.#commitWaiters.add(waiter);\n    // Scopes the backstop timer to this wait so that a commit cancels it rather\n    // than leaving it to fire against a barrier that has already moved on.\n    const poll = new AbortController();\n    const abortPoll = () => poll.abort();\n    signal.addEventListener('abort', abortPoll, {once: true});\n    try {\n      const wokenBy = await Promise.race([\n        committed.promise.then(() => 'commit' as const),\n        this.#sleep(\n          Math.min(this.#barrierPollIntervalMs, remainingMs),\n          poll.signal,\n        ).then(() => 'poll' as const),\n      ]);\n      this.#barrierWakeups.add(1, {'wakeup.source': wokenBy});\n    } finally {\n      this.#commitWaiters.delete(waiter);\n      signal.removeEventListener('abort', abortPoll);\n      poll.abort();\n    }\n  }\n\n  #throwIfAborted(signal: AbortSignal): void {\n    if (this.#closed || signal.aborted) {\n      throw new AbortError('SQLite change-log catchup aborted');\n    }\n  }\n}\n\ntype CommitWaiter = {watermark: string; resolve: () => void};\n\n/**\n * A barrier that gave up on the replica reaching the required head. Both\n * reasons mean \"not yet\", not \"never\", so both end the subscription cleanly\n * for a retry rather than failing it.\n */\nexport class SQLiteChangeLogBarrierError extends Error {}\n\n/** The replica did not reach the required head before the deadline. */\nexport class SQLiteChangeLogBarrierTimeoutError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierTimeoutError';\n}\n\n/** Waiting any longer would have cost more than reconnecting. */\nexport class SQLiteChangeLogBarrierBacklogError extends SQLiteChangeLogBarrierError {\n  readonly name = 'SQLiteChangeLogBarrierBacklogError';\n}\n"],"mappings":";;;;;AAuCA,IAAM,mCAAmC;AAmCzC,IAAM,qBAAkD,EACtD,uBAAsB,aAAY,QAAQ,QAAQ,SAAS,CAAC,EAC9D;;;;;;;;;;AAWA,IAAa,yBAAb,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA,mBAA4B,mBAC1B,eACA,sCACA,0EACF;CACA,2BAAoC,mBAClC,eACA,+CACA,sIAEF;CACA,kBAA2B,mBACzB,eACA,qCACA,iLAGF;CACA,kBAA2B,mBACzB,eACA,qCACA,0CACF;CACA,eAAwB,mBACtB,eACA,kCACA,wDACF;CACA,gBAAyB,mBACvB,eACA,mCACA;EACE,aAAa;EACb,MAAM;CACR,CACF;CACA,mBAA4B,4BAC1B,eACA,sCACA,0CACF;CACA,uBAAgC,4BAC9B,eACA,2CACA,yDACF;CACA,sBAA+B,0BAC7B,eACA,gDACA;EACE,aACE;EACF,MAAM;EACN,kBAAkB;GAChB;GAAG;GAAM;GAAQ;GAAQ;GAAS;GAAW;GAAW;EAC1D;CACF,CACF;CACA,4BAAqB,IAAI,IAAiC;CAC1D,iCAA0B,IAAI,IAAkB;CAChD,UAAU;CAEV,YACE,IACA,WACA,QACA,MACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAK,aAAa;EAClB,KAAK,UAAU;EACf,KAAK,aAAa,KAAK;EACvB,KAAK,oBAAoB,KAAK;EAC9B,KAAK,yBACH,KAAK,yBAAyB;EAChC,KAAK,gBAAgB,KAAK,gBAAgB;EAC1C,KAAK,SAAS,KAAK,SAAS;EAC5B,KAAK,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAK,aAAa,KAAK;CACzB;;;;;;CAOA,MAAM,QACJ,YACA,qBACA,UAAyC,CAAC,GACG;EAC7C,IAAI,KAAK,SAAS;GAChB,WAAW,KAAK,IAAI,WAAW,qCAAqC,CAAC;GACrE,OAAO,EAAC,MAAM,UAAS;EACzB;EAEA,MAAM,QAAQ,IAAI,gBAAgB;EAClC,KAAK,UAAU,IAAI,YAAY,KAAK;EACpC,IAAI;EACJ,IAAI;EACJ,IAAI,YAAY;EAChB,IAAI;GACF,MAAM,KAAK,cAAc,2BAA2B;IAClD,KAAK,gBAAgB,MAAM,MAAM;IAMjC,MAAM,OAAO,KAAK,QAAQ,KAAK,WAAW,SAAS;IACnD,IAAI,KAAK,SAAS,WAAW;KAC3B,wBAAwB,KAAK;KAC7B;IACF;IACA,eAAe,oBAAoB;IACnC,KAAK,WAAW,IAAI,UAAU;IAC9B,YAAY;GACd,CAAC;EACH,SAAS,OAAO;GACd,KAAK,UAAU,OAAO,UAAU;GAChC,IAAI,MAAM,OAAO,SACf,OAAO,EAAC,MAAM,UAAS;GAEzB,IAAI,CAAC,WAAW;IAQd,KAAK,aAAa,cAAc;IAChC,OAAO;KAAC,MAAM;KAAY;IAAK;GACjC;GACA,KAAK,IAAI,QACP,qDAAqD,WAAW,MAChE,KACF;GACA,WAAW,KAAK,KAAK;GACrB,OAAO,EAAC,MAAM,UAAS;EACzB;EACA,IAAI,0BAA0B,KAAA,GAAW;GACvC,KAAK,UAAU,OAAO,UAAU;GAChC,OAAO;IAAC,MAAM;IAAa,cAAc;GAAqB;EAChE;EAEA,KAAU,KACR,YACA,cACA,OACA,QAAQ,OACV;EACA,OAAO,EAAC,MAAM,aAAY;CAC5B;;;;;;;;;;CAWA,kBAAkB,WAAyB;EACzC,KAAK,MAAM,UAAU,KAAK,gBACxB,IAAI,aAAa,OAAO,WAAW;GACjC,KAAK,eAAe,OAAO,MAAM;GACjC,OAAO,QAAQ;EACjB;CAEJ;CAEA,OAAO,YAA8B;EACnC,KAAK,UAAU,IAAI,UAAU,CAAC,EAAE,MAAM;EACtC,KAAK,UAAU,OAAO,UAAU;EAChC,KAAK,WAAW,OAAO,UAAU;CACnC;CAEA,QAAc;EACZ,IAAI,KAAK,SACP;EAEF,KAAK,UAAU;EACf,KAAK,MAAM,CAAC,YAAY,UAAU,KAAK,WAAW;GAChD,MAAM,MAAM;GACZ,KAAK,WAAW,OAAO,UAAU;EACnC;EACA,KAAK,UAAU,MAAM;EACrB,KAAK,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;CAEA,MAAM,KACJ,YACA,cACA,OACA,SACe;EACf,MAAM,EAAC,WAAU;EACjB,MAAM,eAAe,KAAK,KAAK;EAC/B,IAAI,UAAU;EACd,IAAI,mBAAmB;EACvB,IAAI;GACF,MAAM,oBAAoB,KAAK,KAAK;GACpC,MAAM,WAAW,MAAM,KAAK,mBAAmB,cAAc,MAAM;GACnE,MAAM,qBAAqB,KAAK,KAAK,IAAI;GACzC,IAAI,qBAAqB,KAAK,mBAC5B,KAAK,IAAI,OACP,UAAU,mBAAmB,qFACsB,WAAW,IAChE;GAOF,MAAM,WAAW,KAAK,KAAK,IAAI,KAAK;GACpC,MAAM,eAAe,KAAK,KAAK;GAC/B,IAAI;GACJ,IAAI;IACF,OAAO,MAAM,KAAK,aAAa,YAAY,UAAU,UAAU,MAAM;GACvE,UAAU;IAGR,KAAK,qBAAqB,SAAS,KAAK,KAAK,IAAI,YAAY;GAC/D;GACA,KAAK,gBAAgB,MAAM;GAE3B,IAAI,KAAK,SAAS,WAAW;IAC3B,UAAU;IACV,MAAM,UACJ,mCAAmC,KAAK,aAAa,cACvC,WAAW,UAAU;IACrC,KAAK,IAAI,OACP,qCAAqC,WAAW,UAAU,wBAChC,KAAK,aAAa,EAC9C;IACA,WAAW,MAAM,GAA2B,OAAO;IACnD;GACF;GAEA,IAAI,QAAQ;GACZ,IAAI,QAAQ;GACZ,MAAM,QAAQ,KAAK,KAAK;GACxB,IAAI,KAAK,SAAS,SAAS;IACzB,IAAI;IACJ,WAAW,MAAM,WAAW,KAAK,QAAQ,KACvC,WAAW,WACX,KAAK,eACL,KAAK,YACL,MACF,GAAG;KACD,MAAM,YAAY,KAAK,KAAK;KAC5B,MAAM;KACN,MAAM,UAAU,KAAK,KAAK,IAAI;KAC9B,IAAI,mBACF,KAAK,IAAI,UAAU,MAAM,SAAS,QAAQ,GACxC,UAAU,QAAQ,QAAQ,CAAC,EAAE,UAAU,WAAW,GAAG,8CAEvD;KAEF,KAAK,gBAAgB,MAAM;KAC3B,KAAK,MAAM,UAAU,SAAS;MAC5B,oBAAoB,WAAW,QAAQ,MAAM;MAC7C;MACA,SAAS,OAAO,WAAW,OAAO,EAAE;KACtC;KACA,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,CAAC,CAAC,YACxB;IACF;IACA,MAAM;IACN,UAAU;GACZ,OAAO;IACL,UAAU;IACV,KAAK,IAAI,OACP,cAAc,WAAW,GAAG,gBAAgB,WAAW,UAAU,0CACrB,KAAK,cAAc,sCAEjE;GACF;GAEA,KAAK,gBAAgB,MAAM;GAC3B,KAAK,IAAI,OACP,aAAa,WAAW,GAAG,oBAAoB,MAAM,YAC/C,KAAK,KAAK,IAAI,MAAM,KAC5B;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAK,aAAa,IAAI,OAAO,UAAU;GACvC,KAAK,cAAc,IAAI,OAAO,UAAU;GAGxC,WAAgB,YAAY;EAC9B,SAAS,OAAO;GACd,IAAI,OAAO,SACT;GAEF,IAAI,iBAAiB,6BAA6B;IAChD,IAAI,iBAAiB,oCAAoC;KACvD,KAAK,iBAAiB,IAAI,CAAC;KAC3B,UAAU;KACV,KAAK,aAAa,iBAAiB;IACrC,OAME,UAAU;IAUZ,KAAK,IAAI,OACP,2BAA2B,WAAW,GAAG,2BACzC,KACF;IACA,WAAW,MAAM;IACjB;GACF;GACA,UAAU;GACV,KAAK,aAAa,cAAc;GAChC,KAAK,IAAI,QACP,sCAAsC,WAAW,GAAG,eACpD,KACF;GACA,WAAW,KAAK,KAAK;EACvB,UAAU;GACR,mBAAmB,KAAK,IACtB,kBACA,WAAW,SAAS,CAAC,CAAC,YACxB;GACA,MAAM,aAAa;IACjB,gBAAgB;IAChB,UAAU,WAAW;GACvB;GACA,KAAK,gBAAgB,IAAI,GAAG,UAAU;GACtC,KAAK,iBAAiB,SAAS,KAAK,KAAK,IAAI,cAAc,UAAU;GACrE,KAAK,oBAAoB,OAAO,kBAAkB,UAAU;GAC5D,IAAI,KAAK,UAAU,IAAI,UAAU,MAAM,OACrC,KAAK,UAAU,OAAO,UAAU;EAEpC;CACF;;;;;;;;;;CAWA,MAAM,mBACJ,cACA,QACiB;EACjB,IAAI,OAAO,iBAAiB,UAC1B,OAAO;EAET,KAAK,gBAAgB,MAAM;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,QAAQ,KAAK,CACxB,cACA,IAAI,SAAgB,GAAG,WAAW;IAChC,gBACE,OAAO,IAAI,WAAW,mCAAmC,CAAC;IAC5D,OAAO,iBAAiB,SAAS,SAAS,EAAC,MAAM,KAAI,CAAC;GACxD,CAAC,CACH,CAAC;EACH,UAAU;GACR,IAAI,SACF,OAAO,oBAAoB,SAAS,OAAO;EAE/C;CACF;CAEA,MAAM,aACJ,YACA,cACA,UACA,QACoD;EAIpD,MAAM,cAAc,WAAW,gBAAgB;EAC/C,MAAM,UAAU,IAAI,gBAAgB;EACpC,MAAM,qBAAqB,QAAQ,MAAM;EACzC,OAAO,iBAAiB,SAAS,cAAc,EAAC,MAAM,KAAI,CAAC;EAC3D,IAAI,OAAO,SACT,QAAQ,MAAM;EAGhB,IAAI,eAAe,WAAW;EAC9B,IAAI;GACF,MAAM,aAAa,YAAY;IAC7B,OAAO,MAAM;KACX,KAAK,gBAAgB,QAAQ,MAAM;KACnC,MAAM,OAAO,KAAK,QAAQ,KAAK,WAAW,SAAS;KAOnD,IAAI,KAAK,SAAS,aAAa;MAC7B,eAAe,KAAK;MACpB,IAAI,KAAK,iBAAiB,cACxB,OAAO;KAEX;KACA,MAAM,YAAY,WAAW,KAAK,KAAK;KACvC,IAAI,aAAa,GACf,MAAM,IAAI,mCACR,qCAAqC,aAAa,0BACzB,cAC3B;KAEF,MAAM,KAAK,wBACT,cACA,WACA,QAAQ,MACV;IACF;GACF,EAAA,CAAG;GAEH,MAAM,kBAAkB,YAAY,QAAQ,WAAW;IACrD,KAAK,gBAAgB,MAAM;IAG3B,IAAI,CAAC,WAAW,aACd,OAAO,IAAI,cAAqB,CAAC,CAAC;IAEpC,KAAK,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,UAAS,CAAC;IAIxD,KAAK,yBAAyB,IAAI,CAAC;IACnC,MAAM,IAAI,mCACR,0BAA0B,WAAW,GAAG,6DACA,aAAa,0BAC5B,cAC3B;GACF,CAAC;GAED,OAAO,MAAM,QAAQ,KAAK,CAAC,WAAW,eAAe,CAAC;EACxD,UAAU;GACR,OAAO,oBAAoB,SAAS,YAAY;GAChD,QAAQ,MAAM;GACd,YAAY,OAAO;EACrB;CACF;;;;;;;;;CAUA,MAAM,wBACJ,cACA,aACA,QACe;EACf,MAAM,YAAY,SAAe;EACjC,MAAM,SAAS;GAAC,WAAW;GAAc,SAAS,UAAU;EAAO;EACnE,KAAK,eAAe,IAAI,MAAM;EAG9B,MAAM,OAAO,IAAI,gBAAgB;EACjC,MAAM,kBAAkB,KAAK,MAAM;EACnC,OAAO,iBAAiB,SAAS,WAAW,EAAC,MAAM,KAAI,CAAC;EACxD,IAAI;GACF,MAAM,UAAU,MAAM,QAAQ,KAAK,CACjC,UAAU,QAAQ,WAAW,QAAiB,GAC9C,KAAK,OACH,KAAK,IAAI,KAAK,wBAAwB,WAAW,GACjD,KAAK,MACP,CAAC,CAAC,WAAW,MAAe,CAC9B,CAAC;GACD,KAAK,gBAAgB,IAAI,GAAG,EAAC,iBAAiB,QAAO,CAAC;EACxD,UAAU;GACR,KAAK,eAAe,OAAO,MAAM;GACjC,OAAO,oBAAoB,SAAS,SAAS;GAC7C,KAAK,MAAM;EACb;CACF;CAEA,gBAAgB,QAA2B;EACzC,IAAI,KAAK,WAAW,OAAO,SACzB,MAAM,IAAI,WAAW,mCAAmC;CAE5D;AACF;;;;;;AASA,IAAa,8BAAb,cAAiD,MAAM,CAAC;;AAGxD,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB;;AAGA,IAAa,qCAAb,cAAwD,4BAA4B;CAClF,OAAgB;AAClB"}

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js

@@ -19,7 +19,7 @@ var DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE = 1e3;
 * which scales with payload size instead of row count. Normal traffic
 * reaches the row limit first. Wide payloads reach this one.
 */
-var DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE = 32 * 1024 * 1024;
+var DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE = 33554432;
 /** Rows per SQLite catchup batch. */
 var DEFAULT_READ_BATCH_ROWS = 1e3;
 /** Uses separate aggregates so SQLite can use an index for each value. */
@@ -401,7 +401,7 @@ var SQLiteChangeLogComparator = class {
 };
 /** Applies the same reconstruction as Postgres catchup. */
 async function* reconstructBatches(batches) {
-	for await (const batch of batches) yield batch.map(reconstructWatermarkedChange);
+	for await (const batch of batches) yield batch.map(({ watermark, tag, change }) => reconstructWatermarkedChange(watermark, tag, change));
 }
 function minWatermark(a, b) {
 	return a < b ? a : b;

package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js.map

@@ -1 +1 @@
-{"version":3,"file":"sqlite-change-log-comparator.js","names":["#lc","#shard","#changeLogFile","#identity","#pg","#opts","#now","#setTimeoutFn","#clearTimeoutFn","#yield","#compareResults","#compareCycles","#cycleDuration","#scheduleNext","#stopped","#running","#runCycle","#continueWithoutDelay","#rescheduleNext","#timer","#fileGeneration","#cursor","#skipped","#pinBounds","#reconfirm","#compared","#enumerateCommits","#compareTransaction","#readSQLiteBounds"],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-comparator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport {\n  CHANGE_LOG_DB_SCHEMA_VERSION,\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogMeta,\n  type ChangeLogIdentity,\n  type ChangeLogMeta,\n} from '../replicator/change-log-db.ts';\nimport {\n  reconstructWatermarkedChange,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport {\n  digestCatchupRange,\n  type CatchupRangeDigest,\n} from './change-log-compare-digest.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport {isSampledForShard} from './shard-sampling.ts';\nimport {SQLiteChangeLogReader} from './sqlite-change-log-reader.ts';\n\n/** Interval between comparison cycles. */\nconst DEFAULT_COMPARE_INTERVAL_MS = 30_000;\n\n/** Maximum transactions enumerated per cycle. */\nconst DEFAULT_MAX_TRANSACTIONS_PER_CYCLE = 256;\n\n/** Maximum catchup rows read from each store per cycle. */\nconst DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE = 1000;\n\n/**\n * Maximum catchup bytes read from each store per cycle.\n *\n * Rows bound the per-row overhead. Bytes bound the read and hash work,\n * which scales with payload size instead of row count. Normal traffic\n * reaches the row limit first. Wide payloads reach this one.\n */\nconst DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE = 32 * 1024 * 1024;\n\n/** Rows per SQLite catchup batch. */\nconst DEFAULT_READ_BATCH_ROWS = 1000;\n\n/**\n * Result for one committed transaction.\n *\n * `mismatch` covers missing, extra, changed, or reordered rows.\n * `inconclusive` means that store bounds changed during the comparison.\n */\nexport type CompareOutcome =\n  | 'match'\n  | 'mismatch'\n  | 'inconclusive'\n  | 'deferred'\n  | 'oversized';\n\nexport type CompareSkipReason =\n  // The comparator was stopped.\n  | 'stopped'\n  // The writer replaced the file during this cycle.\n  | 'log-rebuilt'\n  // The file is absent, unreadable, or uninitialized.\n  | 'log-unavailable'\n  // The schema version does not match this build.\n  | 'ineligible-schema'\n  // The log belongs to a different replica.\n  | 'ineligible-identity'\n  // The log is younger than the warm-up period.\n  | 'cold-log'\n  // The stores have no complete committed range in common.\n  | 'nothing-to-compare'\n  // The cycle failed. The error was logged and counted.\n  | 'error';\n\nexport type CompareCycleResult =\n  | {readonly kind: 'skipped'; readonly reason: CompareSkipReason}\n  | {\n      readonly kind: 'compared';\n      /** Exclusive lower bound of the compared range. */\n      readonly fromWatermark: string;\n      /** Inclusive upper bound handled. The next cycle resumes from here. */\n      readonly throughWatermark: string;\n      /** Committed transactions handled through `throughWatermark`. */\n      readonly transactions: number;\n      /** Transactions with compared catchup output. */\n      readonly sampled: number;\n      readonly matched: number;\n      readonly mismatched: number;\n      /** Suspects rejected because store bounds changed. */\n      readonly inconclusive: number;\n      /** Sampled transactions deferred to a fresh budget. */\n      readonly deferred: number;\n      /** Sampled transactions that exceed a fresh budget. */\n      readonly oversized: number;\n    };\n\n/** Provides the Postgres side of the catchup comparison. */\nexport interface PGChangeLogRangeReader {\n  getCatchupBounds(): Promise<{\n    minWatermark: string | null;\n    lastWatermark: string;\n  }>;\n  listCommitWatermarks(\n    afterWatermark: string,\n    throughWatermark: string,\n    limit: number,\n  ): Promise<string[]>;\n  readCatchupRange(\n    afterWatermark: string,\n    throughWatermark: string,\n    batchRows?: number | undefined,\n  ): AsyncIterable<ChangeLogEntry[]>;\n}\n\nexport type SQLiteChangeLogCompareOptions = {\n  /** Stable percentage of transactions selected for payload comparison. */\n  readonly comparePercent: number;\n  /** Warm-up time after a seed. The comparator skips a younger log. */\n  readonly retentionMs: number;\n  readonly intervalMs?: number | undefined;\n  readonly maxTransactionsPerCycle?: number | undefined;\n  /** Maximum catchup rows read from each store in one cycle. */\n  readonly maxRowsPerSourcePerCycle?: number | undefined;\n  /** Maximum catchup bytes read from each store in one cycle. */\n  readonly maxBytesPerSourcePerCycle?: number | undefined;\n  readonly readBatchRows?: number | undefined;\n  /** Clock for the warm-up period. Defaults to `Date.now`. */\n  readonly now?: (() => number) | undefined;\n  readonly setTimeoutFn?: typeof setTimeout | undefined;\n  readonly clearTimeoutFn?: typeof clearTimeout | undefined;\n  /** Yields between sampled transactions. Defaults to `setImmediate`. */\n  readonly yieldFn?: (() => Promise<void>) | undefined;\n};\n\n/** Uses separate aggregates so SQLite can use an index for each value. */\nconst SQLITE_BOUNDS_SQL = /*sql*/ `\n  SELECT\n    (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"minWatermark\",\n    (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"headWatermark\"\n`;\n\n/** Lists committed transaction watermarks in `(@after, @through]`. */\nconst SQLITE_COMMITS_SQL = /*sql*/ `\n  SELECT \"watermark\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n   WHERE \"precommit\" IS NOT NULL\n     AND \"watermark\" > @after\n     AND \"watermark\" <= @through\n   ORDER BY \"watermark\"\n   LIMIT @limit\n`;\n\ntype SQLiteBounds = {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\ntype PinnedBounds = {\n  readonly meta: ChangeLogMeta;\n  readonly sqlite: SQLiteBounds;\n  readonly pgMinWatermark: string;\n  readonly pgLastWatermark: string;\n};\n\ntype SampledComparison = {\n  readonly outcome: CompareOutcome;\n  readonly sqliteRowsRead: number;\n  readonly pgRowsRead: number;\n  readonly sqliteBytesRead: number;\n  readonly pgBytesRead: number;\n};\n\n/**\n * Compares SQLite and Postgres catchup output without affecting subscribers.\n *\n * Each cycle compares complete ranges through the lower store head.\n * It checks every transaction for presence and samples payloads.\n * A bounds change makes the affected result `inconclusive`.\n * Postgres remains authoritative.\n */\nexport class SQLiteChangeLogComparator {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeLogFile: string;\n  readonly #identity: ChangeLogIdentity;\n  readonly #pg: PGChangeLogRangeReader;\n  readonly #opts: SQLiteChangeLogCompareOptions;\n  readonly #now: () => number;\n  readonly #setTimeoutFn: typeof setTimeout;\n  readonly #clearTimeoutFn: typeof clearTimeout;\n  readonly #yield: () => Promise<void>;\n\n  readonly #compareResults = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.compare_result',\n    'Outcomes of the dark comparison between what SQLite catchup serves and ' +\n      'what PG catchup serves, per committed transaction. `inconclusive` ' +\n      'means a purge or reseed moved the pinned bounds mid-comparison.',\n  );\n  readonly #compareCycles = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.compare_cycles',\n    'SQLite change-log comparison cycles, by result.',\n  );\n  readonly #cycleDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.compare_cycle_duration',\n    'Duration of one SQLite change-log comparison cycle.',\n  );\n\n  /** Last handled commit watermark. A restart derives it from current bounds. */\n  #cursor: string | undefined;\n  #running: Promise<CompareCycleResult> | undefined;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n  #stopped = false;\n  #continueWithoutDelay = false;\n  #fileGeneration = 0;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    changeLogFile: string,\n    identity: ChangeLogIdentity,\n    pg: PGChangeLogRangeReader,\n    opts: SQLiteChangeLogCompareOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-compare');\n    this.#shard = shard;\n    this.#changeLogFile = changeLogFile;\n    this.#identity = identity;\n    this.#pg = pg;\n    this.#opts = opts;\n    this.#now = opts.now ?? Date.now;\n    this.#setTimeoutFn = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeoutFn = opts.clearTimeoutFn ?? clearTimeout;\n    this.#yield =\n      opts.yieldFn ??\n      (() => new Promise<void>(resolve => setImmediate(resolve)));\n    this.#scheduleNext();\n  }\n\n  /**\n   * Runs one cycle. Concurrent calls share the same promise.\n   * Errors produce `skipped: 'error'`.\n   */\n  compareOnce(): Promise<CompareCycleResult> {\n    if (this.#stopped) {\n      return Promise.resolve({kind: 'skipped', reason: 'stopped'});\n    }\n    if (this.#running === undefined) {\n      this.#running = this.#runCycle()\n        .then(result => {\n          if (this.#continueWithoutDelay) {\n            this.#rescheduleNext(0);\n          }\n          return result;\n        })\n        .finally(() => {\n          this.#running = undefined;\n        });\n    }\n    return this.#running;\n  }\n\n  stop(): void {\n    this.#stopped = true;\n    if (this.#timer !== undefined) {\n      this.#clearTimeoutFn(this.#timer);\n      this.#timer = undefined;\n    }\n  }\n\n  /** Invalidates cycles that can still read a replaced file. */\n  invalidate(): void {\n    this.#fileGeneration++;\n    this.#cursor = undefined;\n    this.#continueWithoutDelay = false;\n  }\n\n  #scheduleNext(\n    delayMs = this.#opts.intervalMs ?? DEFAULT_COMPARE_INTERVAL_MS,\n  ): void {\n    if (this.#stopped || this.#timer !== undefined) {\n      return;\n    }\n    this.#timer = this.#setTimeoutFn(() => {\n      this.#timer = undefined;\n      void this.compareOnce().finally(() => this.#scheduleNext());\n    }, delayMs);\n  }\n\n  #rescheduleNext(delayMs: number): void {\n    if (this.#timer !== undefined) {\n      this.#clearTimeoutFn(this.#timer);\n      this.#timer = undefined;\n    }\n    this.#scheduleNext(delayMs);\n  }\n\n  async #runCycle(): Promise<CompareCycleResult> {\n    const startedAt = performance.now();\n    const fileGeneration = this.#fileGeneration;\n    this.#continueWithoutDelay = false;\n    let db: Database | undefined;\n    let reader: SQLiteChangeLogReader | undefined;\n    try {\n      try {\n        // A readonly handle does not create a missing file.\n        db = new Database(this.#lc, this.#changeLogFile, {readonly: true});\n      } catch {\n        return this.#skipped('log-unavailable');\n      }\n\n      const pinned = await this.#pinBounds(db);\n      if (fileGeneration !== this.#fileGeneration) {\n        return this.#skipped('log-rebuilt');\n      }\n      if (typeof pinned === 'string') {\n        return this.#skipped(pinned);\n      }\n      const {meta} = pinned;\n\n      const from = maxWatermark(\n        this.#cursor ?? meta.seedWatermark,\n        // Exclude the synthetic SQLite seed transaction.\n        // The Postgres seed has no `precommit`, so enumeration also excludes it.\n        meta.seedWatermark,\n        // A store cannot serve the transaction at its minimum watermark.\n        pinned.pgMinWatermark,\n        pinned.sqlite.minWatermark,\n      );\n      const through = minWatermark(\n        pinned.pgLastWatermark,\n        pinned.sqlite.headWatermark,\n      );\n      if (through <= from) {\n        return this.#skipped('nothing-to-compare');\n      }\n\n      let transactions = 0;\n      let sampled = 0;\n      const outcomes: Record<CompareOutcome, number> = {\n        match: 0,\n        mismatch: 0,\n        inconclusive: 0,\n        deferred: 0,\n        oversized: 0,\n      };\n      const record = (watermark: string, outcome: CompareOutcome) => {\n        this.#compareResults.add(1, {outcome});\n        outcomes[outcome]++;\n        if (outcome === 'mismatch') {\n          // Log only the watermark because payloads contain customer data.\n          this.#lc.error?.(\n            'SQLite change-log catchup output diverged from Postgres',\n            {sqliteChangeLogCompare: {watermark}},\n          );\n        }\n      };\n\n      reader = new SQLiteChangeLogReader(this.#lc, this.#changeLogFile);\n      const plan = reader.plan(from);\n      if (plan.kind === 'not-ready') {\n        return this.#skipped('log-unavailable');\n      }\n      if (plan.kind === 'ahead') {\n        return this.#skipped('nothing-to-compare');\n      }\n      if (plan.kind === 'too-old') {\n        // A fixed lower bound makes this result a mismatch.\n        // A changed lower bound makes it inconclusive.\n        const reconfirmed = await this.#reconfirm(db, pinned, from);\n        const outcome =\n          fileGeneration === this.#fileGeneration\n            ? reconfirmed\n            : 'inconclusive';\n        record(from, outcome);\n        if (outcome === 'inconclusive') {\n          // Keep the cursor unchanged because the bounds moved.\n          return this.#compared(from, from, transactions, sampled, outcomes);\n        }\n        // Positional reads can continue after a missing boundary.\n        // Advancing avoids reporting the same boundary in every cycle.\n      }\n\n      const limit = Math.max(\n        1,\n        this.#opts.maxTransactionsPerCycle ??\n          DEFAULT_MAX_TRANSACTIONS_PER_CYCLE,\n      );\n      const union = await this.#enumerateCommits(db, from, through, limit);\n      if (fileGeneration !== this.#fileGeneration) {\n        return this.#skipped('log-rebuilt');\n      }\n      const maxRowsPerSource = Math.max(\n        1,\n        this.#opts.maxRowsPerSourcePerCycle ??\n          DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE,\n      );\n      const maxBytesPerSource = Math.max(\n        1,\n        this.#opts.maxBytesPerSourcePerCycle ??\n          DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE,\n      );\n\n      let sqliteRowsRead = 0;\n      let pgRowsRead = 0;\n      let sqliteBytesRead = 0;\n      let pgBytesRead = 0;\n      let previousBoundary = from;\n      let handledThrough = from;\n      for (const {watermark, inSqlite, inPg} of union) {\n        if (this.#stopped) {\n          break;\n        }\n        let stopAfterTransaction = false;\n        if (!inSqlite || !inPg) {\n          // Check every transaction for presence because this needs no payload read.\n          const outcome = await this.#reconfirm(db, pinned, watermark);\n          record(\n            watermark,\n            fileGeneration === this.#fileGeneration ? outcome : 'inconclusive',\n          );\n        } else if (\n          isSampledForShard(this.#shard, watermark, this.#opts.comparePercent)\n        ) {\n          const sqliteRowsRemaining = maxRowsPerSource - sqliteRowsRead;\n          const pgRowsRemaining = maxRowsPerSource - pgRowsRead;\n          // A read may pass the byte budget by its last row, so this\n          // compares against zero rather than checking for equality.\n          const sqliteBytesRemaining = maxBytesPerSource - sqliteBytesRead;\n          const pgBytesRemaining = maxBytesPerSource - pgBytesRead;\n          if (\n            sqliteRowsRemaining === 0 ||\n            pgRowsRemaining === 0 ||\n            sqliteBytesRemaining <= 0 ||\n            pgBytesRemaining <= 0\n          ) {\n            record(watermark, 'deferred');\n            break;\n          }\n          sampled++;\n          const comparison = await this.#compareTransaction(\n            db,\n            reader,\n            pinned,\n            previousBoundary,\n            watermark,\n            sqliteRowsRemaining,\n            pgRowsRemaining,\n            sqliteBytesRemaining,\n            pgBytesRemaining,\n            maxRowsPerSource,\n            maxBytesPerSource,\n          );\n          sqliteRowsRead += comparison.sqliteRowsRead;\n          pgRowsRead += comparison.pgRowsRead;\n          sqliteBytesRead += comparison.sqliteBytesRead;\n          pgBytesRead += comparison.pgBytesRead;\n          const outcome =\n            fileGeneration === this.#fileGeneration\n              ? comparison.outcome\n              : 'inconclusive';\n          record(watermark, outcome);\n          if (outcome === 'inconclusive') {\n            break;\n          }\n          if (outcome === 'deferred') {\n            break;\n          }\n          if (outcome === 'oversized') {\n            // Advance past a transaction that cannot fit in a fresh budget.\n            stopAfterTransaction = true;\n          }\n          // Yield between sampled reads so other stream work can continue.\n          await this.#yield();\n        }\n        previousBoundary = watermark;\n        handledThrough = watermark;\n        transactions++;\n        if (stopAfterTransaction) {\n          break;\n        }\n      }\n\n      if (\n        !this.#stopped &&\n        fileGeneration === this.#fileGeneration &&\n        outcomes.inconclusive === 0\n      ) {\n        this.#cursor = handledThrough;\n        this.#continueWithoutDelay = handledThrough < through;\n      }\n      return this.#compared(\n        from,\n        handledThrough,\n        transactions,\n        sampled,\n        outcomes,\n      );\n    } catch (e) {\n      this.#lc.warn?.('error comparing the SQLite change log', e);\n      return this.#skipped('error');\n    } finally {\n      reader?.close();\n      db?.close();\n      this.#cycleDuration.recordMs(performance.now() - startedAt);\n    }\n  }\n\n  /** Checks log eligibility and reads the bounds from both stores. */\n  async #pinBounds(db: Database): Promise<PinnedBounds | CompareSkipReason> {\n    let meta: ChangeLogMeta;\n    try {\n      meta = readChangeLogMeta(db);\n    } catch {\n      // The writer has not initialized the log.\n      return 'log-unavailable';\n    }\n    if (meta.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n      return 'ineligible-schema';\n    }\n    const {epoch, generation, replicaID} = this.#identity;\n    if (\n      meta.epoch !== epoch ||\n      meta.generation !== generation ||\n      meta.replicaID !== replicaID\n    ) {\n      return 'ineligible-identity';\n    }\n    if (this.#now() - meta.seededAtMs < this.#opts.retentionMs) {\n      return 'cold-log';\n    }\n    const sqlite = this.#readSQLiteBounds(db);\n    if (sqlite === undefined) {\n      return 'log-unavailable';\n    }\n    const {minWatermark, lastWatermark} = await this.#pg.getCatchupBounds();\n    if (minWatermark === null) {\n      // Only a new replica has an empty Postgres change log.\n      return 'nothing-to-compare';\n    }\n    return {\n      meta,\n      sqlite,\n      pgMinWatermark: minWatermark,\n      pgLastWatermark: lastWatermark,\n    };\n  }\n\n  #readSQLiteBounds(db: Database): SQLiteBounds | undefined {\n    const bounds = db\n      .prepare(SQLITE_BOUNDS_SQL)\n      .get<{minWatermark: string | null; headWatermark: string | null}>();\n    return bounds.minWatermark === null || bounds.headWatermark === null\n      ? undefined\n      : {\n          minWatermark: bounds.minWatermark,\n          headWatermark: bounds.headWatermark,\n        };\n  }\n\n  /**\n   * Merges committed transaction watermarks from both stores.\n   * If a list reaches the limit, the result ends at the last complete common range.\n   */\n  async #enumerateCommits(\n    db: Database,\n    from: string,\n    through: string,\n    limit: number,\n  ): Promise<{watermark: string; inSqlite: boolean; inPg: boolean}[]> {\n    let sqlite: string[] = db\n      .prepare(SQLITE_COMMITS_SQL)\n      .all<{watermark: string}>({after: from, through, limit: limit + 1})\n      .map(({watermark}) => watermark);\n    let pg = await this.#pg.listCommitWatermarks(from, through, limit + 1);\n\n    let effectiveThrough = through;\n    if (sqlite.length > limit) {\n      sqlite.length = limit;\n      effectiveThrough = minWatermark(effectiveThrough, sqlite[limit - 1]);\n    }\n    if (pg.length > limit) {\n      pg.length = limit;\n      effectiveThrough = minWatermark(effectiveThrough, pg[limit - 1]);\n    }\n    if (effectiveThrough !== through) {\n      sqlite = sqlite.filter(w => w <= effectiveThrough);\n      pg = pg.filter(w => w <= effectiveThrough);\n    }\n\n    const union: {watermark: string; inSqlite: boolean; inPg: boolean}[] = [];\n    let s = 0;\n    let p = 0;\n    while (s < sqlite.length || p < pg.length) {\n      const sw = s < sqlite.length ? sqlite[s] : undefined;\n      const pw = p < pg.length ? pg[p] : undefined;\n      if (sw !== undefined && (pw === undefined || sw < pw)) {\n        union.push({watermark: sw, inSqlite: true, inPg: false});\n        s++;\n      } else if (pw !== undefined && (sw === undefined || pw < sw)) {\n        union.push({watermark: pw, inSqlite: false, inPg: true});\n        p++;\n      } else if (sw !== undefined) {\n        union.push({watermark: sw, inSqlite: true, inPg: true});\n        s++;\n        p++;\n      }\n    }\n    return union;\n  }\n\n  /**\n   * Compares the output in `(previousBoundary, watermark]`.\n   * Positional reads do not require either store to contain the boundary row.\n   */\n  async #compareTransaction(\n    db: Database,\n    reader: SQLiteChangeLogReader,\n    pinned: PinnedBounds,\n    previousBoundary: string,\n    watermark: string,\n    sqliteMaxRows: number,\n    pgMaxRows: number,\n    sqliteMaxBytes: number,\n    pgMaxBytes: number,\n    maxRowsPerSource: number,\n    maxBytesPerSource: number,\n  ): Promise<SampledComparison> {\n    let sqlite: CatchupRangeDigest;\n    let pg: CatchupRangeDigest;\n    try {\n      const readBatchRows = this.#opts.readBatchRows ?? DEFAULT_READ_BATCH_ROWS;\n      sqlite = await digestCatchupRange(\n        reader.read(\n          previousBoundary,\n          watermark,\n          Math.min(readBatchRows, sqliteMaxRows),\n          undefined,\n          sqliteMaxRows,\n        ),\n        watermark,\n        sqliteMaxRows,\n        sqliteMaxBytes,\n      );\n      pg = await digestCatchupRange(\n        reconstructBatches(\n          this.#pg.readCatchupRange(previousBoundary, watermark, pgMaxRows),\n        ),\n        watermark,\n        pgMaxRows,\n        pgMaxBytes,\n      );\n    } catch (e) {\n      // A failed catchup read makes the transaction suspect.\n      this.#lc.warn?.(\n        `error reading transaction ${watermark} for comparison`,\n        e,\n      );\n      return {\n        outcome: await this.#reconfirm(db, pinned, watermark),\n        sqliteRowsRead: 0,\n        pgRowsRead: 0,\n        sqliteBytesRead: 0,\n        pgBytesRead: 0,\n      };\n    }\n    const read = {\n      sqliteRowsRead: sqlite.rows,\n      pgRowsRead: pg.rows,\n      sqliteBytesRead: sqlite.bytes,\n      pgBytesRead: pg.bytes,\n    };\n    if (sqlite.limitReached || pg.limitReached) {\n      // Skip a transaction that cannot fit in a fresh budget.\n      // Defer one that does not fit only in the remaining budget.\n      // A budget is fresh when neither dimension has been spent.\n      const sqliteFresh =\n        sqliteMaxRows === maxRowsPerSource &&\n        sqliteMaxBytes === maxBytesPerSource;\n      const pgFresh =\n        pgMaxRows === maxRowsPerSource && pgMaxBytes === maxBytesPerSource;\n      const outcome =\n        (sqlite.limitReached && sqliteFresh) || (pg.limitReached && pgFresh)\n          ? 'oversized'\n          : 'deferred';\n      return {outcome, ...read};\n    }\n    if (sqlite.digest === pg.digest) {\n      return {outcome: 'match', ...read};\n    }\n    return {\n      outcome: await this.#reconfirm(db, pinned, watermark),\n      ...read,\n    };\n  }\n\n  /**\n   * Rechecks a suspect result against current store bounds.\n   * A moved bound, changed seed, failed read, or lower head makes the result inconclusive.\n   */\n  async #reconfirm(\n    db: Database,\n    pinned: PinnedBounds,\n    watermark: string,\n  ): Promise<'mismatch' | 'inconclusive'> {\n    try {\n      const meta = readChangeLogMeta(db);\n      if (\n        meta.seedWatermark !== pinned.meta.seedWatermark ||\n        meta.seededAtMs !== pinned.meta.seededAtMs\n      ) {\n        return 'inconclusive';\n      }\n      const sqlite = this.#readSQLiteBounds(db);\n      const {minWatermark, lastWatermark} = await this.#pg.getCatchupBounds();\n      if (sqlite === undefined || minWatermark === null) {\n        return 'inconclusive';\n      }\n      if (\n        (sqlite.minWatermark > pinned.sqlite.minWatermark &&\n          watermark <= sqlite.minWatermark) ||\n        (minWatermark > pinned.pgMinWatermark && watermark <= minWatermark) ||\n        watermark > sqlite.headWatermark ||\n        watermark > lastWatermark\n      ) {\n        return 'inconclusive';\n      }\n      return 'mismatch';\n    } catch {\n      return 'inconclusive';\n    }\n  }\n\n  #skipped(reason: CompareSkipReason): CompareCycleResult {\n    this.#compareCycles.add(1, {result: reason});\n    if (reason === 'error') {\n      // The caller logged the error.\n    } else if (reason !== 'nothing-to-compare') {\n      this.#lc.debug?.(`skipping SQLite change-log comparison: ${reason}`);\n    }\n    return {kind: 'skipped', reason};\n  }\n\n  #compared(\n    fromWatermark: string,\n    throughWatermark: string,\n    transactions: number,\n    sampled: number,\n    outcomes: Record<CompareOutcome, number>,\n  ): CompareCycleResult {\n    this.#compareCycles.add(1, {result: 'compared'});\n    const result = {\n      kind: 'compared',\n      fromWatermark,\n      throughWatermark,\n      transactions,\n      sampled,\n      matched: outcomes.match,\n      mismatched: outcomes.mismatch,\n      inconclusive: outcomes.inconclusive,\n      deferred: outcomes.deferred,\n      oversized: outcomes.oversized,\n    } as const;\n    this.#lc.debug?.('SQLite change-log comparison cycle', {\n      sqliteChangeLogCompare: result,\n    });\n    return result;\n  }\n}\n\n/** Applies the same reconstruction as Postgres catchup. */\nasync function* reconstructBatches(\n  batches: AsyncIterable<ChangeLogEntry[]>,\n): AsyncIterable<WatermarkedChange[]> {\n  for await (const batch of batches) {\n    yield batch.map(reconstructWatermarkedChange);\n  }\n}\n\nfunction minWatermark(a: string, b: string): string {\n  return a < b ? a : b;\n}\n\nfunction maxWatermark(...watermarks: string[]): string {\n  let max = watermarks[0];\n  for (const w of watermarks) {\n    if (w > max) {\n      max = w;\n    }\n  }\n  return max;\n}\n"],"mappings":";;;;;;;;;AA2BA,IAAM,8BAA8B;;AAGpC,IAAM,qCAAqC;;AAG3C,IAAM,wCAAwC;;;;;;;;AAS9C,IAAM,yCAAyC,KAAK,OAAO;;AAG3D,IAAM,0BAA0B;;AA8FhC,IAAM,oBAA4B;;qCAEG,wBAAwB;;qCAExB,wBAAwB;;;;AAK7D,IAAM,qBAA6B;6BACN,wBAAwB;;;;;;;;;;;;;;;AAoCrD,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,kBAA2B,mBACzB,WACA,oCACA,0MAGF;CACA,iBAA0B,mBACxB,WACA,oCACA,iDACF;CACA,iBAA0B,4BACxB,WACA,4CACA,qDACF;;CAGA;CACA;CACA;CACA,WAAW;CACX,wBAAwB;CACxB,kBAAkB;CAElB,YACE,IACA,OACA,eACA,UACA,IACA,MACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAKC,SAAS;EACd,KAAKC,iBAAiB;EACtB,KAAKC,YAAY;EACjB,KAAKC,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAKC,gBAAgB,KAAK,gBAAgB;EAC1C,KAAKC,kBAAkB,KAAK,kBAAkB;EAC9C,KAAKC,SACH,KAAK,kBACE,IAAI,SAAc,YAAW,aAAa,OAAO,CAAC;EAC3D,KAAKI,cAAc;CACrB;;;;;CAMA,cAA2C;EACzC,IAAI,KAAKC,UACP,OAAO,QAAQ,QAAQ;GAAC,MAAM;GAAW,QAAQ;EAAS,CAAC;EAE7D,IAAI,KAAKC,aAAa,KAAA,GACpB,KAAKA,WAAW,KAAKC,UAAU,EAC5B,MAAK,WAAU;GACd,IAAI,KAAKC,uBACP,KAAKC,gBAAgB,CAAC;GAExB,OAAO;EACT,CAAC,EACA,cAAc;GACb,KAAKH,WAAW,KAAA;EAClB,CAAC;EAEL,OAAO,KAAKA;CACd;CAEA,OAAa;EACX,KAAKD,WAAW;EAChB,IAAI,KAAKK,WAAW,KAAA,GAAW;GAC7B,KAAKX,gBAAgB,KAAKW,MAAM;GAChC,KAAKA,SAAS,KAAA;EAChB;CACF;;CAGA,aAAmB;EACjB,KAAKC;EACL,KAAKC,UAAU,KAAA;EACf,KAAKJ,wBAAwB;CAC/B;CAEA,cACE,UAAU,KAAKZ,MAAM,cAAc,6BAC7B;EACN,IAAI,KAAKS,YAAY,KAAKK,WAAW,KAAA,GACnC;EAEF,KAAKA,SAAS,KAAKZ,oBAAoB;GACrC,KAAKY,SAAS,KAAA;GACd,KAAU,YAAY,EAAE,cAAc,KAAKN,cAAc,CAAC;EAC5D,GAAG,OAAO;CACZ;CAEA,gBAAgB,SAAuB;EACrC,IAAI,KAAKM,WAAW,KAAA,GAAW;GAC7B,KAAKX,gBAAgB,KAAKW,MAAM;GAChC,KAAKA,SAAS,KAAA;EAChB;EACA,KAAKN,cAAc,OAAO;CAC5B;CAEA,MAAMG,YAAyC;EAC7C,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,iBAAiB,KAAKI;EAC5B,KAAKH,wBAAwB;EAC7B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IAEF,KAAK,IAAI,SAAS,KAAKjB,KAAK,KAAKE,gBAAgB,EAAC,UAAU,KAAI,CAAC;GACnE,QAAQ;IACN,OAAO,KAAKoB,SAAS,iBAAiB;GACxC;GAEA,MAAM,SAAS,MAAM,KAAKC,WAAW,EAAE;GACvC,IAAI,mBAAmB,KAAKH,iBAC1B,OAAO,KAAKE,SAAS,aAAa;GAEpC,IAAI,OAAO,WAAW,UACpB,OAAO,KAAKA,SAAS,MAAM;GAE7B,MAAM,EAAC,SAAQ;GAEf,MAAM,OAAO,aACX,KAAKD,WAAW,KAAK,eAGrB,KAAK,eAEL,OAAO,gBACP,OAAO,OAAO,YAChB;GACA,MAAM,UAAU,aACd,OAAO,iBACP,OAAO,OAAO,aAChB;GACA,IAAI,WAAW,MACb,OAAO,KAAKC,SAAS,oBAAoB;GAG3C,IAAI,eAAe;GACnB,IAAI,UAAU;GACd,MAAM,WAA2C;IAC/C,OAAO;IACP,UAAU;IACV,cAAc;IACd,UAAU;IACV,WAAW;GACb;GACA,MAAM,UAAU,WAAmB,YAA4B;IAC7D,KAAKZ,gBAAgB,IAAI,GAAG,EAAC,QAAO,CAAC;IACrC,SAAS;IACT,IAAI,YAAY,YAEd,KAAKV,IAAI,QACP,2DACA,EAAC,wBAAwB,EAAC,UAAS,EAAC,CACtC;GAEJ;GAEA,SAAS,IAAI,sBAAsB,KAAKA,KAAK,KAAKE,cAAc;GAChE,MAAM,OAAO,OAAO,KAAK,IAAI;GAC7B,IAAI,KAAK,SAAS,aAChB,OAAO,KAAKoB,SAAS,iBAAiB;GAExC,IAAI,KAAK,SAAS,SAChB,OAAO,KAAKA,SAAS,oBAAoB;GAE3C,IAAI,KAAK,SAAS,WAAW;IAG3B,MAAM,cAAc,MAAM,KAAKE,WAAW,IAAI,QAAQ,IAAI;IAC1D,MAAM,UACJ,mBAAmB,KAAKJ,kBACpB,cACA;IACN,OAAO,MAAM,OAAO;IACpB,IAAI,YAAY,gBAEd,OAAO,KAAKK,UAAU,MAAM,MAAM,cAAc,SAAS,QAAQ;GAIrE;GAEA,MAAM,QAAQ,KAAK,IACjB,GACA,KAAKpB,MAAM,2BACT,kCACJ;GACA,MAAM,QAAQ,MAAM,KAAKqB,kBAAkB,IAAI,MAAM,SAAS,KAAK;GACnE,IAAI,mBAAmB,KAAKN,iBAC1B,OAAO,KAAKE,SAAS,aAAa;GAEpC,MAAM,mBAAmB,KAAK,IAC5B,GACA,KAAKjB,MAAM,4BACT,qCACJ;GACA,MAAM,oBAAoB,KAAK,IAC7B,GACA,KAAKA,MAAM,6BACT,sCACJ;GAEA,IAAI,iBAAiB;GACrB,IAAI,aAAa;GACjB,IAAI,kBAAkB;GACtB,IAAI,cAAc;GAClB,IAAI,mBAAmB;GACvB,IAAI,iBAAiB;GACrB,KAAK,MAAM,EAAC,WAAW,UAAU,UAAS,OAAO;IAC/C,IAAI,KAAKS,UACP;IAEF,IAAI,uBAAuB;IAC3B,IAAI,CAAC,YAAY,CAAC,MAAM;KAEtB,MAAM,UAAU,MAAM,KAAKU,WAAW,IAAI,QAAQ,SAAS;KAC3D,OACE,WACA,mBAAmB,KAAKJ,kBAAkB,UAAU,cACtD;IACF,OAAO,IACL,kBAAkB,KAAKnB,QAAQ,WAAW,KAAKI,MAAM,cAAc,GACnE;KACA,MAAM,sBAAsB,mBAAmB;KAC/C,MAAM,kBAAkB,mBAAmB;KAG3C,MAAM,uBAAuB,oBAAoB;KACjD,MAAM,mBAAmB,oBAAoB;KAC7C,IACE,wBAAwB,KACxB,oBAAoB,KACpB,wBAAwB,KACxB,oBAAoB,GACpB;MACA,OAAO,WAAW,UAAU;MAC5B;KACF;KACA;KACA,MAAM,aAAa,MAAM,KAAKsB,oBAC5B,IACA,QACA,QACA,kBACA,WACA,qBACA,iBACA,sBACA,kBACA,kBACA,iBACF;KACA,kBAAkB,WAAW;KAC7B,cAAc,WAAW;KACzB,mBAAmB,WAAW;KAC9B,eAAe,WAAW;KAC1B,MAAM,UACJ,mBAAmB,KAAKP,kBACpB,WAAW,UACX;KACN,OAAO,WAAW,OAAO;KACzB,IAAI,YAAY,gBACd;KAEF,IAAI,YAAY,YACd;KAEF,IAAI,YAAY,aAEd,uBAAuB;KAGzB,MAAM,KAAKX,OAAO;IACpB;IACA,mBAAmB;IACnB,iBAAiB;IACjB;IACA,IAAI,sBACF;GAEJ;GAEA,IACE,CAAC,KAAKK,YACN,mBAAmB,KAAKM,mBACxB,SAAS,iBAAiB,GAC1B;IACA,KAAKC,UAAU;IACf,KAAKJ,wBAAwB,iBAAiB;GAChD;GACA,OAAO,KAAKQ,UACV,MACA,gBACA,cACA,SACA,QACF;EACF,SAAS,GAAG;GACV,KAAKzB,IAAI,OAAO,yCAAyC,CAAC;GAC1D,OAAO,KAAKsB,SAAS,OAAO;EAC9B,UAAU;GACR,QAAQ,MAAM;GACd,IAAI,MAAM;GACV,KAAKV,eAAe,SAAS,YAAY,IAAI,IAAI,SAAS;EAC5D;CACF;;CAGA,MAAMW,WAAW,IAAyD;EACxE,IAAI;EACJ,IAAI;GACF,OAAO,kBAAkB,EAAE;EAC7B,QAAQ;GAEN,OAAO;EACT;EACA,IAAI,KAAK,kBAAA,GACP,OAAO;EAET,MAAM,EAAC,OAAO,YAAY,cAAa,KAAKpB;EAC5C,IACE,KAAK,UAAU,SACf,KAAK,eAAe,cACpB,KAAK,cAAc,WAEnB,OAAO;EAET,IAAI,KAAKG,KAAK,IAAI,KAAK,aAAa,KAAKD,MAAM,aAC7C,OAAO;EAET,MAAM,SAAS,KAAKuB,kBAAkB,EAAE;EACxC,IAAI,WAAW,KAAA,GACb,OAAO;EAET,MAAM,EAAC,cAAc,kBAAiB,MAAM,KAAKxB,IAAI,iBAAiB;EACtE,IAAI,iBAAiB,MAEnB,OAAO;EAET,OAAO;GACL;GACA;GACA,gBAAgB;GAChB,iBAAiB;EACnB;CACF;CAEA,kBAAkB,IAAwC;EACxD,MAAM,SAAS,GACZ,QAAQ,iBAAiB,EACzB,IAAiE;EACpE,OAAO,OAAO,iBAAiB,QAAQ,OAAO,kBAAkB,OAC5D,KAAA,IACA;GACE,cAAc,OAAO;GACrB,eAAe,OAAO;EACxB;CACN;;;;;CAMA,MAAMsB,kBACJ,IACA,MACA,SACA,OACkE;EAClE,IAAI,SAAmB,GACpB,QAAQ,kBAAkB,EAC1B,IAAyB;GAAC,OAAO;GAAM;GAAS,OAAO,QAAQ;EAAC,CAAC,EACjE,KAAK,EAAC,gBAAe,SAAS;EACjC,IAAI,KAAK,MAAM,KAAKtB,IAAI,qBAAqB,MAAM,SAAS,QAAQ,CAAC;EAErE,IAAI,mBAAmB;EACvB,IAAI,OAAO,SAAS,OAAO;GACzB,OAAO,SAAS;GAChB,mBAAmB,aAAa,kBAAkB,OAAO,QAAQ,EAAE;EACrE;EACA,IAAI,GAAG,SAAS,OAAO;GACrB,GAAG,SAAS;GACZ,mBAAmB,aAAa,kBAAkB,GAAG,QAAQ,EAAE;EACjE;EACA,IAAI,qBAAqB,SAAS;GAChC,SAAS,OAAO,QAAO,MAAK,KAAK,gBAAgB;GACjD,KAAK,GAAG,QAAO,MAAK,KAAK,gBAAgB;EAC3C;EAEA,MAAM,QAAiE,CAAC;EACxE,IAAI,IAAI;EACR,IAAI,IAAI;EACR,OAAO,IAAI,OAAO,UAAU,IAAI,GAAG,QAAQ;GACzC,MAAM,KAAK,IAAI,OAAO,SAAS,OAAO,KAAK,KAAA;GAC3C,MAAM,KAAK,IAAI,GAAG,SAAS,GAAG,KAAK,KAAA;GACnC,IAAI,OAAO,KAAA,MAAc,OAAO,KAAA,KAAa,KAAK,KAAK;IACrD,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAM,MAAM;IAAK,CAAC;IACvD;GACF,OAAO,IAAI,OAAO,KAAA,MAAc,OAAO,KAAA,KAAa,KAAK,KAAK;IAC5D,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAO,MAAM;IAAI,CAAC;IACvD;GACF,OAAO,IAAI,OAAO,KAAA,GAAW;IAC3B,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAM,MAAM;IAAI,CAAC;IACtD;IACA;GACF;EACF;EACA,OAAO;CACT;;;;;CAMA,MAAMuB,oBACJ,IACA,QACA,QACA,kBACA,WACA,eACA,WACA,gBACA,YACA,kBACA,mBAC4B;EAC5B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,MAAM,gBAAgB,KAAKtB,MAAM,iBAAiB;GAClD,SAAS,MAAM,mBACb,OAAO,KACL,kBACA,WACA,KAAK,IAAI,eAAe,aAAa,GACrC,KAAA,GACA,aACF,GACA,WACA,eACA,cACF;GACA,KAAK,MAAM,mBACT,mBACE,KAAKD,IAAI,iBAAiB,kBAAkB,WAAW,SAAS,CAClE,GACA,WACA,WACA,UACF;EACF,SAAS,GAAG;GAEV,KAAKJ,IAAI,OACP,6BAA6B,UAAU,kBACvC,CACF;GACA,OAAO;IACL,SAAS,MAAM,KAAKwB,WAAW,IAAI,QAAQ,SAAS;IACpD,gBAAgB;IAChB,YAAY;IACZ,iBAAiB;IACjB,aAAa;GACf;EACF;EACA,MAAM,OAAO;GACX,gBAAgB,OAAO;GACvB,YAAY,GAAG;GACf,iBAAiB,OAAO;GACxB,aAAa,GAAG;EAClB;EACA,IAAI,OAAO,gBAAgB,GAAG,cAAc;GAI1C,MAAM,cACJ,kBAAkB,oBAClB,mBAAmB;GACrB,MAAM,UACJ,cAAc,oBAAoB,eAAe;GAKnD,OAAO;IAAC,SAHL,OAAO,gBAAgB,eAAiB,GAAG,gBAAgB,UACxD,cACA;IACW,GAAG;GAAI;EAC1B;EACA,IAAI,OAAO,WAAW,GAAG,QACvB,OAAO;GAAC,SAAS;GAAS,GAAG;EAAI;EAEnC,OAAO;GACL,SAAS,MAAM,KAAKA,WAAW,IAAI,QAAQ,SAAS;GACpD,GAAG;EACL;CACF;;;;;CAMA,MAAMA,WACJ,IACA,QACA,WACsC;EACtC,IAAI;GACF,MAAM,OAAO,kBAAkB,EAAE;GACjC,IACE,KAAK,kBAAkB,OAAO,KAAK,iBACnC,KAAK,eAAe,OAAO,KAAK,YAEhC,OAAO;GAET,MAAM,SAAS,KAAKI,kBAAkB,EAAE;GACxC,MAAM,EAAC,cAAc,kBAAiB,MAAM,KAAKxB,IAAI,iBAAiB;GACtE,IAAI,WAAW,KAAA,KAAa,iBAAiB,MAC3C,OAAO;GAET,IACG,OAAO,eAAe,OAAO,OAAO,gBACnC,aAAa,OAAO,gBACrB,eAAe,OAAO,kBAAkB,aAAa,gBACtD,YAAY,OAAO,iBACnB,YAAY,eAEZ,OAAO;GAET,OAAO;EACT,QAAQ;GACN,OAAO;EACT;CACF;CAEA,SAAS,QAA+C;EACtD,KAAKO,eAAe,IAAI,GAAG,EAAC,QAAQ,OAAM,CAAC;EAC3C,IAAI,WAAW,SAAS,CAExB,OAAO,IAAI,WAAW,sBACpB,KAAKX,IAAI,QAAQ,0CAA0C,QAAQ;EAErE,OAAO;GAAC,MAAM;GAAW;EAAM;CACjC;CAEA,UACE,eACA,kBACA,cACA,SACA,UACoB;EACpB,KAAKW,eAAe,IAAI,GAAG,EAAC,QAAQ,WAAU,CAAC;EAC/C,MAAM,SAAS;GACb,MAAM;GACN;GACA;GACA;GACA;GACA,SAAS,SAAS;GAClB,YAAY,SAAS;GACrB,cAAc,SAAS;GACvB,UAAU,SAAS;GACnB,WAAW,SAAS;EACtB;EACA,KAAKX,IAAI,QAAQ,sCAAsC,EACrD,wBAAwB,OAC1B,CAAC;EACD,OAAO;CACT;AACF;;AAGA,gBAAgB,mBACd,SACoC;CACpC,WAAW,MAAM,SAAS,SACxB,MAAM,MAAM,IAAI,4BAA4B;AAEhD;AAEA,SAAS,aAAa,GAAW,GAAmB;CAClD,OAAO,IAAI,IAAI,IAAI;AACrB;AAEA,SAAS,aAAa,GAAG,YAA8B;CACrD,IAAI,MAAM,WAAW;CACrB,KAAK,MAAM,KAAK,YACd,IAAI,IAAI,KACN,MAAM;CAGV,OAAO;AACT"}
+{"version":3,"file":"sqlite-change-log-comparator.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/sqlite-change-log-comparator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateLatencyHistogram,\n} from '../../observability/metrics.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport {\n  CHANGE_LOG_DB_SCHEMA_VERSION,\n  CHANGE_LOG_STREAM_TABLE,\n  readChangeLogMeta,\n  type ChangeLogIdentity,\n  type ChangeLogMeta,\n} from '../replicator/change-log-db.ts';\nimport {\n  reconstructWatermarkedChange,\n  type ChangeLogEntry,\n} from './change-log-codec.ts';\nimport {\n  digestCatchupRange,\n  type CatchupRangeDigest,\n} from './change-log-compare-digest.ts';\nimport type {WatermarkedChange} from './change-streamer.ts';\nimport {isSampledForShard} from './shard-sampling.ts';\nimport {SQLiteChangeLogReader} from './sqlite-change-log-reader.ts';\n\n/** Interval between comparison cycles. */\nconst DEFAULT_COMPARE_INTERVAL_MS = 30_000;\n\n/** Maximum transactions enumerated per cycle. */\nconst DEFAULT_MAX_TRANSACTIONS_PER_CYCLE = 256;\n\n/** Maximum catchup rows read from each store per cycle. */\nconst DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE = 1000;\n\n/**\n * Maximum catchup bytes read from each store per cycle.\n *\n * Rows bound the per-row overhead. Bytes bound the read and hash work,\n * which scales with payload size instead of row count. Normal traffic\n * reaches the row limit first. Wide payloads reach this one.\n */\nconst DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE = 32 * 1024 * 1024;\n\n/** Rows per SQLite catchup batch. */\nconst DEFAULT_READ_BATCH_ROWS = 1000;\n\n/**\n * Result for one committed transaction.\n *\n * `mismatch` covers missing, extra, changed, or reordered rows.\n * `inconclusive` means that store bounds changed during the comparison.\n */\nexport type CompareOutcome =\n  | 'match'\n  | 'mismatch'\n  | 'inconclusive'\n  | 'deferred'\n  | 'oversized';\n\nexport type CompareSkipReason =\n  // The comparator was stopped.\n  | 'stopped'\n  // The writer replaced the file during this cycle.\n  | 'log-rebuilt'\n  // The file is absent, unreadable, or uninitialized.\n  | 'log-unavailable'\n  // The schema version does not match this build.\n  | 'ineligible-schema'\n  // The log belongs to a different replica.\n  | 'ineligible-identity'\n  // The log is younger than the warm-up period.\n  | 'cold-log'\n  // The stores have no complete committed range in common.\n  | 'nothing-to-compare'\n  // The cycle failed. The error was logged and counted.\n  | 'error';\n\nexport type CompareCycleResult =\n  | {readonly kind: 'skipped'; readonly reason: CompareSkipReason}\n  | {\n      readonly kind: 'compared';\n      /** Exclusive lower bound of the compared range. */\n      readonly fromWatermark: string;\n      /** Inclusive upper bound handled. The next cycle resumes from here. */\n      readonly throughWatermark: string;\n      /** Committed transactions handled through `throughWatermark`. */\n      readonly transactions: number;\n      /** Transactions with compared catchup output. */\n      readonly sampled: number;\n      readonly matched: number;\n      readonly mismatched: number;\n      /** Suspects rejected because store bounds changed. */\n      readonly inconclusive: number;\n      /** Sampled transactions deferred to a fresh budget. */\n      readonly deferred: number;\n      /** Sampled transactions that exceed a fresh budget. */\n      readonly oversized: number;\n    };\n\n/** Provides the Postgres side of the catchup comparison. */\nexport interface PGChangeLogRangeReader {\n  getCatchupBounds(): Promise<{\n    minWatermark: string | null;\n    lastWatermark: string;\n  }>;\n  listCommitWatermarks(\n    afterWatermark: string,\n    throughWatermark: string,\n    limit: number,\n  ): Promise<string[]>;\n  readCatchupRange(\n    afterWatermark: string,\n    throughWatermark: string,\n    batchRows?: number | undefined,\n  ): AsyncIterable<ChangeLogEntry[]>;\n}\n\nexport type SQLiteChangeLogCompareOptions = {\n  /** Stable percentage of transactions selected for payload comparison. */\n  readonly comparePercent: number;\n  /** Warm-up time after a seed. The comparator skips a younger log. */\n  readonly retentionMs: number;\n  readonly intervalMs?: number | undefined;\n  readonly maxTransactionsPerCycle?: number | undefined;\n  /** Maximum catchup rows read from each store in one cycle. */\n  readonly maxRowsPerSourcePerCycle?: number | undefined;\n  /** Maximum catchup bytes read from each store in one cycle. */\n  readonly maxBytesPerSourcePerCycle?: number | undefined;\n  readonly readBatchRows?: number | undefined;\n  /** Clock for the warm-up period. Defaults to `Date.now`. */\n  readonly now?: (() => number) | undefined;\n  readonly setTimeoutFn?: typeof setTimeout | undefined;\n  readonly clearTimeoutFn?: typeof clearTimeout | undefined;\n  /** Yields between sampled transactions. Defaults to `setImmediate`. */\n  readonly yieldFn?: (() => Promise<void>) | undefined;\n};\n\n/** Uses separate aggregates so SQLite can use an index for each value. */\nconst SQLITE_BOUNDS_SQL = /*sql*/ `\n  SELECT\n    (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"minWatermark\",\n    (SELECT max(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n      AS \"headWatermark\"\n`;\n\n/** Lists committed transaction watermarks in `(@after, @through]`. */\nconst SQLITE_COMMITS_SQL = /*sql*/ `\n  SELECT \"watermark\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n   WHERE \"precommit\" IS NOT NULL\n     AND \"watermark\" > @after\n     AND \"watermark\" <= @through\n   ORDER BY \"watermark\"\n   LIMIT @limit\n`;\n\ntype SQLiteBounds = {\n  readonly minWatermark: string;\n  readonly headWatermark: string;\n};\n\ntype PinnedBounds = {\n  readonly meta: ChangeLogMeta;\n  readonly sqlite: SQLiteBounds;\n  readonly pgMinWatermark: string;\n  readonly pgLastWatermark: string;\n};\n\ntype SampledComparison = {\n  readonly outcome: CompareOutcome;\n  readonly sqliteRowsRead: number;\n  readonly pgRowsRead: number;\n  readonly sqliteBytesRead: number;\n  readonly pgBytesRead: number;\n};\n\n/**\n * Compares SQLite and Postgres catchup output without affecting subscribers.\n *\n * Each cycle compares complete ranges through the lower store head.\n * It checks every transaction for presence and samples payloads.\n * A bounds change makes the affected result `inconclusive`.\n * Postgres remains authoritative.\n */\nexport class SQLiteChangeLogComparator {\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeLogFile: string;\n  readonly #identity: ChangeLogIdentity;\n  readonly #pg: PGChangeLogRangeReader;\n  readonly #opts: SQLiteChangeLogCompareOptions;\n  readonly #now: () => number;\n  readonly #setTimeoutFn: typeof setTimeout;\n  readonly #clearTimeoutFn: typeof clearTimeout;\n  readonly #yield: () => Promise<void>;\n\n  readonly #compareResults = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.compare_result',\n    'Outcomes of the dark comparison between what SQLite catchup serves and ' +\n      'what PG catchup serves, per committed transaction. `inconclusive` ' +\n      'means a purge or reseed moved the pinned bounds mid-comparison.',\n  );\n  readonly #compareCycles = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.compare_cycles',\n    'SQLite change-log comparison cycles, by result.',\n  );\n  readonly #cycleDuration = getOrCreateLatencyHistogram(\n    'replica',\n    'sqlite_change_log.compare_cycle_duration',\n    'Duration of one SQLite change-log comparison cycle.',\n  );\n\n  /** Last handled commit watermark. A restart derives it from current bounds. */\n  #cursor: string | undefined;\n  #running: Promise<CompareCycleResult> | undefined;\n  #timer: ReturnType<typeof setTimeout> | undefined;\n  #stopped = false;\n  #continueWithoutDelay = false;\n  #fileGeneration = 0;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    changeLogFile: string,\n    identity: ChangeLogIdentity,\n    pg: PGChangeLogRangeReader,\n    opts: SQLiteChangeLogCompareOptions,\n  ) {\n    this.#lc = lc.withContext('component', 'sqlite-change-log-compare');\n    this.#shard = shard;\n    this.#changeLogFile = changeLogFile;\n    this.#identity = identity;\n    this.#pg = pg;\n    this.#opts = opts;\n    this.#now = opts.now ?? Date.now;\n    this.#setTimeoutFn = opts.setTimeoutFn ?? setTimeout;\n    this.#clearTimeoutFn = opts.clearTimeoutFn ?? clearTimeout;\n    this.#yield =\n      opts.yieldFn ??\n      (() => new Promise<void>(resolve => setImmediate(resolve)));\n    this.#scheduleNext();\n  }\n\n  /**\n   * Runs one cycle. Concurrent calls share the same promise.\n   * Errors produce `skipped: 'error'`.\n   */\n  compareOnce(): Promise<CompareCycleResult> {\n    if (this.#stopped) {\n      return Promise.resolve({kind: 'skipped', reason: 'stopped'});\n    }\n    if (this.#running === undefined) {\n      this.#running = this.#runCycle()\n        .then(result => {\n          if (this.#continueWithoutDelay) {\n            this.#rescheduleNext(0);\n          }\n          return result;\n        })\n        .finally(() => {\n          this.#running = undefined;\n        });\n    }\n    return this.#running;\n  }\n\n  stop(): void {\n    this.#stopped = true;\n    if (this.#timer !== undefined) {\n      this.#clearTimeoutFn(this.#timer);\n      this.#timer = undefined;\n    }\n  }\n\n  /** Invalidates cycles that can still read a replaced file. */\n  invalidate(): void {\n    this.#fileGeneration++;\n    this.#cursor = undefined;\n    this.#continueWithoutDelay = false;\n  }\n\n  #scheduleNext(\n    delayMs = this.#opts.intervalMs ?? DEFAULT_COMPARE_INTERVAL_MS,\n  ): void {\n    if (this.#stopped || this.#timer !== undefined) {\n      return;\n    }\n    this.#timer = this.#setTimeoutFn(() => {\n      this.#timer = undefined;\n      void this.compareOnce().finally(() => this.#scheduleNext());\n    }, delayMs);\n  }\n\n  #rescheduleNext(delayMs: number): void {\n    if (this.#timer !== undefined) {\n      this.#clearTimeoutFn(this.#timer);\n      this.#timer = undefined;\n    }\n    this.#scheduleNext(delayMs);\n  }\n\n  async #runCycle(): Promise<CompareCycleResult> {\n    const startedAt = performance.now();\n    const fileGeneration = this.#fileGeneration;\n    this.#continueWithoutDelay = false;\n    let db: Database | undefined;\n    let reader: SQLiteChangeLogReader | undefined;\n    try {\n      try {\n        // A readonly handle does not create a missing file.\n        db = new Database(this.#lc, this.#changeLogFile, {readonly: true});\n      } catch {\n        return this.#skipped('log-unavailable');\n      }\n\n      const pinned = await this.#pinBounds(db);\n      if (fileGeneration !== this.#fileGeneration) {\n        return this.#skipped('log-rebuilt');\n      }\n      if (typeof pinned === 'string') {\n        return this.#skipped(pinned);\n      }\n      const {meta} = pinned;\n\n      const from = maxWatermark(\n        this.#cursor ?? meta.seedWatermark,\n        // Exclude the synthetic SQLite seed transaction.\n        // The Postgres seed has no `precommit`, so enumeration also excludes it.\n        meta.seedWatermark,\n        // A store cannot serve the transaction at its minimum watermark.\n        pinned.pgMinWatermark,\n        pinned.sqlite.minWatermark,\n      );\n      const through = minWatermark(\n        pinned.pgLastWatermark,\n        pinned.sqlite.headWatermark,\n      );\n      if (through <= from) {\n        return this.#skipped('nothing-to-compare');\n      }\n\n      let transactions = 0;\n      let sampled = 0;\n      const outcomes: Record<CompareOutcome, number> = {\n        match: 0,\n        mismatch: 0,\n        inconclusive: 0,\n        deferred: 0,\n        oversized: 0,\n      };\n      const record = (watermark: string, outcome: CompareOutcome) => {\n        this.#compareResults.add(1, {outcome});\n        outcomes[outcome]++;\n        if (outcome === 'mismatch') {\n          // Log only the watermark because payloads contain customer data.\n          this.#lc.error?.(\n            'SQLite change-log catchup output diverged from Postgres',\n            {sqliteChangeLogCompare: {watermark}},\n          );\n        }\n      };\n\n      reader = new SQLiteChangeLogReader(this.#lc, this.#changeLogFile);\n      const plan = reader.plan(from);\n      if (plan.kind === 'not-ready') {\n        return this.#skipped('log-unavailable');\n      }\n      if (plan.kind === 'ahead') {\n        return this.#skipped('nothing-to-compare');\n      }\n      if (plan.kind === 'too-old') {\n        // A fixed lower bound makes this result a mismatch.\n        // A changed lower bound makes it inconclusive.\n        const reconfirmed = await this.#reconfirm(db, pinned, from);\n        const outcome =\n          fileGeneration === this.#fileGeneration\n            ? reconfirmed\n            : 'inconclusive';\n        record(from, outcome);\n        if (outcome === 'inconclusive') {\n          // Keep the cursor unchanged because the bounds moved.\n          return this.#compared(from, from, transactions, sampled, outcomes);\n        }\n        // Positional reads can continue after a missing boundary.\n        // Advancing avoids reporting the same boundary in every cycle.\n      }\n\n      const limit = Math.max(\n        1,\n        this.#opts.maxTransactionsPerCycle ??\n          DEFAULT_MAX_TRANSACTIONS_PER_CYCLE,\n      );\n      const union = await this.#enumerateCommits(db, from, through, limit);\n      if (fileGeneration !== this.#fileGeneration) {\n        return this.#skipped('log-rebuilt');\n      }\n      const maxRowsPerSource = Math.max(\n        1,\n        this.#opts.maxRowsPerSourcePerCycle ??\n          DEFAULT_MAX_ROWS_PER_SOURCE_PER_CYCLE,\n      );\n      const maxBytesPerSource = Math.max(\n        1,\n        this.#opts.maxBytesPerSourcePerCycle ??\n          DEFAULT_MAX_BYTES_PER_SOURCE_PER_CYCLE,\n      );\n\n      let sqliteRowsRead = 0;\n      let pgRowsRead = 0;\n      let sqliteBytesRead = 0;\n      let pgBytesRead = 0;\n      let previousBoundary = from;\n      let handledThrough = from;\n      for (const {watermark, inSqlite, inPg} of union) {\n        if (this.#stopped) {\n          break;\n        }\n        let stopAfterTransaction = false;\n        if (!inSqlite || !inPg) {\n          // Check every transaction for presence because this needs no payload read.\n          const outcome = await this.#reconfirm(db, pinned, watermark);\n          record(\n            watermark,\n            fileGeneration === this.#fileGeneration ? outcome : 'inconclusive',\n          );\n        } else if (\n          isSampledForShard(this.#shard, watermark, this.#opts.comparePercent)\n        ) {\n          const sqliteRowsRemaining = maxRowsPerSource - sqliteRowsRead;\n          const pgRowsRemaining = maxRowsPerSource - pgRowsRead;\n          // A read may pass the byte budget by its last row, so this\n          // compares against zero rather than checking for equality.\n          const sqliteBytesRemaining = maxBytesPerSource - sqliteBytesRead;\n          const pgBytesRemaining = maxBytesPerSource - pgBytesRead;\n          if (\n            sqliteRowsRemaining === 0 ||\n            pgRowsRemaining === 0 ||\n            sqliteBytesRemaining <= 0 ||\n            pgBytesRemaining <= 0\n          ) {\n            record(watermark, 'deferred');\n            break;\n          }\n          sampled++;\n          const comparison = await this.#compareTransaction(\n            db,\n            reader,\n            pinned,\n            previousBoundary,\n            watermark,\n            sqliteRowsRemaining,\n            pgRowsRemaining,\n            sqliteBytesRemaining,\n            pgBytesRemaining,\n            maxRowsPerSource,\n            maxBytesPerSource,\n          );\n          sqliteRowsRead += comparison.sqliteRowsRead;\n          pgRowsRead += comparison.pgRowsRead;\n          sqliteBytesRead += comparison.sqliteBytesRead;\n          pgBytesRead += comparison.pgBytesRead;\n          const outcome =\n            fileGeneration === this.#fileGeneration\n              ? comparison.outcome\n              : 'inconclusive';\n          record(watermark, outcome);\n          if (outcome === 'inconclusive') {\n            break;\n          }\n          if (outcome === 'deferred') {\n            break;\n          }\n          if (outcome === 'oversized') {\n            // Advance past a transaction that cannot fit in a fresh budget.\n            stopAfterTransaction = true;\n          }\n          // Yield between sampled reads so other stream work can continue.\n          await this.#yield();\n        }\n        previousBoundary = watermark;\n        handledThrough = watermark;\n        transactions++;\n        if (stopAfterTransaction) {\n          break;\n        }\n      }\n\n      if (\n        !this.#stopped &&\n        fileGeneration === this.#fileGeneration &&\n        outcomes.inconclusive === 0\n      ) {\n        this.#cursor = handledThrough;\n        this.#continueWithoutDelay = handledThrough < through;\n      }\n      return this.#compared(\n        from,\n        handledThrough,\n        transactions,\n        sampled,\n        outcomes,\n      );\n    } catch (e) {\n      this.#lc.warn?.('error comparing the SQLite change log', e);\n      return this.#skipped('error');\n    } finally {\n      reader?.close();\n      db?.close();\n      this.#cycleDuration.recordMs(performance.now() - startedAt);\n    }\n  }\n\n  /** Checks log eligibility and reads the bounds from both stores. */\n  async #pinBounds(db: Database): Promise<PinnedBounds | CompareSkipReason> {\n    let meta: ChangeLogMeta;\n    try {\n      meta = readChangeLogMeta(db);\n    } catch {\n      // The writer has not initialized the log.\n      return 'log-unavailable';\n    }\n    if (meta.schemaVersion !== CHANGE_LOG_DB_SCHEMA_VERSION) {\n      return 'ineligible-schema';\n    }\n    const {epoch, generation, replicaID} = this.#identity;\n    if (\n      meta.epoch !== epoch ||\n      meta.generation !== generation ||\n      meta.replicaID !== replicaID\n    ) {\n      return 'ineligible-identity';\n    }\n    if (this.#now() - meta.seededAtMs < this.#opts.retentionMs) {\n      return 'cold-log';\n    }\n    const sqlite = this.#readSQLiteBounds(db);\n    if (sqlite === undefined) {\n      return 'log-unavailable';\n    }\n    const {minWatermark, lastWatermark} = await this.#pg.getCatchupBounds();\n    if (minWatermark === null) {\n      // Only a new replica has an empty Postgres change log.\n      return 'nothing-to-compare';\n    }\n    return {\n      meta,\n      sqlite,\n      pgMinWatermark: minWatermark,\n      pgLastWatermark: lastWatermark,\n    };\n  }\n\n  #readSQLiteBounds(db: Database): SQLiteBounds | undefined {\n    const bounds = db\n      .prepare(SQLITE_BOUNDS_SQL)\n      .get<{minWatermark: string | null; headWatermark: string | null}>();\n    return bounds.minWatermark === null || bounds.headWatermark === null\n      ? undefined\n      : {\n          minWatermark: bounds.minWatermark,\n          headWatermark: bounds.headWatermark,\n        };\n  }\n\n  /**\n   * Merges committed transaction watermarks from both stores.\n   * If a list reaches the limit, the result ends at the last complete common range.\n   */\n  async #enumerateCommits(\n    db: Database,\n    from: string,\n    through: string,\n    limit: number,\n  ): Promise<{watermark: string; inSqlite: boolean; inPg: boolean}[]> {\n    let sqlite: string[] = db\n      .prepare(SQLITE_COMMITS_SQL)\n      .all<{watermark: string}>({after: from, through, limit: limit + 1})\n      .map(({watermark}) => watermark);\n    let pg = await this.#pg.listCommitWatermarks(from, through, limit + 1);\n\n    let effectiveThrough = through;\n    if (sqlite.length > limit) {\n      sqlite.length = limit;\n      effectiveThrough = minWatermark(effectiveThrough, sqlite[limit - 1]);\n    }\n    if (pg.length > limit) {\n      pg.length = limit;\n      effectiveThrough = minWatermark(effectiveThrough, pg[limit - 1]);\n    }\n    if (effectiveThrough !== through) {\n      sqlite = sqlite.filter(w => w <= effectiveThrough);\n      pg = pg.filter(w => w <= effectiveThrough);\n    }\n\n    const union: {watermark: string; inSqlite: boolean; inPg: boolean}[] = [];\n    let s = 0;\n    let p = 0;\n    while (s < sqlite.length || p < pg.length) {\n      const sw = s < sqlite.length ? sqlite[s] : undefined;\n      const pw = p < pg.length ? pg[p] : undefined;\n      if (sw !== undefined && (pw === undefined || sw < pw)) {\n        union.push({watermark: sw, inSqlite: true, inPg: false});\n        s++;\n      } else if (pw !== undefined && (sw === undefined || pw < sw)) {\n        union.push({watermark: pw, inSqlite: false, inPg: true});\n        p++;\n      } else if (sw !== undefined) {\n        union.push({watermark: sw, inSqlite: true, inPg: true});\n        s++;\n        p++;\n      }\n    }\n    return union;\n  }\n\n  /**\n   * Compares the output in `(previousBoundary, watermark]`.\n   * Positional reads do not require either store to contain the boundary row.\n   */\n  async #compareTransaction(\n    db: Database,\n    reader: SQLiteChangeLogReader,\n    pinned: PinnedBounds,\n    previousBoundary: string,\n    watermark: string,\n    sqliteMaxRows: number,\n    pgMaxRows: number,\n    sqliteMaxBytes: number,\n    pgMaxBytes: number,\n    maxRowsPerSource: number,\n    maxBytesPerSource: number,\n  ): Promise<SampledComparison> {\n    let sqlite: CatchupRangeDigest;\n    let pg: CatchupRangeDigest;\n    try {\n      const readBatchRows = this.#opts.readBatchRows ?? DEFAULT_READ_BATCH_ROWS;\n      sqlite = await digestCatchupRange(\n        reader.read(\n          previousBoundary,\n          watermark,\n          Math.min(readBatchRows, sqliteMaxRows),\n          undefined,\n          sqliteMaxRows,\n        ),\n        watermark,\n        sqliteMaxRows,\n        sqliteMaxBytes,\n      );\n      pg = await digestCatchupRange(\n        reconstructBatches(\n          this.#pg.readCatchupRange(previousBoundary, watermark, pgMaxRows),\n        ),\n        watermark,\n        pgMaxRows,\n        pgMaxBytes,\n      );\n    } catch (e) {\n      // A failed catchup read makes the transaction suspect.\n      this.#lc.warn?.(\n        `error reading transaction ${watermark} for comparison`,\n        e,\n      );\n      return {\n        outcome: await this.#reconfirm(db, pinned, watermark),\n        sqliteRowsRead: 0,\n        pgRowsRead: 0,\n        sqliteBytesRead: 0,\n        pgBytesRead: 0,\n      };\n    }\n    const read = {\n      sqliteRowsRead: sqlite.rows,\n      pgRowsRead: pg.rows,\n      sqliteBytesRead: sqlite.bytes,\n      pgBytesRead: pg.bytes,\n    };\n    if (sqlite.limitReached || pg.limitReached) {\n      // Skip a transaction that cannot fit in a fresh budget.\n      // Defer one that does not fit only in the remaining budget.\n      // A budget is fresh when neither dimension has been spent.\n      const sqliteFresh =\n        sqliteMaxRows === maxRowsPerSource &&\n        sqliteMaxBytes === maxBytesPerSource;\n      const pgFresh =\n        pgMaxRows === maxRowsPerSource && pgMaxBytes === maxBytesPerSource;\n      const outcome =\n        (sqlite.limitReached && sqliteFresh) || (pg.limitReached && pgFresh)\n          ? 'oversized'\n          : 'deferred';\n      return {outcome, ...read};\n    }\n    if (sqlite.digest === pg.digest) {\n      return {outcome: 'match', ...read};\n    }\n    return {\n      outcome: await this.#reconfirm(db, pinned, watermark),\n      ...read,\n    };\n  }\n\n  /**\n   * Rechecks a suspect result against current store bounds.\n   * A moved bound, changed seed, failed read, or lower head makes the result inconclusive.\n   */\n  async #reconfirm(\n    db: Database,\n    pinned: PinnedBounds,\n    watermark: string,\n  ): Promise<'mismatch' | 'inconclusive'> {\n    try {\n      const meta = readChangeLogMeta(db);\n      if (\n        meta.seedWatermark !== pinned.meta.seedWatermark ||\n        meta.seededAtMs !== pinned.meta.seededAtMs\n      ) {\n        return 'inconclusive';\n      }\n      const sqlite = this.#readSQLiteBounds(db);\n      const {minWatermark, lastWatermark} = await this.#pg.getCatchupBounds();\n      if (sqlite === undefined || minWatermark === null) {\n        return 'inconclusive';\n      }\n      if (\n        (sqlite.minWatermark > pinned.sqlite.minWatermark &&\n          watermark <= sqlite.minWatermark) ||\n        (minWatermark > pinned.pgMinWatermark && watermark <= minWatermark) ||\n        watermark > sqlite.headWatermark ||\n        watermark > lastWatermark\n      ) {\n        return 'inconclusive';\n      }\n      return 'mismatch';\n    } catch {\n      return 'inconclusive';\n    }\n  }\n\n  #skipped(reason: CompareSkipReason): CompareCycleResult {\n    this.#compareCycles.add(1, {result: reason});\n    if (reason === 'error') {\n      // The caller logged the error.\n    } else if (reason !== 'nothing-to-compare') {\n      this.#lc.debug?.(`skipping SQLite change-log comparison: ${reason}`);\n    }\n    return {kind: 'skipped', reason};\n  }\n\n  #compared(\n    fromWatermark: string,\n    throughWatermark: string,\n    transactions: number,\n    sampled: number,\n    outcomes: Record<CompareOutcome, number>,\n  ): CompareCycleResult {\n    this.#compareCycles.add(1, {result: 'compared'});\n    const result = {\n      kind: 'compared',\n      fromWatermark,\n      throughWatermark,\n      transactions,\n      sampled,\n      matched: outcomes.match,\n      mismatched: outcomes.mismatch,\n      inconclusive: outcomes.inconclusive,\n      deferred: outcomes.deferred,\n      oversized: outcomes.oversized,\n    } as const;\n    this.#lc.debug?.('SQLite change-log comparison cycle', {\n      sqliteChangeLogCompare: result,\n    });\n    return result;\n  }\n}\n\n/** Applies the same reconstruction as Postgres catchup. */\nasync function* reconstructBatches(\n  batches: AsyncIterable<ChangeLogEntry[]>,\n): AsyncIterable<WatermarkedChange[]> {\n  for await (const batch of batches) {\n    yield batch.map(({watermark, tag, change}) =>\n      reconstructWatermarkedChange(watermark, tag, change),\n    );\n  }\n}\n\nfunction minWatermark(a: string, b: string): string {\n  return a < b ? a : b;\n}\n\nfunction maxWatermark(...watermarks: string[]): string {\n  let max = watermarks[0];\n  for (const w of watermarks) {\n    if (w > max) {\n      max = w;\n    }\n  }\n  return max;\n}\n"],"mappings":";;;;;;;;;AA2BA,IAAM,8BAA8B;;AAGpC,IAAM,qCAAqC;;AAG3C,IAAM,wCAAwC;;;;;;;;AAS9C,IAAM,yCAAyC;;AAG/C,IAAM,0BAA0B;;AA8FhC,IAAM,oBAA4B;;qCAEG,wBAAwB;;qCAExB,wBAAwB;;;;AAK7D,IAAM,qBAA6B;6BACN,wBAAwB;;;;;;;;;;;;;;;AAoCrD,IAAa,4BAAb,MAAuC;CACrC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,kBAA2B,mBACzB,WACA,oCACA,0MAGF;CACA,iBAA0B,mBACxB,WACA,oCACA,iDACF;CACA,iBAA0B,4BACxB,WACA,4CACA,qDACF;;CAGA;CACA;CACA;CACA,WAAW;CACX,wBAAwB;CACxB,kBAAkB;CAElB,YACE,IACA,OACA,eACA,UACA,IACA,MACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,2BAA2B;EAClE,KAAK,SAAS;EACd,KAAK,iBAAiB;EACtB,KAAK,YAAY;EACjB,KAAK,MAAM;EACX,KAAK,QAAQ;EACb,KAAK,OAAO,KAAK,OAAO,KAAK;EAC7B,KAAK,gBAAgB,KAAK,gBAAgB;EAC1C,KAAK,kBAAkB,KAAK,kBAAkB;EAC9C,KAAK,SACH,KAAK,kBACE,IAAI,SAAc,YAAW,aAAa,OAAO,CAAC;EAC3D,KAAK,cAAc;CACrB;;;;;CAMA,cAA2C;EACzC,IAAI,KAAK,UACP,OAAO,QAAQ,QAAQ;GAAC,MAAM;GAAW,QAAQ;EAAS,CAAC;EAE7D,IAAI,KAAK,aAAa,KAAA,GACpB,KAAK,WAAW,KAAK,UAAU,CAAC,CAC7B,MAAK,WAAU;GACd,IAAI,KAAK,uBACP,KAAK,gBAAgB,CAAC;GAExB,OAAO;EACT,CAAC,CAAC,CACD,cAAc;GACb,KAAK,WAAW,KAAA;EAClB,CAAC;EAEL,OAAO,KAAK;CACd;CAEA,OAAa;EACX,KAAK,WAAW;EAChB,IAAI,KAAK,WAAW,KAAA,GAAW;GAC7B,KAAK,gBAAgB,KAAK,MAAM;GAChC,KAAK,SAAS,KAAA;EAChB;CACF;;CAGA,aAAmB;EACjB,KAAK;EACL,KAAK,UAAU,KAAA;EACf,KAAK,wBAAwB;CAC/B;CAEA,cACE,UAAU,KAAK,MAAM,cAAc,6BAC7B;EACN,IAAI,KAAK,YAAY,KAAK,WAAW,KAAA,GACnC;EAEF,KAAK,SAAS,KAAK,oBAAoB;GACrC,KAAK,SAAS,KAAA;GACd,KAAU,YAAY,CAAC,CAAC,cAAc,KAAK,cAAc,CAAC;EAC5D,GAAG,OAAO;CACZ;CAEA,gBAAgB,SAAuB;EACrC,IAAI,KAAK,WAAW,KAAA,GAAW;GAC7B,KAAK,gBAAgB,KAAK,MAAM;GAChC,KAAK,SAAS,KAAA;EAChB;EACA,KAAK,cAAc,OAAO;CAC5B;CAEA,MAAM,YAAyC;EAC7C,MAAM,YAAY,YAAY,IAAI;EAClC,MAAM,iBAAiB,KAAK;EAC5B,KAAK,wBAAwB;EAC7B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IAEF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,gBAAgB,EAAC,UAAU,KAAI,CAAC;GACnE,QAAQ;IACN,OAAO,KAAK,SAAS,iBAAiB;GACxC;GAEA,MAAM,SAAS,MAAM,KAAK,WAAW,EAAE;GACvC,IAAI,mBAAmB,KAAK,iBAC1B,OAAO,KAAK,SAAS,aAAa;GAEpC,IAAI,OAAO,WAAW,UACpB,OAAO,KAAK,SAAS,MAAM;GAE7B,MAAM,EAAC,SAAQ;GAEf,MAAM,OAAO,aACX,KAAK,WAAW,KAAK,eAGrB,KAAK,eAEL,OAAO,gBACP,OAAO,OAAO,YAChB;GACA,MAAM,UAAU,aACd,OAAO,iBACP,OAAO,OAAO,aAChB;GACA,IAAI,WAAW,MACb,OAAO,KAAK,SAAS,oBAAoB;GAG3C,IAAI,eAAe;GACnB,IAAI,UAAU;GACd,MAAM,WAA2C;IAC/C,OAAO;IACP,UAAU;IACV,cAAc;IACd,UAAU;IACV,WAAW;GACb;GACA,MAAM,UAAU,WAAmB,YAA4B;IAC7D,KAAK,gBAAgB,IAAI,GAAG,EAAC,QAAO,CAAC;IACrC,SAAS,QAAQ;IACjB,IAAI,YAAY,YAEd,KAAK,IAAI,QACP,2DACA,EAAC,wBAAwB,EAAC,UAAS,EAAC,CACtC;GAEJ;GAEA,SAAS,IAAI,sBAAsB,KAAK,KAAK,KAAK,cAAc;GAChE,MAAM,OAAO,OAAO,KAAK,IAAI;GAC7B,IAAI,KAAK,SAAS,aAChB,OAAO,KAAK,SAAS,iBAAiB;GAExC,IAAI,KAAK,SAAS,SAChB,OAAO,KAAK,SAAS,oBAAoB;GAE3C,IAAI,KAAK,SAAS,WAAW;IAG3B,MAAM,cAAc,MAAM,KAAK,WAAW,IAAI,QAAQ,IAAI;IAC1D,MAAM,UACJ,mBAAmB,KAAK,kBACpB,cACA;IACN,OAAO,MAAM,OAAO;IACpB,IAAI,YAAY,gBAEd,OAAO,KAAK,UAAU,MAAM,MAAM,cAAc,SAAS,QAAQ;GAIrE;GAEA,MAAM,QAAQ,KAAK,IACjB,GACA,KAAK,MAAM,2BACT,kCACJ;GACA,MAAM,QAAQ,MAAM,KAAK,kBAAkB,IAAI,MAAM,SAAS,KAAK;GACnE,IAAI,mBAAmB,KAAK,iBAC1B,OAAO,KAAK,SAAS,aAAa;GAEpC,MAAM,mBAAmB,KAAK,IAC5B,GACA,KAAK,MAAM,4BACT,qCACJ;GACA,MAAM,oBAAoB,KAAK,IAC7B,GACA,KAAK,MAAM,6BACT,sCACJ;GAEA,IAAI,iBAAiB;GACrB,IAAI,aAAa;GACjB,IAAI,kBAAkB;GACtB,IAAI,cAAc;GAClB,IAAI,mBAAmB;GACvB,IAAI,iBAAiB;GACrB,KAAK,MAAM,EAAC,WAAW,UAAU,UAAS,OAAO;IAC/C,IAAI,KAAK,UACP;IAEF,IAAI,uBAAuB;IAC3B,IAAI,CAAC,YAAY,CAAC,MAAM;KAEtB,MAAM,UAAU,MAAM,KAAK,WAAW,IAAI,QAAQ,SAAS;KAC3D,OACE,WACA,mBAAmB,KAAK,kBAAkB,UAAU,cACtD;IACF,OAAO,IACL,kBAAkB,KAAK,QAAQ,WAAW,KAAK,MAAM,cAAc,GACnE;KACA,MAAM,sBAAsB,mBAAmB;KAC/C,MAAM,kBAAkB,mBAAmB;KAG3C,MAAM,uBAAuB,oBAAoB;KACjD,MAAM,mBAAmB,oBAAoB;KAC7C,IACE,wBAAwB,KACxB,oBAAoB,KACpB,wBAAwB,KACxB,oBAAoB,GACpB;MACA,OAAO,WAAW,UAAU;MAC5B;KACF;KACA;KACA,MAAM,aAAa,MAAM,KAAK,oBAC5B,IACA,QACA,QACA,kBACA,WACA,qBACA,iBACA,sBACA,kBACA,kBACA,iBACF;KACA,kBAAkB,WAAW;KAC7B,cAAc,WAAW;KACzB,mBAAmB,WAAW;KAC9B,eAAe,WAAW;KAC1B,MAAM,UACJ,mBAAmB,KAAK,kBACpB,WAAW,UACX;KACN,OAAO,WAAW,OAAO;KACzB,IAAI,YAAY,gBACd;KAEF,IAAI,YAAY,YACd;KAEF,IAAI,YAAY,aAEd,uBAAuB;KAGzB,MAAM,KAAK,OAAO;IACpB;IACA,mBAAmB;IACnB,iBAAiB;IACjB;IACA,IAAI,sBACF;GAEJ;GAEA,IACE,CAAC,KAAK,YACN,mBAAmB,KAAK,mBACxB,SAAS,iBAAiB,GAC1B;IACA,KAAK,UAAU;IACf,KAAK,wBAAwB,iBAAiB;GAChD;GACA,OAAO,KAAK,UACV,MACA,gBACA,cACA,SACA,QACF;EACF,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,yCAAyC,CAAC;GAC1D,OAAO,KAAK,SAAS,OAAO;EAC9B,UAAU;GACR,QAAQ,MAAM;GACd,IAAI,MAAM;GACV,KAAK,eAAe,SAAS,YAAY,IAAI,IAAI,SAAS;EAC5D;CACF;;CAGA,MAAM,WAAW,IAAyD;EACxE,IAAI;EACJ,IAAI;GACF,OAAO,kBAAkB,EAAE;EAC7B,QAAQ;GAEN,OAAO;EACT;EACA,IAAI,KAAK,kBAAA,GACP,OAAO;EAET,MAAM,EAAC,OAAO,YAAY,cAAa,KAAK;EAC5C,IACE,KAAK,UAAU,SACf,KAAK,eAAe,cACpB,KAAK,cAAc,WAEnB,OAAO;EAET,IAAI,KAAK,KAAK,IAAI,KAAK,aAAa,KAAK,MAAM,aAC7C,OAAO;EAET,MAAM,SAAS,KAAK,kBAAkB,EAAE;EACxC,IAAI,WAAW,KAAA,GACb,OAAO;EAET,MAAM,EAAC,cAAc,kBAAiB,MAAM,KAAK,IAAI,iBAAiB;EACtE,IAAI,iBAAiB,MAEnB,OAAO;EAET,OAAO;GACL;GACA;GACA,gBAAgB;GAChB,iBAAiB;EACnB;CACF;CAEA,kBAAkB,IAAwC;EACxD,MAAM,SAAS,GACZ,QAAQ,iBAAiB,CAAC,CAC1B,IAAiE;EACpE,OAAO,OAAO,iBAAiB,QAAQ,OAAO,kBAAkB,OAC5D,KAAA,IACA;GACE,cAAc,OAAO;GACrB,eAAe,OAAO;EACxB;CACN;;;;;CAMA,MAAM,kBACJ,IACA,MACA,SACA,OACkE;EAClE,IAAI,SAAmB,GACpB,QAAQ,kBAAkB,CAAC,CAC3B,IAAyB;GAAC,OAAO;GAAM;GAAS,OAAO,QAAQ;EAAC,CAAC,CAAC,CAClE,KAAK,EAAC,gBAAe,SAAS;EACjC,IAAI,KAAK,MAAM,KAAK,IAAI,qBAAqB,MAAM,SAAS,QAAQ,CAAC;EAErE,IAAI,mBAAmB;EACvB,IAAI,OAAO,SAAS,OAAO;GACzB,OAAO,SAAS;GAChB,mBAAmB,aAAa,kBAAkB,OAAO,QAAQ,EAAE;EACrE;EACA,IAAI,GAAG,SAAS,OAAO;GACrB,GAAG,SAAS;GACZ,mBAAmB,aAAa,kBAAkB,GAAG,QAAQ,EAAE;EACjE;EACA,IAAI,qBAAqB,SAAS;GAChC,SAAS,OAAO,QAAO,MAAK,KAAK,gBAAgB;GACjD,KAAK,GAAG,QAAO,MAAK,KAAK,gBAAgB;EAC3C;EAEA,MAAM,QAAiE,CAAC;EACxE,IAAI,IAAI;EACR,IAAI,IAAI;EACR,OAAO,IAAI,OAAO,UAAU,IAAI,GAAG,QAAQ;GACzC,MAAM,KAAK,IAAI,OAAO,SAAS,OAAO,KAAK,KAAA;GAC3C,MAAM,KAAK,IAAI,GAAG,SAAS,GAAG,KAAK,KAAA;GACnC,IAAI,OAAO,KAAA,MAAc,OAAO,KAAA,KAAa,KAAK,KAAK;IACrD,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAM,MAAM;IAAK,CAAC;IACvD;GACF,OAAO,IAAI,OAAO,KAAA,MAAc,OAAO,KAAA,KAAa,KAAK,KAAK;IAC5D,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAO,MAAM;IAAI,CAAC;IACvD;GACF,OAAO,IAAI,OAAO,KAAA,GAAW;IAC3B,MAAM,KAAK;KAAC,WAAW;KAAI,UAAU;KAAM,MAAM;IAAI,CAAC;IACtD;IACA;GACF;EACF;EACA,OAAO;CACT;;;;;CAMA,MAAM,oBACJ,IACA,QACA,QACA,kBACA,WACA,eACA,WACA,gBACA,YACA,kBACA,mBAC4B;EAC5B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,MAAM,gBAAgB,KAAK,MAAM,iBAAiB;GAClD,SAAS,MAAM,mBACb,OAAO,KACL,kBACA,WACA,KAAK,IAAI,eAAe,aAAa,GACrC,KAAA,GACA,aACF,GACA,WACA,eACA,cACF;GACA,KAAK,MAAM,mBACT,mBACE,KAAK,IAAI,iBAAiB,kBAAkB,WAAW,SAAS,CAClE,GACA,WACA,WACA,UACF;EACF,SAAS,GAAG;GAEV,KAAK,IAAI,OACP,6BAA6B,UAAU,kBACvC,CACF;GACA,OAAO;IACL,SAAS,MAAM,KAAK,WAAW,IAAI,QAAQ,SAAS;IACpD,gBAAgB;IAChB,YAAY;IACZ,iBAAiB;IACjB,aAAa;GACf;EACF;EACA,MAAM,OAAO;GACX,gBAAgB,OAAO;GACvB,YAAY,GAAG;GACf,iBAAiB,OAAO;GACxB,aAAa,GAAG;EAClB;EACA,IAAI,OAAO,gBAAgB,GAAG,cAAc;GAI1C,MAAM,cACJ,kBAAkB,oBAClB,mBAAmB;GACrB,MAAM,UACJ,cAAc,oBAAoB,eAAe;GAKnD,OAAO;IAAC,SAHL,OAAO,gBAAgB,eAAiB,GAAG,gBAAgB,UACxD,cACA;IACW,GAAG;GAAI;EAC1B;EACA,IAAI,OAAO,WAAW,GAAG,QACvB,OAAO;GAAC,SAAS;GAAS,GAAG;EAAI;EAEnC,OAAO;GACL,SAAS,MAAM,KAAK,WAAW,IAAI,QAAQ,SAAS;GACpD,GAAG;EACL;CACF;;;;;CAMA,MAAM,WACJ,IACA,QACA,WACsC;EACtC,IAAI;GACF,MAAM,OAAO,kBAAkB,EAAE;GACjC,IACE,KAAK,kBAAkB,OAAO,KAAK,iBACnC,KAAK,eAAe,OAAO,KAAK,YAEhC,OAAO;GAET,MAAM,SAAS,KAAK,kBAAkB,EAAE;GACxC,MAAM,EAAC,cAAc,kBAAiB,MAAM,KAAK,IAAI,iBAAiB;GACtE,IAAI,WAAW,KAAA,KAAa,iBAAiB,MAC3C,OAAO;GAET,IACG,OAAO,eAAe,OAAO,OAAO,gBACnC,aAAa,OAAO,gBACrB,eAAe,OAAO,kBAAkB,aAAa,gBACtD,YAAY,OAAO,iBACnB,YAAY,eAEZ,OAAO;GAET,OAAO;EACT,QAAQ;GACN,OAAO;EACT;CACF;CAEA,SAAS,QAA+C;EACtD,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,OAAM,CAAC;EAC3C,IAAI,WAAW,SAAS,CAExB,OAAO,IAAI,WAAW,sBACpB,KAAK,IAAI,QAAQ,0CAA0C,QAAQ;EAErE,OAAO;GAAC,MAAM;GAAW;EAAM;CACjC;CAEA,UACE,eACA,kBACA,cACA,SACA,UACoB;EACpB,KAAK,eAAe,IAAI,GAAG,EAAC,QAAQ,WAAU,CAAC;EAC/C,MAAM,SAAS;GACb,MAAM;GACN;GACA;GACA;GACA;GACA,SAAS,SAAS;GAClB,YAAY,SAAS;GACrB,cAAc,SAAS;GACvB,UAAU,SAAS;GACnB,WAAW,SAAS;EACtB;EACA,KAAK,IAAI,QAAQ,sCAAsC,EACrD,wBAAwB,OAC1B,CAAC;EACD,OAAO;CACT;AACF;;AAGA,gBAAgB,mBACd,SACoC;CACpC,WAAW,MAAM,SAAS,SACxB,MAAM,MAAM,KAAK,EAAC,WAAW,KAAK,aAChC,6BAA6B,WAAW,KAAK,MAAM,CACrD;AAEJ;AAEA,SAAS,aAAa,GAAW,GAAmB;CAClD,OAAO,IAAI,IAAI,IAAI;AACrB;AAEA,SAAS,aAAa,GAAG,YAA8B;CACrD,IAAI,MAAM,WAAW;CACrB,KAAK,MAAM,KAAK,YACd,IAAI,IAAI,KACN,MAAM;CAGV,OAAO;AACT"}