@rocicorp/zero 1.12.0-canary.5 → 1.12.0-canary.7

package/out/zero-cache/src/services/change-source/pg/fork-request.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fork-request.js","names":["v"],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/fork-request.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {toStateVersionString} from './lsn.ts';\n\n/**\n * A fork request is a (non-transactional) logical message that a new\n * replication-manager emits once it has created the replication slot of a\n * replica that it is about to fork from a running replica, i.e. restore from\n * its backup.\n *\n * The fork waits for the running replica to have backed up (and acked) its\n * replication slot past the new slot's `consistentPoint`. A running\n * replication-manager otherwise backs up at its regular (e.g. 15 second)\n * cadence, which would delay the fork by as much. Upon receiving a fork\n * request in its replication stream, it instead backs up through the\n * `consistentPoint` as soon as its replica reaches it (see\n * `syncBackupThrough` in the change source protocol).\n *\n * The message is non-transactional so that it is streamed, and handled, as\n * soon as it is decoded, ahead of the transaction that inserts the new\n * replica's row (which reaches the running replica past the\n * `consistentPoint`). Replication-managers that do not recognize the prefix\n * ignore it, as they do any non-transactional message of another prefix.\n */\nconst forkRequestMessageSchema = v.object({\n  replicaID: v.string(),\n  slot: v.string(),\n  consistentPoint: v.string(),\n});\n\n/** A fork request as it is emitted (see {@link emitForkRequest}). */\nexport type ForkRequest = v.Infer<typeof forkRequestMessageSchema>;\n\n/**\n * Parses a {@link ForkRequest}, converting its `consistentPoint` (an LSN) to\n * the watermark through which to back up (`syncBackupThrough`). A malformed\n * LSN fails the parse, like any other malformed request, so that the request\n * is ignored rather than failing the replication stream.\n */\nexport const forkRequestSchema = forkRequestMessageSchema.chain(request => {\n  try {\n    return v.ok({\n      ...request,\n      syncBackupThrough: toStateVersionString(request.consistentPoint),\n    });\n  } catch (e) {\n    return v.err(\n      `malformed consistentPoint \"${request.consistentPoint}\": ${String(e)}`,\n    );\n  }\n});\n\nexport type ParsedForkRequest = v.Infer<typeof forkRequestSchema>;\n\nexport function forkRequestMessagePrefix({appID, shardNum}: ShardID) {\n  return `${appID}/${shardNum}/fork/v1`;\n}\n\n/**\n * Emits a {@link ForkRequest}. This only expedites the fork, which is correct\n * without it, so failures (e.g. a missing EXECUTE privilege on\n * `pg_logical_emit_message`) are logged rather than thrown.\n *\n * The message is flushed on PG 17+. Earlier versions lack the `flush`\n * parameter, in which case the subsequent commit of the new replica's row\n * flushes it.\n */\nexport async function emitForkRequest(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  request: ForkRequest,\n): Promise<void> {\n  const prefix = forkRequestMessagePrefix(shard);\n  const content = JSON.stringify(request);\n  try {\n    const [{pgVersion}] = await sql<{pgVersion: number}[]>`\n      SELECT current_setting('server_version_num')::int AS \"pgVersion\"`;\n    if (pgVersion >= PG_17) {\n      await sql`SELECT pg_logical_emit_message(\n        false, ${prefix}::text, ${content}::text, true)`;\n    } else {\n      await sql`SELECT pg_logical_emit_message(\n        false, ${prefix}::text, ${content}::text)`;\n    }\n    lc.info?.(`requested fork at ${request.consistentPoint}`, request);\n  } catch (e) {\n    lc.warn?.(\n      `unable to emit fork request. The fork waits for the next regular ` +\n        `backup of the replica it forks.`,\n      e,\n    );\n  }\n}\n"],"mappings":";;;;;;;;;AA0CA,IAAa,oBAfoBA,eAAE,OAAO;CACxC,WAAWA,eAAE,OAAO;CACpB,MAAMA,eAAE,OAAO;CACf,iBAAiBA,eAAE,OAAO;AAC5B,CAWiC,CAAA,CAAyB,OAAM,YAAW;CACzE,IAAI;EACF,OAAOA,eAAE,GAAG;GACV,GAAG;GACH,mBAAmB,qBAAqB,QAAQ,eAAe;EACjE,CAAC;CACH,SAAS,GAAG;EACV,OAAOA,eAAE,IACP,8BAA8B,QAAQ,gBAAgB,KAAK,OAAO,CAAC,GACrE;CACF;AACF,CAAC;AAID,SAAgB,yBAAyB,EAAC,OAAO,YAAoB;CACnE,OAAO,GAAG,MAAM,GAAG,SAAS;AAC9B;;;;;;;;;;AAWA,eAAsB,gBACpB,IACA,KACA,OACA,SACe;CACf,MAAM,SAAS,yBAAyB,KAAK;CAC7C,MAAM,UAAU,KAAK,UAAU,OAAO;CACtC,IAAI;EACF,MAAM,CAAC,EAAC,eAAc,MAAM,GAA0B;;EAEtD,IAAI,aAAA,MACF,MAAM,GAAG;iBACE,OAAO,UAAU,QAAQ;OAEpC,MAAM,GAAG;iBACE,OAAO,UAAU,QAAQ;EAEtC,GAAG,OAAO,qBAAqB,QAAQ,mBAAmB,OAAO;CACnE,SAAS,GAAG;EACV,GAAG,OACD,oGAEA,CACF;CACF;AACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -10,6 +10,7 @@ import { createReplica, getReplicaState, getRestoreCandidates, metadataPublicati
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import "../../../types/pg-versions.js";
 import { createReplicationSessionFor, endSession, extractConnectionConfig, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
+import { emitForkRequest } from "./fork-request.js";
 import { defu } from "defu";
 import postgres from "postgres";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE, PG_LOCK_NOT_AVAILABLE } from "@drdgvhbh/postgres-error-codes";
@@ -219,6 +220,11 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaI
 				dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
 				lsn: String(toBigInt(slot.consistent_point))
 			});
+			if (stage === 2) await emitForkRequest(lc, sql, shard, {
+				replicaID,
+				slot: slot.slot_name,
+				consistentPoint: slot.consistent_point
+			});
 			const replica = await runTx(sql, async (tx) => {
 				await createReplica(tx, shard, replicaID, slot.slot_name, epoch, stage === 0 ? toStateVersionString(slot.consistent_point) : "", backupOptions, stage);
 				return must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`);

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB;;AAG/B,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,EAAA,CACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC;AAI7C,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,CAAC,CAC/B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {emitForkRequest} from './fork-request.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      if (stage === Restore) {\n        // Expedites the fork, which waits on the source replica's backup of\n        // the new slot's position (see fork-request.ts). Emitted before the\n        // replica's row is inserted, which flushes it, and outside of that\n        // transaction so that a failure to emit it does not fail the fork.\n        await emitForkRequest(lc, sql, shard, {\n          replicaID,\n          slot: slot.slot_name,\n          consistentPoint: slot.consistent_point,\n        });\n      }\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAyFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB;;AAG/B,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,EAAA,CACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC;AAI7C,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,IAAI,UAAA,GAKF,MAAM,gBAAgB,IAAI,KAAK,OAAO;IACpC;IACA,MAAM,KAAK;IACX,iBAAiB,KAAK;GACxB,CAAC;GAEH,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,CAAC,CAC/B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -768,6 +768,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>]>;

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts

@@ -14,6 +14,7 @@ export declare const downstreamStatusSchema: v.ObjectType<{
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>;
 export type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;
 export declare const downstreamStatusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
@@ -26,6 +27,7 @@ export declare const downstreamStatusMessageSchema: v.TupleType<[v.Type<"status"
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;
@@ -45,6 +47,7 @@ export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">,
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
     commitTimeMs: v.Optional<number>;

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;aAQjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}
+{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;aAajC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.js

@@ -9,7 +9,8 @@ import { changeSourceReportSchema } from "../../../replicator/reporter/report-sc
 */
 var downstreamStatusSchema = valita_exports.object({
 	ack: valita_exports.boolean().optional(() => true),
-	lagReport: changeSourceReportSchema.optional()
+	lagReport: changeSourceReportSchema.optional(),
+	syncBackupThrough: valita_exports.string().optional()
 });
 var downstreamStatusMessageSchema = valita_exports.tuple([
 	valita_exports.literal("status"),

package/out/zero-cache/src/services/change-source/protocol/current/status.js.map

@@ -1 +1 @@
-{"version":3,"file":"status.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  // Indicates whether the status message should be echoed back\n  // in an upstream status message once the consumer has successfully\n  // processed/persisted all preceding changes.\n  ack: v.boolean().optional(() => true),\n\n  // Contains a lag report for recording end to end latency metrics.\n  lagReport: changeSourceReportSchema.optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyBA,eAAE,OAAO;CAI7C,KAAKA,eAAE,QAAQ,CAAC,CAAC,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;AAC/C,CAAC;AAID,IAAa,gCAAgCA,eAAE,MAAM;CACnDA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8BA,eAAE,MAAM;CACjDA,eAAE,QAAQ,QAAQ;CAClBA,eAAE,MAAM,wBAAwB,YAAY;CAC5CA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}
+{"version":3,"file":"status.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/**\n * The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n */\nexport const downstreamStatusSchema = v.object({\n  // Indicates whether the status message should be echoed back\n  // in an upstream status message once the consumer has successfully\n  // processed/persisted all preceding changes.\n  ack: v.boolean().optional(() => true),\n\n  // Contains a lag report for recording end to end latency metrics.\n  lagReport: changeSourceReportSchema.optional(),\n\n  // Requests that the consumer back up its changes through this watermark\n  // promptly, rather than at its regular backup cadence, as the change source\n  // is waiting on the resulting ack (e.g. to fork a new replica from it).\n  syncBackupThrough: v.string().optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  downstreamStatusSchema,\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n */\nexport const upstreamStatusMessageSchema = v.tuple([\n  v.literal('status'),\n  v.union(downstreamStatusSchema, commitSchema),\n  v.object({watermark: v.string()}),\n]);\n\n/**\n * Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n *\n * A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n *\n * The `zero-cache` sends StatusMessages to the ChangeSource:\n *\n * * when it has processed a `Commit` received from the ChangeSource,\n *   unless the `Begin` message specified `skipAck`.\n *\n * * when it receives a `StatusMessage` and all preceding `Commit` messages\n *   have been processed\n *\n * This allows the ChangeSource to clean up change log entries appropriately.\n *\n * Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n  typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyBA,eAAE,OAAO;CAI7C,KAAKA,eAAE,QAAQ,CAAC,CAAC,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;CAK7C,mBAAmBA,eAAE,OAAO,CAAC,CAAC,SAAS;AACzC,CAAC;AAID,IAAa,gCAAgCA,eAAE,MAAM;CACnDA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8BA,eAAE,MAAM;CACjDA,eAAE,QAAQ,QAAQ;CAClBA,eAAE,MAAM,wBAAwB,YAAY;CAC5CA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts

@@ -10,6 +10,7 @@ export declare const changeSourceUpstreamSchema: v.TupleType<[v.Type<"status">,
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
     commitTimeMs: v.Optional<number>;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -4,17 +4,17 @@ import { LitestreamSyncRequester } from "./litestream-sync-requester.js";
 import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
 import { ReplicaPoller } from "./replica-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
-function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState }) {
+function createBackupCleanupMonitor({ lc, config, replicaFile, verifyBackupState }) {
 	const { litestream, replica } = config;
 	const backupURL = backupDestinationURL(litestream);
-	let stream;
-	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
-	else if (config.litestream.backupUsingV5) stream = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 }).start();
+	let poller;
+	if (!backupURL) poller = new ReplicaPoller(lc, replicaFile);
+	else if (config.litestream.backupUsingV5) poller = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 });
 	else {
 		const { port: metricsPort } = litestream;
-		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
+		poller = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file)));
 	}
-	return new BackupMonitor(lc, stream, changeStreamer, replicaFile);
+	return new BackupMonitor(lc, poller, replicaFile);
 }
 //#endregion
 export { createBackupCleanupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAwBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,CAAC,CAAC,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,CAAC,CAAC,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,CAAC,CAAC,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {\n  backupDestinationURL,\n  getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {BackupMonitor, type BackupPoller} from './backup-monitor.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const backupURL = backupDestinationURL(litestream);\n\n  let poller: BackupPoller;\n\n  if (!backupURL) {\n    poller = new ReplicaPoller(lc, replicaFile);\n  } else if (config.litestream.backupUsingV5) {\n    poller = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    });\n  } else {\n    const {port: metricsPort} = litestream;\n    poller = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    );\n  }\n\n  return new BackupMonitor(lc, poller, replicaFile);\n}\n"],"mappings":";;;;;;AAqBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW;MACrC,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC;MACI;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD;CACF;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,WAAW;AAClD"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -1,15 +1,23 @@
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { getOrCreateGauge } from "../../observability/metrics.js";
+import { Subscription } from "../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
 //#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+/**
+* Relays the watermarks backed up by a {@link BackupPoller} to the
+* change-streamer, which consumes them (see {@link backups}) and requests
+* prompt backups (see {@link requestSync}) as needed.
+*/
 var BackupMonitor = class {
 	id = "backup-monitor";
 	#lc;
-	#watermarks;
-	#changeStreamer;
+	#poller;
 	#replicaFile;
 	#firstBackupReceived = resolver();
+	#backups = Subscription.create({ coalesce: (curr) => curr });
+	#watermarks;
+	#stopped = false;
 	#latestBackup;
 	#firstBackupResolved = false;
 	/**
@@ -19,36 +27,61 @@ var BackupMonitor = class {
 	* "any watermark" rather than hanging startup forever.
 	*/
 	#coverageTarget;
-	constructor(lc, watermarks, changeStreamer, replicaFile) {
+	constructor(lc, poller, replicaFile) {
 		this.#lc = lc;
-		this.#watermarks = watermarks;
-		this.#changeStreamer = changeStreamer;
+		this.#poller = poller;
 		this.#replicaFile = replicaFile;
 	}
 	firstBackupReceived() {
 		return this.#firstBackupReceived.promise;
 	}
+	/**
+	* The stream of backed-up watermarks, each later than the last, coalesced
+	* to the latest one that has not been consumed. It ends when the monitor
+	* stops.
+	*/
+	backups() {
+		return this.#backups;
+	}
+	/**
+	* Requests that the replica be backed up through `watermark` promptly, if
+	* the poller supports it (see {@link BackupPoller.requestSync}). A no-op
+	* unless the monitor is running.
+	*/
+	requestSync(watermark) {
+		if (!this.#watermarks || this.#stopped || this.#latestBackup && this.#latestBackup.watermark >= watermark) return;
+		if (this.#poller.requestSync) {
+			this.#lc.info?.(`requesting backup through ${watermark}`);
+			this.#poller.requestSync(watermark);
+		}
+	}
 	async run() {
+		if (this.#stopped) return;
 		this.#lc.info?.("starting backup monitor");
 		this.#initBackupLagMetric();
 		this.#coverageTarget = this.#readReplicaStateVersion();
-		for await (const backedUp of this.#watermarks) {
-			if (this.#latestBackup) {
-				if (backedUp.watermark < this.#latestBackup.watermark) {
-					this.#lc.warn?.(`ignoring earlier backup watermark ${backedUp.watermark}`, { backedUp });
-					continue;
-				}
-				if (backedUp.watermark === this.#latestBackup.watermark) {
-					this.#lc.debug?.(`ignoring redundant backup watermark ${backedUp.watermark}`, { backedUp });
-					continue;
+		this.#watermarks = this.#poller.start();
+		try {
+			for await (const backedUp of this.#watermarks) {
+				if (this.#latestBackup) {
+					if (backedUp.watermark < this.#latestBackup.watermark) {
+						this.#lc.warn?.(`ignoring earlier backup watermark ${backedUp.watermark}`, { backedUp });
+						continue;
+					}
+					if (backedUp.watermark === this.#latestBackup.watermark) {
+						this.#lc.debug?.(`ignoring redundant backup watermark ${backedUp.watermark}`, { backedUp });
+						continue;
+					}
 				}
+				this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, { backedUp });
+				this.#latestBackup = backedUp;
+				this.#backups.push(backedUp);
+				this.#checkFirstBackupCovers(backedUp);
 			}
-			this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, { backedUp });
-			this.#latestBackup = backedUp;
-			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
-			this.#checkFirstBackupCovers(backedUp);
+			this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
+		} finally {
+			this.#backups.cancel();
 		}
-		this.#lc.info?.("watermark stream closed. BackupMonitor stopped.");
 	}
 	/**
 	* Resolves {@link firstBackupReceived} once the backup actually covers the
@@ -90,7 +123,9 @@ var BackupMonitor = class {
 		}
 	}
 	stop() {
-		this.#watermarks.cancel();
+		this.#stopped = true;
+		this.#watermarks?.cancel();
+		this.#backups.cancel();
 		return promiseVoid;
 	}
 	#initBackupLagMetric() {

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,kBAAkB;EACvB,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAK,aAAa;GAC7C,IAAI,KAAK,eAAe;IACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;KACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;KACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAK,gBAAgB;GACrB,KAAK,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAK,wBAAwB,QAAQ;EACvC;EACA,KAAK,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {SingletonService} from '../service.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\n/**\n * Determines which watermarks of the replica are backed up, by whatever means\n * the backup (or its absence) allows.\n */\nexport interface BackupPoller {\n  /**\n   * Starts polling, returning the stream of backed-up watermarks. Must only\n   * be called once.\n   */\n  start(): Source<BackedUpWatermark>;\n\n  /**\n   * Requests that the replica be backed up through `watermark` promptly,\n   * rather than at the poller's regular cadence, once the replica has reached\n   * it. The resulting watermark is emitted on the stream as usual. Pollers\n   * that cannot initiate backups do not implement this.\n   */\n  requestSync?(watermark: string): void;\n}\n\n/**\n * The interface through which the change-streamer consumes backed-up\n * watermarks and requests prompt backups (see {@link BackupMonitor}).\n */\nexport type BackupSource = Pick<BackupMonitor, 'backups' | 'requestSync'>;\n\n/**\n * Relays the watermarks backed up by a {@link BackupPoller} to the\n * change-streamer, which consumes them (see {@link backups}) and requests\n * prompt backups (see {@link requestSync}) as needed.\n */\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #poller: BackupPoller;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n  // Backed-up watermarks only move forward, so the consumer only needs the\n  // latest one. Coalescing keeps at most one outstanding, so that the monitor\n  // neither waits on, nor queues for, the consumer.\n  readonly #backups = Subscription.create<BackedUpWatermark>({\n    coalesce: curr => curr,\n  });\n\n  #watermarks: Source<BackedUpWatermark> | undefined;\n  #stopped = false;\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(lc: LogContext, poller: BackupPoller, replicaFile: string) {\n    this.#lc = lc;\n    this.#poller = poller;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  /**\n   * The stream of backed-up watermarks, each later than the last, coalesced\n   * to the latest one that has not been consumed. It ends when the monitor\n   * stops.\n   */\n  backups(): Source<BackedUpWatermark> {\n    return this.#backups;\n  }\n\n  /**\n   * Requests that the replica be backed up through `watermark` promptly, if\n   * the poller supports it (see {@link BackupPoller.requestSync}). A no-op\n   * unless the monitor is running.\n   */\n  requestSync(watermark: string) {\n    if (\n      !this.#watermarks ||\n      this.#stopped ||\n      (this.#latestBackup && this.#latestBackup.watermark >= watermark)\n    ) {\n      return;\n    }\n    if (this.#poller.requestSync) {\n      this.#lc.info?.(`requesting backup through ${watermark}`);\n      this.#poller.requestSync(watermark);\n    }\n  }\n\n  async run() {\n    if (this.#stopped) {\n      return;\n    }\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    this.#watermarks = this.#poller.start();\n    try {\n      for await (const backedUp of this.#watermarks) {\n        if (this.#latestBackup) {\n          if (backedUp.watermark < this.#latestBackup.watermark) {\n            this.#lc.warn?.(\n              `ignoring earlier backup watermark ${backedUp.watermark}`,\n              {backedUp},\n            );\n            continue;\n          }\n          if (backedUp.watermark === this.#latestBackup.watermark) {\n            this.#lc.debug?.(\n              `ignoring redundant backup watermark ${backedUp.watermark}`,\n              {backedUp},\n            );\n            continue;\n          }\n        }\n        this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n          backedUp,\n        });\n        this.#latestBackup = backedUp;\n        this.#backups.push(backedUp);\n        this.#checkFirstBackupCovers(backedUp);\n      }\n      this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n    } finally {\n      this.#backups.cancel();\n    }\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#stopped = true;\n    this.#watermarks?.cancel();\n    this.#backups.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AA8CA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA,uBAAgC,SAAS;CAIzC,WAAoB,aAAa,OAA0B,EACzD,WAAU,SAAQ,KACpB,CAAC;CAED;CACA,WAAW;CACX;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YAAY,IAAgB,QAAsB,aAAqB;EACrE,KAAK,MAAM;EACX,KAAK,UAAU;EACf,KAAK,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAK,qBAAqB;CACnC;;;;;;CAOA,UAAqC;EACnC,OAAO,KAAK;CACd;;;;;;CAOA,YAAY,WAAmB;EAC7B,IACE,CAAC,KAAK,eACN,KAAK,YACJ,KAAK,iBAAiB,KAAK,cAAc,aAAa,WAEvD;EAEF,IAAI,KAAK,QAAQ,aAAa;GAC5B,KAAK,IAAI,OAAO,6BAA6B,WAAW;GACxD,KAAK,QAAQ,YAAY,SAAS;EACpC;CACF;CAEA,MAAM,MAAM;EACV,IAAI,KAAK,UACP;EAEF,KAAK,IAAI,OAAO,yBAAyB;EACzC,KAAK,qBAAqB;EAC1B,KAAK,kBAAkB,KAAK,yBAAyB;EAErD,KAAK,cAAc,KAAK,QAAQ,MAAM;EACtC,IAAI;GACF,WAAW,MAAM,YAAY,KAAK,aAAa;IAC7C,IAAI,KAAK,eAAe;KACtB,IAAI,SAAS,YAAY,KAAK,cAAc,WAAW;MACrD,KAAK,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;MACA;KACF;KACA,IAAI,SAAS,cAAc,KAAK,cAAc,WAAW;MACvD,KAAK,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;MACA;KACF;IACF;IACA,KAAK,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;IACD,KAAK,gBAAgB;IACrB,KAAK,SAAS,KAAK,QAAQ;IAC3B,KAAK,wBAAwB,QAAQ;GACvC;GACA,KAAK,IAAI,OAAO,iDAAiD;EACnE,UAAU;GACR,KAAK,SAAS,OAAO;EACvB;CACF;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAK,sBACP;EAEF,MAAM,SAAS,KAAK;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAK,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAK,uBAAuB;EAC5B,KAAK,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,CAAC,CACpE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAK,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAK,WAAW;EAChB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO;EACrB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,CAAC,CAAC,aAAY,MAAK;GAClB,MAAM,eAAe,KAAK;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAK,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAK,KAAK,KAAK,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,CAAC,CACnE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAK,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -202,6 +202,7 @@ var ChangeStreamerImpl = class {
 	#purgeScheduler;
 	#comparator;
 	#acker;
+	#backupMonitor;
 	#initializer;
 	#readRouter;
 	#autoReset;
@@ -216,7 +217,7 @@ var ChangeStreamerImpl = class {
 	#reservationDemotions = getOrCreateCounter("replication", "sqlite_change_log.reservation_demotions", "Snapshot reservations demoted from SQLite to PG because the change log could not cover the backup being restored. With PG retired these followers have no fallback, so this is the rate at which one would instead have to wait for a later backup.");
 	#reservationConfirmDelays = getOrCreateCounter("replication", "sqlite_change_log.reservation_confirm_delays", "Snapshot reservations whose confirmation was deferred because the selected source's change-log minimum was later than the backup watermark. Counted once per reservation, by that source.");
 	#latestLagReportCommitTimeMs = 0;
-	#lagReportingDisabled = resolver();
+	#lagReports = resolver();
 	#backupWatermark;
 	#pgPurgedWatermark = "";
 	/**
@@ -330,6 +331,11 @@ var ChangeStreamerImpl = class {
 		this.#purgeLock = initialPurgeLock;
 		this.#autoReset = autoReset;
 		this.#state = new RunningState(this.id, void 0, setTimeoutFn);
+		this.#backupMonitor = opts.backupMonitor;
+		if (this.#backupMonitor) this.#trackBackups(this.#backupMonitor);
+	}
+	async #trackBackups(monitor) {
+		for await (const { watermark } of monitor.backups()) this.#trackBackupWatermark(watermark);
 	}
 	/**
 	* Whether the legacy Postgres change log is enabled, which is determined
@@ -342,7 +348,7 @@ var ChangeStreamerImpl = class {
 		this.#lc.info?.("starting change stream");
 		this.#forwarder.startProgressMonitor();
 		const lagReportInit = await this.#source.startLagReporter();
-		this.#lagReportingDisabled.resolve(!lagReportInit);
+		this.#lagReports.resolve(lagReportInit ? { request: () => this.#source.requestLagReport?.() } : void 0);
 		if (lagReportInit) this.#latestLagReportCommitTimeMs = lagReportInit.firstCommitTimeMs;
 		const storer = this.#storer;
 		let takeoverPending = storer !== null;
@@ -372,13 +378,16 @@ var ChangeStreamerImpl = class {
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
 				this.#acker.reset(stream.acks, lastWatermark);
-				for await (const change of stream.changes) {
+				streamLoop: for await (const change of stream.changes) {
 					this.#acker.trackDownstream(change);
 					const [type, msg] = change;
-					if (type === "status" && watermark === null && this.#mayRestart()) break;
 					switch (type) {
 						case "status":
-							if (watermark === null) storer?.status(change[2].watermark);
+							if (msg.syncBackupThrough) this.#backupMonitor?.requestSync(msg.syncBackupThrough);
+							if (watermark === null) {
+								if (this.#mayRestart()) break streamLoop;
+								storer?.status(change[2].watermark);
+							}
 							if (msg.lagReport && msg.lagReport.lastTimings.commitTimeMs >= this.#latestLagReportCommitTimeMs) {
 								this.#latestLagReportCommitTimeMs = msg.lagReport.lastTimings.commitTimeMs;
 								this.#forwarder.sendStatus({
@@ -478,7 +487,7 @@ var ChangeStreamerImpl = class {
 		const lc = this.#lc.withContext("subscriber", id);
 		const subscriber = new Subscriber(protocolVersion, id, mode, watermark, downstream, {
 			wsBatched,
-			lagReportingDisabled: this.#lagReportingDisabled.promise,
+			lagReports: this.#lagReports.promise,
 			drainable: mode === "serving" && ctx.drainable,
 			...this.#backfills.subscriberOptions(lc, ctx)
 		});
@@ -616,7 +625,17 @@ var ChangeStreamerImpl = class {
 	drain() {
 		return this.#drainTracker.drain();
 	}
-	trackBackupWatermark(watermark) {
+	/**
+	* Tracks the watermark up to which the replica has been backed up. This
+	* serves two purposes:
+	* - It serves as the maximum watermark up to which the change-log can be
+	*   purged. Note that the change-streamer also takes into account the
+	*   positions of its subscribers and snapshot reservations when purging
+	*   changes in the change-log.
+	* - In RMv2, this ACK's the upstream change-source to allow its buffer of
+	*   changes (e.g. the replication_slot) to advance.
+	*/
+	#trackBackupWatermark(watermark) {
 		this.#backupWatermark = watermark;
 		this.#acker.trackBackup(watermark);
 		this.#requestSQLitePurge("deferred");
@@ -759,7 +778,7 @@ var ChangeStreamerImpl = class {
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#drainTracker.stop();
-		this.#lagReportingDisabled.resolve(false);
+		this.#lagReports.resolve({ request: () => {} });
 		this.#stream?.changes.cancel();
 		this.#purgeScheduler?.stop();
 		this.#comparator?.stop();