@rocicorp/zero 1.11.0-canary.8 → 1.11.0-canary.9

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  createReplica,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: must be false if pgVersion < PG_17. Caller must verify.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          toStateVersionString(slot.consistent_point),\n          backupOptions,\n        );\n\n        return {slot, capturedSnapshot, initialSession};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Deletes \"old\" replicas (i.e. those with a lower rank than the current)\n * and attempts to drop replication slots that are not associated with any\n * replica.\n *\n * If a slot could not be dropped because there is still an active subscriber,\n * it will be reflected in the `draining` count that is returned. When there\n * are draining slots, the method should be retried until all orphaned slots\n * have been dropped.\n */\nexport async function dropOldReplicasAndSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  beforeRank: bigint,\n): Promise<{dropped: number; active: number; draining: number}> {\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n  const oldReplicas = await sql`\n    SELECT id, rank::float8, slot, version, \"initialSyncContext\", \"subscriberContext\"\n     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};\n  `;\n  if (oldReplicas.length) {\n    lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, {oldReplicas});\n    await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;\n  }\n\n  return dropUnclaimedSlots(lc, sql, shard);\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;AA4DA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,WAAW,eAAe;CAI1C,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,WACA,UACA,eACA,iBACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAG1D,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,gBAAgB,GAC1C,aACF;IAEA,OAAO;KAAC;KAAM;KAAkB;IAAc;GAChD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,KACA,OACA,YAC8D;CAC9D,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAC/C,MAAM,cAAc,MAAM,GAAG;;YAEnB,IAAI,aAAa,EAAE,gBAAgB,WAAW;;CAExD,IAAI,YAAY,QAAQ;EACtB,GAAG,OAAO,YAAY,YAAY,OAAO,kBAAkB,EAAC,YAAW,CAAC;EACxE,MAAM,GAAG,eAAe,IAAI,aAAa,EAAE,gBAAgB;CAC7D;CAEA,OAAO,mBAAmB,IAAI,KAAK,KAAK;AAC1C;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  createReplica,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: must be false if pgVersion < PG_17. Caller must verify.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          toStateVersionString(slot.consistent_point),\n          backupOptions,\n        );\n\n        return {slot, capturedSnapshot, initialSession};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;AA4DA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,WAAW,eAAe;CAI1C,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,WACA,UACA,eACA,iBACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAG1D,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,gBAAgB,GAC1C,aACF;IAEA,OAAO;KAAC;KAAM;KAAkB;IAAc;GAChD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -11,7 +11,7 @@ import { streamIn, streamInWithSize, streamOut, streamOutStringified } from "../
 import { discoverChangeStreamerAddress } from "./schema/tables.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket from "ws";
+import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 4;
@@ -116,11 +116,11 @@ var ChangeStreamerHttpClient = class {
 		return uri;
 	}
 	async reserveSnapshot(taskID) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		return streamIn(this.#lc, ws, snapshotMessageSchema);
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
 		return streamInWithSize(this.#lc, ws, downstreamSchema);
 	}
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#subscribe","#reserveSnapshot","#receiveWebsocket","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamInWithSize,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SizedDownstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?: Pick<NormalizedZeroConfig, 'adminPassword'> | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {adminPassword: undefined},\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    void readinessGate.then(() => {\n      if (startupDelayMs > 0) {\n        // In RMv1, starting the change-streamer forcibly shuts down the\n        // previous change-streamer, causing view-syncers to reconnect.\n        // If this replication-manager has just started, the routing layer may\n        // not have registered it with DNS, as that only happens after it\n        // confirms health checks. To minimize downtime, the takeover is\n        // delayed for the configured startupDelayMs _after_ beginning to\n        // advertise readiness.\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      } else {\n        // In RMv2, this is not necessary because a new RM does not kill the\n        // old one; it is started as soon as possible.\n        this.#ensureChangeStreamerStarted('no startup delay configured');\n      }\n    });\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamInWithSize(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    logsChangeStream: params.getBoolean('logsChangeStream'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    initial: ctx.initial ? 'true' : 'false',\n    logsChangeStream: ctx.logsChangeStream ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU,EAAC,eAAe,KAAA,EAAS,GACxC,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKG,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKJ,gBAAgB,yBAAyB,MAAM;GAC5D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKI,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKJ,gBAAgB,UAAU,GAAG;GAC3D,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKK,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKP,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKO,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKN;EAC3D,cAAmB,WAAW;GAC5B,IAAI,iBAAiB,GAAG;IAQtB,KAAKD,IAAI,OACP,WAAW,eAAe,qCAC5B;IACA,KAAK,OAAO,iBAER,KAAKM,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;GACF,OAGE,KAAKA,6BAA6B,6BAA6B;EAEnE,CAAC;CACH;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKL,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKQ,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKT,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKW,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKX,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAA0D;EAIxE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKW,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,iBAAiB,KAAKX,KAAK,IAAI,gBAAgB;CACxD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;EAIpC,kBAAkB,OAAO,WAAW,kBAAkB;CACxD;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,SAAS,IAAI,UAAU,SAAS;EAChC,kBAAkB,IAAI,mBAAmB,SAAS;CACpD,CAAC;AACH"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#subscribe","#reserveSnapshot","#receiveWebsocket","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamInWithSize,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService, type Options as HttpOptions} from '../http-service.ts';\nimport {handleProfzRequest} from '../profz.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type SizedDownstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 4;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = HttpOptions & {\n  startupDelayMs: number;\n  config?: Pick<NormalizedZeroConfig, 'adminPassword'> | undefined;\n  getProfileWorker?: (() => Promise<Worker>) | undefined;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      fastify.get('/profz', (req, res) =>\n        handleProfzRequest(\n          lc,\n          opts.config ?? {adminPassword: undefined},\n          req,\n          res,\n          opts.getProfileWorker,\n          undefined,\n          'change-streamer',\n        ),\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = async (ws: WebSocket, req: RequestHeaders) => {\n    this.#ensureChangeStreamerStarted('incoming snapshot reservation');\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        await this.#changeStreamer.startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs, readinessGate = promiseVoid} = this.#opts;\n    void readinessGate.then(() => {\n      if (startupDelayMs > 0) {\n        // In RMv1, starting the change-streamer forcibly shuts down the\n        // previous change-streamer, causing view-syncers to reconnect.\n        // If this replication-manager has just started, the routing layer may\n        // not have registered it with DNS, as that only happens after it\n        // confirms health checks. To minimize downtime, the takeover is\n        // delayed for the configured startupDelayMs _after_ beginning to\n        // advertise readiness.\n        this.#lc.info?.(\n          `waiting ${startupDelayMs}ms before taking over the change log`,\n        );\n        this._state.setTimeout(\n          () =>\n            this.#ensureChangeStreamerStarted(\n              `startup delay elapsed (${startupDelayMs} ms)`,\n            ),\n          startupDelayMs,\n        );\n      } else {\n        // In RMv2, this is not necessary because a new RM does not kill the\n        // old one; it is started as soon as possible.\n        this.#ensureChangeStreamerStarted('no startup delay configured');\n      }\n    });\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<SizedDownstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamInWithSize(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', true),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n    // Absent for subscribers that predate the parameter, which is the safe\n    // default: the barrier falls back to polling rather than waiting on an\n    // ACK that would never be attributed to a writer.\n    logsChangeStream: params.getBoolean('logsChangeStream'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    initial: ctx.initial ? 'true' : 'false',\n    logsChangeStream: ctx.logsChangeStream ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKG,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,QAAQ,IAAI,WAAW,KAAK,QAC1B,mBACE,IACA,KAAK,UAAU,EAAC,eAAe,KAAA,EAAS,GACxC,KACA,KACA,KAAK,kBACL,KAAA,GACA,iBACF,CACF;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKL,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;CACzB;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,mBAA4B,OAAO,IAAe,QAAwB;EACxE,KAAKG,6BAA6B,+BAA+B;EACjE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,MAAM,KAAKJ,gBAAgB,yBAAyB,MAAM;GAC5D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKI,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKJ,gBAAgB,UAAU,GAAG;GAC3D,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKK,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKP,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKO,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,gBAAgB,gBAAgB,gBAAe,KAAKN;EAC3D,cAAmB,WAAW;GAC5B,IAAI,iBAAiB,GAAG;IAQtB,KAAKD,IAAI,OACP,WAAW,eAAe,qCAC5B;IACA,KAAK,OAAO,iBAER,KAAKM,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;GACF,OAGE,KAAKA,6BAA6B,6BAA6B;EAEnE,CAAC;CACH;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKL,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKQ,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKT,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKW,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKX,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAA0D;EAIxE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKW,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,iBAAiB,KAAKX,KAAK,IAAI,gBAAgB;CACxD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,IAAI;EACjC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;EAIpC,kBAAkB,OAAO,WAAW,kBAAkB;CACxD;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,SAAS,IAAI,UAAU,SAAS;EAChC,kBAAkB,IAAI,mBAAmB,SAAS;CACpD,CAAC;AACH"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -334,7 +334,7 @@ var ChangeStreamerImpl = class {
 				this.#stream = stream;
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
 				watermark = null;
-				this.#acker.reset(stream.acks);
+				this.#acker.reset(stream.acks, this.#pgChangeLogEnabled ? "" : lastWatermark);
 				for await (const change of stream.changes) {
 					this.#acker.trackDownstream(change);
 					const [type, msg] = change;
@@ -371,16 +371,15 @@ var ChangeStreamerImpl = class {
 						json
 					];
 					unflushedBytes += json.length;
-					if (unflushedBytes < flushBytesThreshold) {
-						this.#forwarder.forward(entry);
-						this.#recordForwardedTransactionBoundary(type, entry[0]);
-					} else {
-						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
-						this.#recordForwardedTransactionBoundary(type, entry[0]);
-						await promiseOrAbort(forwarded, stream.changes.signal, this.#state.signal);
+					let flowControl;
+					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
+					else flowControl = this.#forwarder.forwardWithFlowControl(entry);
+					this.#recordForwardedTransactionBoundary(type, entry[0]);
+					if (type === "commit" || type === "rollback") watermark = null;
+					if (flowControl) {
+						await promiseOrAbort(flowControl, stream.changes.signal, this.#state.signal);
 						unflushedBytes = 0;
 					}
-					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#pgChangeLogEnabled ? this.#storer.readyForMore() : void 0;
 					if (readyForMore) await promiseOrAbort(readyForMore, stream.changes.signal, this.#state.signal);
 				}