@rocicorp/zero 1.11.0-canary.26 → 1.11.0-canary.28

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;AAoBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(Math.min(10 * 2 ** i, 1000));\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Discard the streamed messages. Otherwise the paused socket backs up the\n  // wal_sender (e.g. with keepalives and logical messages) for the duration\n  // of the initial sync / restore, and a wal_sender blocked on writing to the\n  // client does not exit (and release the slot) when it is terminated by the\n  // takeover; it instead blocks on sending the termination error to the\n  // client, until signaled again.\n  readable.resume();\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;AAoBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,KAAK,IAAI,KAAK,KAAK,GAAG,GAAI,CAAC;IACvC;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAOvD,SAAS,OAAO;CAGhB,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts

@@ -27,8 +27,23 @@ export type CreateSlotSpec = {
     failover?: boolean;
     temporary?: boolean;
     lockTimeout?: number;
+    diagnosticsDB?: PostgresDB | undefined;
 };
-export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, }: CreateSlotSpec): Promise<ReplicationSlot>;
+export declare const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60000;
+export declare function createReplicationSlot(lc: LogContext, session: postgres.Sql, { slotName, failover, temporary, lockTimeout, diagnosticsDB, }: CreateSlotSpec): Promise<ReplicationSlot>;
+/**
+ * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting
+ * on: the walsender's wait event and blocking pids, and the oldest
+ * transactions holding an xid. Slot creation waits for every such transaction
+ * on the instance (in all databases), each with its own lock_timeout, so a
+ * chain of overlapping transactions can exceed the client-side timeout
+ * without the lock_timeout ever firing.
+ *
+ * Query text is deliberately omitted since it can contain row values.
+ *
+ * Exported for testing.
+ */
+export declare function logSlotCreationBlockers(lc: LogContext, sql: PostgresDB, walsenderPID: number, slotName: string): Promise<void>;
 /**
  * Replica and slot creation involves two sessions for proper coordination
  * with other replica management logic:

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAIrC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAQtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CA8C1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CA+GnC;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAsD9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
+{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAKrC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAQtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;IAIrB,aAAa,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACxC,CAAC;AAUF,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAUzD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,EACpC,aAAa,GACd,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAiD1B;AAKD;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,iBAmCjB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAmHnC;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAsD9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -11,25 +11,70 @@ import { orTimeout } from "../../../types/timeout.js";
 import { createReplicationSessionFor, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
-var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 3e4;
+var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 6e4;
 var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
-async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS }) {
+async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS, diagnosticsDB }) {
 	await session.unsafe(`SET lock_timeout = ${lockTimeout}`);
-	const { pgVersion } = (await session.unsafe(`
-      SELECT current_setting('server_version_num') as "pgVersion";
+	const { pgVersion, pid: walsenderPID } = (await session.unsafe(`
+      SELECT current_setting('server_version_num') as "pgVersion",
+             pg_backend_pid() as "pid";
   `))[0];
 	const maybeTemporary = temporary ? "TEMPORARY" : "";
 	const options = failover && pgVersion >= 17e4 ? "(FAILOVER)" : "";
 	const raced = await orTimeout(session.unsafe(`
     CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`), CREATE_REPLICATION_SLOT_TIMEOUT_MS);
 	if (raced === "timed-out") {
+		if (diagnosticsDB) await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);
 		session.end().catch((e) => lc.warn?.(`Error closing timed out replication slot session`, e));
-		throw new Error(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. Crashing to force a clean restart.`);
+		throw new Error(`Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`);
 	}
 	const [slot] = raced;
 	lc.info?.(`Created replication slot ${slotName}`, slot);
 	return slot;
 }
+var SLOT_DIAGNOSTICS_TIMEOUT_MS = 5e3;
+var SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;
+/**
+* Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting
+* on: the walsender's wait event and blocking pids, and the oldest
+* transactions holding an xid. Slot creation waits for every such transaction
+* on the instance (in all databases), each with its own lock_timeout, so a
+* chain of overlapping transactions can exceed the client-side timeout
+* without the lock_timeout ever firing.
+*
+* Query text is deliberately omitted since it can contain row values.
+*
+* Exported for testing.
+*/
+async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
+	try {
+		const result = await orTimeout(Promise.all([sql`
+          SELECT pid, wait_event_type as "waitEventType", wait_event as "waitEvent",
+                 pg_blocking_pids(pid) as "blockingPIDs"
+            FROM pg_stat_activity WHERE pid = ${walsenderPID}`, sql`
+          SELECT pid, datname, usename, application_name as "applicationName",
+                 backend_type as "backendType", state,
+                 wait_event_type as "waitEventType", wait_event as "waitEvent",
+                 backend_xid::text as xid,
+                 xact_start::text as "xactStart",
+                 extract(epoch from now() - xact_start)::float8 as "xactAgeSec"
+            FROM pg_stat_activity
+            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}
+            ORDER BY xact_start NULLS LAST
+            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`]), SLOT_DIAGNOSTICS_TIMEOUT_MS);
+		if (result === "timed-out") {
+			lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);
+			return;
+		}
+		const [[walsender], transactions] = result;
+		lc.warn?.(`Replication slot ${slotName} creation blocked`, {
+			walsender: walsender ?? null,
+			transactions
+		});
+	} catch (e) {
+		lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);
+	}
+}
 /**
 * Replica and slot creation involves two sessions for proper coordination
 * with other replica management logic:
@@ -108,7 +153,8 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaI
 				}
 				const slot = await createReplicationSlot(lc, replicationSession, {
 					slotName,
-					failover
+					failover,
+					diagnosticsDB: sql
 				});
 				const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
 				const initialSession = await keepSlotActiveUntilTakenOver(lc, replicationSession, slot.slot_name, metadataPublicationName(shard.appID, shard.shardNum), toBigInt(slot.consistent_point));
@@ -136,7 +182,7 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaI
 			if (slotName) {
 				lc.warn?.(`deleting slot ${slotName} due to error`, e);
 				await replicationSession.end();
-				await sql`SELECT pg_drop_replication_slot(${slotName})`;
+				await dropUnclaimedSlots(lc, sql, shard);
 			}
 			throw e;
 		}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion} = (\n    await session.unsafe<{pgVersion: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        if (stage === InitialSync) {\n          // With the lock acquired, ensure that only one initial sync is\n          // active at a time. getRestoreCandidates() orders its results\n          // with stage = InitialSync (and active = true) first.\n          const others = await getRestoreCandidates(lc, tx, shard, epoch);\n          if (others.length) {\n            const [{id, stage, active}] = others;\n            if (stage === InitialSync && active) {\n              throw new AutoResetSignal(\n                `another replica (${id}) is performing initial sync`,\n              );\n            }\n          }\n        }\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n\n        const replica = must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n\n        return {slot, capturedSnapshot, initialSession, replica};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      return {\n        slot,\n        replica,\n        reservation: await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(lsn),\n        ),\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    await replicationSession.end();\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AA8EA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,eACL,MAAM,QAAQ,OAA8B;;GAE7C,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,IAAI,OAAO,KAAK;KAC9D,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAOA,OAAO;KAAC;KAAM;KAAkB;KAAgB,SALhC,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBAGyB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,aACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAO9E,OAAO;IACL;IACA,SAPc,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aAKrB;IACA,aAAa,MAAM,6BACjB,IACA,oBACA,MACA,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,GAAG,CACd;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,MAAM,mBAAmB,IAAI;EAC7B,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    if (diagnosticsDB) {\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        if (stage === InitialSync) {\n          // With the lock acquired, ensure that only one initial sync is\n          // active at a time. getRestoreCandidates() orders its results\n          // with stage = InitialSync (and active = true) first.\n          const others = await getRestoreCandidates(lc, tx, shard, epoch);\n          if (others.length) {\n            const [{id, stage, active}] = others;\n            if (stage === InitialSync && active) {\n              throw new AutoResetSignal(\n                `another replica (${id}) is performing initial sync`,\n              );\n            }\n          }\n        }\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n          diagnosticsDB: sql,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n\n        const replica = must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n\n        return {slot, capturedSnapshot, initialSession, replica};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong. The\n      // slot may not exist (e.g. if its creation failed), and its name may\n      // have been reused since the lock was released, so leave it to\n      // dropUnclaimedSlots(), which only drops slots without a replica.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      return {\n        slot,\n        replica,\n        reservation: await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(lsn),\n        ),\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    await replicationSession.end();\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AAmFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EACzB,IAAI,eACF,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAIzE,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,IAAI,OAAO,KAAK;KAC9D,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;KACA,eAAe;IACjB,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAOA,OAAO;KAAC;KAAM;KAAkB;KAAgB,SALhC,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBAGyB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAKA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,aACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAO9E,OAAO;IACL;IACA,SAPc,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aAKrB;IACA,aAAa,MAAM,6BACjB,IACA,oBACA,MACA,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,GAAG,CACd;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,MAAM,mBAAmB,IAAI;EAC7B,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-source/pg/schema/shard.d.ts

@@ -49,7 +49,7 @@ declare const replicaSchema: v.ObjectType<Omit<{
     epoch: v.Type<number>;
     generation: v.Type<string>;
     stage: v.Type<0 | 1 | 2>;
-    backupPath: v.Type<string | null>;
+    backupPath: v.UnionType<[v.Type<null>, v.Type<string>]>;
     backupV5: v.Type<boolean>;
 }, "initialSchema" | "initialSyncContext" | "subscriberContext"> & {
     initialSchema: v.Type<{
@@ -118,8 +118,8 @@ declare const replicaSchema: v.ObjectType<Omit<{
             replicaIdentityColumns: string[];
         }[];
     }>;
-    initialSyncContext: v.Type<Readonly<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>> | null>;
-    subscriberContext: v.Type<Readonly<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>> | null>;
+    initialSyncContext: v.UnionType<[v.Type<null>, v.Type<Readonly<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>]>;
+    subscriberContext: v.UnionType<[v.Type<null>, v.Type<Readonly<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>>]>;
 }, undefined>;
 export type Replica = v.Infer<typeof replicaSchema>;
 declare const replicaStateSchema: v.ObjectType<Omit<{
@@ -129,7 +129,7 @@ declare const replicaStateSchema: v.ObjectType<Omit<{
     epoch: v.Type<number>;
     generation: v.Type<string>;
     stage: v.Type<0 | 1 | 2>;
-    backupPath: v.Type<string | null>;
+    backupPath: v.UnionType<[v.Type<null>, v.Type<string>]>;
     backupV5: v.Type<boolean>;
 }, "active" | "confirmedFlushLsn"> & {
     active: v.Type<boolean>;

package/out/zero-cache/src/services/change-source/protocol/current/data.d.ts

@@ -67,7 +67,7 @@ export declare const updateSchema: v.ObjectType<{
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>;
 export declare const deleteSchema: v.ObjectType<{
@@ -351,7 +351,7 @@ export declare const dataChangeSchema: v.UnionType<[v.ObjectType<{
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts

@@ -31,7 +31,7 @@ declare const data: v.TupleType<[v.Type<"data">, v.UnionType<[v.UnionType<[v.Obj
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -284,7 +284,7 @@ export declare const changeStreamDataSchema: v.UnionType<[v.TupleType<[v.Type<"b
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;
@@ -539,7 +539,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
             type?: "default" | "full" | "index" | "nothing" | undefined;
         };
     }>;
-    key: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue> | null>;
+    key: v.UnionType<[v.Type<null>, v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     new: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"delete">;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts

@@ -41,10 +41,9 @@ export declare const backfillRequestSchema: v.ObjectType<{
         schema: v.Type<string>;
         name: v.Type<string>;
     }, "metadata"> & {
-        metadata: v.Type<{
-            [x: string]: import("../../../../../../shared/src/bigint-json.ts").JSONValue;
-            rowKey: Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>;
-        } | null>;
+        metadata: v.UnionType<[v.Type<null>, v.ObjectType<{
+            rowKey: v.Type<Record<string, import("../../../../../../shared/src/bigint-json.ts").JSONValue>>;
+        }, v.Type<import("../../../../../../shared/src/bigint-json.ts").JSONValue>>]>;
     }, undefined>;
     columns: v.Type<Record<string, {
         id: Record<string, import("./json.ts").JSONValue | undefined>;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAiB/B,YACE,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM,EAMpB;IAED,mBAAmB,kBAElB;IAEK,GAAG,kBAsBR;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGpB;CA+BF"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAiB/B,YACE,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,EACrC,cAAc,EAAE,qBAAqB,EACrC,WAAW,EAAE,MAAM,EAMpB;IAED,mBAAmB,kBAElB;IAEK,GAAG,kBA8BR;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAGpB;CA+BF"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.js

@@ -35,15 +35,15 @@ var BackupMonitor = class {
 		for await (const backedUp of this.#watermarks) {
 			if (this.#latestBackup) {
 				if (backedUp.watermark < this.#latestBackup.watermark) {
-					this.#lc.warn?.(`ignoring earlier backup watermark`, { backedUp });
+					this.#lc.warn?.(`ignoring earlier backup watermark ${backedUp.watermark}`, { backedUp });
 					continue;
 				}
 				if (backedUp.watermark === this.#latestBackup.watermark) {
-					this.#lc.debug?.(`ignoring redundant backup watermark`, { backedUp });
+					this.#lc.debug?.(`ignoring redundant backup watermark ${backedUp.watermark}`, { backedUp });
 					continue;
 				}
 			}
-			this.#lc.info?.(`received backup watermark`, { backedUp });
+			this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, { backedUp });
 			this.#latestBackup = backedUp;
 			this.#changeStreamer.trackBackupWatermark(backedUp.watermark);
 			this.#checkFirstBackupCovers(backedUp);

package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debuggin; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(`ignoring earlier backup watermark`, {backedUp});\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(`ignoring redundant backup watermark`, {backedUp});\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark`, {backedUp});\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OAAO,qCAAqC,EAAC,SAAQ,CAAC;KAC/D;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QAAQ,uCAAuC,EAAC,SAAQ,CAAC;KAClE;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,EAAC,SAAQ,CAAC;GACvD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"backup-monitor.js","names":["#lc","#watermarks","#changeStreamer","#replicaFile","#firstBackupReceived","#initBackupLagMetric","#coverageTarget","#readReplicaStateVersion","#latestBackup","#checkFirstBackupCovers","#firstBackupResolved"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {getOrCreateGauge} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {SingletonService} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nexport type BackedUpWatermark = {\n  watermark: string;\n  writeTimeMs?: number | undefined; // optional for debugging; available on v5\n  backupTimeMs: number;\n};\n\nexport class BackupMonitor implements SingletonService {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #watermarks: Source<BackedUpWatermark>;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #replicaFile: string;\n  readonly #firstBackupReceived = resolver();\n\n  #latestBackup: BackedUpWatermark | undefined;\n  #firstBackupResolved = false;\n  /**\n   * The replica's `stateVersion` when the monitor started, i.e. the point the\n   * backup has to reach before it covers what this task is about to serve.\n   * `undefined` when the replica could not be read, which degrades the gate to\n   * \"any watermark\" rather than hanging startup forever.\n   */\n  #coverageTarget: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    watermarks: Source<BackedUpWatermark>,\n    changeStreamer: ChangeStreamerService,\n    replicaFile: string,\n  ) {\n    this.#lc = lc;\n    this.#watermarks = watermarks;\n    this.#changeStreamer = changeStreamer;\n    this.#replicaFile = replicaFile;\n  }\n\n  firstBackupReceived() {\n    return this.#firstBackupReceived.promise;\n  }\n\n  async run() {\n    this.#lc.info?.('starting backup monitor');\n    this.#initBackupLagMetric();\n    this.#coverageTarget = this.#readReplicaStateVersion();\n\n    for await (const backedUp of this.#watermarks) {\n      if (this.#latestBackup) {\n        if (backedUp.watermark < this.#latestBackup.watermark) {\n          this.#lc.warn?.(\n            `ignoring earlier backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n        if (backedUp.watermark === this.#latestBackup.watermark) {\n          this.#lc.debug?.(\n            `ignoring redundant backup watermark ${backedUp.watermark}`,\n            {backedUp},\n          );\n          continue;\n        }\n      }\n      this.#lc.info?.(`received backup watermark ${backedUp.watermark}`, {\n        backedUp,\n      });\n      this.#latestBackup = backedUp;\n      this.#changeStreamer.trackBackupWatermark(backedUp.watermark);\n      this.#checkFirstBackupCovers(backedUp);\n    }\n    this.#lc.info?.('watermark stream closed. BackupMonitor stopped.');\n  }\n\n  /**\n   * Resolves {@link firstBackupReceived} once the backup actually covers the\n   * replica, rather than on the first watermark of any value.\n   *\n   * A new backup destination starts empty and is filled by re-uploading the\n   * local LTX chain, so the first watermarks read back out of it are real but\n   * far behind the replica. Releasing the readiness gate on one of those lets\n   * the task serve against a backup that does not yet cover it, which is what\n   * demotes a restoring view-syncer to Postgres catchup.\n   *\n   * The target is pinned at startup rather than compared against the replica's\n   * current position, which keeps the gate satisfiable under sustained writes.\n   */\n  #checkFirstBackupCovers(backedUp: BackedUpWatermark) {\n    if (this.#firstBackupResolved) {\n      return;\n    }\n    const target = this.#coverageTarget;\n    if (target !== undefined && backedUp.watermark < target) {\n      this.#lc.info?.(`backup does not yet cover the replica`, {\n        backedUp,\n        coverageTarget: target,\n      });\n      return;\n    }\n    this.#firstBackupResolved = true;\n    this.#firstBackupReceived.resolve();\n  }\n\n  #readReplicaStateVersion(): string | undefined {\n    let db;\n    try {\n      db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      const {stateVersion} = db\n        .prepare(/*sql*/ `SELECT stateVersion FROM \"_zero.replicationState\"`)\n        .get<{stateVersion: string}>();\n      return stateVersion;\n    } catch (e) {\n      // Without a target the gate degrades to its previous behavior. That is\n      // strictly better than blocking startup on a replica we cannot read.\n      this.#lc.warn?.(\n        `unable to read the replica's stateVersion; ` +\n          `the initial backup gate will accept the first watermark`,\n        e,\n      );\n      return undefined;\n    } finally {\n      db?.close();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#watermarks.cancel();\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(o => {\n      const latestBackup = this.#latestBackup;\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.backupTimeMs);\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;AAeA,IAAa,gBAAb,MAAuD;CACrD,KAAc;CACd;CACA;CACA;CACA;CACA,uBAAgC,SAAS;CAEzC;CACA,uBAAuB;;;;;;;CAOvB;CAEA,YACE,IACA,YACA,gBACA,aACA;EACA,KAAKA,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,kBAAkB;EACvB,KAAKC,eAAe;CACtB;CAEA,sBAAsB;EACpB,OAAO,KAAKC,qBAAqB;CACnC;CAEA,MAAM,MAAM;EACV,KAAKJ,IAAI,OAAO,yBAAyB;EACzC,KAAKK,qBAAqB;EAC1B,KAAKC,kBAAkB,KAAKC,yBAAyB;EAErD,WAAW,MAAM,YAAY,KAAKN,aAAa;GAC7C,IAAI,KAAKO,eAAe;IACtB,IAAI,SAAS,YAAY,KAAKA,cAAc,WAAW;KACrD,KAAKR,IAAI,OACP,qCAAqC,SAAS,aAC9C,EAAC,SAAQ,CACX;KACA;IACF;IACA,IAAI,SAAS,cAAc,KAAKQ,cAAc,WAAW;KACvD,KAAKR,IAAI,QACP,uCAAuC,SAAS,aAChD,EAAC,SAAQ,CACX;KACA;IACF;GACF;GACA,KAAKA,IAAI,OAAO,6BAA6B,SAAS,aAAa,EACjE,SACF,CAAC;GACD,KAAKQ,gBAAgB;GACrB,KAAKN,gBAAgB,qBAAqB,SAAS,SAAS;GAC5D,KAAKO,wBAAwB,QAAQ;EACvC;EACA,KAAKT,IAAI,OAAO,iDAAiD;CACnE;;;;;;;;;;;;;;CAeA,wBAAwB,UAA6B;EACnD,IAAI,KAAKU,sBACP;EAEF,MAAM,SAAS,KAAKJ;EACpB,IAAI,WAAW,KAAA,KAAa,SAAS,YAAY,QAAQ;GACvD,KAAKN,IAAI,OAAO,yCAAyC;IACvD;IACA,gBAAgB;GAClB,CAAC;GACD;EACF;EACA,KAAKU,uBAAuB;EAC5B,KAAKN,qBAAqB,QAAQ;CACpC;CAEA,2BAA+C;EAC7C,IAAI;EACJ,IAAI;GACF,KAAK,IAAI,SAAS,KAAKJ,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GAC/D,MAAM,EAAC,iBAAgB,GACpB,QAAgB,mDAAmD,EACnE,IAA4B;GAC/B,OAAO;EACT,SAAS,GAAG;GAGV,KAAKH,IAAI,OACP,sGAEA,CACF;GACA;EACF,UAAU;GACR,IAAI,MAAM;EACZ;CACF;CAEA,OAAsB;EACpB,KAAKC,YAAY,OAAO;EACxB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,aAAY,MAAK;GAClB,MAAM,eAAe,KAAKO;GAC1B,IAAI,CAAC,cAAc;IACjB,KAAKR,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKG,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,YAAY;IACrE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKH,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,KAAK,EACV,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAIzD,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAIL,KAAK,cAAc,EAEnB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AAexB,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EACH,IAAI,CAAC,oBAAoB,EAAE,eAAe,GAAG,kBAAkB,CAAC,GAChE,SAAS,CAAC;IACd,gBAAgB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuCtC;IAiLD,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACvC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,wBACX,YAAW,cAAc,EAAE,gBAAgB,EAAE,mBAAmB;;IAOhE,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAC7C,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAwBD;IAEK,SAAS,CACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAI7C;IAED;;;;;;;OAOG;IAEG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC7C,QAAQ,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;KAC3C,CAAC,CAgBD;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAiB3E"}
+{"version":3,"file":"change-streamer-http.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAC,KAAK,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAKL,KAAK,KAAK,EACV,KAAK,MAAM,EACZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAIzD,OAAO,EAAC,WAAW,EAAE,KAAK,OAAO,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG5E,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,iBAAiB,EACvB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAIL,KAAK,cAAc,EAEnB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AAmBxB,KAAK,OAAO,GAAG,WAAW,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EACH,IAAI,CAAC,oBAAoB,EAAE,eAAe,GAAG,kBAAkB,CAAC,GAChE,SAAS,CAAC;IACd,gBAAgB,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;CACxD,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,WAAW;;IACvD,QAAQ,CAAC,EAAE,iCAAiC;IAK5C,YACE,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,qBAAqB,EAuCtC;IAiLD,UAAmB,QAAQ,IAAI,IAAI,CA2BlC;IAED,UAAyB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAIhD;CACF;AAED,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACvC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,CACP,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAChD;AAED,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,wBACX,YAAW,cAAc,EAAE,gBAAgB,EAAE,mBAAmB;;IAOhE,YACE,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,MAAM,EAChB,iBAAiB,EAAE,MAAM,GAAG,SAAS,EAWtC;IAmBK,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAC7C,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,mBAAmB,CAAC;KAC/B,CAAC,CAwBD;IAEK,SAAS,CACb,GAAG,EAAE,iBAAiB,GACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAI7C;IAED;;;;;;;OAOG;IAEG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC7C,QAAQ,EAAE,YAAY,CAAC,iBAAiB,CAAC,CAAC;KAC3C,CAAC,CAuBD;CACF;AAED,KAAK,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;AAE/D,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAiB3E"}