@rocicorp/zero 1.11.0 → 1.11.2

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first","#worker","#closedPromise","#sigtermTimer","#stopped"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {Readable} from 'node:stream';\nimport {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport type {SlotKeeperConfig} from '../../../../server/slot-keeper.ts';\nimport {SLOT_KEEPER_URL} from '../../../../server/worker-urls.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {childWorker, type Worker} from '../../../../types/processes.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nexport const DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nexport async function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(Math.min(10 * 2 ** i, 1000));\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport class SlotReservationHandle extends Readable {\n  readonly #worker: Worker;\n  readonly #closedPromise: Promise<void>;\n  #stopped = false;\n  #sigtermTimer?: NodeJS.Timeout | undefined;\n\n  constructor(worker: Worker) {\n    super({read() {}});\n    this.#worker = worker;\n    this.#closedPromise = new Promise(resolve => {\n      worker.once('close', () => {\n        clearTimeout(this.#sigtermTimer);\n        resolve();\n      });\n    });\n    // Suppress unhandled error events if destroyed with an error\n    this.on('error', () => {});\n    worker.on('close', () => {\n      if (!this.destroyed) {\n        this.destroy();\n      }\n    });\n    worker.on('error', err => {\n      if (!this.destroyed) {\n        this.destroy(err instanceof Error ? err : new Error(String(err)));\n      }\n    });\n  }\n\n  override _destroy(\n    error: Error | null,\n    callback: (error?: Error | null) => void,\n  ): void {\n    if (!this.#stopped) {\n      this.#stopped = true;\n      try {\n        this.#worker.send(['stop', {}]);\n      } catch {\n        // ignore\n      }\n      this.#sigtermTimer = setTimeout(() => {\n        try {\n          this.#worker.kill('SIGTERM');\n        } catch {}\n      }, 5_000);\n      this.#sigtermTimer.unref();\n    }\n    callback(error);\n  }\n\n  async end(timeoutMs = 5_000): Promise<void> {\n    this.destroy();\n    const closed = await Promise.race([\n      this.#closedPromise.then(() => true),\n      sleep(timeoutMs).then(() => false),\n    ]);\n    if (!closed) {\n      try {\n        this.#worker.kill('SIGKILL');\n      } catch {}\n      // Await worker closure after forced kill so that the replication session\n      // is completely terminated before any subsequent slot drop attempt.\n      await Promise.race([this.#closedPromise, sleep(2_000)]);\n    }\n  }\n}\n\nexport async function endSession(session?: Readable): Promise<void> {\n  if (!session) {\n    return;\n  }\n  if ('end' in session && typeof session.end === 'function') {\n    await (session as {end: () => Promise<void>}).end();\n  } else {\n    session.destroy();\n  }\n}\n\nexport type PgConnectionConfig = {\n  host?: string | string[] | undefined;\n  port?: number | number[] | undefined;\n  path?: string | undefined;\n  database?: string | undefined;\n  user?: string | undefined;\n  pass?: string | null | undefined;\n  ssl?:\n    | boolean\n    | string\n    | {rejectUnauthorized?: boolean | undefined}\n    | undefined;\n};\n\nexport function extractConnectionConfig(\n  session: postgres.Sql,\n): PgConnectionConfig {\n  const opts = session.options;\n  return {\n    host: opts.host,\n    port: opts.port,\n    path: (opts as {path?: string | undefined}).path,\n    database: opts.database,\n    user: opts.user,\n    pass: opts.pass,\n    ssl:\n      typeof opts.ssl === 'object' && opts.ssl !== null\n        ? {\n            rejectUnauthorized: (\n              opts.ssl as {rejectUnauthorized?: boolean | undefined}\n            ).rejectUnauthorized,\n          }\n        : (opts.ssl as boolean | string | undefined),\n  };\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * Runs in a separate process with its own event loop and connection reconnect\n * loop to prevent slot drop races during heavy initial sync table copy or\n * SQLite index building.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  config: SlotKeeperConfig,\n): Promise<Readable> {\n  const {slot, lsn} = config;\n  lc.info?.(`spawning slot-keeper process for ${slot} at lsn ${lsn} ...`);\n  const worker = childWorker(SLOT_KEEPER_URL, process.env);\n  worker.onMessageType('init', () => {\n    worker.send(['start', config]);\n  });\n\n  // Wait for the slot keeper to establish replication before returning\n  await new Promise<void>((resolve, reject) => {\n    const onReady = () => {\n      cleanup();\n      resolve();\n    };\n    const onError = (err: unknown) => {\n      cleanup();\n      reject(err);\n    };\n    const onClose = (code: number) => {\n      cleanup();\n      reject(\n        new Error(\n          `slot-keeper for ${slot} exited with code ${code} before becoming ready`,\n        ),\n      );\n    };\n    function cleanup() {\n      worker.off('error', onError);\n      worker.off('close', onClose);\n    }\n    worker.onceMessageType('ready', onReady);\n    worker.once('error', onError);\n    worker.once('close', onClose);\n  });\n\n  lc.info?.(`slot-keeper active for ${slot}`);\n  return new SlotReservationHandle(worker);\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nexport function makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAsCA,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAA,GACA,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAsB,uBACpB,IACA,SACA,MACA,cACA,KACA,cAAA,GACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,KAAK,IAAI,KAAK,KAAK,GAAG,GAAI,CAAC;IACvC;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,wBAAb,cAA2C,SAAS;CAClD;CACA;CACA,WAAW;CACX;CAEA,YAAY,QAAgB;EAC1B,MAAM,EAAC,OAAO,CAAC,EAAC,CAAC;EACjB,KAAKK,UAAU;EACf,KAAKC,iBAAiB,IAAI,SAAQ,YAAW;GAC3C,OAAO,KAAK,eAAe;IACzB,aAAa,KAAKC,aAAa;IAC/B,QAAQ;GACV,CAAC;EACH,CAAC;EAED,KAAK,GAAG,eAAe,CAAC,CAAC;EACzB,OAAO,GAAG,eAAe;GACvB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ;EAEjB,CAAC;EACD,OAAO,GAAG,UAAS,QAAO;GACxB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;EAEpE,CAAC;CACH;CAEA,SACE,OACA,UACM;EACN,IAAI,CAAC,KAAKC,UAAU;GAClB,KAAKA,WAAW;GAChB,IAAI;IACF,KAAKH,QAAQ,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;GAChC,QAAQ,CAER;GACA,KAAKE,gBAAgB,iBAAiB;IACpC,IAAI;KACF,KAAKF,QAAQ,KAAK,SAAS;IAC7B,QAAQ,CAAC;GACX,GAAG,GAAK;GACR,KAAKE,cAAc,MAAM;EAC3B;EACA,SAAS,KAAK;CAChB;CAEA,MAAM,IAAI,YAAY,KAAsB;EAC1C,KAAK,QAAQ;EAKb,IAAI,CAAC,MAJgB,QAAQ,KAAK,CAChC,KAAKD,eAAe,WAAW,IAAI,GACnC,MAAM,SAAS,EAAE,WAAW,KAAK,CACnC,CAAC,GACY;GACX,IAAI;IACF,KAAKD,QAAQ,KAAK,SAAS;GAC7B,QAAQ,CAAC;GAGT,MAAM,QAAQ,KAAK,CAAC,KAAKC,gBAAgB,MAAM,GAAK,CAAC,CAAC;EACxD;CACF;AACF;AAEA,eAAsB,WAAW,SAAmC;CAClE,IAAI,CAAC,SACH;CAEF,IAAI,SAAS,WAAW,OAAO,QAAQ,QAAQ,YAC7C,MAAO,QAAuC,IAAI;MAElD,QAAQ,QAAQ;AAEpB;AAgBA,SAAgB,wBACd,SACoB;CACpB,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAO,KAAqC;EAC5C,UAAU,KAAK;EACf,MAAM,KAAK;EACX,MAAM,KAAK;EACX,KACE,OAAO,KAAK,QAAQ,YAAY,KAAK,QAAQ,OACzC,EACE,oBACE,KAAK,IACL,mBACJ,IACC,KAAK;CACd;AACF;;;;;;;;;;AAWA,eAAsB,6BACpB,IACA,QACmB;CACnB,MAAM,EAAC,MAAM,QAAO;CACpB,GAAG,OAAO,oCAAoC,KAAK,UAAU,IAAI,KAAK;CACtE,MAAM,SAAS,YAAY,iBAAiB,QAAQ,GAAG;CACvD,OAAO,cAAc,cAAc;EACjC,OAAO,KAAK,CAAC,SAAS,MAAM,CAAC;CAC/B,CAAC;CAGD,MAAM,IAAI,SAAe,SAAS,WAAW;EAC3C,MAAM,gBAAgB;GACpB,QAAQ;GACR,QAAQ;EACV;EACA,MAAM,WAAW,QAAiB;GAChC,QAAQ;GACR,OAAO,GAAG;EACZ;EACA,MAAM,WAAW,SAAiB;GAChC,QAAQ;GACR,uBACE,IAAI,MACF,mBAAmB,KAAK,oBAAoB,KAAK,uBACnD,CACF;EACF;EACA,SAAS,UAAU;GACjB,OAAO,IAAI,SAAS,OAAO;GAC3B,OAAO,IAAI,SAAS,OAAO;EAC7B;EACA,OAAO,gBAAgB,SAAS,OAAO;EACvC,OAAO,KAAK,SAAS,OAAO;EAC5B,OAAO,KAAK,SAAS,OAAO;CAC9B,CAAC;CAED,GAAG,OAAO,0BAA0B,MAAM;CAC1C,OAAO,IAAI,sBAAsB,MAAM;AACzC;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAgB,QAAQ,KAAqB;CAC3C,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {Readable} from 'node:stream';\nimport {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport type {SlotKeeperConfig} from '../../../../server/slot-keeper.ts';\nimport {SLOT_KEEPER_URL} from '../../../../server/worker-urls.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {childWorker, type Worker} from '../../../../types/processes.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nexport const DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nexport async function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE + 1,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(Math.min(10 * 2 ** i, 1000));\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport class SlotReservationHandle extends Readable {\n  readonly #worker: Worker;\n  readonly #closedPromise: Promise<void>;\n  #stopped = false;\n  #sigtermTimer?: NodeJS.Timeout | undefined;\n\n  constructor(worker: Worker) {\n    super({read() {}});\n    this.#worker = worker;\n    this.#closedPromise = new Promise(resolve => {\n      worker.once('close', () => {\n        clearTimeout(this.#sigtermTimer);\n        resolve();\n      });\n    });\n    // Suppress unhandled error events if destroyed with an error\n    this.on('error', () => {});\n    worker.on('close', () => {\n      if (!this.destroyed) {\n        this.destroy();\n      }\n    });\n    worker.on('error', err => {\n      if (!this.destroyed) {\n        this.destroy(err instanceof Error ? err : new Error(String(err)));\n      }\n    });\n  }\n\n  override _destroy(\n    error: Error | null,\n    callback: (error?: Error | null) => void,\n  ): void {\n    if (!this.#stopped) {\n      this.#stopped = true;\n      try {\n        this.#worker.send(['stop', {}]);\n      } catch {\n        // ignore\n      }\n      this.#sigtermTimer = setTimeout(() => {\n        try {\n          this.#worker.kill('SIGTERM');\n        } catch {}\n      }, 5_000);\n      this.#sigtermTimer.unref();\n    }\n    callback(error);\n  }\n\n  async end(timeoutMs = 5_000): Promise<void> {\n    this.destroy();\n    const closed = await Promise.race([\n      this.#closedPromise.then(() => true),\n      sleep(timeoutMs).then(() => false),\n    ]);\n    if (!closed) {\n      try {\n        this.#worker.kill('SIGKILL');\n      } catch {}\n      // Await worker closure after forced kill so that the replication session\n      // is completely terminated before any subsequent slot drop attempt.\n      await Promise.race([this.#closedPromise, sleep(2_000)]);\n    }\n  }\n}\n\nexport async function endSession(session?: Readable): Promise<void> {\n  if (!session) {\n    return;\n  }\n  if ('end' in session && typeof session.end === 'function') {\n    await (session as {end: () => Promise<void>}).end();\n  } else {\n    session.destroy();\n  }\n}\n\nexport type PgConnectionConfig = {\n  host?: string | string[] | undefined;\n  port?: number | number[] | undefined;\n  path?: string | undefined;\n  database?: string | undefined;\n  user?: string | undefined;\n  pass?: string | null | undefined;\n  ssl?:\n    | boolean\n    | string\n    | {rejectUnauthorized?: boolean | undefined}\n    | undefined;\n};\n\nexport function extractConnectionConfig(\n  session: postgres.Sql,\n): PgConnectionConfig {\n  const opts = session.options;\n  return {\n    host: opts.host,\n    port: opts.port,\n    path: (opts as {path?: string | undefined}).path,\n    database: opts.database,\n    user: opts.user,\n    pass: opts.pass,\n    ssl:\n      typeof opts.ssl === 'object' && opts.ssl !== null\n        ? {\n            rejectUnauthorized: (\n              opts.ssl as {rejectUnauthorized?: boolean | undefined}\n            ).rejectUnauthorized,\n          }\n        : (opts.ssl as boolean | string | undefined),\n  };\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * Runs in a separate process with its own event loop and connection reconnect\n * loop to prevent slot drop races during heavy initial sync table copy or\n * SQLite index building.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  config: SlotKeeperConfig,\n): Promise<Readable> {\n  const {slot, lsn} = config;\n  lc.info?.(`spawning slot-keeper process for ${slot} at lsn ${lsn} ...`);\n  const worker = childWorker(SLOT_KEEPER_URL, process.env);\n  worker.onMessageType('init', () => {\n    worker.send(['start', config]);\n  });\n\n  // Wait for the slot keeper to establish replication before returning\n  await new Promise<void>((resolve, reject) => {\n    const onReady = () => {\n      cleanup();\n      resolve();\n    };\n    const onError = (err: unknown) => {\n      cleanup();\n      reject(err);\n    };\n    const onClose = (code: number) => {\n      cleanup();\n      reject(\n        new Error(\n          `slot-keeper for ${slot} exited with code ${code} before becoming ready`,\n        ),\n      );\n    };\n    function cleanup() {\n      worker.off('error', onError);\n      worker.off('close', onClose);\n    }\n    worker.onceMessageType('ready', onReady);\n    worker.once('error', onError);\n    worker.once('close', onClose);\n  });\n\n  lc.info?.(`slot-keeper active for ${slot}`);\n  return new SlotReservationHandle(worker);\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nexport function makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAsCA,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAA,GACA,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAChB,MAAM,cAAc,MAAM,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC;CACvE,MAAM,SAAS,IAAI,eAAe,WAAW;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,CAAC,CACA,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAK,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAK,MAAM,GAAG,YAAY,eAAe,WAAW,KAAK,MAAM,CAAC;EAChE,KAAK,cAAc;EACnB,KAAK,qBAAqB;EAC1B,KAAK,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAK,IAAI,OACP,YAAY,KAAK,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAK,OACP;EAEF,MAAM,QAAQ,KAAK;EACnB,KAAK,SAAS;EACd,IAAI,IAAI,QAAQ,aAEZ;OAAA,MAAM,KAAK,UACV,SAAS,QAAQ,KAAK,oBAEvB;EAAA;EAGJ,KAAK,QAAQ;EACb,MAAM,UAAU,MAAM,KAAK;EAC3B,uBAAuB,CAAC,CAAC,SAAS,OAAO;EACzC,KAAK,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAK,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAK,SAAS,KAAK,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG;AAC5D;AAEA,eAAsB,uBACpB,IACA,SACA,MACA,cACA,KACA,cAAsB,GACtB;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,CAAC,CACA,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,KAAK,IAAI,KAAK,KAAK,GAAG,GAAI,CAAC;IACvC;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,IAAa,wBAAb,cAA2C,SAAS;CAClD;CACA;CACA,WAAW;CACX;CAEA,YAAY,QAAgB;EAC1B,MAAM,EAAC,OAAO,CAAC,EAAC,CAAC;EACjB,KAAK,UAAU;EACf,KAAK,iBAAiB,IAAI,SAAQ,YAAW;GAC3C,OAAO,KAAK,eAAe;IACzB,aAAa,KAAK,aAAa;IAC/B,QAAQ;GACV,CAAC;EACH,CAAC;EAED,KAAK,GAAG,eAAe,CAAC,CAAC;EACzB,OAAO,GAAG,eAAe;GACvB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ;EAEjB,CAAC;EACD,OAAO,GAAG,UAAS,QAAO;GACxB,IAAI,CAAC,KAAK,WACR,KAAK,QAAQ,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;EAEpE,CAAC;CACH;CAEA,SACE,OACA,UACM;EACN,IAAI,CAAC,KAAK,UAAU;GAClB,KAAK,WAAW;GAChB,IAAI;IACF,KAAK,QAAQ,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;GAChC,QAAQ,CAER;GACA,KAAK,gBAAgB,iBAAiB;IACpC,IAAI;KACF,KAAK,QAAQ,KAAK,SAAS;IAC7B,QAAQ,CAAC;GACX,GAAG,GAAK;GACR,KAAK,cAAc,MAAM;EAC3B;EACA,SAAS,KAAK;CAChB;CAEA,MAAM,IAAI,YAAY,KAAsB;EAC1C,KAAK,QAAQ;EAKb,IAAI,CAAC,MAJgB,QAAQ,KAAK,CAChC,KAAK,eAAe,WAAW,IAAI,GACnC,MAAM,SAAS,CAAC,CAAC,WAAW,KAAK,CACnC,CAAC,GACY;GACX,IAAI;IACF,KAAK,QAAQ,KAAK,SAAS;GAC7B,QAAQ,CAAC;GAGT,MAAM,QAAQ,KAAK,CAAC,KAAK,gBAAgB,MAAM,GAAK,CAAC,CAAC;EACxD;CACF;AACF;AAEA,eAAsB,WAAW,SAAmC;CAClE,IAAI,CAAC,SACH;CAEF,IAAI,SAAS,WAAW,OAAO,QAAQ,QAAQ,YAC7C,MAAO,QAAuC,IAAI;MAElD,QAAQ,QAAQ;AAEpB;AAgBA,SAAgB,wBACd,SACoB;CACpB,MAAM,OAAO,QAAQ;CACrB,OAAO;EACL,MAAM,KAAK;EACX,MAAM,KAAK;EACX,MAAO,KAAqC;EAC5C,UAAU,KAAK;EACf,MAAM,KAAK;EACX,MAAM,KAAK;EACX,KACE,OAAO,KAAK,QAAQ,YAAY,KAAK,QAAQ,OACzC,EACE,oBACE,KAAK,IACL,mBACJ,IACC,KAAK;CACd;AACF;;;;;;;;;;AAWA,eAAsB,6BACpB,IACA,QACmB;CACnB,MAAM,EAAC,MAAM,QAAO;CACpB,GAAG,OAAO,oCAAoC,KAAK,UAAU,IAAI,KAAK;CACtE,MAAM,SAAS,YAAY,iBAAiB,QAAQ,GAAG;CACvD,OAAO,cAAc,cAAc;EACjC,OAAO,KAAK,CAAC,SAAS,MAAM,CAAC;CAC/B,CAAC;CAGD,MAAM,IAAI,SAAe,SAAS,WAAW;EAC3C,MAAM,gBAAgB;GACpB,QAAQ;GACR,QAAQ;EACV;EACA,MAAM,WAAW,QAAiB;GAChC,QAAQ;GACR,OAAO,GAAG;EACZ;EACA,MAAM,WAAW,SAAiB;GAChC,QAAQ;GACR,uBACE,IAAI,MACF,mBAAmB,KAAK,oBAAoB,KAAK,uBACnD,CACF;EACF;EACA,SAAS,UAAU;GACjB,OAAO,IAAI,SAAS,OAAO;GAC3B,OAAO,IAAI,SAAS,OAAO;EAC7B;EACA,OAAO,gBAAgB,SAAS,OAAO;EACvC,OAAO,KAAK,SAAS,OAAO;EAC5B,OAAO,KAAK,SAAS,OAAO;CAC9B,CAAC;CAED,GAAG,OAAO,0BAA0B,MAAM;CAC1C,OAAO,IAAI,sBAAsB,MAAM;AACzC;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAgB,QAAQ,KAAqB;CAC3C,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-source/pg/lsn.js.map

@@ -1 +1 @@
-{"version":3,"file":"lsn.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/lsn.ts"],"sourcesContent":["import {assert} from '../../../../../shared/src/asserts.ts';\nimport {\n  majorVersionToString,\n  stateVersionFromString,\n} from '../../../types/state-version.ts';\nimport type {Change} from '../protocol/current/data.ts';\n\n/**\n * Parsing and conversion utilities for the pg_lsn Type, which represents\n * the \"Log Sequence Number\" used as a monotonic progress marker for logical\n * replication from a PostgresSQL database.\n *\n * The LSN is a 64-bit integer represented in logical replication as two\n * hexadecimal numbers (up to 8 digits each) separated by a slash. This is\n * converted to a LexiVersion and used as DB-agnostic version in change log,\n * invalidation index, and row version in the tables of the sync replica.\n */\nexport type LSN = string;\n\nexport type RecordType = Change['tag'];\n\nexport function toBigInt(lsn: LSN): bigint {\n  const parts = lsn.split('/');\n  assert(parts.length === 2, `Malformed LSN: \"${lsn}\"`);\n  const high = BigInt(`0x${parts[0]}`);\n  const low = BigInt(`0x${parts[1]}`);\n  const val = (high << 32n) + low;\n  return val;\n}\n\nexport function toStateVersionString(lsn: LSN): string {\n  return majorVersionToString(toBigInt(lsn));\n}\n\n/** The LSN is tracked by the `major` component of the StateVersion. */\nexport function fromStateVersionString(ver: string): LSN {\n  const {major} = stateVersionFromString(ver);\n  return fromBigInt(major);\n}\n\nexport function fromBigInt(val: bigint): LSN {\n  const high = val >> 32n;\n  const low = val & 0xffffffffn;\n  return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;\n}\n"],"mappings":";;;AAqBA,SAAgB,SAAS,KAAkB;CACzC,MAAM,QAAQ,IAAI,MAAM,GAAG;CAC3B,OAAO,MAAM,WAAW,GAAG,mBAAmB,IAAI,EAAE;CACpD,MAAM,OAAO,OAAO,KAAK,MAAM,IAAI;CACnC,MAAM,MAAM,OAAO,KAAK,MAAM,IAAI;CAElC,QADa,QAAQ,OAAO;AAE9B;AAEA,SAAgB,qBAAqB,KAAkB;CACrD,OAAO,qBAAqB,SAAS,GAAG,CAAC;AAC3C;;AAGA,SAAgB,uBAAuB,KAAkB;CACvD,MAAM,EAAC,UAAS,uBAAuB,GAAG;CAC1C,OAAO,WAAW,KAAK;AACzB;AAEA,SAAgB,WAAW,KAAkB;CAC3C,MAAM,OAAO,OAAO;CACpB,MAAM,MAAM,MAAM;CAClB,OAAO,GAAG,KAAK,SAAS,EAAE,EAAE,YAAY,EAAE,GAAG,IAAI,SAAS,EAAE,EAAE,YAAY;AAC5E"}
+{"version":3,"file":"lsn.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/lsn.ts"],"sourcesContent":["import {assert} from '../../../../../shared/src/asserts.ts';\nimport {\n  majorVersionToString,\n  stateVersionFromString,\n} from '../../../types/state-version.ts';\nimport type {Change} from '../protocol/current/data.ts';\n\n/**\n * Parsing and conversion utilities for the pg_lsn Type, which represents\n * the \"Log Sequence Number\" used as a monotonic progress marker for logical\n * replication from a PostgresSQL database.\n *\n * The LSN is a 64-bit integer represented in logical replication as two\n * hexadecimal numbers (up to 8 digits each) separated by a slash. This is\n * converted to a LexiVersion and used as DB-agnostic version in change log,\n * invalidation index, and row version in the tables of the sync replica.\n */\nexport type LSN = string;\n\nexport type RecordType = Change['tag'];\n\nexport function toBigInt(lsn: LSN): bigint {\n  const parts = lsn.split('/');\n  assert(parts.length === 2, `Malformed LSN: \"${lsn}\"`);\n  const high = BigInt(`0x${parts[0]}`);\n  const low = BigInt(`0x${parts[1]}`);\n  const val = (high << 32n) + low;\n  return val;\n}\n\nexport function toStateVersionString(lsn: LSN): string {\n  return majorVersionToString(toBigInt(lsn));\n}\n\n/** The LSN is tracked by the `major` component of the StateVersion. */\nexport function fromStateVersionString(ver: string): LSN {\n  const {major} = stateVersionFromString(ver);\n  return fromBigInt(major);\n}\n\nexport function fromBigInt(val: bigint): LSN {\n  const high = val >> 32n;\n  const low = val & 0xffffffffn;\n  return `${high.toString(16).toUpperCase()}/${low.toString(16).toUpperCase()}`;\n}\n"],"mappings":";;;AAqBA,SAAgB,SAAS,KAAkB;CACzC,MAAM,QAAQ,IAAI,MAAM,GAAG;CAC3B,OAAO,MAAM,WAAW,GAAG,mBAAmB,IAAI,EAAE;CACpD,MAAM,OAAO,OAAO,KAAK,MAAM,IAAI;CACnC,MAAM,MAAM,OAAO,KAAK,MAAM,IAAI;CAElC,QADa,QAAQ,OAAO;AAE9B;AAEA,SAAgB,qBAAqB,KAAkB;CACrD,OAAO,qBAAqB,SAAS,GAAG,CAAC;AAC3C;;AAGA,SAAgB,uBAAuB,KAAkB;CACvD,MAAM,EAAC,UAAS,uBAAuB,GAAG;CAC1C,OAAO,WAAW,KAAK;AACzB;AAEA,SAAgB,WAAW,KAAkB;CAC3C,MAAM,OAAO,OAAO;CACpB,MAAM,MAAM,MAAM;CAClB,OAAO,GAAG,KAAK,SAAS,EAAE,CAAC,CAAC,YAAY,EAAE,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC,YAAY;AAC5E"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js

@@ -2,8 +2,8 @@ import { replicationSlotPrefix } from "./schema/shard.js";
 import "../../../types/pg-versions.js";
 import { dropInactiveSlotsAndReplicas } from "./replication-slots.js";
 //#region ../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts
-var DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 300 * 1e3;
-var DEFAULT_POLL_INTERVAL_MS = 60 * 1e3;
+var DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 3e5;
+var DEFAULT_POLL_INTERVAL_MS = 6e4;
 var ReplicationSlotCleanupMonitor = class {
 	#lc;
 	/** The shard constrains which slots are monitored (e.g. zero_0_*). */

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slot-cleanup-monitor.js","names":["#lc","#shard","#upstream","#currentSlot","#inactiveSlotCleanupTimeoutMs","#pollIntervalMs","#firstInactiveTimes","#pollTimer","#poll"],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {dropInactiveSlotsAndReplicas} from './replication-slots.ts';\nimport {replicationSlotPrefix} from './schema/shard.ts';\n\ntype Options = {\n  inactiveSlotCleanupTimeoutMs?: number | undefined;\n  pollIntervalMs?: number | undefined;\n};\n\nconst DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 5 * 60 * 1000;\nconst DEFAULT_POLL_INTERVAL_MS = 60 * 1000;\n\nexport class ReplicationSlotCleanupMonitor {\n  readonly #lc: LogContext;\n\n  /** The shard constrains which slots are monitored (e.g. zero_0_*). */\n  readonly #shard: ShardID;\n  readonly #upstream: PostgresDB;\n  /**\n   * This slot that this task is using. Cleanup of slots is predicated on\n   * this slot remaining active, in order to guarantee at least one healthy\n   * slot in the shard.\n   */\n  readonly #currentSlot: string;\n\n  /**\n   * How long a slot is inactive (while the currentSlot is active) before it\n   * is eligible to be cleaned up. Because new slots are immediately connected\n   * to, even while an initial-sync or replica-restore is in progress, slots\n   * should only be inactive if no server is using it, or if a server was\n   * disconnected (e.g. a pg restart) and is in the process of reconnecting\n   * to it (see `keepSlotActiveUntilTakenOver`).\n   *\n   * As such, this timeout should be long enough to account for a server\n   * reconnecting to upstream. Note again that the cleanup monitor only runs\n   * while at least one replication slot is active, guaranteeing that upstream\n   * replication is working.\n   */\n  readonly #inactiveSlotCleanupTimeoutMs: number;\n  readonly #pollIntervalMs: number;\n\n  /** Manually tracked inactive times for PG < 17. null for PG 17+ */\n  readonly #firstInactiveTimes: Map<string, Date> | null;\n\n  #pollTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    upstream: PostgresDB,\n    upstreamPgServerVersionNum: number,\n    currentSlot: string,\n    {inactiveSlotCleanupTimeoutMs, pollIntervalMs}: Options = {},\n  ) {\n    this.#lc = lc.withContext('component', 'replication-slot-cleanup-monitor');\n    this.#shard = shard;\n    this.#upstream = upstream;\n    this.#currentSlot = currentSlot;\n    this.#inactiveSlotCleanupTimeoutMs =\n      inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;\n    this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n    this.#firstInactiveTimes =\n      upstreamPgServerVersionNum >= PG_17 ? null : new Map();\n  }\n\n  start() {\n    if (this.#pollTimer) {\n      this.#lc.warn?.(`monitor already started`);\n      return;\n    }\n    this.#pollTimer = setInterval(\n      () =>\n        this.#poll(new Date()).catch(e =>\n          this.#lc.warn?.(`error checking replication slots`, e),\n        ),\n      this.#pollIntervalMs,\n    );\n  }\n\n  stop() {\n    clearInterval(this.#pollTimer);\n    this.#pollTimer = undefined;\n    this.#firstInactiveTimes?.clear();\n  }\n\n  async #poll(now: Date) {\n    if (!this.#pollTimer) {\n      return; // concurrently stopped\n    }\n    const slots = await this.getSlotsToCleanup(now);\n    if (slots.length) {\n      await dropInactiveSlotsAndReplicas(\n        this.#lc,\n        this.#upstream,\n        this.#shard,\n        slots,\n      );\n    }\n  }\n\n  async getSlotsToCleanup(now: Date): Promise<string[]> {\n    const slotPoolPrefix = replicationSlotPrefix(this.#shard);\n\n    const slotsToCleanUp: string[] = [];\n    const trackedInactiveTimes = this.#firstInactiveTimes;\n\n    if (trackedInactiveTimes) {\n      // For PG <17, each poll tracks and updates earliest time the slot was\n      // observed to be inactive.\n      const shardSlots = await this.#upstream<{slot: string; active: boolean}[]>\n      /*sql*/ `\n        SELECT slot_name AS slot, active FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `;\n      for (const {slot, active} of shardSlots) {\n        if (slot === this.#currentSlot && !active) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        if (active) {\n          trackedInactiveTimes.delete(slot);\n        } else {\n          const inactiveSince = trackedInactiveTimes.get(slot);\n          if (!inactiveSince) {\n            trackedInactiveTimes.set(slot, now);\n          } else if (\n            now.getTime() - inactiveSince.getTime() >=\n            this.#inactiveSlotCleanupTimeoutMs\n          ) {\n            this.#lc.info?.(\n              `slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`,\n            );\n            slotsToCleanUp.push(slot);\n          }\n        }\n      }\n    } else {\n      // PG 17+ is simpler as inactivity time is tracked in the table.\n      const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;\n      const inactiveSlots = await this.#upstream<\n        {slot: string; inactiveSince: number}[]\n      > /*sql*/ `\n        SELECT slot_name AS slot, inactive_since AS \"inactiveSince\" FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'}\n            AND active = false\n            AND inactive_since IS NOT NULL;\n      `;\n      for (const {slot, inactiveSince} of inactiveSlots) {\n        if (slot === this.#currentSlot) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        // The slot has been inactive since at or before the expiration cutoff\n        // (now - timeout), i.e. inactive for at least the timeout duration.\n        if (inactiveSince <= expiration) {\n          this.#lc.info?.(\n            `slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`,\n          );\n          slotsToCleanUp.push(slot);\n        }\n      }\n    }\n    return slotsToCleanUp;\n  }\n}\n"],"mappings":";;;;AAYA,IAAM,2CAA2C,MAAS;AAC1D,IAAM,2BAA2B,KAAK;AAEtC,IAAa,gCAAb,MAA2C;CACzC;;CAGA;CACA;;;;;;CAMA;;;;;;;;;;;;;;CAeA;CACA;;CAGA;CAEA;CAEA,YACE,IACA,OACA,UACA,4BACA,aACA,EAAC,8BAA8B,mBAA2B,CAAC,GAC3D;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kCAAkC;EACzE,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,eAAe;EACpB,KAAKC,gCACH,gCAAgC;EAClC,KAAKC,kBAAkB,kBAAkB;EACzC,KAAKC,sBACH,8BAAA,OAAsC,uBAAO,IAAI,IAAI;CACzD;CAEA,QAAQ;EACN,IAAI,KAAKC,YAAY;GACnB,KAAKP,IAAI,OAAO,yBAAyB;GACzC;EACF;EACA,KAAKO,aAAa,kBAEd,KAAKC,sBAAM,IAAI,KAAK,CAAC,EAAE,OAAM,MAC3B,KAAKR,IAAI,OAAO,oCAAoC,CAAC,CACvD,GACF,KAAKK,eACP;CACF;CAEA,OAAO;EACL,cAAc,KAAKE,UAAU;EAC7B,KAAKA,aAAa,KAAA;EAClB,KAAKD,qBAAqB,MAAM;CAClC;CAEA,MAAME,MAAM,KAAW;EACrB,IAAI,CAAC,KAAKD,YACR;EAEF,MAAM,QAAQ,MAAM,KAAK,kBAAkB,GAAG;EAC9C,IAAI,MAAM,QACR,MAAM,6BACJ,KAAKP,KACL,KAAKE,WACL,KAAKD,QACL,KACF;CAEJ;CAEA,MAAM,kBAAkB,KAA8B;EACpD,MAAM,iBAAiB,sBAAsB,KAAKA,MAAM;EAExD,MAAM,iBAA2B,CAAC;EAClC,MAAM,uBAAuB,KAAKK;EAElC,IAAI,sBAAsB;GAGxB,MAAM,aAAa,MAAM,KAAKJ,SACtB;;iCAEmB,iBAAiB,IAAI;;GAEhD,KAAK,MAAM,EAAC,MAAM,YAAW,YAAY;IACvC,IAAI,SAAS,KAAKC,gBAAgB,CAAC,QAAQ;KACzC,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IACA,IAAI,QACF,qBAAqB,OAAO,IAAI;SAC3B;KACL,MAAM,gBAAgB,qBAAqB,IAAI,IAAI;KACnD,IAAI,CAAC,eACH,qBAAqB,IAAI,MAAM,GAAG;UAC7B,IACL,IAAI,QAAQ,IAAI,cAAc,QAAQ,KACtC,KAAKC,+BACL;MACA,KAAKJ,IAAI,OACP,QAAQ,KAAK,2BAA2B,cAAc,YAAY,EAAE,6BACtE;MACA,eAAe,KAAK,IAAI;KAC1B;IACF;GACF;EACF,OAAO;GAEL,MAAM,aAAa,IAAI,QAAQ,IAAI,KAAKI;GACxC,MAAM,gBAAgB,MAAM,KAAKF,SAEvB;;iCAEiB,iBAAiB,IAAI;;;;GAIhD,KAAK,MAAM,EAAC,MAAM,mBAAkB,eAAe;IACjD,IAAI,SAAS,KAAKC,cAAc;KAC9B,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IAGA,IAAI,iBAAiB,YAAY;KAC/B,KAAKH,IAAI,OACP,QAAQ,KAAK,2BAA2B,IAAI,KAAK,aAAa,EAAE,YAAY,EAAE,6BAChF;KACA,eAAe,KAAK,IAAI;IAC1B;GACF;EACF;EACA,OAAO;CACT;AACF"}
+{"version":3,"file":"replication-slot-cleanup-monitor.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {dropInactiveSlotsAndReplicas} from './replication-slots.ts';\nimport {replicationSlotPrefix} from './schema/shard.ts';\n\ntype Options = {\n  inactiveSlotCleanupTimeoutMs?: number | undefined;\n  pollIntervalMs?: number | undefined;\n};\n\nconst DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 5 * 60 * 1000;\nconst DEFAULT_POLL_INTERVAL_MS = 60 * 1000;\n\nexport class ReplicationSlotCleanupMonitor {\n  readonly #lc: LogContext;\n\n  /** The shard constrains which slots are monitored (e.g. zero_0_*). */\n  readonly #shard: ShardID;\n  readonly #upstream: PostgresDB;\n  /**\n   * This slot that this task is using. Cleanup of slots is predicated on\n   * this slot remaining active, in order to guarantee at least one healthy\n   * slot in the shard.\n   */\n  readonly #currentSlot: string;\n\n  /**\n   * How long a slot is inactive (while the currentSlot is active) before it\n   * is eligible to be cleaned up. Because new slots are immediately connected\n   * to, even while an initial-sync or replica-restore is in progress, slots\n   * should only be inactive if no server is using it, or if a server was\n   * disconnected (e.g. a pg restart) and is in the process of reconnecting\n   * to it (see `keepSlotActiveUntilTakenOver`).\n   *\n   * As such, this timeout should be long enough to account for a server\n   * reconnecting to upstream. Note again that the cleanup monitor only runs\n   * while at least one replication slot is active, guaranteeing that upstream\n   * replication is working.\n   */\n  readonly #inactiveSlotCleanupTimeoutMs: number;\n  readonly #pollIntervalMs: number;\n\n  /** Manually tracked inactive times for PG < 17. null for PG 17+ */\n  readonly #firstInactiveTimes: Map<string, Date> | null;\n\n  #pollTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    upstream: PostgresDB,\n    upstreamPgServerVersionNum: number,\n    currentSlot: string,\n    {inactiveSlotCleanupTimeoutMs, pollIntervalMs}: Options = {},\n  ) {\n    this.#lc = lc.withContext('component', 'replication-slot-cleanup-monitor');\n    this.#shard = shard;\n    this.#upstream = upstream;\n    this.#currentSlot = currentSlot;\n    this.#inactiveSlotCleanupTimeoutMs =\n      inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;\n    this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n    this.#firstInactiveTimes =\n      upstreamPgServerVersionNum >= PG_17 ? null : new Map();\n  }\n\n  start() {\n    if (this.#pollTimer) {\n      this.#lc.warn?.(`monitor already started`);\n      return;\n    }\n    this.#pollTimer = setInterval(\n      () =>\n        this.#poll(new Date()).catch(e =>\n          this.#lc.warn?.(`error checking replication slots`, e),\n        ),\n      this.#pollIntervalMs,\n    );\n  }\n\n  stop() {\n    clearInterval(this.#pollTimer);\n    this.#pollTimer = undefined;\n    this.#firstInactiveTimes?.clear();\n  }\n\n  async #poll(now: Date) {\n    if (!this.#pollTimer) {\n      return; // concurrently stopped\n    }\n    const slots = await this.getSlotsToCleanup(now);\n    if (slots.length) {\n      await dropInactiveSlotsAndReplicas(\n        this.#lc,\n        this.#upstream,\n        this.#shard,\n        slots,\n      );\n    }\n  }\n\n  async getSlotsToCleanup(now: Date): Promise<string[]> {\n    const slotPoolPrefix = replicationSlotPrefix(this.#shard);\n\n    const slotsToCleanUp: string[] = [];\n    const trackedInactiveTimes = this.#firstInactiveTimes;\n\n    if (trackedInactiveTimes) {\n      // For PG <17, each poll tracks and updates earliest time the slot was\n      // observed to be inactive.\n      const shardSlots = await this.#upstream<{slot: string; active: boolean}[]>\n      /*sql*/ `\n        SELECT slot_name AS slot, active FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `;\n      for (const {slot, active} of shardSlots) {\n        if (slot === this.#currentSlot && !active) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        if (active) {\n          trackedInactiveTimes.delete(slot);\n        } else {\n          const inactiveSince = trackedInactiveTimes.get(slot);\n          if (!inactiveSince) {\n            trackedInactiveTimes.set(slot, now);\n          } else if (\n            now.getTime() - inactiveSince.getTime() >=\n            this.#inactiveSlotCleanupTimeoutMs\n          ) {\n            this.#lc.info?.(\n              `slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`,\n            );\n            slotsToCleanUp.push(slot);\n          }\n        }\n      }\n    } else {\n      // PG 17+ is simpler as inactivity time is tracked in the table.\n      const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;\n      const inactiveSlots = await this.#upstream<\n        {slot: string; inactiveSince: number}[]\n      > /*sql*/ `\n        SELECT slot_name AS slot, inactive_since AS \"inactiveSince\" FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'}\n            AND active = false\n            AND inactive_since IS NOT NULL;\n      `;\n      for (const {slot, inactiveSince} of inactiveSlots) {\n        if (slot === this.#currentSlot) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        // The slot has been inactive since at or before the expiration cutoff\n        // (now - timeout), i.e. inactive for at least the timeout duration.\n        if (inactiveSince <= expiration) {\n          this.#lc.info?.(\n            `slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`,\n          );\n          slotsToCleanUp.push(slot);\n        }\n      }\n    }\n    return slotsToCleanUp;\n  }\n}\n"],"mappings":";;;;AAYA,IAAM,2CAA2C;AACjD,IAAM,2BAA2B;AAEjC,IAAa,gCAAb,MAA2C;CACzC;;CAGA;CACA;;;;;;CAMA;;;;;;;;;;;;;;CAeA;CACA;;CAGA;CAEA;CAEA,YACE,IACA,OACA,UACA,4BACA,aACA,EAAC,8BAA8B,mBAA2B,CAAC,GAC3D;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,kCAAkC;EACzE,KAAK,SAAS;EACd,KAAK,YAAY;EACjB,KAAK,eAAe;EACpB,KAAK,gCACH,gCAAgC;EAClC,KAAK,kBAAkB,kBAAkB;EACzC,KAAK,sBACH,8BAAA,OAAsC,uBAAO,IAAI,IAAI;CACzD;CAEA,QAAQ;EACN,IAAI,KAAK,YAAY;GACnB,KAAK,IAAI,OAAO,yBAAyB;GACzC;EACF;EACA,KAAK,aAAa,kBAEd,KAAK,sBAAM,IAAI,KAAK,CAAC,CAAC,CAAC,OAAM,MAC3B,KAAK,IAAI,OAAO,oCAAoC,CAAC,CACvD,GACF,KAAK,eACP;CACF;CAEA,OAAO;EACL,cAAc,KAAK,UAAU;EAC7B,KAAK,aAAa,KAAA;EAClB,KAAK,qBAAqB,MAAM;CAClC;CAEA,MAAM,MAAM,KAAW;EACrB,IAAI,CAAC,KAAK,YACR;EAEF,MAAM,QAAQ,MAAM,KAAK,kBAAkB,GAAG;EAC9C,IAAI,MAAM,QACR,MAAM,6BACJ,KAAK,KACL,KAAK,WACL,KAAK,QACL,KACF;CAEJ;CAEA,MAAM,kBAAkB,KAA8B;EACpD,MAAM,iBAAiB,sBAAsB,KAAK,MAAM;EAExD,MAAM,iBAA2B,CAAC;EAClC,MAAM,uBAAuB,KAAK;EAElC,IAAI,sBAAsB;GAGxB,MAAM,aAAa,MAAM,KAAK,SACtB;;iCAEmB,iBAAiB,IAAI;;GAEhD,KAAK,MAAM,EAAC,MAAM,YAAW,YAAY;IACvC,IAAI,SAAS,KAAK,gBAAgB,CAAC,QAAQ;KACzC,KAAK,IAAI,OACP,gBAAgB,KAAK,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IACA,IAAI,QACF,qBAAqB,OAAO,IAAI;SAC3B;KACL,MAAM,gBAAgB,qBAAqB,IAAI,IAAI;KACnD,IAAI,CAAC,eACH,qBAAqB,IAAI,MAAM,GAAG;UAC7B,IACL,IAAI,QAAQ,IAAI,cAAc,QAAQ,KACtC,KAAK,+BACL;MACA,KAAK,IAAI,OACP,QAAQ,KAAK,2BAA2B,cAAc,YAAY,EAAE,6BACtE;MACA,eAAe,KAAK,IAAI;KAC1B;IACF;GACF;EACF,OAAO;GAEL,MAAM,aAAa,IAAI,QAAQ,IAAI,KAAK;GACxC,MAAM,gBAAgB,MAAM,KAAK,SAEvB;;iCAEiB,iBAAiB,IAAI;;;;GAIhD,KAAK,MAAM,EAAC,MAAM,mBAAkB,eAAe;IACjD,IAAI,SAAS,KAAK,cAAc;KAC9B,KAAK,IAAI,OACP,gBAAgB,KAAK,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IAGA,IAAI,iBAAiB,YAAY;KAC/B,KAAK,IAAI,OACP,QAAQ,KAAK,2BAA2B,IAAI,KAAK,aAAa,CAAC,CAAC,YAAY,EAAE,6BAChF;KACA,eAAe,KAAK,IAAI;IAC1B;GACF;EACF;EACA,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slot-health.js

@@ -67,6 +67,6 @@ async function queryReplicationSlotHealth(db, slot) {
     WHERE slot_name = ${slot}`)[0];
 }
 //#endregion
-export { registerReplicationSlotHealthMetrics };
+export { queryReplicationSlotHealth, registerReplicationSlotHealthMetrics, slotHealthStatus };
 
 //# sourceMappingURL=replication-slot-health.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-health.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slot-health.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-health.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\n\nconst SLOT_HEALTH_STATUSES = [\n  'ok',\n  'unreserved',\n  'lost',\n  'missing',\n  'unknown',\n] as const;\n\nexport type SlotHealthStatus = (typeof SLOT_HEALTH_STATUSES)[number];\n\nexport type SlotHealthRow = {\n  readonly restartLSN: string | null;\n  readonly walStatus: string | null;\n  readonly retainedWalBytes: number | null;\n  readonly safeWalBytes: number | null;\n};\n\nexport function slotHealthStatus(\n  row: Pick<SlotHealthRow, 'restartLSN' | 'walStatus'> | undefined,\n): SlotHealthStatus {\n  if (row === undefined) {\n    return 'missing';\n  }\n  if (row.restartLSN === null || row.walStatus === 'lost') {\n    return 'lost';\n  }\n  switch (row.walStatus) {\n    case 'reserved':\n    case 'extended':\n      return 'ok';\n    case 'unreserved':\n      return 'unreserved';\n    default:\n      return 'unknown';\n  }\n}\n\nexport function registerReplicationSlotHealthMetrics(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  isStopped: () => boolean,\n): void {\n  getOrCreateGauge('replication', 'slot_health', {\n    description: 'Health of the active logical replication slot.',\n    unit: '1',\n  }).addCallback(async result => {\n    if (isStopped()) {\n      return;\n    }\n\n    let status: SlotHealthStatus;\n    try {\n      status = slotHealthStatus(await queryReplicationSlotHealth(db, slot));\n    } catch (e) {\n      lc.warn?.(`error querying replication slot health`, e);\n      status = 'unknown';\n    }\n\n    for (const s of SLOT_HEALTH_STATUSES) {\n      result.observe(s === status ? 1 : 0, {slot, status: s});\n    }\n  });\n\n  getOrCreateGauge('replication', 'slot_retained_wal_bytes', {\n    description: 'WAL bytes retained by the active logical replication slot.',\n    unit: 'By',\n  }).addCallback(result =>\n    observeSlotBytes(lc, db, slot, isStopped, result, 'retainedWalBytes'),\n  );\n\n  getOrCreateGauge('replication', 'slot_safe_wal_bytes', {\n    description:\n      'WAL bytes the active logical replication slot can retain before loss.',\n    unit: 'By',\n  }).addCallback(result =>\n    observeSlotBytes(lc, db, slot, isStopped, result, 'safeWalBytes'),\n  );\n}\n\nasync function observeSlotBytes(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  isStopped: () => boolean,\n  result: ObservableResult,\n  field: 'retainedWalBytes' | 'safeWalBytes',\n): Promise<void> {\n  if (isStopped()) {\n    return;\n  }\n  try {\n    const row = await queryReplicationSlotHealth(db, slot);\n    const value = row?.[field];\n    if (typeof value === 'number' && Number.isFinite(value)) {\n      result.observe(value, {slot});\n    }\n  } catch (e) {\n    lc.warn?.(`error querying replication slot ${field}`, e);\n  }\n}\n\nexport async function queryReplicationSlotHealth(\n  db: PostgresDB,\n  slot: string,\n): Promise<SlotHealthRow | undefined> {\n  const rows = await db<SlotHealthRow[]>`\n    SELECT\n      restart_lsn AS \"restartLSN\",\n      wal_status AS \"walStatus\",\n      CASE\n        WHEN restart_lsn IS NULL THEN NULL\n        ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)::float8\n      END AS \"retainedWalBytes\",\n      safe_wal_size::float8 AS \"safeWalBytes\"\n    FROM pg_replication_slots\n    WHERE slot_name = ${slot}`;\n  return rows[0];\n}\n"],"mappings":";;AAKA,IAAM,uBAAuB;CAC3B;CACA;CACA;CACA;CACA;AACF;AAWA,SAAgB,iBACd,KACkB;CAClB,IAAI,QAAQ,KAAA,GACV,OAAO;CAET,IAAI,IAAI,eAAe,QAAQ,IAAI,cAAc,QAC/C,OAAO;CAET,QAAQ,IAAI,WAAZ;EACE,KAAK;EACL,KAAK,YACH,OAAO;EACT,KAAK,cACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,SAAgB,qCACd,IACA,IACA,MACA,WACM;CACN,iBAAiB,eAAe,eAAe;EAC7C,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,OAAM,WAAU;EAC7B,IAAI,UAAU,GACZ;EAGF,IAAI;EACJ,IAAI;GACF,SAAS,iBAAiB,MAAM,2BAA2B,IAAI,IAAI,CAAC;EACtE,SAAS,GAAG;GACV,GAAG,OAAO,0CAA0C,CAAC;GACrD,SAAS;EACX;EAEA,KAAK,MAAM,KAAK,sBACd,OAAO,QAAQ,MAAM,SAAS,IAAI,GAAG;GAAC;GAAM,QAAQ;EAAC,CAAC;CAE1D,CAAC;CAED,iBAAiB,eAAe,2BAA2B;EACzD,aAAa;EACb,MAAM;CACR,CAAC,EAAE,aAAY,WACb,iBAAiB,IAAI,IAAI,MAAM,WAAW,QAAQ,kBAAkB,CACtE;CAEA,iBAAiB,eAAe,uBAAuB;EACrD,aACE;EACF,MAAM;CACR,CAAC,EAAE,aAAY,WACb,iBAAiB,IAAI,IAAI,MAAM,WAAW,QAAQ,cAAc,CAClE;AACF;AAEA,eAAe,iBACb,IACA,IACA,MACA,WACA,QACA,OACe;CACf,IAAI,UAAU,GACZ;CAEF,IAAI;EAEF,MAAM,SAAQ,MADI,2BAA2B,IAAI,IAAI,KACjC;EACpB,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GACpD,OAAO,QAAQ,OAAO,EAAC,KAAI,CAAC;CAEhC,SAAS,GAAG;EACV,GAAG,OAAO,mCAAmC,SAAS,CAAC;CACzD;AACF;AAEA,eAAsB,2BACpB,IACA,MACoC;CAYpC,QAAO,MAXY,EAAmB;;;;;;;;;;wBAUhB,QACV;AACd"}
+{"version":3,"file":"replication-slot-health.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-health.ts"],"sourcesContent":["import type {ObservableResult} from '@opentelemetry/api';\nimport type {LogContext} from '@rocicorp/logger';\nimport {getOrCreateGauge} from '../../../observability/metrics.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\n\nconst SLOT_HEALTH_STATUSES = [\n  'ok',\n  'unreserved',\n  'lost',\n  'missing',\n  'unknown',\n] as const;\n\nexport type SlotHealthStatus = (typeof SLOT_HEALTH_STATUSES)[number];\n\nexport type SlotHealthRow = {\n  readonly restartLSN: string | null;\n  readonly walStatus: string | null;\n  readonly retainedWalBytes: number | null;\n  readonly safeWalBytes: number | null;\n};\n\nexport function slotHealthStatus(\n  row: Pick<SlotHealthRow, 'restartLSN' | 'walStatus'> | undefined,\n): SlotHealthStatus {\n  if (row === undefined) {\n    return 'missing';\n  }\n  if (row.restartLSN === null || row.walStatus === 'lost') {\n    return 'lost';\n  }\n  switch (row.walStatus) {\n    case 'reserved':\n    case 'extended':\n      return 'ok';\n    case 'unreserved':\n      return 'unreserved';\n    default:\n      return 'unknown';\n  }\n}\n\nexport function registerReplicationSlotHealthMetrics(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  isStopped: () => boolean,\n): void {\n  getOrCreateGauge('replication', 'slot_health', {\n    description: 'Health of the active logical replication slot.',\n    unit: '1',\n  }).addCallback(async result => {\n    if (isStopped()) {\n      return;\n    }\n\n    let status: SlotHealthStatus;\n    try {\n      status = slotHealthStatus(await queryReplicationSlotHealth(db, slot));\n    } catch (e) {\n      lc.warn?.(`error querying replication slot health`, e);\n      status = 'unknown';\n    }\n\n    for (const s of SLOT_HEALTH_STATUSES) {\n      result.observe(s === status ? 1 : 0, {slot, status: s});\n    }\n  });\n\n  getOrCreateGauge('replication', 'slot_retained_wal_bytes', {\n    description: 'WAL bytes retained by the active logical replication slot.',\n    unit: 'By',\n  }).addCallback(result =>\n    observeSlotBytes(lc, db, slot, isStopped, result, 'retainedWalBytes'),\n  );\n\n  getOrCreateGauge('replication', 'slot_safe_wal_bytes', {\n    description:\n      'WAL bytes the active logical replication slot can retain before loss.',\n    unit: 'By',\n  }).addCallback(result =>\n    observeSlotBytes(lc, db, slot, isStopped, result, 'safeWalBytes'),\n  );\n}\n\nasync function observeSlotBytes(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  isStopped: () => boolean,\n  result: ObservableResult,\n  field: 'retainedWalBytes' | 'safeWalBytes',\n): Promise<void> {\n  if (isStopped()) {\n    return;\n  }\n  try {\n    const row = await queryReplicationSlotHealth(db, slot);\n    const value = row?.[field];\n    if (typeof value === 'number' && Number.isFinite(value)) {\n      result.observe(value, {slot});\n    }\n  } catch (e) {\n    lc.warn?.(`error querying replication slot ${field}`, e);\n  }\n}\n\nexport async function queryReplicationSlotHealth(\n  db: PostgresDB,\n  slot: string,\n): Promise<SlotHealthRow | undefined> {\n  const rows = await db<SlotHealthRow[]>`\n    SELECT\n      restart_lsn AS \"restartLSN\",\n      wal_status AS \"walStatus\",\n      CASE\n        WHEN restart_lsn IS NULL THEN NULL\n        ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)::float8\n      END AS \"retainedWalBytes\",\n      safe_wal_size::float8 AS \"safeWalBytes\"\n    FROM pg_replication_slots\n    WHERE slot_name = ${slot}`;\n  return rows[0];\n}\n"],"mappings":";;AAKA,IAAM,uBAAuB;CAC3B;CACA;CACA;CACA;CACA;AACF;AAWA,SAAgB,iBACd,KACkB;CAClB,IAAI,QAAQ,KAAA,GACV,OAAO;CAET,IAAI,IAAI,eAAe,QAAQ,IAAI,cAAc,QAC/C,OAAO;CAET,QAAQ,IAAI,WAAZ;EACE,KAAK;EACL,KAAK,YACH,OAAO;EACT,KAAK,cACH,OAAO;EACT,SACE,OAAO;CACX;AACF;AAEA,SAAgB,qCACd,IACA,IACA,MACA,WACM;CACN,iBAAiB,eAAe,eAAe;EAC7C,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,OAAM,WAAU;EAC7B,IAAI,UAAU,GACZ;EAGF,IAAI;EACJ,IAAI;GACF,SAAS,iBAAiB,MAAM,2BAA2B,IAAI,IAAI,CAAC;EACtE,SAAS,GAAG;GACV,GAAG,OAAO,0CAA0C,CAAC;GACrD,SAAS;EACX;EAEA,KAAK,MAAM,KAAK,sBACd,OAAO,QAAQ,MAAM,SAAS,IAAI,GAAG;GAAC;GAAM,QAAQ;EAAC,CAAC;CAE1D,CAAC;CAED,iBAAiB,eAAe,2BAA2B;EACzD,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,WACb,iBAAiB,IAAI,IAAI,MAAM,WAAW,QAAQ,kBAAkB,CACtE;CAEA,iBAAiB,eAAe,uBAAuB;EACrD,aACE;EACF,MAAM;CACR,CAAC,CAAC,CAAC,aAAY,WACb,iBAAiB,IAAI,IAAI,MAAM,WAAW,QAAQ,cAAc,CAClE;AACF;AAEA,eAAe,iBACb,IACA,IACA,MACA,WACA,QACA,OACe;CACf,IAAI,UAAU,GACZ;CAEF,IAAI;EAEF,MAAM,SAAQ,MADI,2BAA2B,IAAI,IAAI,EAAA,GACjC;EACpB,IAAI,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,GACpD,OAAO,QAAQ,OAAO,EAAC,KAAI,CAAC;CAEhC,SAAS,GAAG;EACV,GAAG,OAAO,mCAAmC,SAAS,CAAC;CACzD;AACF;AAEA,eAAsB,2BACpB,IACA,MACoC;CAYpC,QAAO,MAXY,EAAmB;;;;;;;;;;wBAUhB,OAAA,CACV;AACd"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -15,7 +15,7 @@ import postgres from "postgres";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE, PG_LOCK_NOT_AVAILABLE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/replication-slots.ts
 var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 6e4;
-var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
+var SERVER_LOCK_TIMEOUT_MS = 59e3;
 /** Thrown when the client-side timeout for creating a slot is reached. */
 var SlotCreationTimeoutError = class extends Error {
 	name = "SlotCreationTimeoutError";
@@ -94,7 +94,7 @@ async function logSlotCreationBlockers(lc, sql, walsenderPID, slotName) {
 function isSlotCreationBlocked(e) {
 	return isPostgresError(e, PG_LOCK_NOT_AVAILABLE) || e instanceof SlotCreationTimeoutError || e instanceof Error && e.code === "CONNECTION_CLOSED";
 }
-var DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 1800 * 1e3;
+var DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 18e5;
 var DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1e3;
 /**
 * Creates a dedicated (non-pooled) session for holding the session-level
@@ -396,6 +396,6 @@ function replicationSlotManagementLock(shard) {
 	return `replication-slot-management:${shard.appID}_${shard.shardNum}`;
 }
 //#endregion
-export { claimSlotForResumption, createReplicaAndSlot, createReplicationSlot, dropInactiveSlotsAndReplicas };
+export { CREATE_REPLICATION_SLOT_TIMEOUT_MS, SlotCreationTimeoutError, claimSlotForResumption, createReplicaAndSlot, createReplicationSlot, dropInactiveSlotsAndReplicas, logSlotCreationBlockers, slotPoolSuffix };
 
 //# sourceMappingURL=replication-slots.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB,qCAAqC;;AAGpE,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC,OAAU;AAIvD,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n  PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  endSession,\n  extractConnectionConfig,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n\n  // A (non-replication) connection to the same upstream instance, used to\n  // log what slot creation was waiting on if it times out.\n  diagnosticsDB?: PostgresDB | undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. */\nexport class SlotCreationTimeoutError extends Error {\n  readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n    diagnosticsDB,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion, pid: walsenderPID} = (\n    await session.unsafe<{pgVersion: number; pid: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\",\n             pg_backend_pid() as \"pid\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  try {\n    const raced = await orTimeout(\n      createSlot,\n      CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n    );\n    if (raced === 'timed-out') {\n      throw new SlotCreationTimeoutError(\n        `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n      );\n    }\n    const [slot] = raced;\n    lc.info?.(`Created replication slot ${slotName}`, slot);\n    return slot;\n  } catch (e) {\n    if (\n      diagnosticsDB &&\n      (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n        e instanceof SlotCreationTimeoutError)\n    ) {\n      // After a lock_timeout, the walsender is no longer waiting, but the\n      // transactions it was waiting on are likely still running. After the\n      // client-side timeout, it may still be waiting on them.\n      await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n    }\n    throw e;\n  }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/**\n * Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n *\n * Query text is deliberately omitted since it can contain row values.\n *\n * Exported for testing.\n */\nexport async function logSlotCreationBlockers(\n  lc: LogContext,\n  sql: PostgresDB,\n  walsenderPID: number,\n  slotName: string,\n) {\n  try {\n    const result = await orTimeout(\n      Promise.all([\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 pg_blocking_pids(pid) as \"blockingPIDs\"\n            FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n        sql<JSONObject[]> /*sql*/ `\n          SELECT pid, datname, usename, application_name as \"applicationName\",\n                 backend_type as \"backendType\", state,\n                 wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n                 backend_xid::text as xid,\n                 xact_start::text as \"xactStart\",\n                 extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n            FROM pg_stat_activity\n            WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n            ORDER BY xact_start NULLS LAST\n            LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n      ]),\n      SLOT_DIAGNOSTICS_TIMEOUT_MS,\n    );\n    if (result === 'timed-out') {\n      lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n      return;\n    }\n    const [[walsender], transactions] = result;\n    lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n      walsender: walsender ?? null,\n      transactions,\n    });\n  } catch (e) {\n    lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n  }\n}\n\n/**\n * Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n */\nfunction isSlotCreationBlocked(e: unknown): boolean {\n  return (\n    isPostgresError(e, PG_LOCK_NOT_AVAILABLE) ||\n    e instanceof SlotCreationTimeoutError ||\n    (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n  );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 * 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n  // For overriding in tests.\n  lockTimeout?: number | undefined;\n  maxBlockedMs?: number | undefined;\n  retryDelayMs?: number | undefined;\n};\n\n/**\n * Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n */\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {['application_name']: applicationName},\n      },\n      // See createReplicationSessionFor() for why this cast is necessary.\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\n/**\n * Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n *\n * * A dedicated session acquires a (session-level) advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots. The lock is held\n *   outside of a transaction so that the session is not \"idle in\n *   transaction\" while slot creation waits on upstream transactions,\n *   which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n  {\n    lockTimeout,\n    maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n    retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n  }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n  const start = Date.now();\n  let addedReplicationRole = false;\n  let cleanedUpForSlotLimit = false;\n\n  for (let attempt = 1; ; attempt++) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    // Note: The replicationSession is used to create the replication slot\n    // and closed immediately afterwards, or on error. A new session is used\n    // for each attempt, since a failed attempt closes it.\n    const replicationSession = createReplicationSessionFor(sql, sessionName);\n    const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n    // Ending the lock session releases the replication slot management lock.\n    // The sessions are ended with a timeout in case a hung slot creation\n    // (e.g. after a network partition) is still pending.\n    const endSessions = () =>\n      Promise.allSettled([\n        replicationSession.end({timeout: 5}),\n        lockSession.end({timeout: 5}),\n      ]);\n\n    let slotName: string | undefined;\n    let initialSession: Readable | undefined;\n    let creatingSlot = false;\n    try {\n      await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n      if (stage === InitialSync) {\n        // With the lock acquired, ensure that only one initial sync is\n        // active at a time. getRestoreCandidates() orders its results\n        // with stage = InitialSync (and active = true) first.\n        const others = await getRestoreCandidates(sql, shard, epoch);\n        lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n        if (others.length) {\n          const [{id, stage, active}] = others;\n          if (stage === InitialSync && active) {\n            throw new AutoResetSignal(\n              `another replica (${id}) is performing initial sync`,\n            );\n          }\n        }\n      }\n\n      // Pick an available slotName from the slotPoolPrefix pool.\n      const names = await sql<{name: string}[]> /*sql*/ `\n        SELECT slot_name as name FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `.values();\n      const inUse = new Set(names.flat());\n      for (let next = 0; ; next++) {\n        const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n        if (!inUse.has(candidateName)) {\n          slotName = candidateName;\n          break;\n        }\n      }\n\n      creatingSlot = true;\n      const slot = await createReplicationSlot(lc, replicationSession, {\n        slotName,\n        failover,\n        lockTimeout,\n        diagnosticsDB: sql,\n      });\n      creatingSlot = false;\n      const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n      await replicationSession.end({timeout: 5});\n      initialSession = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot: slot.slot_name,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(slot.consistent_point)),\n      });\n\n      const replica = await runTx(sql, async tx => {\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n        return must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n      });\n\n      return {slot, capturedSnapshot, initialSession, replica};\n    } catch (e) {\n      // Release the lock (and end the attempt's sessions) before handling the\n      // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n      // session.\n      await endSessions();\n      if (\n        !addedReplicationRole &&\n        isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n      ) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        addedReplicationRole = true;\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (\n        !cleanedUpForSlotLimit &&\n        isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n      ) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        cleanedUpForSlotLimit = true;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      if (initialSession) {\n        try {\n          await endSession(initialSession);\n        } catch {}\n      }\n      // The slot may not exist (e.g. if its creation failed), and its name\n      // may be reused once the lock is released, so any created slot is\n      // left to dropUnclaimedSlots(), which only drops slots without a\n      // replica. This runs at the start of the next attempt.\n      const blockedMs = Date.now() - start;\n      if (\n        creatingSlot &&\n        isSlotCreationBlocked(e) &&\n        blockedMs < maxBlockedMs\n      ) {\n        lc.warn?.(\n          `Replication slot ${slotName} creation blocked by upstream transactions ` +\n            `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n          e,\n        );\n        await sleep(retryDelayMs * (1 + 4 * Math.random()));\n        continue;\n      }\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await dropUnclaimedSlots(lc, sql, shard);\n      }\n      throw e;\n    } finally {\n      await endSessions();\n    }\n  }\n}\n\n/**\n * Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n *\n * The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  _sessionName: string,\n  {id: replicaID, slot}: Pick<ReplicaState, 'id' | 'slot'>,\n): Promise<ReservedSlot | null> {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  let reservation: Readable | undefined;\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<{active: boolean; lsn: string}[]> /*sql*/ `\n      SELECT active, confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n        return null;\n      }\n      const [{active, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      reservation = await keepSlotActiveUntilTakenOver(lc, {\n        db: extractConnectionConfig(sql),\n        slot,\n        dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n        lsn: String(toBigInt(lsn)),\n      });\n      return {\n        slot,\n        replica,\n        reservation,\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    if (reservation) {\n      try {\n        await endSession(reservation);\n      } catch {}\n    }\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    // Conversely, delete replicas whose slots no longer exist, e.g. because\n    // they were dropped outside of zero-cache's control (such as in an\n    // upstream failover). Slot names are reused, so such rows would otherwise\n    // be associated with a new slot of the same name. This is done (and\n    // committed) before the slot is created, both so that the rows are never\n    // associated with it, and because CREATE_REPLICATION_SLOT waits for\n    // transactions that have written (i.e. been assigned an xid).\n    const orphaned = await tx<{id: string}[]> /*sql*/ `\n      DELETE FROM ${tx(replicasTable)} replica\n        WHERE NOT EXISTS (\n          SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n        )\n        RETURNING id;\n    `;\n    if (orphaned.length) {\n      lc.warn?.(`deleted replicas whose slots no longer exist`, {\n        replicas: orphaned.map(({id}) => id),\n      });\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB;;AAG/B,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,EAAA,CACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC;AAI7C,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,CAAC,CAC/B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js

@@ -334,6 +334,6 @@ function dropEventTriggerStatements(appID, shardID) {
   `;
 }
 //#endregion
-export { createEventFunctionStatements, createEventTriggerStatements, replicationEventSchema };
+export { TAGS, createEventFunctionStatements, createEventTriggerStatements, ddlEventSchema, ddlStartEventSchema, ddlUpdateEventSchema, dropEventTriggerStatements, replicationEventSchema, schemaSnapshotEventSchema };
 
 //# sourceMappingURL=ddl.js.map

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js.map

@@ -1 +1 @@
-{"version":3,"file":"ddl.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {publishedSchema, publishedSchemaQuery} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\n// In protocol v2 (planned, not yet emitted), \"ddlStart\" events that are not\n// associated with a schema change will be context-only, omitting the\n// `schema` snapshot entirely. This avoids bloating the WAL with (large)\n// redundant schema snapshots for DDL commands that do not affect the\n// published schema, e.g. the CREATE/ALTER/DROP TABLE sub-commands executed\n// by REFRESH MATERIALIZED VIEW CONCURRENTLY.\n//\n// This release already accepts v2 events so that the (subsequent) release\n// that upgrades the triggers to emit them remains rollback safe with respect\n// to this one.\nconst versionSchema = v.literalUnion(PROTOCOL_VERSION, 2);\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\nexport const ddlEventSchema = triggerEvent.extend({\n  version: versionSchema,\n  event: v.object({tag: v.string()}),\n  // Maps the OID of each published table to the `attnum`s of published\n  // columns that were created in the (upstream) transaction that emitted\n  // the event. Such columns are guaranteed to have the column default in\n  // all pre-existing rows, and can thus be replicated without backfill if\n  // the default value itself is replicable. Tables whose publication\n  // entries (e.g. column lists) were modified in the same transaction are\n  // excluded, since a newly *published* (as opposed to newly *created*)\n  // column may hold arbitrary values in existing rows.\n  //\n  // The field is absent in messages from older versions of the upstream\n  // functions (in which case backfill decisions fall back to the command\n  // tag heuristic), and `null` when there are no such columns.\n  newColumns: v.record(v.array(v.number())).nullable().optional(),\n});\n\n/**\n * A {@link DdlStartEvent} message is emitted before every DDL event,\n * containing the command `tag`.\n *\n * In most cases, the `DdlStartEvent` itself will not be associated with a\n * schema change, in which case the event is context-only (in protocol v2,\n * planned, both `previousSchema` and `schema` will be absent; in v1,\n * `schema` is set and `previousSchema` is `null`). The message is still\n * emitted to provide the command `tag` context in case an immediately\n * following `DdlStartEvent` tag is emitted with a schema change (which can\n * happen when another event trigger results in a nested ddl statement).\n *\n * In such cases, the `previousSchema` and `schema` fields of the latter event\n * are used to determine the necessary schema change operations (as they are\n * with `ddlUpdate` and `schemaSnapshot` events), and the `tag` of the\n * preceding start event indicates the command that precipitated the schema\n * change (e.g. a CREATE vs ALTER) to determine whether a backfill is\n * necessary.\n */\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n  // Set (along with `previousSchema`) only if the ddlStart event itself is\n  // associated with a schema change. Absent in context-only (protocol v2,\n  // planned) events. v1 events always contain the current `schema`.\n  schema: publishedSchema.optional(),\n  // For ddlStart messages, previousSchema is `null` (v1) or absent\n  // (protocol v2, planned) if there was no change in schema detected.\n  previousSchema: publishedSchema.nullable().optional(),\n  // For backwards compatibility with previous versions of the trigger,\n  // default an absent `event` field with a semantic equivalent. This\n  // field override can be removed in a version that is rollback safe\n  // with 1.4.0.\n  event: v.object({tag: v.string()}).optional(() => ({tag: 'UNKNOWN'})),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * A {@link DdlUpdateEvent} is emitted if there was a change in the schema.\n * It always contains `previousSchema` and (current) `schema` fields, leaving\n * it to the receiver to compute the necessary schema change operations.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  // ddlUpdate messages are only emitted if the schema changed. `schema`\n  // contains the current (i.e. post-change) snapshot.\n  schema: publishedSchema,\n  // The `previousSchema` contains the schema before the change.\n  //\n  // In 1.5.0 it is always set, and can be made non-optional when\n  // rollback safe.\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit `previousSchema` and `schema` snapshots when a difference\n * is detected. This is a workaround provided to support detection of schema\n * changes from `ALTER PUBLICATION` commands on supabase, which does not fire\n * event triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by trailing the publication change with a\n * `COMMENT ON PUBLICATION` statement, e.g.\n *\n * ```sql\n * BEGIN;\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * Note that it is fine to invoke `COMMENT ON PUBLICATION` statements\n * on a database that *does* support event triggers on\n * `ALTER PUBLICATION` statements, as it will simply be a no-op.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n  type: v.literal('schemaSnapshot'),\n  schema: publishedSchema,\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n  schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n// pg_advisory_xact_lock key for serializing ddl statements in order to\n// produce correct schema change diffs.\nconst DDL_SERIALIZATION_LOCK = 0x3c6b8468f1bac0b0n;\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nexport function createEventFunctionStatements(\n  shard: ShardConfig,\n  includePartialIndexes = true,\n) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT COALESCE(current_query(), 'current_query() returned NULL') AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(reason TEXT, tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE '${appID}_${shardNum} ignoring % % %', reason, tag, \n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Note: DROP and CREATE to upgrade from v20 to v21 because the\n-- return type has changed. This can be simplified to CREATE OR REPLACE\n-- once 1.5.0 is rollback safe.\nDROP FUNCTION IF EXISTS ${schema}.schema_specs();\nCREATE FUNCTION ${schema}.schema_specs()\nRETURNS JSON \nSTABLE\nAS $$\n  ${publishedSchemaQuery(publications, includePartialIndexes)}\n$$ LANGUAGE sql;\n\n\n-- Stores the most recent published schema\nCREATE TABLE IF NOT EXISTS ${schema}.\"publishedSchema\" (\n  current JSON,\n  exists BOOL PRIMARY KEY DEFAULT true CHECK (exists)\n);\n\nINSERT INTO ${schema}.\"publishedSchema\" (current) VALUES (${schema}.schema_specs())\n  ON CONFLICT (exists) DO \n  UPDATE SET current = excluded.current;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas(event_type text, tag text, target record)\nRETURNS void AS $$\nDECLARE\n  prev_schema_specs JSON;\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  SELECT current FROM ${schema}.\"publishedSchema\" INTO prev_schema_specs;\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n  \n  IF prev_schema_specs::text != schema_specs::text THEN\n    UPDATE ${schema}.\"publishedSchema\" SET current = schema_specs;\n  ELSIF event_type = 'ddlStart' THEN\n    -- ddlStart events are always be emitted to allow the zero-cache\n    -- to track the context of the current command tag in the face of\n    -- nested event triggers (e.g. start->start->end->end).\n    prev_schema_specs = NULL;\n  ELSIF event_type = 'ddlUpdate' THEN\n    -- TODO: fold 'schemaSnapshot' into this condition too (i.e. make it \"ELSE\")\n    -- when 1.5.0 is rollback safe. Until then, noop schemaSnapshots are sent\n    -- for compatibility with 1.0.0 ~ 1.4.0.\n    PERFORM ${schema}.notice_ignore('noop', tag, target);\n    RETURN;\n  END IF;\n\n  SELECT json_build_object(\n    'type', event_type,\n    'version', ${PROTOCOL_VERSION},\n    'previousSchema', prev_schema_specs,\n    'schema', schema_specs,\n    'event', json_build_object('tag', tag),\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, '${appID}/${shardNum}/ddl', message);\n\n  RAISE NOTICE 'Emitted ${appID}_${shardNum} % for % %', event_type, tag,\n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Hook/workaround to manually trigger replication of schema changes on DBs \n-- that do not support/allow event triggers.\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas()\nRETURNS void AS $$\nBEGIN\n  PERFORM ${schema}.update_schemas('schemaSnapshot', 'MANUAL', NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  -- serialize DDL statements to compute correct schema change diffs\n  PERFORM pg_advisory_xact_lock(${DDL_SERIALIZATION_LOCK});\n  PERFORM ${schema}.update_schemas('ddlStart', TG_TAG, NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end()\nRETURNS event_trigger AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs JSON;\n  message TEXT;\n  event TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n  SELECT true INTO relevant;\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND TG_TAG != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication' THEN\n    SELECT 1 WHERE target.object_identity = ANY (publications)\n      INTO relevant;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF TG_TAG LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    relevant := NULL;\n  END IF;\n\n  IF relevant IS NULL THEN\n    PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n    RETURN;\n  END IF;\n\n  IF TG_TAG = 'COMMENT' THEN\n    -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n    IF target.object_type != 'publication' THEN\n      PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n      RETURN;\n    END IF;\n    PERFORM ${schema}.update_schemas('schemaSnapshot', TG_TAG, target);\n  ELSE\n    PERFORM ${schema}.update_schemas('ddlUpdate', TG_TAG, target);\n  END IF;\n\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    /*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_end`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit([...TAGS, 'COMMENT'])})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_end();\n`,\n  ];\n\n  // Drop legacy functions / triggers.\n  triggers.push(\n    `DROP FUNCTION IF EXISTS ${schema}.emit_ddl_end(text) CASCADE;`,\n    `DROP FUNCTION IF EXISTS ${schema}.notice_ignore(text, record);`,\n  );\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(`DROP FUNCTION IF EXISTS ${schema}.emit_${tagID}() CASCADE;`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  return /*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_end_${shardID}`)};\n  `;\n}\n"],"mappings":";;;;;;AAyBA,IAAM,gBAAgB,aAAA,GAAiC,CAAC;AAMxD,IAAa,iBAJQ,eAAE,OAAO,EAC5B,SAAS,eAAE,OAAO,EAAC,OAAO,eAAE,OAAO,EAAC,CAAC,EAAE,KAAK,eAAE,OAAO,CAAC,EACxD,CAE8B,EAAa,OAAO;CAChD,SAAS;CACT,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;CAajC,YAAY,eAAE,OAAO,eAAE,MAAM,eAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS;AAChE,CAAC;;;;;;;;;;;;;;;;;;;;AAqBD,IAAa,sBAAsB,eAAe,OAAO;CACvD,MAAM,eAAE,QAAQ,UAAU;CAI1B,QAAQ,gBAAgB,SAAS;CAGjC,gBAAgB,gBAAgB,SAAS,EAAE,SAAS;CAKpD,OAAO,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC,EAAE,gBAAgB,EAAC,KAAK,UAAS,EAAE;AACtE,CAAC;;;;;;AASD,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAM,eAAE,QAAQ,WAAW;CAG3B,QAAQ;CAKR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;;;;;;;;;;;;;;;;;;;;;;;AA0BD,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAM,eAAE,QAAQ,gBAAgB;CAChC,QAAQ;CACR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;AAID,IAAa,yBAAyB,eAAE,MACtC,qBACA,sBACA,yBACF;AAMA,SAAS,OAAO,UAAkB;CAChC,QAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,QAAQ,CAAC;AAC3D;AAIA,IAAM,yBAAyB;;;;;;;;;;;;;;;;;;AAmB/B,SAAgB,8BACd,OACA,wBAAwB,MACxB;CACA,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;kBAGlB,MAAM,GAAG,SAAS;;;;;;;;;0BASV,OAAO;kBACf,OAAO;;;;IAIrB,qBAAqB,cAAc,qBAAqB,EAAE;;;;;6BAKjC,OAAO;;;;;cAKtB,OAAO,uCAAuC,OAAO;;;;;6BAKtC,OAAO;;;;;;;wBAOZ,OAAO;WACpB,OAAO;;;aAGL,OAAO;;;;;;;;;;cAUN,OAAO;;;;;;;;;;iBAUJ,OAAO;;;2CAGmB,MAAM,GAAG,SAAS;;0BAEnC,MAAM,GAAG,SAAS;;;;;;;;6BAQf,OAAO;;;YAGxB,OAAO;;;;;6BAKU,OAAO;;;;;;;kCAOF,uBAAuB;YAC7C,OAAO;;;;;6BAKU,OAAO;;;;;;;;;;0BAUV,QAAI,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;cAGT,OAAO;;cAEP,OAAO;;;;;;AAMrB;AAGA,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAgB,6BAA6B,OAAoB;CAE/D,OAAO,MAAM,aAAa,QAAQ,sCAAsC;CAKxE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,QAAQ;CAC/B,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CAEvC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,QAAQ,GAC9C;uBACW,QAAQ,GAAG,MAAM,WAAW,EAAE;;iBAEpC,QAAI,IAAI,EAAE;sBACL,OAAO;;uBAEN,QAAQ,GAAG,MAAM,SAAS,EAAE;;iBAElC,QAAI,CAAC,GAAG,MAAM,SAAS,CAAC,EAAE;sBACrB,OAAO;CAE3B;CAGA,SAAS,KACP,2BAA2B,OAAO,+BAClC,2BAA2B,OAAO,8BACpC;CACA,KAAK,MAAM,OAAO,CAAC,GAAG,MAAM,SAAS,GAAG;EACtC,MAAM,QAAQ,IAAI,YAAY,EAAE,QAAQ,KAAK,GAAG;EAChD,SAAS,KAAK,2BAA2B,OAAO,QAAQ,MAAM,YAAY;CAC5E;CACA,OAAO,SAAS,KAAK,EAAE;AACzB;AAGA,SAAgB,2BACd,OACA,SACA;CACA,OAAe;mCACkB,GAAG,GAAG,MAAM,aAAa,SAAS,EAAE;mCACpC,GAAG,GAAG,MAAM,WAAW,SAAS,EAAE;;AAErE"}
+{"version":3,"file":"ddl.js","names":["v.literalUnion","v","lit"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/ddl.ts"],"sourcesContent":["import {literal as lit} from 'pg-format';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {upstreamSchema, type ShardConfig} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {publishedSchema, publishedSchemaQuery} from './published.ts';\n\n// Sent in the 'version' tag of \"ddlStart\" and \"ddlUpdate\" event messages.\n// This is used to ensure that the message constructed in the upstream\n// Trigger function is compatible with the code processing it in the zero-cache.\n//\n// Increment this when changing the format of the contents of the \"ddl\" events.\n// This will allow old / incompatible code to detect the change and abort.\nexport const PROTOCOL_VERSION = 1;\n\n// In protocol v2 (planned, not yet emitted), \"ddlStart\" events that are not\n// associated with a schema change will be context-only, omitting the\n// `schema` snapshot entirely. This avoids bloating the WAL with (large)\n// redundant schema snapshots for DDL commands that do not affect the\n// published schema, e.g. the CREATE/ALTER/DROP TABLE sub-commands executed\n// by REFRESH MATERIALIZED VIEW CONCURRENTLY.\n//\n// This release already accepts v2 events so that the (subsequent) release\n// that upgrades the triggers to emit them remains rollback safe with respect\n// to this one.\nconst versionSchema = v.literalUnion(PROTOCOL_VERSION, 2);\n\nconst triggerEvent = v.object({\n  context: v.object({query: v.string()}).rest(v.string()),\n});\n\nexport const ddlEventSchema = triggerEvent.extend({\n  version: versionSchema,\n  event: v.object({tag: v.string()}),\n  // Maps the OID of each published table to the `attnum`s of published\n  // columns that were created in the (upstream) transaction that emitted\n  // the event. Such columns are guaranteed to have the column default in\n  // all pre-existing rows, and can thus be replicated without backfill if\n  // the default value itself is replicable. Tables whose publication\n  // entries (e.g. column lists) were modified in the same transaction are\n  // excluded, since a newly *published* (as opposed to newly *created*)\n  // column may hold arbitrary values in existing rows.\n  //\n  // The field is absent in messages from older versions of the upstream\n  // functions (in which case backfill decisions fall back to the command\n  // tag heuristic), and `null` when there are no such columns.\n  newColumns: v.record(v.array(v.number())).nullable().optional(),\n});\n\n/**\n * A {@link DdlStartEvent} message is emitted before every DDL event,\n * containing the command `tag`.\n *\n * In most cases, the `DdlStartEvent` itself will not be associated with a\n * schema change, in which case the event is context-only (in protocol v2,\n * planned, both `previousSchema` and `schema` will be absent; in v1,\n * `schema` is set and `previousSchema` is `null`). The message is still\n * emitted to provide the command `tag` context in case an immediately\n * following `DdlStartEvent` tag is emitted with a schema change (which can\n * happen when another event trigger results in a nested ddl statement).\n *\n * In such cases, the `previousSchema` and `schema` fields of the latter event\n * are used to determine the necessary schema change operations (as they are\n * with `ddlUpdate` and `schemaSnapshot` events), and the `tag` of the\n * preceding start event indicates the command that precipitated the schema\n * change (e.g. a CREATE vs ALTER) to determine whether a backfill is\n * necessary.\n */\nexport const ddlStartEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlStart'),\n  // Set (along with `previousSchema`) only if the ddlStart event itself is\n  // associated with a schema change. Absent in context-only (protocol v2,\n  // planned) events. v1 events always contain the current `schema`.\n  schema: publishedSchema.optional(),\n  // For ddlStart messages, previousSchema is `null` (v1) or absent\n  // (protocol v2, planned) if there was no change in schema detected.\n  previousSchema: publishedSchema.nullable().optional(),\n  // For backwards compatibility with previous versions of the trigger,\n  // default an absent `event` field with a semantic equivalent. This\n  // field override can be removed in a version that is rollback safe\n  // with 1.4.0.\n  event: v.object({tag: v.string()}).optional(() => ({tag: 'UNKNOWN'})),\n});\n\nexport type DdlStartEvent = v.Infer<typeof ddlStartEventSchema>;\n\n/**\n * A {@link DdlUpdateEvent} is emitted if there was a change in the schema.\n * It always contains `previousSchema` and (current) `schema` fields, leaving\n * it to the receiver to compute the necessary schema change operations.\n */\nexport const ddlUpdateEventSchema = ddlEventSchema.extend({\n  type: v.literal('ddlUpdate'),\n  // ddlUpdate messages are only emitted if the schema changed. `schema`\n  // contains the current (i.e. post-change) snapshot.\n  schema: publishedSchema,\n  // The `previousSchema` contains the schema before the change.\n  //\n  // In 1.5.0 it is always set, and can be made non-optional when\n  // rollback safe.\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type DdlUpdateEvent = v.Infer<typeof ddlUpdateEventSchema>;\n\n/**\n * The `schemaSnapshot` message is a snapshot of a schema taken in response to\n * a `COMMENT ON PUBLICATION` command, which is a hook recognized by zero\n * to manually emit `previousSchema` and `schema` snapshots when a difference\n * is detected. This is a workaround provided to support detection of schema\n * changes from `ALTER PUBLICATION` commands on supabase, which does not fire\n * event triggers for them (https://github.com/supabase/supautils/issues/123).\n *\n * The hook is exercised by trailing the publication change with a\n * `COMMENT ON PUBLICATION` statement, e.g.\n *\n * ```sql\n * BEGIN;\n * ALTER PUBLICATION my_publication ...;\n * COMMENT ON PUBLICATION my_publication IS 'whatever';\n * COMMIT;\n * ```\n *\n * Note that it is fine to invoke `COMMENT ON PUBLICATION` statements\n * on a database that *does* support event triggers on\n * `ALTER PUBLICATION` statements, as it will simply be a no-op.\n */\nexport const schemaSnapshotEventSchema = ddlEventSchema.extend({\n  type: v.literal('schemaSnapshot'),\n  schema: publishedSchema,\n  previousSchema: publishedSchema.optional(),\n});\n\nexport type SchemaSnapshotEvent = v.Infer<typeof schemaSnapshotEventSchema>;\n\nexport const replicationEventSchema = v.union(\n  ddlStartEventSchema,\n  ddlUpdateEventSchema,\n  schemaSnapshotEventSchema,\n);\n\nexport type ReplicationEvent = v.Infer<typeof replicationEventSchema>;\n\n// Creates a function that appends `_{shard-num}` to the input and\n// quotes the result to be a valid identifier.\nfunction append(shardNum: number) {\n  return (name: string) => id(name + '_' + String(shardNum));\n}\n\n// pg_advisory_xact_lock key for serializing ddl statements in order to\n// produce correct schema change diffs.\nconst DDL_SERIALIZATION_LOCK = 0x3c6b8468f1bac0b0n;\n\n/**\n * Event trigger functions contain the core logic that are invoked by triggers.\n *\n * Note that although many of these functions can theoretically be parameterized and\n * shared across shards, it is advantageous to keep the functions in each shard\n * isolated from each other in order to avoid the complexity of shared-function\n * versioning.\n *\n * In a sense, shards (and their triggers and functions) should be thought of as\n * execution environments that can be updated at different schedules. If per-shard\n * triggers called into shared functions, we would have to consider versioning the\n * functions when changing their behavior, backwards compatibility, removal of\n * unused versions, etc. (not unlike versioning of npm packages).\n *\n * Instead, we opt for the simplicity and isolation of having each shard\n * completely own (and maintain) the entirety of its trigger/function stack.\n */\nexport function createEventFunctionStatements(\n  shard: ShardConfig,\n  includePartialIndexes = true,\n) {\n  const {appID, shardNum, publications} = shard;\n  const schema = id(upstreamSchema(shard)); // e.g. \"{APP_ID}_{SHARD_ID}\"\n  return /*sql*/ `\nCREATE SCHEMA IF NOT EXISTS ${schema};\n\nCREATE OR REPLACE FUNCTION ${schema}.get_trigger_context()\nRETURNS record AS $$\nDECLARE\n  result record;\nBEGIN\n  SELECT COALESCE(current_query(), 'current_query() returned NULL') AS \"query\" into result;\n  RETURN result;\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.notice_ignore(reason TEXT, tag TEXT, target record)\nRETURNS void AS $$\nBEGIN\n  RAISE NOTICE '${appID}_${shardNum} ignoring % % %', reason, tag, \n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Note: DROP and CREATE to upgrade from v20 to v21 because the\n-- return type has changed. This can be simplified to CREATE OR REPLACE\n-- once 1.5.0 is rollback safe.\nDROP FUNCTION IF EXISTS ${schema}.schema_specs();\nCREATE FUNCTION ${schema}.schema_specs()\nRETURNS JSON \nSTABLE\nAS $$\n  ${publishedSchemaQuery(publications, includePartialIndexes)}\n$$ LANGUAGE sql;\n\n\n-- Stores the most recent published schema\nCREATE TABLE IF NOT EXISTS ${schema}.\"publishedSchema\" (\n  current JSON,\n  exists BOOL PRIMARY KEY DEFAULT true CHECK (exists)\n);\n\nINSERT INTO ${schema}.\"publishedSchema\" (current) VALUES (${schema}.schema_specs())\n  ON CONFLICT (exists) DO \n  UPDATE SET current = excluded.current;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas(event_type text, tag text, target record)\nRETURNS void AS $$\nDECLARE\n  prev_schema_specs JSON;\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  SELECT current FROM ${schema}.\"publishedSchema\" INTO prev_schema_specs;\n  SELECT ${schema}.schema_specs() INTO schema_specs;\n  \n  IF prev_schema_specs::text != schema_specs::text THEN\n    UPDATE ${schema}.\"publishedSchema\" SET current = schema_specs;\n  ELSIF event_type = 'ddlStart' THEN\n    -- ddlStart events are always be emitted to allow the zero-cache\n    -- to track the context of the current command tag in the face of\n    -- nested event triggers (e.g. start->start->end->end).\n    prev_schema_specs = NULL;\n  ELSIF event_type = 'ddlUpdate' THEN\n    -- TODO: fold 'schemaSnapshot' into this condition too (i.e. make it \"ELSE\")\n    -- when 1.5.0 is rollback safe. Until then, noop schemaSnapshots are sent\n    -- for compatibility with 1.0.0 ~ 1.4.0.\n    PERFORM ${schema}.notice_ignore('noop', tag, target);\n    RETURN;\n  END IF;\n\n  SELECT json_build_object(\n    'type', event_type,\n    'version', ${PROTOCOL_VERSION},\n    'previousSchema', prev_schema_specs,\n    'schema', schema_specs,\n    'event', json_build_object('tag', tag),\n    'context', ${schema}.get_trigger_context()\n  ) INTO message;\n\n  PERFORM pg_logical_emit_message(true, '${appID}/${shardNum}/ddl', message);\n\n  RAISE NOTICE 'Emitted ${appID}_${shardNum} % for % %', event_type, tag,\n    COALESCE(row_to_json(target)::text, '');\nEND\n$$ LANGUAGE plpgsql;\n\n\n-- Hook/workaround to manually trigger replication of schema changes on DBs \n-- that do not support/allow event triggers.\nCREATE OR REPLACE FUNCTION ${schema}.update_schemas()\nRETURNS void AS $$\nBEGIN\n  PERFORM ${schema}.update_schemas('schemaSnapshot', 'MANUAL', NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_start()\nRETURNS event_trigger AS $$\nDECLARE\n  schema_specs JSON;\n  message TEXT;\nBEGIN\n  -- serialize DDL statements to compute correct schema change diffs\n  PERFORM pg_advisory_xact_lock(${DDL_SERIALIZATION_LOCK});\n  PERFORM ${schema}.update_schemas('ddlStart', TG_TAG, NULL);\nEND\n$$ LANGUAGE plpgsql;\n\n\nCREATE OR REPLACE FUNCTION ${schema}.emit_ddl_end()\nRETURNS event_trigger AS $$\nDECLARE\n  publications TEXT[];\n  target RECORD;\n  relevant RECORD;\n  schema_specs JSON;\n  message TEXT;\n  event TEXT;\nBEGIN\n  publications := ARRAY[${lit(publications)}];\n\n  SELECT objid, object_type, object_identity \n    FROM pg_event_trigger_ddl_commands() \n    LIMIT 1 INTO target;\n\n  -- Filter DDL updates that are not relevant to the shard (i.e. publications) when possible.\n  SELECT true INTO relevant;\n\n  -- Note: ALTER TABLE statements may *remove* the table from the set of published\n  --       tables, and there is no way to determine if the table \"used to be\" in the\n  --       set. Thus, all ALTER TABLE statements must produce a ddl update, similar to\n  --       any DROP * statement.\n  IF (target.object_type = 'table' AND TG_TAG != 'ALTER TABLE') \n     OR target.object_type = 'table column' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'index' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_indexes as ind ON ind.schemaname = ns.nspname AND ind.indexname = c.relname\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = ind.tablename\n      WHERE c.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication relation' THEN\n    SELECT pb.pubname FROM pg_publication_rel AS rel\n      JOIN pg_publication AS pb ON pb.oid = rel.prpubid\n      WHERE rel.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'publication namespace' THEN\n    SELECT pb.pubname FROM pg_publication_namespace AS ns\n      JOIN pg_publication AS pb ON pb.oid = ns.pnpubid\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications) \n      INTO relevant;\n\n  ELSIF target.object_type = 'schema' THEN\n    SELECT ns.nspname AS \"schema\", c.relname AS \"name\" FROM pg_class AS c\n      JOIN pg_namespace AS ns ON c.relnamespace = ns.oid\n      JOIN pg_publication_tables AS pb ON pb.schemaname = ns.nspname AND pb.tablename = c.relname\n      WHERE ns.oid = target.objid AND pb.pubname = ANY (publications)\n      INTO relevant;\n\n  ELSIF target.object_type = 'publication' THEN\n    SELECT 1 WHERE target.object_identity = ANY (publications)\n      INTO relevant;\n\n  -- no-op CREATE IF NOT EXIST statements\n  ELSIF TG_TAG LIKE 'CREATE %' AND target.object_type IS NULL THEN\n    relevant := NULL;\n  END IF;\n\n  IF relevant IS NULL THEN\n    PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n    RETURN;\n  END IF;\n\n  IF TG_TAG = 'COMMENT' THEN\n    -- Only make schemaSnapshots for COMMENT ON PUBLICATION\n    IF target.object_type != 'publication' THEN\n      PERFORM ${schema}.notice_ignore('irrelevant', TG_TAG, target);\n      RETURN;\n    END IF;\n    PERFORM ${schema}.update_schemas('schemaSnapshot', TG_TAG, target);\n  ELSE\n    PERFORM ${schema}.update_schemas('ddlUpdate', TG_TAG, target);\n  END IF;\n\nEND\n$$ LANGUAGE plpgsql;\n`;\n}\n\n// Exported for testing.\nexport const TAGS = [\n  'CREATE TABLE',\n  'ALTER TABLE',\n  'CREATE INDEX',\n  'DROP TABLE',\n  'DROP INDEX',\n  'ALTER PUBLICATION',\n  'ALTER SCHEMA',\n] as const;\n\nexport function createEventTriggerStatements(shard: ShardConfig) {\n  // Better to assert here than get a cryptic syntax error from Postgres.\n  assert(shard.publications.length, `shard publications must be non-empty`);\n\n  // Unlike functions, which are namespaced in shard-specific schemas,\n  // EVENT TRIGGER names are in the global namespace and thus must include\n  // the appID and shardNum.\n  const {appID, shardNum} = shard;\n  const sharded = append(shardNum);\n  const schema = id(upstreamSchema(shard));\n\n  const triggers = [\n    dropEventTriggerStatements(shard.appID, shard.shardNum),\n    /*sql*/ `\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_start`)}\n  ON ddl_command_start\n  WHEN TAG IN (${lit(TAGS)})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_start();\n\nCREATE EVENT TRIGGER ${sharded(`${appID}_ddl_end`)}\n  ON ddl_command_end\n  WHEN TAG IN (${lit([...TAGS, 'COMMENT'])})\n  EXECUTE PROCEDURE ${schema}.emit_ddl_end();\n`,\n  ];\n\n  // Drop legacy functions / triggers.\n  triggers.push(\n    `DROP FUNCTION IF EXISTS ${schema}.emit_ddl_end(text) CASCADE;`,\n    `DROP FUNCTION IF EXISTS ${schema}.notice_ignore(text, record);`,\n  );\n  for (const tag of [...TAGS, 'COMMENT']) {\n    const tagID = tag.toLowerCase().replace(' ', '_');\n    triggers.push(`DROP FUNCTION IF EXISTS ${schema}.emit_${tagID}() CASCADE;`);\n  }\n  return triggers.join('');\n}\n\n// Exported for testing.\nexport function dropEventTriggerStatements(\n  appID: string,\n  shardID: string | number,\n) {\n  return /*sql*/ `\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_start_${shardID}`)};\n    DROP EVENT TRIGGER IF EXISTS ${id(`${appID}_ddl_end_${shardID}`)};\n  `;\n}\n"],"mappings":";;;;;;AAyBA,IAAM,gBAAgBA,aAAAA,GAAiC,CAAC;AAMxD,IAAa,iBAJQC,eAAE,OAAO,EAC5B,SAASA,eAAE,OAAO,EAAC,OAAOA,eAAE,OAAO,EAAC,CAAC,CAAC,CAAC,KAAKA,eAAE,OAAO,CAAC,EACxD,CAE8B,CAAA,CAAa,OAAO;CAChD,SAAS;CACT,OAAOA,eAAE,OAAO,EAAC,KAAKA,eAAE,OAAO,EAAC,CAAC;CAajC,YAAYA,eAAE,OAAOA,eAAE,MAAMA,eAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;AAChE,CAAC;;;;;;;;;;;;;;;;;;;;AAqBD,IAAa,sBAAsB,eAAe,OAAO;CACvD,MAAMA,eAAE,QAAQ,UAAU;CAI1B,QAAQ,gBAAgB,SAAS;CAGjC,gBAAgB,gBAAgB,SAAS,CAAC,CAAC,SAAS;CAKpD,OAAOA,eAAE,OAAO,EAAC,KAAKA,eAAE,OAAO,EAAC,CAAC,CAAC,CAAC,gBAAgB,EAAC,KAAK,UAAS,EAAE;AACtE,CAAC;;;;;;AASD,IAAa,uBAAuB,eAAe,OAAO;CACxD,MAAMA,eAAE,QAAQ,WAAW;CAG3B,QAAQ;CAKR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;;;;;;;;;;;;;;;;;;;;;;;AA0BD,IAAa,4BAA4B,eAAe,OAAO;CAC7D,MAAMA,eAAE,QAAQ,gBAAgB;CAChC,QAAQ;CACR,gBAAgB,gBAAgB,SAAS;AAC3C,CAAC;AAID,IAAa,yBAAyBA,eAAE,MACtC,qBACA,sBACA,yBACF;AAMA,SAAS,OAAO,UAAkB;CAChC,QAAQ,SAAiB,GAAG,OAAO,MAAM,OAAO,QAAQ,CAAC;AAC3D;AAIA,IAAM,yBAAyB;;;;;;;;;;;;;;;;;;AAmB/B,SAAgB,8BACd,OACA,wBAAwB,MACxB;CACA,MAAM,EAAC,OAAO,UAAU,iBAAgB;CACxC,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CACvC,OAAe;8BACa,OAAO;;6BAER,OAAO;;;;;;;;;;;6BAWP,OAAO;;;kBAGlB,MAAM,GAAG,SAAS;;;;;;;;;0BASV,OAAO;kBACf,OAAO;;;;IAIrB,qBAAqB,cAAc,qBAAqB,EAAE;;;;;6BAKjC,OAAO;;;;;cAKtB,OAAO,uCAAuC,OAAO;;;;;6BAKtC,OAAO;;;;;;;wBAOZ,OAAO;WACpB,OAAO;;;aAGL,OAAO;;;;;;;;;;cAUN,OAAO;;;;;;;;;;iBAUJ,OAAO;;;2CAGmB,MAAM,GAAG,SAAS;;0BAEnC,MAAM,GAAG,SAAS;;;;;;;;6BAQf,OAAO;;;YAGxB,OAAO;;;;;6BAKU,OAAO;;;;;;;kCAOF,uBAAuB;YAC7C,OAAO;;;;;6BAKU,OAAO;;;;;;;;;;0BAUVC,QAAI,YAAY,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cA0D9B,OAAO;;;;;;;gBAOL,OAAO;;;cAGT,OAAO;;cAEP,OAAO;;;;;;AAMrB;AAGA,IAAa,OAAO;CAClB;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAgB,6BAA6B,OAAoB;CAE/D,OAAO,MAAM,aAAa,QAAQ,sCAAsC;CAKxE,MAAM,EAAC,OAAO,aAAY;CAC1B,MAAM,UAAU,OAAO,QAAQ;CAC/B,MAAM,SAAS,GAAG,eAAe,KAAK,CAAC;CAEvC,MAAM,WAAW,CACf,2BAA2B,MAAM,OAAO,MAAM,QAAQ,GAC9C;uBACW,QAAQ,GAAG,MAAM,WAAW,EAAE;;iBAEpCA,QAAI,IAAI,EAAE;sBACL,OAAO;;uBAEN,QAAQ,GAAG,MAAM,SAAS,EAAE;;iBAElCA,QAAI,CAAC,GAAG,MAAM,SAAS,CAAC,EAAE;sBACrB,OAAO;CAE3B;CAGA,SAAS,KACP,2BAA2B,OAAO,+BAClC,2BAA2B,OAAO,8BACpC;CACA,KAAK,MAAM,OAAO,CAAC,GAAG,MAAM,SAAS,GAAG;EACtC,MAAM,QAAQ,IAAI,YAAY,CAAC,CAAC,QAAQ,KAAK,GAAG;EAChD,SAAS,KAAK,2BAA2B,OAAO,QAAQ,MAAM,YAAY;CAC5E;CACA,OAAO,SAAS,KAAK,EAAE;AACzB;AAGA,SAAgB,2BACd,OACA,SACA;CACA,OAAe;mCACkB,GAAG,GAAG,MAAM,aAAa,SAAS,EAAE;mCACpC,GAAG,GAAG,MAAM,WAAW,SAAS,EAAE;;AAErE"}