@rocicorp/zero 1.11.0-canary.21 → 1.11.0-canary.23

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":["#lc","#start","#restartLSN","#confirmedFlushLSN","#startTime","#done","#first"],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {getOrCreateHistogram} from '../../../../observability/metrics.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt, toBigInt, type LSN} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport function createReplicationSessionFor(\n  db: PostgresDB,\n  applicationName: string,\n) {\n  return postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n}\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n  inboundTimeoutOverrideMs?: number | undefined,\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = createReplicationSessionFor(db, applicationName);\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs, inboundTimeoutOverrideMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms. ` +\n          `Inbound timeout: ${inboundTimeoutMs}ms` +\n          (inboundTimeoutMs === walSenderTimeoutMs * 2\n            ? ''\n            : ' (configured override)')\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const decode = await InitialDecodeTracker.start(lc, session, slot, lsn);\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> inbound timeout ${inboundTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      const msg = parseStreamMessage(lc, buffer, parser);\n      if (msg) {\n        decode.observe(msg, lastReceivedTime);\n      }\n      return msg;\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\nconst INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S = [\n  0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1200, 2400, 3600, 7200,\n];\n\nfunction initialDecodeHistogram() {\n  return getOrCreateHistogram('replication', 'stream_initial_decode', {\n    description:\n      'Time from START_REPLICATION until upstream has decoded the WAL ' +\n      \"between the slot's restart_lsn and the start of the stream\",\n    unit: 's',\n    bucketBoundaries: INITIAL_DECODE_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n\n/**\n * Measures how long upstream takes to decode the WAL between the slot's\n * `restart_lsn` and the start of the stream, during which the stream\n * delivers nothing but keepalives. This span can be large, e.g. for a slot\n * that was created (or last acked) long before the requested start LSN.\n *\n * The walsender streams from `max(lsn, confirmed_flush_lsn)`, but must\n * decode from `restart_lsn` to rebuild its transaction state. The decode is\n * considered complete upon the first message at or past the start:\n * * any XLogData message, since only transactions committed after the\n *   start are sent, or\n * * a keepalive at or past the start. Keepalives report the decoding\n *   position, except for the first one of the stream, which reports the\n *   slot's `confirmed_flush_lsn` and is thus ignored if it is at or past the\n *   start.\n *\n * Note that on an idle upstream, a walsender whose start is already\n * confirmed may not send a keepalive until other WAL is written or the\n * periodic keepalive (`wal_sender_timeout / 2`) is due, so the measured\n * time is an upper bound when completed by a keepalive.\n */\nexport class InitialDecodeTracker {\n  static async start(\n    lc: LogContext,\n    session: postgres.Sql,\n    slot: string,\n    lsn: bigint,\n  ): Promise<InitialDecodeTracker> {\n    let restartLSN: bigint | undefined;\n    let confirmedFlushLSN: bigint | undefined;\n    try {\n      // Replication sessions only support the simple query protocol, so the\n      // slot name is inlined as a literal.\n      const [row] = await session\n        .unsafe<{restartLSN: LSN | null; confirmedFlushLSN: LSN | null}[]>(\n          `SELECT restart_lsn AS \"restartLSN\",\n                  confirmed_flush_lsn AS \"confirmedFlushLSN\"\n             FROM pg_replication_slots\n             WHERE slot_name = ${lit(slot)}`,\n        )\n        .simple();\n      restartLSN = row?.restartLSN ? toBigInt(row.restartLSN) : undefined;\n      confirmedFlushLSN = row?.confirmedFlushLSN\n        ? toBigInt(row.confirmedFlushLSN)\n        : undefined;\n    } catch (e) {\n      lc.warn?.(`unable to read position of replication slot ${slot}`, e);\n    }\n    return new InitialDecodeTracker(\n      lc,\n      lsn,\n      restartLSN,\n      confirmedFlushLSN,\n      Date.now(),\n    );\n  }\n\n  readonly #lc: LogContext;\n  readonly #start: bigint;\n  readonly #restartLSN: bigint | undefined;\n  readonly #confirmedFlushLSN: bigint | undefined;\n  readonly #startTime: number;\n  #first = true;\n  #done = false;\n\n  constructor(\n    lc: LogContext,\n    lsn: bigint,\n    restartLSN: bigint | undefined,\n    confirmedFlushLSN: bigint | undefined,\n    startTime: number,\n  ) {\n    // The walsender silently starts at confirmed_flush_lsn if the\n    // requested lsn is behind it.\n    this.#start =\n      confirmedFlushLSN !== undefined && confirmedFlushLSN > lsn\n        ? confirmedFlushLSN\n        : lsn;\n    this.#lc = lc.withContext('decodeStart', fromBigInt(this.#start));\n    this.#restartLSN = restartLSN;\n    this.#confirmedFlushLSN = confirmedFlushLSN;\n    this.#startTime = startTime;\n\n    if (restartLSN !== undefined) {\n      this.#lc.info?.(\n        `decoding ${this.#start - restartLSN} bytes of WAL from ` +\n          `restart_lsn ${fromBigInt(restartLSN)} to the start of the stream`,\n      );\n    }\n  }\n\n  /**\n   * Observes a received message, returning the elapsed decode time (ms) if\n   * this message completed the initial decode.\n   */\n  observe([lsn, msg]: StreamMessage, now: number): number | undefined {\n    if (this.#done) {\n      return undefined;\n    }\n    const first = this.#first;\n    this.#first = false;\n    if (msg.tag === 'keepalive') {\n      if (\n        lsn < this.#start ||\n        (first && lsn === this.#confirmedFlushLSN) // reports confirmed_flush\n      ) {\n        return undefined;\n      }\n    }\n    this.#done = true;\n    const elapsed = now - this.#startTime;\n    initialDecodeHistogram().recordMs(elapsed);\n    this.#lc.info?.(\n      `upstream decoded the WAL to the start of the stream in ` +\n        `${elapsed} ms (${msg.tag} at ${fromBigInt(lsn)}` +\n        (this.#restartLSN === undefined\n          ? ')'\n          : `, ${this.#start - this.#restartLSN} bytes from restart_lsn)`),\n      {elapsed},\n    );\n    return elapsed;\n  }\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * An `inboundTimeoutOverrideMs` (when positive) replaces the default\n * `2x wal_sender_timeout` inbound threshold. The two timeouts protect\n * different parties with asymmetric false-positive costs: `wal_sender_timeout`\n * is the server reaping dead clients (a false positive there costs a cheap\n * reconnect), while the inbound watchdog tears down the stream and replays the\n * in-flight transaction — expensive, and self-defeating when the wal sender is\n * legitimately silent for long stretches (e.g. decoding through WAL from\n * unpublished tables, or assembling a large transaction that pgoutput only\n * sends at commit). Operators who don't own the server setting (e.g. managed\n * or operator-provisioned Postgres with an aggressive cluster-wide\n * `wal_sender_timeout`) can widen the client-side fuse independently.\n *\n * The override has no effect when `wal_sender_timeout` is disabled (0): the\n * server then sends no periodic keepalives, so inbound silence is normal even\n * on an idle healthy connection and any silence-based watchdog would\n * false-positive.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(\n  walSenderTimeoutMs: number,\n  inboundTimeoutOverrideMs?: number | undefined,\n): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.). A 0 / negative / NaN override\n    // falls back to the default rather than degenerating the threshold.\n    inboundTimeoutMs:\n      inboundTimeoutOverrideMs !== undefined && inboundTimeoutOverrideMs > 0\n        ? inboundTimeoutOverrideMs\n        : walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\n/**\n * Starts a \"dummy\" replication session for the specified `slot` which:\n * * does not consume any records\n * * keeps the session alive by sending status messages at the starting LSN\n *\n * This essentially reserves the replication slot for this process by marking\n * it as \"active\" in pg_replication_slots, allowing replication slot cleanup\n * logic to distinguish a new-but-initializing replication slot from one that\n * has been abandoned / orphaned.\n *\n * When initialization (e.g. initial-sync) has finished, the server should take\n * over the replication slot using pg_terminate_backend, which will close the\n * session.\n *\n * Note that caller should not close the supplied `session`, as that will\n * also terminate the dummy replication session.\n */\nexport async function keepSlotActiveUntilTakenOver(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  dummyPublication: string,\n  lsn: bigint,\n) {\n  const [readable, writeable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    [dummyPublication],\n    lsn,\n    1,\n  );\n  lc.info?.(`keeping ${slot} active until taken over ...`);\n  // Send periodic status messages upstream to indicate that the session is\n  // alive, but keep the confirmed_flush_lsn at the initial lsn.\n  const keepalive = setInterval(() => writeable.write(makeAck(lsn)), 10_000);\n\n  readable.on('close', async () => {\n    lc.info?.(`released initial reservation of ${slot}`);\n    clearInterval(keepalive);\n    try {\n      await session.end();\n      lc.info?.(`ended session`);\n    } catch (e) {\n      lc.warn?.(`error ending session`, e);\n    }\n  });\n  readable.once('error', e => {\n    lc.info?.(`closing replication session to ${session.options.host}`, e);\n    readable.destroy();\n  });\n  return readable;\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;AAoBA,IAAM,6CAA6C;AAcnD,SAAgB,4BACd,IACA,iBACA;CACA,OAAO,SACL,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;AACF;AAEA,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAClB,0BAIC;CACD,MAAM,UAAU,4BAA4B,IAAI,eAAe;CAQ/D,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,oBAAoB,wBAAwB;CACvE,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,uBAChD,iBAAiB,OACpC,qBAAqB,qBAAqB,IACvC,KACA,4BACN,gGAEN;CAEA,MAAM,SAAS,MAAM,qBAAqB,MAAM,IAAI,SAAS,MAAM,GAAG;CACtE,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,wBAC5C,iBAAiB,uDAE3C;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,MAAM,MAAM,mBAAmB,IAAI,QAAQ,MAAM;GACjD,IAAI,KACF,OAAO,QAAQ,KAAK,gBAAgB;GAEtC,OAAO;EACT;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;AAEA,IAAM,wCAAwC;CAC5C;CAAK;CAAK;CAAG;CAAG;CAAG;CAAI;CAAI;CAAI;CAAK;CAAK;CAAK;CAAM;CAAM;CAAM;AAClE;AAEA,SAAS,yBAAyB;CAChC,OAAO,qBAAqB,eAAe,yBAAyB;EAClE,aACE;EAEF,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH;;;;;;;;;;;;;;;;;;;;;;AAuBA,IAAa,uBAAb,MAAa,qBAAqB;CAChC,aAAa,MACX,IACA,SACA,MACA,KAC+B;EAC/B,IAAI;EACJ,IAAI;EACJ,IAAI;GAGF,MAAM,CAAC,OAAO,MAAM,QACjB,OACC;;;iCAGuB,IAAI,IAAI,GACjC,EACC,OAAO;GACV,aAAa,KAAK,aAAa,SAAS,IAAI,UAAU,IAAI,KAAA;GAC1D,oBAAoB,KAAK,oBACrB,SAAS,IAAI,iBAAiB,IAC9B,KAAA;EACN,SAAS,GAAG;GACV,GAAG,OAAO,+CAA+C,QAAQ,CAAC;EACpE;EACA,OAAO,IAAI,qBACT,IACA,KACA,YACA,mBACA,KAAK,IAAI,CACX;CACF;CAEA;CACA;CACA;CACA;CACA;CACA,SAAS;CACT,QAAQ;CAER,YACE,IACA,KACA,YACA,mBACA,WACA;EAGA,KAAKC,SACH,sBAAsB,KAAA,KAAa,oBAAoB,MACnD,oBACA;EACN,KAAKD,MAAM,GAAG,YAAY,eAAe,WAAW,KAAKC,MAAM,CAAC;EAChE,KAAKC,cAAc;EACnB,KAAKC,qBAAqB;EAC1B,KAAKC,aAAa;EAElB,IAAI,eAAe,KAAA,GACjB,KAAKJ,IAAI,OACP,YAAY,KAAKC,SAAS,WAAW,iCACpB,WAAW,UAAU,EAAE,4BAC1C;CAEJ;;;;;CAMA,QAAQ,CAAC,KAAK,MAAqB,KAAiC;EAClE,IAAI,KAAKI,OACP;EAEF,MAAM,QAAQ,KAAKC;EACnB,KAAKA,SAAS;EACd,IAAI,IAAI,QAAQ;OAEZ,MAAM,KAAKL,UACV,SAAS,QAAQ,KAAKE,oBAEvB;EAAA;EAGJ,KAAKE,QAAQ;EACb,MAAM,UAAU,MAAM,KAAKD;EAC3B,uBAAuB,EAAE,SAAS,OAAO;EACzC,KAAKJ,IAAI,OACP,0DACK,QAAQ,OAAO,IAAI,IAAI,MAAM,WAAW,GAAG,OAC7C,KAAKE,gBAAgB,KAAA,IAClB,MACA,KAAK,KAAKD,SAAS,KAAKC,YAAY,4BAC1C,EAAC,QAAO,CACV;EACA,OAAO;CACT;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,SAAgB,uBACd,oBACA,0BAMA;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAMA,kBACE,6BAA6B,KAAA,KAAa,2BAA2B,IACjE,2BACA,qBAAqB;EAE3B,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;;;;;;;;;;;;;;;;;;AAmBA,eAAsB,6BACpB,IACA,SACA,MACA,kBACA,KACA;CACA,MAAM,CAAC,UAAU,aAAa,MAAM,uBAClC,IACA,SACA,MACA,CAAC,gBAAgB,GACjB,KACA,CACF;CACA,GAAG,OAAO,WAAW,KAAK,6BAA6B;CAGvD,MAAM,YAAY,kBAAkB,UAAU,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAM;CAEzE,SAAS,GAAG,SAAS,YAAY;EAC/B,GAAG,OAAO,mCAAmC,MAAM;EACnD,cAAc,SAAS;EACvB,IAAI;GACF,MAAM,QAAQ,IAAI;GAClB,GAAG,OAAO,eAAe;EAC3B,SAAS,GAAG;GACV,GAAG,OAAO,wBAAwB,CAAC;EACrC;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,GAAG,OAAO,kCAAkC,QAAQ,QAAQ,QAAQ,CAAC;EACrE,SAAS,QAAQ;CACnB,CAAC;CACD,OAAO;AACT;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.d.ts

@@ -49,6 +49,12 @@ export declare class BackfillStateTracker {
      * state is never ahead of that of aligned subscribers.
      */
     track(change: ChangeStreamData): void;
+    /**
+     * Requests a restart of the change stream (at the next transaction
+     * boundary), e.g. to start it with BackfillRequests that it was not
+     * started with.
+     */
+    requestRestart(reason: string): void;
     /** Logs and records a (planned) restart for the {@link restartReason}. */
     recordRestart(): void;
     /**

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-state-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAK9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAEnF,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,iBAAiB,EACjB,WAAW,GAAG,WAAW,GAAG,mBAAmB,CAChD,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,oBAAoB;;IA6C/B,YAAY,EAAE,EAAE,UAAU,EAEzB;IAED;;;;;OAKG;IACH,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED;;;;;;;;;OASG;IACH,WAAW,CAAC,mBAAmB,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAyBrE;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,QAG7B;IAED,0EAA0E;IAC1E,aAAa,SAGZ;IAED;;;OAGG;IACH,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,iBAAiB,GACrB,yBAAyB,CAqB3B;IAED;;;;OAIG;IACH,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,QAMtD;CAsBF"}
+{"version":3,"file":"backfill-state-tracker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAK9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,iDAAiD,CAAC;AACtF,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAEnF,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAEnE,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,iBAAiB,EACjB,WAAW,GAAG,WAAW,GAAG,mBAAmB,CAChD,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,oBAAoB;;IA6C/B,YAAY,EAAE,EAAE,UAAU,EAEzB;IAED;;;;;OAKG;IACH,IAAI,aAAa,IAAI,MAAM,GAAG,IAAI,CAEjC;IAED;;;;;;;;;OASG;IACH,WAAW,CAAC,mBAAmB,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAyBrE;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,EAAE,gBAAgB,QAG7B;IAED;;;;OAIG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,QAE5B;IAED,0EAA0E;IAC1E,aAAa,SAGZ;IAED;;;OAGG;IACH,iBAAiB,CACf,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,iBAAiB,GACrB,yBAAyB,CAqB3B;IAED;;;;OAIG;IACH,QAAQ,CACN,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,YAAY,CAAC,MAAM,GAAG,kBAAkB,CAAC,QAMtD;CAsBF"}

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.js

@@ -88,6 +88,14 @@ var BackfillStateTracker = class {
 		this.#sessionBackfills.apply(change);
 		this.#logBackfills.apply(change);
 	}
+	/**
+	* Requests a restart of the change stream (at the next transaction
+	* boundary), e.g. to start it with BackfillRequests that it was not
+	* started with.
+	*/
+	requestRestart(reason) {
+		this.#restartReason ??= reason;
+	}
 	/** Logs and records a (planned) restart for the {@link restartReason}. */
 	recordRestart() {
 		this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);

package/out/zero-cache/src/services/change-streamer/backfill-state-tracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"backfill-state-tracker.js","names":["#lc","#subscribers","#restarts","#unexpectedIgnores","#restartReason","#logBackfills","#sessionBackfills","#checkCoverage"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  getSuperset,\n  withResumePoint,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {SubscriberContext} from './change-streamer.ts';\nimport type {Subscriber, SubscriberOptions} from './subscriber.ts';\n\nexport type SubscriberBackfillOptions = Pick<\n  SubscriberOptions,\n  'backfills' | 'onAligned' | 'onBackfillIgnored'\n>;\n\n/**\n * Tracks the pending backfills (and their progress) of the change log, the\n * subscribers, and the current change stream, in order to determine:\n *\n * * the BackfillRequests with which to start a change stream, i.e. the\n *   superset of the pending backfills of the change log and all aligned\n *   subscribers, starting from the earliest progress, and\n * * whether (and why) the change stream must be restarted, i.e. when a\n *   newly aligned subscriber's pending backfills are not covered by the\n *   current stream.\n *\n * The change-streamer calls {@link startStream()} whenever it starts a change\n * stream, {@link track()} for every change that it forwards, and restarts\n * the stream at the next transaction boundary whenever a\n * {@link restartReason} is set.\n */\nexport class BackfillStateTracker {\n  readonly #lc: LogContext;\n\n  /**\n   * The pending backfills of the change log itself, i.e. those of a\n   * (hypothetical) subscriber that has applied every change in the log.\n   * Whenever a stream is started, it is reset from the change log's cookie\n   * jar (which is consistent with the stream's starting watermark), carrying\n   * over the progress tracked in memory (as the cookie jar is\n   * progress-agnostic). It stands in for subscribers that do not report their\n   * pending backfills (i.e. protocol < v8), and ensures that backfills that\n   * are pending in the change log continue to be requested.\n   */\n  #logBackfills = new BackfillState();\n\n  /**\n   * The backfill progress of the current change stream (i.e. \"session\"),\n   * seeded from the BackfillRequests with which the stream was started.\n   * This determines whether a (newly aligned) subscriber's pending backfills\n   * are covered by the stream.\n   */\n  #sessionBackfills = new BackfillState();\n\n  /** Subscribers that report their pending backfills (protocol v8+). */\n  readonly #subscribers = new Set<Subscriber>();\n\n  /**\n   * Set (to the reason) when the change stream must be restarted with\n   * new BackfillRequests.\n   */\n  #restartReason: string | null = null;\n\n  readonly #restarts = getOrCreateCounter(\n    'replication',\n    'backfill_stream_restarts',\n    'Count of change stream restarts initiated to rewind backfills for ' +\n      'subscribers whose pending backfills were not covered by the stream',\n  );\n  readonly #unexpectedIgnores = getOrCreateCounter(\n    'replication',\n    'backfill_unexpected_ignores',\n    'Count of backfill messages unexpectedly ignored (for some columns) by ' +\n      'an aligned subscriber. The expected count is 0.',\n  );\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  /**\n   * The reason for which the change stream must be restarted, or `null` if\n   * it need not be. The restart should happen at the next transaction\n   * boundary, i.e. upon the next `commit`, or `status` message outside of a\n   * transaction.\n   */\n  get restartReason(): string | null {\n    return this.#restartReason;\n  }\n\n  /**\n   * Computes the BackfillRequests with which to start a new change stream:\n   * the superset of the pending backfills of the change log and of all\n   * aligned subscribers, all of which are consistent with the stream's\n   * starting watermark since streams are (re)started at transaction\n   * boundaries.\n   *\n   * @param logBackfillRequests The BackfillRequests from the change log's\n   *        cookie jar, consistent with the stream's starting watermark.\n   */\n  startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[] {\n    // The cookie jar can be behind #logBackfills (e.g. if the storer failed\n    // to persist a forwarded commit), in which case the new stream\n    // re-delivers those changes.\n    this.#logBackfills = new BackfillState(\n      this.#logBackfills.withProgress(logBackfillRequests),\n    );\n\n    const aligned = [...this.#subscribers].filter(s => s.aligned);\n    const requests = getSuperset(\n      this.#logBackfills.requests(),\n      ...aligned.map(s => must(s.backfills).requests()),\n    ).map(withResumePoint);\n\n    this.#sessionBackfills = new BackfillState(requests);\n    // Aligned subscribers are accounted for in the requests.\n    this.#restartReason = null;\n    if (requests.length) {\n      this.#lc.info?.(\n        `starting change stream with ${requests.length} backfill requests ` +\n          `(from the change log and ${aligned.length} subscribers)`,\n        {requests},\n      );\n    }\n    return requests;\n  }\n\n  /**\n   * Tracks a change that is forwarded to subscribers. This must be called\n   * (synchronously) before the change is forwarded, so that the stream's\n   * state is never ahead of that of aligned subscribers.\n   */\n  track(change: ChangeStreamData) {\n    this.#sessionBackfills.apply(change);\n    this.#logBackfills.apply(change);\n  }\n\n  /** Logs and records a (planned) restart for the {@link restartReason}. */\n  recordRestart() {\n    this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);\n    this.#restarts.add(1);\n  }\n\n  /**\n   * Returns the backfill-related {@link SubscriberOptions} for a new\n   * subscriber, which must subsequently be passed to {@link register()}.\n   */\n  subscriberOptions(\n    lc: LogContext,\n    ctx: SubscriberContext,\n  ): SubscriberBackfillOptions {\n    const backfills = newBackfillState(lc, ctx);\n    if (!backfills) {\n      return {};\n    }\n    return {\n      backfills,\n      onAligned: subscriber => {\n        if (this.#subscribers.has(subscriber)) {\n          this.#checkCoverage(subscriber);\n        }\n      },\n      onBackfillIgnored: columns => {\n        if (this.#restartReason === null) {\n          lc.warn?.(\n            `aligned subscriber ignored backfill data for ${columns.join(', ')}`,\n          );\n          this.#unexpectedIgnores.add(1);\n        }\n      },\n    };\n  }\n\n  /**\n   * Registers a subscriber (created with {@link subscriberOptions()}) until\n   * its `downstream` is closed. This is a no-op for subscribers that do not\n   * report their pending backfills.\n   */\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (subscriber.backfills) {\n      this.#subscribers.add(subscriber);\n      downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));\n    }\n  }\n\n  /**\n   * Checks whether the pending backfills of a newly aligned subscriber are\n   * covered by the current stream, requesting a restart of the stream if not.\n   *\n   * This can be checked as soon as the subscriber is aligned (even in the\n   * middle of a transaction), since both the subscriber's and the stream's\n   * BackfillStates compare their committed state, and the stream's state is\n   * never updated with a change before the change is forwarded to (aligned)\n   * subscribers. The restart itself happens at the next transaction boundary.\n   */\n  #checkCoverage(subscriber: Subscriber) {\n    const uncovered = must(subscriber.backfills).uncovered(\n      this.#sessionBackfills,\n    );\n    if (uncovered.length) {\n      this.#restartReason ??=\n        `subscriber ${subscriber.id} needs backfills that are not ` +\n        `covered by the current stream: ${uncovered.join(', ')}`;\n    }\n  }\n}\n\nfunction newBackfillState(\n  lc: LogContext,\n  ctx: SubscriberContext,\n): BackfillState | undefined {\n  if (ctx.backfills === undefined) {\n    return undefined; // protocol < v8\n  }\n  try {\n    return new BackfillState(ctx.backfills);\n  } catch (e) {\n    lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAAkC;CAChC;;;;;;;;;;;CAYA,gBAAgB,IAAI,cAAc;;;;;;;CAQlC,oBAAoB,IAAI,cAAc;;CAGtC,+BAAwB,IAAI,IAAgB;;;;;CAM5C,iBAAgC;CAEhC,YAAqB,mBACnB,eACA,4BACA,sIAEF;CACA,qBAA8B,mBAC5B,eACA,+BACA,uHAEF;CAEA,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;;;;;;;CAQA,IAAI,gBAA+B;EACjC,OAAO,KAAKI;CACd;;;;;;;;;;;CAYA,YAAY,qBAA2D;EAIrE,KAAKC,gBAAgB,IAAI,cACvB,KAAKA,cAAc,aAAa,mBAAmB,CACrD;EAEA,MAAM,UAAU,CAAC,GAAG,KAAKJ,YAAY,EAAE,QAAO,MAAK,EAAE,OAAO;EAC5D,MAAM,WAAW,YACf,KAAKI,cAAc,SAAS,GAC5B,GAAG,QAAQ,KAAI,MAAK,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAClD,EAAE,IAAI,eAAe;EAErB,KAAKC,oBAAoB,IAAI,cAAc,QAAQ;EAEnD,KAAKF,iBAAiB;EACtB,IAAI,SAAS,QACX,KAAKJ,IAAI,OACP,+BAA+B,SAAS,OAAO,8CACjB,QAAQ,OAAO,gBAC7C,EAAC,SAAQ,CACX;EAEF,OAAO;CACT;;;;;;CAOA,MAAM,QAA0B;EAC9B,KAAKM,kBAAkB,MAAM,MAAM;EACnC,KAAKD,cAAc,MAAM,MAAM;CACjC;;CAGA,gBAAgB;EACd,KAAKL,IAAI,OAAO,6BAA6B,KAAKI,gBAAgB;EAClE,KAAKF,UAAU,IAAI,CAAC;CACtB;;;;;CAMA,kBACE,IACA,KAC2B;EAC3B,MAAM,YAAY,iBAAiB,IAAI,GAAG;EAC1C,IAAI,CAAC,WACH,OAAO,CAAC;EAEV,OAAO;GACL;GACA,YAAW,eAAc;IACvB,IAAI,KAAKD,aAAa,IAAI,UAAU,GAClC,KAAKM,eAAe,UAAU;GAElC;GACA,oBAAmB,YAAW;IAC5B,IAAI,KAAKH,mBAAmB,MAAM;KAChC,GAAG,OACD,gDAAgD,QAAQ,KAAK,IAAI,GACnE;KACA,KAAKD,mBAAmB,IAAI,CAAC;IAC/B;GACF;EACF;CACF;;;;;;CAOA,SACE,YACA,YACA;EACA,IAAI,WAAW,WAAW;GACxB,KAAKF,aAAa,IAAI,UAAU;GAChC,WAAW,sBAAsB,KAAKA,aAAa,OAAO,UAAU,CAAC;EACvE;CACF;;;;;;;;;;;CAYA,eAAe,YAAwB;EACrC,MAAM,YAAY,KAAK,WAAW,SAAS,EAAE,UAC3C,KAAKK,iBACP;EACA,IAAI,UAAU,QACZ,KAAKF,mBACH,cAAc,WAAW,GAAG,+DACM,UAAU,KAAK,IAAI;CAE3D;AACF;AAEA,SAAS,iBACP,IACA,KAC2B;CAC3B,IAAI,IAAI,cAAc,KAAA,GACpB;CAEF,IAAI;EACF,OAAO,IAAI,cAAc,IAAI,SAAS;CACxC,SAAS,GAAG;EACV,GAAG,OAAO,qCAAqC,IAAI,MAAM,CAAC;EAC1D;CACF;AACF"}
+{"version":3,"file":"backfill-state-tracker.js","names":["#lc","#subscribers","#restarts","#unexpectedIgnores","#restartReason","#logBackfills","#sessionBackfills","#checkCoverage"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backfill-state-tracker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Subscription} from '../../types/subscription.ts';\nimport {\n  getSuperset,\n  withResumePoint,\n} from '../change-source/protocol/backfill-progress.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {BackfillState} from './backfill-state.ts';\nimport type {PreSerializedBatch} from './broadcast.ts';\nimport type {SubscriberContext} from './change-streamer.ts';\nimport type {Subscriber, SubscriberOptions} from './subscriber.ts';\n\nexport type SubscriberBackfillOptions = Pick<\n  SubscriberOptions,\n  'backfills' | 'onAligned' | 'onBackfillIgnored'\n>;\n\n/**\n * Tracks the pending backfills (and their progress) of the change log, the\n * subscribers, and the current change stream, in order to determine:\n *\n * * the BackfillRequests with which to start a change stream, i.e. the\n *   superset of the pending backfills of the change log and all aligned\n *   subscribers, starting from the earliest progress, and\n * * whether (and why) the change stream must be restarted, i.e. when a\n *   newly aligned subscriber's pending backfills are not covered by the\n *   current stream.\n *\n * The change-streamer calls {@link startStream()} whenever it starts a change\n * stream, {@link track()} for every change that it forwards, and restarts\n * the stream at the next transaction boundary whenever a\n * {@link restartReason} is set.\n */\nexport class BackfillStateTracker {\n  readonly #lc: LogContext;\n\n  /**\n   * The pending backfills of the change log itself, i.e. those of a\n   * (hypothetical) subscriber that has applied every change in the log.\n   * Whenever a stream is started, it is reset from the change log's cookie\n   * jar (which is consistent with the stream's starting watermark), carrying\n   * over the progress tracked in memory (as the cookie jar is\n   * progress-agnostic). It stands in for subscribers that do not report their\n   * pending backfills (i.e. protocol < v8), and ensures that backfills that\n   * are pending in the change log continue to be requested.\n   */\n  #logBackfills = new BackfillState();\n\n  /**\n   * The backfill progress of the current change stream (i.e. \"session\"),\n   * seeded from the BackfillRequests with which the stream was started.\n   * This determines whether a (newly aligned) subscriber's pending backfills\n   * are covered by the stream.\n   */\n  #sessionBackfills = new BackfillState();\n\n  /** Subscribers that report their pending backfills (protocol v8+). */\n  readonly #subscribers = new Set<Subscriber>();\n\n  /**\n   * Set (to the reason) when the change stream must be restarted with\n   * new BackfillRequests.\n   */\n  #restartReason: string | null = null;\n\n  readonly #restarts = getOrCreateCounter(\n    'replication',\n    'backfill_stream_restarts',\n    'Count of change stream restarts initiated to rewind backfills for ' +\n      'subscribers whose pending backfills were not covered by the stream',\n  );\n  readonly #unexpectedIgnores = getOrCreateCounter(\n    'replication',\n    'backfill_unexpected_ignores',\n    'Count of backfill messages unexpectedly ignored (for some columns) by ' +\n      'an aligned subscriber. The expected count is 0.',\n  );\n\n  constructor(lc: LogContext) {\n    this.#lc = lc;\n  }\n\n  /**\n   * The reason for which the change stream must be restarted, or `null` if\n   * it need not be. The restart should happen at the next transaction\n   * boundary, i.e. upon the next `commit`, or `status` message outside of a\n   * transaction.\n   */\n  get restartReason(): string | null {\n    return this.#restartReason;\n  }\n\n  /**\n   * Computes the BackfillRequests with which to start a new change stream:\n   * the superset of the pending backfills of the change log and of all\n   * aligned subscribers, all of which are consistent with the stream's\n   * starting watermark since streams are (re)started at transaction\n   * boundaries.\n   *\n   * @param logBackfillRequests The BackfillRequests from the change log's\n   *        cookie jar, consistent with the stream's starting watermark.\n   */\n  startStream(logBackfillRequests: BackfillRequest[]): BackfillRequest[] {\n    // The cookie jar can be behind #logBackfills (e.g. if the storer failed\n    // to persist a forwarded commit), in which case the new stream\n    // re-delivers those changes.\n    this.#logBackfills = new BackfillState(\n      this.#logBackfills.withProgress(logBackfillRequests),\n    );\n\n    const aligned = [...this.#subscribers].filter(s => s.aligned);\n    const requests = getSuperset(\n      this.#logBackfills.requests(),\n      ...aligned.map(s => must(s.backfills).requests()),\n    ).map(withResumePoint);\n\n    this.#sessionBackfills = new BackfillState(requests);\n    // Aligned subscribers are accounted for in the requests.\n    this.#restartReason = null;\n    if (requests.length) {\n      this.#lc.info?.(\n        `starting change stream with ${requests.length} backfill requests ` +\n          `(from the change log and ${aligned.length} subscribers)`,\n        {requests},\n      );\n    }\n    return requests;\n  }\n\n  /**\n   * Tracks a change that is forwarded to subscribers. This must be called\n   * (synchronously) before the change is forwarded, so that the stream's\n   * state is never ahead of that of aligned subscribers.\n   */\n  track(change: ChangeStreamData) {\n    this.#sessionBackfills.apply(change);\n    this.#logBackfills.apply(change);\n  }\n\n  /**\n   * Requests a restart of the change stream (at the next transaction\n   * boundary), e.g. to start it with BackfillRequests that it was not\n   * started with.\n   */\n  requestRestart(reason: string) {\n    this.#restartReason ??= reason;\n  }\n\n  /** Logs and records a (planned) restart for the {@link restartReason}. */\n  recordRestart() {\n    this.#lc.info?.(`restarting change stream: ${this.#restartReason}`);\n    this.#restarts.add(1);\n  }\n\n  /**\n   * Returns the backfill-related {@link SubscriberOptions} for a new\n   * subscriber, which must subsequently be passed to {@link register()}.\n   */\n  subscriberOptions(\n    lc: LogContext,\n    ctx: SubscriberContext,\n  ): SubscriberBackfillOptions {\n    const backfills = newBackfillState(lc, ctx);\n    if (!backfills) {\n      return {};\n    }\n    return {\n      backfills,\n      onAligned: subscriber => {\n        if (this.#subscribers.has(subscriber)) {\n          this.#checkCoverage(subscriber);\n        }\n      },\n      onBackfillIgnored: columns => {\n        if (this.#restartReason === null) {\n          lc.warn?.(\n            `aligned subscriber ignored backfill data for ${columns.join(', ')}`,\n          );\n          this.#unexpectedIgnores.add(1);\n        }\n      },\n    };\n  }\n\n  /**\n   * Registers a subscriber (created with {@link subscriberOptions()}) until\n   * its `downstream` is closed. This is a no-op for subscribers that do not\n   * report their pending backfills.\n   */\n  register(\n    subscriber: Subscriber,\n    downstream: Subscription<string | PreSerializedBatch>,\n  ) {\n    if (subscriber.backfills) {\n      this.#subscribers.add(subscriber);\n      downstream.addCloseHandler(() => this.#subscribers.delete(subscriber));\n    }\n  }\n\n  /**\n   * Checks whether the pending backfills of a newly aligned subscriber are\n   * covered by the current stream, requesting a restart of the stream if not.\n   *\n   * This can be checked as soon as the subscriber is aligned (even in the\n   * middle of a transaction), since both the subscriber's and the stream's\n   * BackfillStates compare their committed state, and the stream's state is\n   * never updated with a change before the change is forwarded to (aligned)\n   * subscribers. The restart itself happens at the next transaction boundary.\n   */\n  #checkCoverage(subscriber: Subscriber) {\n    const uncovered = must(subscriber.backfills).uncovered(\n      this.#sessionBackfills,\n    );\n    if (uncovered.length) {\n      this.#restartReason ??=\n        `subscriber ${subscriber.id} needs backfills that are not ` +\n        `covered by the current stream: ${uncovered.join(', ')}`;\n    }\n  }\n}\n\nfunction newBackfillState(\n  lc: LogContext,\n  ctx: SubscriberContext,\n): BackfillState | undefined {\n  if (ctx.backfills === undefined) {\n    return undefined; // protocol < v8\n  }\n  try {\n    return new BackfillState(ctx.backfills);\n  } catch (e) {\n    lc.warn?.(`invalid backfills from subscriber ${ctx.id}`, e);\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoCA,IAAa,uBAAb,MAAkC;CAChC;;;;;;;;;;;CAYA,gBAAgB,IAAI,cAAc;;;;;;;CAQlC,oBAAoB,IAAI,cAAc;;CAGtC,+BAAwB,IAAI,IAAgB;;;;;CAM5C,iBAAgC;CAEhC,YAAqB,mBACnB,eACA,4BACA,sIAEF;CACA,qBAA8B,mBAC5B,eACA,+BACA,uHAEF;CAEA,YAAY,IAAgB;EAC1B,KAAKA,MAAM;CACb;;;;;;;CAQA,IAAI,gBAA+B;EACjC,OAAO,KAAKI;CACd;;;;;;;;;;;CAYA,YAAY,qBAA2D;EAIrE,KAAKC,gBAAgB,IAAI,cACvB,KAAKA,cAAc,aAAa,mBAAmB,CACrD;EAEA,MAAM,UAAU,CAAC,GAAG,KAAKJ,YAAY,EAAE,QAAO,MAAK,EAAE,OAAO;EAC5D,MAAM,WAAW,YACf,KAAKI,cAAc,SAAS,GAC5B,GAAG,QAAQ,KAAI,MAAK,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC,CAClD,EAAE,IAAI,eAAe;EAErB,KAAKC,oBAAoB,IAAI,cAAc,QAAQ;EAEnD,KAAKF,iBAAiB;EACtB,IAAI,SAAS,QACX,KAAKJ,IAAI,OACP,+BAA+B,SAAS,OAAO,8CACjB,QAAQ,OAAO,gBAC7C,EAAC,SAAQ,CACX;EAEF,OAAO;CACT;;;;;;CAOA,MAAM,QAA0B;EAC9B,KAAKM,kBAAkB,MAAM,MAAM;EACnC,KAAKD,cAAc,MAAM,MAAM;CACjC;;;;;;CAOA,eAAe,QAAgB;EAC7B,KAAKD,mBAAmB;CAC1B;;CAGA,gBAAgB;EACd,KAAKJ,IAAI,OAAO,6BAA6B,KAAKI,gBAAgB;EAClE,KAAKF,UAAU,IAAI,CAAC;CACtB;;;;;CAMA,kBACE,IACA,KAC2B;EAC3B,MAAM,YAAY,iBAAiB,IAAI,GAAG;EAC1C,IAAI,CAAC,WACH,OAAO,CAAC;EAEV,OAAO;GACL;GACA,YAAW,eAAc;IACvB,IAAI,KAAKD,aAAa,IAAI,UAAU,GAClC,KAAKM,eAAe,UAAU;GAElC;GACA,oBAAmB,YAAW;IAC5B,IAAI,KAAKH,mBAAmB,MAAM;KAChC,GAAG,OACD,gDAAgD,QAAQ,KAAK,IAAI,GACnE;KACA,KAAKD,mBAAmB,IAAI,CAAC;IAC/B;GACF;EACF;CACF;;;;;;CAOA,SACE,YACA,YACA;EACA,IAAI,WAAW,WAAW;GACxB,KAAKF,aAAa,IAAI,UAAU;GAChC,WAAW,sBAAsB,KAAKA,aAAa,OAAO,UAAU,CAAC;EACvE;CACF;;;;;;;;;;;CAYA,eAAe,YAAwB;EACrC,MAAM,YAAY,KAAK,WAAW,SAAS,EAAE,UAC3C,KAAKK,iBACP;EACA,IAAI,UAAU,QACZ,KAAKF,mBACH,cAAc,WAAW,GAAG,+DACM,UAAU,KAAK,IAAI;CAE3D;AACF;AAEA,SAAS,iBACP,IACA,KAC2B;CAC3B,IAAI,IAAI,cAAc,KAAA,GACpB;CAEF,IAAI;EACF,OAAO,IAAI,cAAc,IAAI,SAAS;CACxC,SAAS,GAAG;EACV,GAAG,OAAO,qCAAqC,IAAI,MAAM,CAAC;EAC1D;CACF;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAyB,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAI5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAgDpD"}
+{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAGpE,OAAO,EAAyB,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,OAAO,EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAG5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,iBAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAyBpD"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -1,29 +1,16 @@
-import { must } from "../../../../shared/src/must.js";
 import { getLastBackupTime } from "../litestream/commands.js";
 import { BackupMonitor } from "./backup-monitor.js";
 import { LitestreamSyncRequester } from "./litestream-sync-requester.js";
 import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
 import { ReplicaPoller } from "./replica-poller.js";
-import { VfsWatermarkPoller } from "./vfs-watermark-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
 function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState }) {
-	const { log, litestream, replica } = config;
+	const { litestream, replica } = config;
 	const { backupURL } = litestream;
 	let stream;
 	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
-	else if (config.litestream.backupUsingV5 && config.litestream.syncRequesterEnabled) stream = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 }).start();
-	else if (config.litestream.backupUsingV5) {
-		const { logLevel, endpoint, region, vfsQueryExecutable, vfsPollIntervalMs: remotePollIntervalMs } = litestream;
-		stream = new VfsWatermarkPoller(lc, replicaFile, {
-			executable: must(vfsQueryExecutable, `litestream-vfs-query-executable must be defined`),
-			remotePollIntervalMs,
-			backupURL,
-			region,
-			endpoint,
-			logLevel,
-			logFormat: log.format
-		}).start();
-	} else {
+	else if (config.litestream.backupUsingV5) stream = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 }).start();
+	else {
 		const { port: metricsPort } = litestream;
 		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
 	}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\nimport {VfsWatermarkPoller} from './vfs-watermark-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {log, litestream, replica} = config;\n  const {backupURL} = litestream;\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (\n    config.litestream.backupUsingV5 &&\n    config.litestream.syncRequesterEnabled\n  ) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else if (config.litestream.backupUsingV5) {\n    const {\n      logLevel,\n      endpoint,\n      region,\n      vfsQueryExecutable,\n      vfsPollIntervalMs: remotePollIntervalMs,\n    } = litestream;\n    stream = new VfsWatermarkPoller(lc, replicaFile, {\n      executable: must(\n        vfsQueryExecutable,\n        `litestream-vfs-query-executable must be defined`,\n      ),\n      remotePollIntervalMs,\n      backupURL,\n      region,\n      endpoint,\n      logLevel,\n      logFormat: log.format,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;;;AAuBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,KAAK,YAAY,YAAW;CACnC,MAAM,EAAC,cAAa;CAEpB,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IACL,OAAO,WAAW,iBAClB,OAAO,WAAW,sBAElB,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,EAAE,MAAM;MACJ,IAAI,OAAO,WAAW,eAAe;EAC1C,MAAM,EACJ,UACA,UACA,QACA,oBACA,mBAAmB,yBACjB;EACJ,SAAS,IAAI,mBAAmB,IAAI,aAAa;GAC/C,YAAY,KACV,oBACA,iDACF;GACA;GACA;GACA;GACA;GACA;GACA,WAAW,IAAI;EACjB,CAAC,EAAE,MAAM;CACX,OAAO;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {type BackedUpWatermark, BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n  type BackupStateVerifier,\n  Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  verifyBackupState?: BackupStateVerifier | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n  const {litestream, replica} = config;\n  const {backupURL} = litestream;\n\n  let stream: Source<BackedUpWatermark>;\n\n  if (!backupURL) {\n    stream = new ReplicaPoller(lc, replicaFile).start();\n  } else if (config.litestream.backupUsingV5) {\n    stream = new LitestreamSyncRequester(lc, replicaFile, {\n      intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n    }).start();\n  } else {\n    const {port: metricsPort} = litestream;\n    stream = new Litestream3PrometheusPoller(\n      lc,\n      replicaFile,\n      backupURL,\n      `http://localhost:${metricsPort}/metrics`,\n      verifyBackupState ??\n        (() => getLastBackupTime(lc, litestream, replica.file)),\n    ).start();\n  }\n\n  return new BackupMonitor(lc, stream, changeStreamer, replicaFile);\n}\n"],"mappings":";;;;;;AAqBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,EAAC,cAAa;CAEpB,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW,EAAE,MAAM;MAC7C,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC,EAAE,MAAM;MACJ;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD,EAAE,MAAM;CACV;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,gBAAgB,WAAW;AAClE"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts

@@ -80,6 +80,23 @@ export type ChangeLogInitializerSources = {
      * an error, since neither says anything about the two stores.
      */
     readonly changeLog: () => Database | undefined;
+    /**
+     * Re-initializes the Postgres change log from the replica's resume point
+     * (taking ownership of it) if it does not contain all of the replica's
+     * changes, re-checking under lock. Typically
+     * `Storer.reinitializeFromReplica()`.
+     *
+     * Supplied only when the change log's head is behind the position of the
+     * replication slot from which replication resumes, i.e. when it may be
+     * missing transactions that the slot will not stream. Otherwise, a replica
+     * that is ahead of the change log (as changes are forwarded before they are
+     * stored) is caught up by the change stream, which resumes from the change
+     * log's head.
+     */
+    readonly reinitializePgChangeLog?: ((params: {
+        watermark: string;
+        cookies: CookieSet;
+    }) => Promise<'reinitialized' | 'maintained'>) | undefined;
 };
 type Opts = {
     /**

package/out/zero-cache/src/services/change-streamer/change-log-initializer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-initializer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAKtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAEL,KAAK,oBAAoB,EAC1B,MAAM,gCAAgC,CAAC;AAQxC,0EAA0E;AAC1E,MAAM,MAAM,wBAAwB,GAAG;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,EAAE,eAAe,EAAE,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,oBAAoB,GAC5B,OAAO,GACP,kBAAkB,GAClB,WAAW,GACX,cAAc,GACd,OAAO,CAAC;AAEZ,MAAM,MAAM,2BAA2B,GAAG;IACxC,yDAAyD;IACzD,QAAQ,CAAC,WAAW,EAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAC9D,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,MAAM,wBAAwB,CAAC;IACjD;;;;;;;OAOG;IACH,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,UAAU,EAAE,oBAAoB,GAAG,SAAS,EAC5C,IAAI,EAAE,MAAM,oBAAoB,KAC7B,oBAAoB,GAAG,SAAS,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,QAAQ,GAAG,SAAS,CAAC;CAChD,CAAC;AAEF,KAAK,IAAI,GAAG;IACV;;;;;;OAMG;IAEH,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC,CAAC;AAcF,qBAAa,oBAAoB;;IAe/B,YACE,EAAE,EAAE,UAAU,EACd,EAAC,mBAAmB,EAAE,eAAe,EAAC,EAAE,IAAI,EAC5C,OAAO,EAAE,2BAA2B,EAUrC;IAED;;;;;OAKG;IACH,cAAc,IAAI,oBAAoB,GAAG,SAAS,CAEjD;IAED;;;;;;OAMG;IACG,UAAU,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAqCpD;CA+GF;AAUD;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,EAAE,EAAE,QAAQ,GACX,wBAAwB,CAc1B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,GAClB,MAAM,wBAAwB,CAahC"}
+{"version":3,"file":"change-log-initializer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAKtD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AACnF,OAAO,EAGL,KAAK,SAAS,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAEL,KAAK,oBAAoB,EAC1B,MAAM,gCAAgC,CAAC;AAQxC,0EAA0E;AAC1E,MAAM,MAAM,wBAAwB,GAAG;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,gBAAgB,EAAE,eAAe,EAAE,CAAC;IAC7C,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,oBAAoB,GAC5B,OAAO,GACP,kBAAkB,GAClB,WAAW,GACX,cAAc,GACd,OAAO,CAAC;AAEZ,MAAM,MAAM,2BAA2B,GAAG;IACxC,yDAAyD;IACzD,QAAQ,CAAC,WAAW,EAAE,MAAM,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAC9D,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,MAAM,wBAAwB,CAAC;IACjD;;;;;;;OAOG;IACH,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,UAAU,EAAE,oBAAoB,GAAG,SAAS,EAC5C,IAAI,EAAE,MAAM,oBAAoB,KAC7B,oBAAoB,GAAG,SAAS,CAAC;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,QAAQ,GAAG,SAAS,CAAC;IAC/C;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,uBAAuB,CAAC,EAC7B,CAAC,CAAC,MAAM,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,SAAS,CAAC;KACpB,KAAK,OAAO,CAAC,eAAe,GAAG,YAAY,CAAC,CAAC,GAC9C,SAAS,CAAC;CACf,CAAC;AAEF,KAAK,IAAI,GAAG;IACV;;;;;;OAMG;IAEH,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC,CAAC;AAcF,qBAAa,oBAAoB;;IAqB/B,YACE,EAAE,EAAE,UAAU,EACd,EAAC,mBAAmB,EAAE,eAAe,EAAC,EAAE,IAAI,EAC5C,OAAO,EAAE,2BAA2B,EAUrC;IAED;;;;;OAKG;IACH,cAAc,IAAI,oBAAoB,GAAG,SAAS,CAEjD;IAED;;;;;;OAMG;IACG,UAAU,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAqCpD;CA2JF;AAUD;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,EAAE,EAAE,QAAQ,GACX,wBAAwB,CAc1B;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,2BAA2B,CACzC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,GAClB,MAAM,wBAAwB,CAahC"}

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js

@@ -29,6 +29,11 @@ var ChangeLogInitializer = class {
 	#sources;
 	#resultCounter = getOrCreateCounter("replica", "sqlite_change_log.init_compare_result", "Comparisons of the replica-derived stream initialization parameters against the Postgres-derived ones, by classification.");
 	#lastComparison;
+	/**
+	* The (restored) replica's resume point that the Postgres change log must
+	* cover, read once. See {@link #pgChangeLog}.
+	*/
+	#restoredReplica;
 	constructor(lc, { initFromPgChangeLog, initFromReplica }, sources) {
 		assert(initFromPgChangeLog || initFromReplica, `At least one of initFromPgChangeLog or initFromReplica must be true`);
 		this.#lc = lc.withContext("component", "change-log-initializer");
@@ -54,7 +59,7 @@ var ChangeLogInitializer = class {
 	*/
 	async initialize() {
 		this.#lastComparison = void 0;
-		const pg = this.#initFromPgChangeLog ? await this.#sources.pgChangeLog() : void 0;
+		const pg = this.#initFromPgChangeLog ? await this.#pgChangeLog() : void 0;
 		const replica = this.#initFromReplica ? this.#readReplica(pg === void 0) : void 0;
 		const reconciled = this.#sources.reconcileChangeLog(pg && {
 			resumeWatermark: pg.lastWatermark,
@@ -72,6 +77,44 @@ var ChangeLogInitializer = class {
 		};
 	}
 	/**
+	* Reads the Postgres change log's initialization parameters, first
+	* re-initializing it from the replica (taking ownership of it) if it may be
+	* missing transactions that the replication slot will not stream, e.g. if it
+	* is no longer maintained after running in high-availability mode. This is
+	* the Postgres counterpart of seeding a new SQLite change log from the
+	* replica.
+	*
+	* This only applies when the change log's head is behind the slot's position
+	* (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is
+	* supplied), and the change log is only re-initialized if its head is also
+	* behind the replica's. Otherwise, it contains every transaction before the
+	* slot's position: one that was acked after being stored would be in the
+	* change log, and one that was acked after being backed up would be in the
+	* replica, which the change log's head covers.
+	*
+	* The replica is read only once, i.e. as it was restored from the backup,
+	* before any stream is started. The live replica is no reference: it is
+	* caught up from the forwarded changes, which can be ahead of what has been
+	* stored, e.g. by the change log's previous owner. Against that fixed
+	* resume point, re-initialization is idempotent: afterwards, the change log's
+	* head (which only advances) covers it, so subsequent stream connections
+	* leave it untouched.
+	*/
+	async #pgChangeLog() {
+		const pg = await this.#sources.pgChangeLog();
+		const reinitialize = this.#sources.reinitializePgChangeLog;
+		if (!reinitialize) return pg;
+		this.#restoredReplica ??= this.#sources.replica();
+		const { lastWatermark, cookies } = this.#restoredReplica;
+		if (pg.lastWatermark >= lastWatermark) return pg;
+		const result = await reinitialize({
+			watermark: lastWatermark,
+			cookies
+		});
+		this.#lc.info?.(`Postgres change log at ${pg.lastWatermark} is behind the replica at ${lastWatermark}: ${result}`);
+		return this.#sources.pgChangeLog();
+	}
+	/**
 	* Compares both sources and records the result. Postgres remains
 	* authoritative when both sources are enabled.
 	*/

package/out/zero-cache/src/services/change-streamer/change-log-initializer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#readReplica","#compare","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#sources.pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkJA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKE,SAAS,YAAY,IAChC,KAAA;EAGJ,MAAM,UAAU,KAAKD,mBACjB,KAAKI,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKH,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKE,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKC,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKR,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKK,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKN,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKM,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKT,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}
+{"version":3,"file":"change-log-initializer.js","names":["#lc","#initFromPgChangeLog","#initFromReplica","#sources","#resultCounter","#lastComparison","#pgChangeLog","#readReplica","#compare","#restoredReplica","#classify","#report"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-log-initializer.ts"],"sourcesContent":["/**\n * Where a stream connection's initialization parameters come from.\n *\n * A change-streamer starts a stream from three things read at one position: the\n * watermark to resume at, the {@link BackfillRequest}s to hand the change\n * source, and the cookie set those requests were derived from. Postgres has\n * always supplied all three (`Storer.getStartStreamInitializationParameters()`).\n * The point of this workstream is that a restored replica can supply them\n * instead, which is what lets the `cdc` schema be decommissioned.\n *\n * This is the two-source stage. Both are computed on every stream connection,\n * compared, and Postgres's is used — so the replica-derived path runs in\n * production, at full rate, before anything depends on it. The flip is\n * `initFromPgChangeLog: false`, not a rewrite.\n *\n * The shape is {@link UpstreamAcker}'s, deliberately: two stores, one output, a\n * flag per store, and the Postgres flag carrying the marker that retires it.\n * That class solved the identical problem for upstream ACKs earlier in this\n * workstream.\n *\n * ### Why the comparison is a fold and not an equality\n *\n * The replica is a subscriber downstream of the change-streamer's write loop,\n * so its state version trails the log's head by however long the replicator is\n * behind. A raw difference between the two cookie sets is therefore the normal\n * case and says nothing. What is checked instead is that the replica's set,\n * carried forward over exactly the schema changes the log holds in the interval,\n * equals Postgres's set at Postgres's watermark. The log holds precisely those\n * transactions, schema changes are rare, and the check is exact rather than\n * approximate — so it also happens to be the strongest available test of the\n * premise that three interpreters of one fold agree.\n */\n\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {BigIntJSON} from '../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {SchemaChange} from '../change-source/protocol/current/data.ts';\nimport {schemaChangeTags} from '../change-source/protocol/current/schema-change-tags.ts';\nimport type {BackfillRequest} from '../change-source/protocol/current/upstream.ts';\nimport {\n  backfillRequestsFrom,\n  foldCookies,\n  type CookieSet,\n} from '../replicator/change-log-cookies.ts';\nimport {\n  CHANGE_LOG_STREAM_TABLE,\n  type ChangeLogResumePoint,\n} from '../replicator/change-log-db.ts';\nimport {\n  readBackfillRequests,\n  readReplicaCookies,\n} from '../replicator/schema/backfilling.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {SQLITE_CHANGE_LOG_BOUNDARY_SQL} from './sqlite-change-log-reader.ts';\n\n/** Everything one stream connection starts from, read at one position. */\nexport type InitializationParameters = {\n  readonly lastWatermark: string;\n  readonly backfillRequests: BackfillRequest[];\n  readonly cookies: CookieSet;\n};\n\n/**\n * - `equal` — the two stores were at the same watermark and held the same\n *   cookies. Nothing had to be folded, which is the strongest of the four.\n * - `equal-after-fold` — the replica trailed, and carrying its set forward over\n *   the log's schema changes in the interval reproduced Postgres's exactly.\n *   The expected steady-state result.\n * - `divergent` — the two disagree at the same position. Alert; investigate\n *   individually.\n * - `inconclusive` — the log does not span the interval, so the fold cannot be\n *   performed. Routine on a cold log, and not a finding.\n * - `error` — the comparison itself failed: the replica could not be read, or\n *   the fold threw. Not one of the four classifications, because it is a\n *   statement about this process rather than about the two stores. Kept\n *   separate rather than folded into `inconclusive` so that a change-streamer\n *   that has silently stopped being able to read its replica is visible before\n *   that read becomes the only source.\n */\nexport type InitComparisonResult =\n  | 'equal'\n  | 'equal-after-fold'\n  | 'divergent'\n  | 'inconclusive'\n  | 'error';\n\nexport type ChangeLogInitializerSources = {\n  /** `Storer.getStartStreamInitializationParameters()`. */\n  readonly pgChangeLog: () => Promise<InitializationParameters>;\n  /** Typically {@link replicaInitializationSource}. */\n  readonly replica: () => InitializationParameters;\n  /**\n   * Reconciles the SQLite change log and returns its current resume point.\n   *\n   * When `resumeFrom` is present, it supplies the resume point. Otherwise, a\n   * valid log uses its own head. A new or invalid log uses `seed`.\n   *\n   * Returns `undefined` when the writer is disabled or unavailable.\n   */\n  readonly reconcileChangeLog: (\n    resumeFrom: ChangeLogResumePoint | undefined,\n    seed: () => ChangeLogResumePoint,\n  ) => ChangeLogResumePoint | undefined;\n  /**\n   * The change log, for the fold. `undefined` before the stream loop's first\n   * reconcile has created it, and again once the writer has failed soft and\n   * deleted it — both of which make the comparison `inconclusive` rather than\n   * an error, since neither says anything about the two stores.\n   */\n  readonly changeLog: () => Database | undefined;\n  /**\n   * Re-initializes the Postgres change log from the replica's resume point\n   * (taking ownership of it) if it does not contain all of the replica's\n   * changes, re-checking under lock. Typically\n   * `Storer.reinitializeFromReplica()`.\n   *\n   * Supplied only when the change log's head is behind the position of the\n   * replication slot from which replication resumes, i.e. when it may be\n   * missing transactions that the slot will not stream. Otherwise, a replica\n   * that is ahead of the change log (as changes are forwarded before they are\n   * stored) is caught up by the change stream, which resumes from the change\n   * log's head.\n   */\n  readonly reinitializePgChangeLog?:\n    | ((params: {\n        watermark: string;\n        cookies: CookieSet;\n      }) => Promise<'reinitialized' | 'maintained'>)\n    | undefined;\n};\n\ntype Opts = {\n  /**\n   * When true, Postgres supplies the resume point and the cookies for a new\n   * change log.\n   *\n   * When false, a valid log resumes from its own head. The replica supplies the\n   * resume point for a new or invalid log.\n   */\n  // TODO: set false when retiring PG\n  readonly initFromPgChangeLog: boolean;\n  /**\n   * Derive the parameters from the replica. When Postgres is enabled, these\n   * parameters are used only for comparison.\n   *\n   * When Postgres is disabled, the replica supplies the seed and the fallback\n   * resume point.\n   */\n  readonly initFromReplica: boolean;\n};\n\n/**\n * The most change-log rows one comparison will scan.\n *\n * The interval is `(replicaWatermark, pgWatermark]`, so it is normally a\n * handful of transactions — but a replicator that has been stalled can leave it\n * arbitrarily large, and this runs on the stream loop between reconciliation and\n * `startStream`, where latency delays the resumption of replication. Over the\n * cap the comparison declines rather than pays: a chronically lagging replicator\n * shows up as `inconclusive`, which is charted, and not as a stall.\n */\nconst MAX_FOLD_SCAN_ROWS = 10_000;\n\nexport class ChangeLogInitializer {\n  readonly #lc: LogContext;\n  readonly #initFromPgChangeLog: boolean;\n  readonly #initFromReplica: boolean;\n  readonly #sources: ChangeLogInitializerSources;\n\n  readonly #resultCounter = getOrCreateCounter(\n    'replica',\n    'sqlite_change_log.init_compare_result',\n    'Comparisons of the replica-derived stream initialization parameters ' +\n      'against the Postgres-derived ones, by classification.',\n  );\n\n  #lastComparison: InitComparisonResult | undefined;\n\n  /**\n   * The (restored) replica's resume point that the Postgres change log must\n   * cover, read once. See {@link #pgChangeLog}.\n   */\n  #restoredReplica: InitializationParameters | undefined;\n\n  constructor(\n    lc: LogContext,\n    {initFromPgChangeLog, initFromReplica}: Opts,\n    sources: ChangeLogInitializerSources,\n  ) {\n    assert(\n      initFromPgChangeLog || initFromReplica,\n      `At least one of initFromPgChangeLog or initFromReplica must be true`,\n    );\n    this.#lc = lc.withContext('component', 'change-log-initializer');\n    this.#initFromPgChangeLog = initFromPgChangeLog;\n    this.#initFromReplica = initFromReplica;\n    this.#sources = sources;\n  }\n\n  /**\n   * Returns the comparison result from the most recent {@link initialize}\n   * call. Returns `undefined` when that call used only one source.\n   *\n   * Tests and metrics use this result. It does not affect initialization.\n   */\n  lastComparison(): InitComparisonResult | undefined {\n    return this.#lastComparison;\n  }\n\n  /**\n   * Returns the initialization parameters for one stream connection after it\n   * reconciles the change log.\n   *\n   * The selected source determines how reconciliation works. Keeping both\n   * operations here hides that choice from the caller.\n   */\n  async initialize(): Promise<InitializationParameters> {\n    this.#lastComparison = undefined;\n    const pg = this.#initFromPgChangeLog\n      ? await this.#pgChangeLog()\n      : undefined;\n    // The replica is required when Postgres is disabled. It seeds a new log and\n    // supplies a resume point when the writer is unavailable.\n    const replica = this.#initFromReplica\n      ? this.#readReplica(pg === undefined)\n      : undefined;\n\n    const reconciled = this.#sources.reconcileChangeLog(\n      pg && {resumeWatermark: pg.lastWatermark, cookies: pg.cookies},\n      () => resumePointOf(must(replica, 'no seed for the SQLite change log')),\n    );\n\n    if (pg) {\n      if (replica) {\n        // Compare only after reconciliation. The fold must read the schema\n        // changes from the replica watermark through the Postgres watermark.\n        // Reconciliation makes sure that the log contains that interval.\n        this.#lastComparison = this.#compare(pg, replica);\n      }\n      return pg;\n    }\n\n    // Use the reconciled log position. If the writer is unavailable, use the\n    // replica position.\n    const resumePoint =\n      reconciled ?? resumePointOf(must(replica, 'no initialization source'));\n    return {\n      lastWatermark: resumePoint.resumeWatermark,\n      cookies: resumePoint.cookies,\n      // Derive the requests from these cookies so that the requests and the\n      // watermark describe the same position.\n      backfillRequests: backfillRequestsFrom(resumePoint.cookies),\n    };\n  }\n\n  /**\n   * Reads the Postgres change log's initialization parameters, first\n   * re-initializing it from the replica (taking ownership of it) if it may be\n   * missing transactions that the replication slot will not stream, e.g. if it\n   * is no longer maintained after running in high-availability mode. This is\n   * the Postgres counterpart of seeding a new SQLite change log from the\n   * replica.\n   *\n   * This only applies when the change log's head is behind the slot's position\n   * (i.e. when {@link ChangeLogInitializerSources.reinitializePgChangeLog} is\n   * supplied), and the change log is only re-initialized if its head is also\n   * behind the replica's. Otherwise, it contains every transaction before the\n   * slot's position: one that was acked after being stored would be in the\n   * change log, and one that was acked after being backed up would be in the\n   * replica, which the change log's head covers.\n   *\n   * The replica is read only once, i.e. as it was restored from the backup,\n   * before any stream is started. The live replica is no reference: it is\n   * caught up from the forwarded changes, which can be ahead of what has been\n   * stored, e.g. by the change log's previous owner. Against that fixed\n   * resume point, re-initialization is idempotent: afterwards, the change log's\n   * head (which only advances) covers it, so subsequent stream connections\n   * leave it untouched.\n   */\n  async #pgChangeLog(): Promise<InitializationParameters> {\n    const pg = await this.#sources.pgChangeLog();\n    const reinitialize = this.#sources.reinitializePgChangeLog;\n    if (!reinitialize) {\n      return pg;\n    }\n    this.#restoredReplica ??= this.#sources.replica();\n    const {lastWatermark, cookies} = this.#restoredReplica;\n    if (pg.lastWatermark >= lastWatermark) {\n      return pg;\n    }\n    const result = await reinitialize({watermark: lastWatermark, cookies});\n    this.#lc.info?.(\n      `Postgres change log at ${pg.lastWatermark} is behind the replica at ` +\n        `${lastWatermark}: ${result}`,\n    );\n    // Either way, the change log has changed since it was read.\n    return this.#sources.pgChangeLog();\n  }\n\n  /**\n   * Compares both sources and records the result. Postgres remains\n   * authoritative when both sources are enabled.\n   */\n  #compare(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    let result: InitComparisonResult;\n    try {\n      result = this.#classify(pg, replica);\n    } catch (e) {\n      this.#lc.warn?.('failed to compare change-log initialization sources', e);\n      result = 'error';\n    }\n    this.#resultCounter.add(1, {result});\n    this.#lc.debug?.('compared change-log initialization sources', {\n      changeLogInitCompare: {\n        result,\n        pgWatermark: pg.lastWatermark,\n        replicaWatermark: replica.lastWatermark,\n      },\n    });\n    return result;\n  }\n\n  #readReplica(required: boolean): InitializationParameters | undefined {\n    try {\n      return this.#sources.replica();\n    } catch (e) {\n      if (required) {\n        throw e;\n      }\n      // Observational until the flip, so a replica that cannot be read must not\n      // stop replication -- but it must not be quiet either, since the flip\n      // makes this read the only source.\n      this.#lc.warn?.(\n        'failed to derive change-log initialization from the replica',\n        e,\n      );\n      this.#resultCounter.add(1, {result: 'error'});\n      return undefined;\n    }\n  }\n\n  #classify(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n  ): InitComparisonResult {\n    if (replica.lastWatermark > pg.lastWatermark) {\n      // Transactions reach subscribers before the storer's commit, so a replica\n      // read taken while one is in flight can be a transaction ahead. The fold\n      // only runs forward, so there is nothing to be said either way.\n      return 'inconclusive';\n    }\n    if (replica.lastWatermark === pg.lastWatermark) {\n      return this.#report(pg, replica, replica.cookies, 'equal');\n    }\n\n    const db = this.#sources.changeLog();\n    if (db === undefined || !spansInterval(db, replica.lastWatermark)) {\n      return 'inconclusive';\n    }\n    const changes = readSchemaChanges(\n      db,\n      replica.lastWatermark,\n      pg.lastWatermark,\n    );\n    if (changes === undefined) {\n      return 'inconclusive'; // over MAX_FOLD_SCAN_ROWS\n    }\n    return this.#report(\n      pg,\n      replica,\n      foldCookies(replica.cookies, changes),\n      'equal-after-fold',\n      changes.length,\n    );\n  }\n\n  #report(\n    pg: InitializationParameters,\n    replica: InitializationParameters,\n    derived: CookieSet,\n    whenEqual: InitComparisonResult,\n    foldedChanges = 0,\n  ): InitComparisonResult {\n    const expected = canonicalizeCookies(pg.cookies);\n    const actual = canonicalizeCookies(derived);\n    if (expected === actual) {\n      return whenEqual;\n    }\n    // Both sets are normally empty and are at most a handful of rows, so they\n    // are logged whole: a divergence is investigated individually (`P§:10`)\n    // and the two renderings are the whole of what there is to look at.\n    this.#lc.error?.(\n      'the replica-derived change-log cookies do not match Postgres',\n      {\n        changeLogInitCompare: {\n          pgWatermark: pg.lastWatermark,\n          replicaWatermark: replica.lastWatermark,\n          foldedChanges,\n          expected,\n          actual,\n        },\n      },\n    );\n    return 'divergent';\n  }\n}\n\nconst resumePointOf = ({\n  lastWatermark,\n  cookies,\n}: InitializationParameters): ChangeLogResumePoint => ({\n  resumeWatermark: lastWatermark,\n  cookies,\n});\n\n/**\n * Reads the replica's initialization parameters in one snapshot, which is\n * invariant 15 on the replica side: the state version and the cookies are only\n * meaningful together.\n */\nexport function readReplicaInitializationParameters(\n  db: Database,\n): InitializationParameters {\n  const runner = new StatementRunner(db);\n  runner.begin(); // deferred, i.e. one read snapshot for the three statements\n  try {\n    return {\n      lastWatermark: getReplicationState(runner).stateVersion,\n      backfillRequests: readBackfillRequests(db),\n      cookies: readReplicaCookies(db),\n    };\n  } finally {\n    if (db.inTransaction) {\n      runner.rollback();\n    }\n  }\n}\n\n/**\n * The replica source for {@link ChangeLogInitializer}, opening and closing the\n * file per stream connection rather than holding a handle on a database another\n * process owns.\n *\n * The handle is read-only, which is invariant 19 stated in the API: the\n * replica's cookie tables are written by the change-processor and by nothing\n * else. Note that a read-only SQLite connection still needs to attach the\n * WAL's `-shm`, so this can fail if it opens at a moment when no writer holds\n * the replica; that surfaces as the `error` classification, which is the point\n * of running the path under observation before anything depends on it.\n */\nexport function replicaInitializationSource(\n  lc: LogContext,\n  replicaFile: string,\n): () => InitializationParameters {\n  return () => {\n    const db = new Database(\n      lc.withContext('component', 'change-log-initializer'),\n      replicaFile,\n      {readonly: true},\n    );\n    try {\n      return readReplicaInitializationParameters(db);\n    } finally {\n      db.close();\n    }\n  };\n}\n\n/**\n * Whether the log holds the complete interval above `fromWatermark`, using the\n * same predicate catchup uses to decide `too-old`: the log both reaches back\n * that far and contains that transaction's `commit` row, so the next row after\n * it is the first change of the interval rather than the middle of a\n * transaction the log only partly holds.\n */\nfunction spansInterval(db: Database, fromWatermark: string): boolean {\n  if (!streamTableExists(db)) {\n    return false;\n  }\n  const row = db\n    .prepare(/*sql*/ `\n      SELECT\n        (SELECT min(\"watermark\") FROM \"${CHANGE_LOG_STREAM_TABLE}\")\n          AS \"minWatermark\",\n        coalesce((${SQLITE_CHANGE_LOG_BOUNDARY_SQL}), 0) AS \"boundaryExists\"\n    `)\n    .get<{minWatermark: string | null; boundaryExists: number}>({\n      fromWatermark,\n    });\n  return (\n    row.minWatermark !== null &&\n    fromWatermark >= row.minWatermark &&\n    row.boundaryExists === 1\n  );\n}\n\nconst SCHEMA_CHANGE_TAG_LIST = schemaChangeTags\n  .map(tag => `'${tag}'`)\n  .join(', ');\n\n/**\n * The log's schema changes over `(after, through]`, in stream order, or\n * `undefined` if the interval is too large to scan (see\n * {@link MAX_FOLD_SCAN_ROWS}).\n *\n * The tag is stored beside the verbatim change so this scan never parses the\n * payloads of ordinary data changes. The row count is still checked first so a\n * stalled replicator cannot make the range scan unbounded.\n *\n * The count is itself capped, at one row past the cap it is deciding. A bare\n * `count(*)` over the interval is `O(interval)`, so the unbounded interval this\n * exists to decline would still be walked in full before being declined —\n * which is the latency the cap is here to avoid, not a cheaper form of it. The\n * `LIMIT` makes the decision `O(MAX_FOLD_SCAN_ROWS)`, and the comparison is\n * unaffected: the subquery returns the true count whenever it is within the\n * cap, and `cap + 1` whenever it is not.\n */\nfunction readSchemaChanges(\n  db: Database,\n  after: string,\n  through: string,\n): SchemaChange[] | undefined {\n  const {rows} = db\n    .prepare(/*sql*/ `\n      SELECT count(*) AS \"rows\" FROM (\n        SELECT 1 FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n          WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          LIMIT ${MAX_FOLD_SCAN_ROWS + 1}\n      )\n    `)\n    .get<{rows: number}>(after, through);\n  if (rows > MAX_FOLD_SCAN_ROWS) {\n    return undefined;\n  }\n  return db\n    .prepare(/*sql*/ `\n      SELECT \"change\" FROM \"${CHANGE_LOG_STREAM_TABLE}\"\n        WHERE \"watermark\" > ? AND \"watermark\" <= ?\n          AND \"tag\" IN (${SCHEMA_CHANGE_TAG_LIST})\n        ORDER BY \"watermark\", \"pos\"\n    `)\n    .all<{change: string}>(after, through)\n    .map(({change}) => BigIntJSON.parse(change) as SchemaChange);\n}\n\nfunction streamTableExists(db: Database): boolean {\n  return (\n    db\n      .prepare(/*sql*/ `\n        SELECT 1 FROM \"sqlite_master\" WHERE \"type\" = 'table' AND \"name\" = ?\n      `)\n      .get<{1: number} | undefined>(CHANGE_LOG_STREAM_TABLE) !== undefined\n  );\n}\n\n/**\n * A canonical rendering of a whole cookie set: rows sorted by primary key,\n * object keys sorted at every depth.\n *\n * Both the equality check and the `divergent` log line fall out of this one\n * value, and neither can be defeated by a difference that is not a difference.\n * Postgres stores the cookie documents as `jsonb`, which does not preserve key\n * order, and the two readers order their rows in SQL in two engines under two\n * collations -- `storer.ts` pins `COLLATE \"C\"` for exactly that reason. Sorting\n * again here means an ordering or key-order regression can never be reported as\n * a divergence, which is a class of false alert that would cost more than it\n * caught.\n */\nfunction canonicalizeCookies(cookies: CookieSet): string {\n  return BigIntJSON.stringify({\n    tableMetadata: byKey(cookies.tableMetadata, c => [c.schema, c.table]).map(\n      c => [c.schema, c.table, sortKeys(c.metadata)],\n    ),\n    backfilling: byKey(cookies.backfilling, c => [\n      c.schema,\n      c.table,\n      c.column,\n    ]).map(c => [c.schema, c.table, c.column, sortKeys(c.backfill)]),\n  });\n}\n\nfunction byKey<T>(rows: readonly T[], key: (row: T) => string[]): T[] {\n  return rows\n    .map(row => [JSON.stringify(key(row)), row] as const)\n    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n    .map(([, row]) => row);\n}\n\nfunction sortKeys(value: unknown): unknown {\n  if (Array.isArray(value)) {\n    return value.map(sortKeys);\n  }\n  if (value === null || typeof value !== 'object') {\n    return value;\n  }\n  return Object.fromEntries(\n    Object.entries(value)\n      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))\n      .map(([k, v]) => [k, sortKeys(v)]),\n  );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAqKA,IAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;CAEA,iBAA0B,mBACxB,WACA,yCACA,2HAEF;CAEA;;;;;CAMA;CAEA,YACE,IACA,EAAC,qBAAqB,mBACtB,SACA;EACA,OACE,uBAAuB,iBACvB,qEACF;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,wBAAwB;EAC/D,KAAKC,uBAAuB;EAC5B,KAAKC,mBAAmB;EACxB,KAAKC,WAAW;CAClB;;;;;;;CAQA,iBAAmD;EACjD,OAAO,KAAKE;CACd;;;;;;;;CASA,MAAM,aAAgD;EACpD,KAAKA,kBAAkB,KAAA;EACvB,MAAM,KAAK,KAAKJ,uBACZ,MAAM,KAAKK,aAAa,IACxB,KAAA;EAGJ,MAAM,UAAU,KAAKJ,mBACjB,KAAKK,aAAa,OAAO,KAAA,CAAS,IAClC,KAAA;EAEJ,MAAM,aAAa,KAAKJ,SAAS,mBAC/B,MAAM;GAAC,iBAAiB,GAAG;GAAe,SAAS,GAAG;EAAO,SACvD,cAAc,KAAK,SAAS,mCAAmC,CAAC,CACxE;EAEA,IAAI,IAAI;GACN,IAAI,SAIF,KAAKE,kBAAkB,KAAKG,SAAS,IAAI,OAAO;GAElD,OAAO;EACT;EAIA,MAAM,cACJ,cAAc,cAAc,KAAK,SAAS,0BAA0B,CAAC;EACvE,OAAO;GACL,eAAe,YAAY;GAC3B,SAAS,YAAY;GAGrB,kBAAkB,qBAAqB,YAAY,OAAO;EAC5D;CACF;;;;;;;;;;;;;;;;;;;;;;;;;CA0BA,MAAMF,eAAkD;EACtD,MAAM,KAAK,MAAM,KAAKH,SAAS,YAAY;EAC3C,MAAM,eAAe,KAAKA,SAAS;EACnC,IAAI,CAAC,cACH,OAAO;EAET,KAAKM,qBAAqB,KAAKN,SAAS,QAAQ;EAChD,MAAM,EAAC,eAAe,YAAW,KAAKM;EACtC,IAAI,GAAG,iBAAiB,eACtB,OAAO;EAET,MAAM,SAAS,MAAM,aAAa;GAAC,WAAW;GAAe;EAAO,CAAC;EACrE,KAAKT,IAAI,OACP,0BAA0B,GAAG,cAAc,4BACtC,cAAc,IAAI,QACzB;EAEA,OAAO,KAAKG,SAAS,YAAY;CACnC;;;;;CAMA,SACE,IACA,SACsB;EACtB,IAAI;EACJ,IAAI;GACF,SAAS,KAAKO,UAAU,IAAI,OAAO;EACrC,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,uDAAuD,CAAC;GACxE,SAAS;EACX;EACA,KAAKI,eAAe,IAAI,GAAG,EAAC,OAAM,CAAC;EACnC,KAAKJ,IAAI,QAAQ,8CAA8C,EAC7D,sBAAsB;GACpB;GACA,aAAa,GAAG;GAChB,kBAAkB,QAAQ;EAC5B,EACF,CAAC;EACD,OAAO;CACT;CAEA,aAAa,UAAyD;EACpE,IAAI;GACF,OAAO,KAAKG,SAAS,QAAQ;EAC/B,SAAS,GAAG;GACV,IAAI,UACF,MAAM;GAKR,KAAKH,IAAI,OACP,+DACA,CACF;GACA,KAAKI,eAAe,IAAI,GAAG,EAAC,QAAQ,QAAO,CAAC;GAC5C;EACF;CACF;CAEA,UACE,IACA,SACsB;EACtB,IAAI,QAAQ,gBAAgB,GAAG,eAI7B,OAAO;EAET,IAAI,QAAQ,kBAAkB,GAAG,eAC/B,OAAO,KAAKO,QAAQ,IAAI,SAAS,QAAQ,SAAS,OAAO;EAG3D,MAAM,KAAK,KAAKR,SAAS,UAAU;EACnC,IAAI,OAAO,KAAA,KAAa,CAAC,cAAc,IAAI,QAAQ,aAAa,GAC9D,OAAO;EAET,MAAM,UAAU,kBACd,IACA,QAAQ,eACR,GAAG,aACL;EACA,IAAI,YAAY,KAAA,GACd,OAAO;EAET,OAAO,KAAKQ,QACV,IACA,SACA,YAAY,QAAQ,SAAS,OAAO,GACpC,oBACA,QAAQ,MACV;CACF;CAEA,QACE,IACA,SACA,SACA,WACA,gBAAgB,GACM;EACtB,MAAM,WAAW,oBAAoB,GAAG,OAAO;EAC/C,MAAM,SAAS,oBAAoB,OAAO;EAC1C,IAAI,aAAa,QACf,OAAO;EAKT,KAAKX,IAAI,QACP,gEACA,EACE,sBAAsB;GACpB,aAAa,GAAG;GAChB,kBAAkB,QAAQ;GAC1B;GACA;GACA;EACF,EACF,CACF;EACA,OAAO;CACT;AACF;AAEA,IAAM,iBAAiB,EACrB,eACA,eACqD;CACrD,iBAAiB;CACjB;AACF;;;;;;AAOA,SAAgB,oCACd,IAC0B;CAC1B,MAAM,SAAS,IAAI,gBAAgB,EAAE;CACrC,OAAO,MAAM;CACb,IAAI;EACF,OAAO;GACL,eAAe,oBAAoB,MAAM,EAAE;GAC3C,kBAAkB,qBAAqB,EAAE;GACzC,SAAS,mBAAmB,EAAE;EAChC;CACF,UAAU;EACR,IAAI,GAAG,eACL,OAAO,SAAS;CAEpB;AACF;;;;;;;;;;;;;AAcA,SAAgB,4BACd,IACA,aACgC;CAChC,aAAa;EACX,MAAM,KAAK,IAAI,SACb,GAAG,YAAY,aAAa,wBAAwB,GACpD,aACA,EAAC,UAAU,KAAI,CACjB;EACA,IAAI;GACF,OAAO,oCAAoC,EAAE;EAC/C,UAAU;GACR,GAAG,MAAM;EACX;CACF;AACF;;;;;;;;AASA,SAAS,cAAc,IAAc,eAAgC;CACnE,IAAI,CAAC,kBAAkB,EAAE,GACvB,OAAO;CAET,MAAM,MAAM,GACT,QAAgB;;yCAEoB,wBAAwB;;oBAE7C,+BAA+B;KAC9C,EACA,IAA2D,EAC1D,cACF,CAAC;CACH,OACE,IAAI,iBAAiB,QACrB,iBAAiB,IAAI,gBACrB,IAAI,mBAAmB;AAE3B;AAEA,IAAM,yBAAyB,iBAC5B,KAAI,QAAO,IAAI,IAAI,EAAE,EACrB,KAAK,IAAI;;;;;;;;;;;;;;;;;;AAmBZ,SAAS,kBACP,IACA,OACA,SAC4B;CAC5B,MAAM,EAAC,SAAQ,GACZ,QAAgB;;yBAEI,wBAAwB;;kBAE/B,qBAAqB,EAAE;;KAEpC,EACA,IAAoB,OAAO,OAAO;CACrC,IAAI,OAAO,oBACT;CAEF,OAAO,GACJ,QAAgB;8BACS,wBAAwB;;0BAE5B,uBAAuB;;KAE5C,EACA,IAAsB,OAAO,OAAO,EACpC,KAAK,EAAC,aAAY,WAAW,MAAM,MAAM,CAAiB;AAC/D;AAEA,SAAS,kBAAkB,IAAuB;CAChD,OACE,GACG,QAAgB;;OAEhB,EACA,IAA6B,uBAAuB,MAAM,KAAA;AAEjE;;;;;;;;;;;;;;AAeA,SAAS,oBAAoB,SAA4B;CACvD,OAAO,WAAW,UAAU;EAC1B,eAAe,MAAM,QAAQ,gBAAe,MAAK,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE,KACpE,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,SAAS,EAAE,QAAQ;EAAC,CAC/C;EACA,aAAa,MAAM,QAAQ,cAAa,MAAK;GAC3C,EAAE;GACF,EAAE;GACF,EAAE;EACJ,CAAC,EAAE,KAAI,MAAK;GAAC,EAAE;GAAQ,EAAE;GAAO,EAAE;GAAQ,SAAS,EAAE,QAAQ;EAAC,CAAC;CACjE,CAAC;AACH;AAEA,SAAS,MAAS,MAAoB,KAAgC;CACpE,OAAO,KACJ,KAAI,QAAO,CAAC,KAAK,UAAU,IAAI,GAAG,CAAC,GAAG,GAAG,CAAU,EACnD,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,GAAG,SAAS,GAAG;AACzB;AAEA,SAAS,SAAS,OAAyB;CACzC,IAAI,MAAM,QAAQ,KAAK,GACrB,OAAO,MAAM,IAAI,QAAQ;CAE3B,IAAI,UAAU,QAAQ,OAAO,UAAU,UACrC,OAAO;CAET,OAAO,OAAO,YACZ,OAAO,QAAQ,KAAK,EACjB,MAAM,CAAC,IAAI,CAAC,OAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAE,EAC/C,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CACrC;AACF"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts

@@ -77,6 +77,13 @@ export type TuningOptions = StorerOptions & {
      * backup are authoritative for all of those responsibilities.
      */
     pgChangeLogEnabled: boolean;
+    /**
+     * Whether the PG change log's head was behind the position of the
+     * replication slot from which replication resumes, in which case the
+     * change log is re-initialized from the (restored) replica if it does not
+     * contain all of the replica's changes (see `PgChangeLogPurgeLocker`).
+     */
+    pgChangeLogBehindSlot?: boolean | undefined;
     flowControlConsensusTimeoutProportion: number;
     flowControlSlowSubscriberGracePeriodMs?: number | undefined;
     sqliteCatchup?: SQLiteCatchupOptions | undefined;

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAE1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAiDhC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAcjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAM3C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAajF,OAAO,EACL,KAAK,qBAAqB,EAE1B,KAAK,iBAAiB,EAEvB,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAGL,KAAK,oCAAoC,EAC1C,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAIL,KAAK,uBAAuB,EAC7B,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAEL,KAAK,4BAA4B,EAClC,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAIrB,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,sEAAsE;IACtE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,iBAAiB,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9D;;;;OAIG;IACH,YAAY,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;IACvD;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,8DAA8D;IAC9D,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CACnE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C;;;;OAIG;IACH,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5C,qCAAqC,EAAE,MAAM,CAAC;IAC9C,sCAAsC,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5D,aAAa,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACjD;;;OAGG;IACH,qBAAqB,CAAC,EAClB,IAAI,CACF,4BAA4B,EAC5B,UAAU,GAAG,YAAY,GAAG,WAAW,CACxC,GACD,SAAS,CAAC;IACd;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oCAAoC,GAAG,SAAS,CAAC;IACxE;;;;;;OAMG;IACH,sBAAsB,CAAC,EACnB,CAAC,6BAA6B,GAAG;QAAC,WAAW,EAAE,MAAM,CAAA;KAAC,CAAC,GACvD,SAAS,CAAC;IACd,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,2BAA2B,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CAiDhC"}