@rocicorp/zero 1.9.0-canary.4 → 1.9.0-canary.5

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n  sourceTerminated: Promise<Error>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        if (sinceLastReceived > inboundTimeoutMs && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> 2x wal_sender_timeout ${walSenderTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const sourceTerminated = resolver<Error>();\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once('close', () => {\n    if (!destroyed) {\n      const error = new Error(\n        `replication stream closed by ${db.options.host}`,\n      );\n      sourceTerminated.resolve(error);\n      lc.warn?.(error.message);\n    }\n  });\n  readable.once('error', e => {\n    if (!destroyed) {\n      sourceTerminated.resolve(e);\n    }\n    // Don't log the shutdown signal. This is the expected way for upstream\n    // to close the connection (and will be logged downstream).\n    (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e);\n  });\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n    sourceTerminated: sourceTerminated.promise,\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(walSenderTimeoutMs: number): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.).\n    inboundTimeoutMs: walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;;AAoBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAKjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,kBAAkB;CAC7C,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,MACtE,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,IAAI,oBAAoB,oBAAoB,CAAC,SAAS,WAAW;GAC/D,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,8BACtC,mBAAmB,uDAEnD;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAChB,MAAM,mBAAmB,SAAgB;CAEzC,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KAAK,eAAe;EAC3B,IAAI,CAAC,WAAW;GACd,MAAM,wBAAQ,IAAI,MAChB,gCAAgC,GAAG,QAAQ,MAC7C;GACA,iBAAiB,QAAQ,KAAK;GAC9B,GAAG,OAAO,MAAM,OAAO;EACzB;CACF,CAAC;CACD,SAAS,KAAK,UAAS,MAAK;EAC1B,IAAI,CAAC,WACH,iBAAiB,QAAQ,CAAC;EAI5B,aAAc,SAAS,iBAAiB,EAAE,SAAS,qBACjD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC;CAChD,CAAC;CAED,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;EACpB,kBAAkB,iBAAiB;CACrC;AACF;;;;;;;;;;;;;;;;AAiBA,SAAgB,uBAAuB,oBAKrC;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAKA,kBAAkB,qBAAqB;EAEvC,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const {\n    enabled: livenessEnabled,\n    manualKeepaliveTimeout,\n    inboundTimeoutMs,\n    timerIntervalMs,\n  } = computeLivenessTimings(walSenderTimeoutMs);\n  lc.info?.(\n    livenessEnabled\n      ? `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n          `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`\n      : `wal_sender_timeout is disabled (0); skipping manual keepalives and ` +\n          `inbound liveness detection.`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Tear down the readable to force a reconnect if the inbound half of the\n  // connection goes silent (our outbound acks keep flowing into the void,\n  // neither side errors, and the change-source would otherwise sit there\n  // forever). See computeLivenessTimings for the derivation of these\n  // thresholds — and why the whole timer is disabled when wal_sender_timeout\n  // is 0.\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = livenessEnabled\n    ? setInterval(() => {\n        const now = Date.now();\n        if (now - lastAckTime > manualKeepaliveTimeout) {\n          sendAck(0n);\n          lc.debug?.(`sent manual keepalive`);\n        }\n        const sinceLastReceived = now - lastReceivedTime;\n        const {resetWindow, timedOut} = evaluateInboundLiveness({\n          sinceLastReceived,\n          inboundTimeoutMs,\n          queued: messages.queued,\n        });\n        if (resetWindow) {\n          // Downstream back-pressure is preventing us from consuming (and thus\n          // receiving) upstream messages. Keep the inbound window fresh so that\n          // when the backlog finally drains we grant a full inboundTimeoutMs\n          // before concluding the connection is dead — otherwise a long stall\n          // would leave lastReceivedTime stale and fire this watchdog on the\n          // first post-drain tick, tearing down a healthy connection.\n          lastReceivedTime = now;\n        } else if (timedOut && !readable.destroyed) {\n          lc.warn?.(\n            `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n              `(> 2x wal_sender_timeout ${walSenderTimeoutMs}ms). Destroying ` +\n              `replication stream to force reconnect.`,\n          );\n          readable.destroy(\n            new Error(\n              `replication stream inbound timeout after ${sinceLastReceived}ms`,\n            ),\n          );\n        }\n      }, timerIntervalMs)\n    : undefined;\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Derives the timings for the replication liveness timer from the server's\n * `wal_sender_timeout` (in milliseconds).\n *\n * Postgres treats `wal_sender_timeout = 0` as \"disabled\": it will neither\n * terminate an idle wal sender nor emit the periodic keepalives it otherwise\n * sends at roughly `wal_sender_timeout / 2`. Every timing below is derived from\n * that interval, so a disabled timeout disables the liveness machinery entirely\n * (`enabled: false`). Otherwise the derived thresholds would all collapse to 0,\n * inverting the intended meaning: the inbound watchdog would fire on the very\n * first tick and destroy the stream, while the timer itself would busy-spin at\n * a 0ms interval — together producing a continuous reconnect storm.\n *\n * https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n */\nexport function computeLivenessTimings(walSenderTimeoutMs: number): {\n  enabled: boolean;\n  manualKeepaliveTimeout: number;\n  inboundTimeoutMs: number;\n  timerIntervalMs: number;\n} {\n  // Guard with `!(x > 0)` so that 0, negative, and NaN values all disable the\n  // timer rather than producing degenerate (zero / NaN) thresholds.\n  if (!(walSenderTimeoutMs > 0)) {\n    return {\n      enabled: false,\n      manualKeepaliveTimeout: 0,\n      inboundTimeoutMs: 0,\n      timerIntervalMs: 0,\n    };\n  }\n  // Send a manual keepalive if nothing has been sent in the last 75% of the\n  // wal_sender_timeout, so Postgres never times us out under back-pressure.\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  return {\n    enabled: true,\n    manualKeepaliveTimeout,\n    // Force a reconnect after 2x wal_sender_timeout — i.e. two consecutive\n    // missed keepalives — without any inbound message, to avoid spurious\n    // reconnects from a single delayed keepalive (PG scheduling jitter under\n    // load, our own event-loop stalls, etc.).\n    inboundTimeoutMs: walSenderTimeoutMs * 2,\n    // Poll at 1/5th of the manual keepalive interval.\n    timerIntervalMs: manualKeepaliveTimeout / 5,\n  };\n}\n\n/**\n * Pure decision for the inbound-liveness watchdog, given the time since the\n * last inbound message, the timeout threshold, and the number of messages\n * currently queued downstream (see {@link SourceWithPendingQueue.queued}).\n *\n * A non-empty downstream queue means we have received data we haven't yet\n * consumed: the reason we aren't receiving *more* is our own back-pressure, not\n * upstream silence. In that case we `resetWindow` (treat the backlog as\n * liveness evidence) rather than counting the stall against the connection.\n * This both suppresses false timeouts during back-pressure and — crucially —\n * keeps `lastReceivedTime` fresh so the drain transition (queue emptying after\n * a long stall) doesn't fire the watchdog against a stale timestamp on the very\n * next tick.\n *\n * Only when the queue is empty (consumption has caught up, so reading — and\n * thus keepalives — should be flowing again) does an over-threshold gap\n * indicate a genuinely dead inbound half.\n */\nexport function evaluateInboundLiveness({\n  sinceLastReceived,\n  inboundTimeoutMs,\n  queued,\n}: {\n  sinceLastReceived: number;\n  inboundTimeoutMs: number;\n  queued: number;\n}): {resetWindow: boolean; timedOut: boolean} {\n  if (queued > 0) {\n    return {resetWindow: true, timedOut: false};\n  }\n  return {resetWindow: false, timedOut: sinceLastReceived > inboundTimeoutMs};\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,EACJ,SAAS,iBACT,wBACA,kBACA,oBACE,uBAAuB,kBAAkB;CAC7C,GAAG,OACD,kBACI,uBAAuB,mBAAmB,gDACK,uBAAuB,MACtE,gGAEN;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAQA,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAClB,kBAAkB;EAChB,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,MAAM,EAAC,aAAa,aAAY,wBAAwB;GACtD;GACA;GACA,QAAQ,SAAS;EACnB,CAAC;EACD,IAAI,aAOF,mBAAmB;OACd,IAAI,YAAY,CAAC,SAAS,WAAW;GAC1C,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,8BACtC,mBAAmB,uDAEnD;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,eAAe,IAClB,KAAA;CAEJ,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;;;;;;;;;;;AAiBA,SAAgB,uBAAuB,oBAKrC;CAGA,IAAI,EAAE,qBAAqB,IACzB,OAAO;EACL,SAAS;EACT,wBAAwB;EACxB,kBAAkB;EAClB,iBAAiB;CACnB;CAIF,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,OAAO;EACL,SAAS;EACT;EAKA,kBAAkB,qBAAqB;EAEvC,iBAAiB,yBAAyB;CAC5C;AACF;;;;;;;;;;;;;;;;;;;AAoBA,SAAgB,wBAAwB,EACtC,mBACA,kBACA,UAK4C;CAC5C,IAAI,SAAS,GACX,OAAO;EAAC,aAAa;EAAM,UAAU;CAAK;CAE5C,OAAO;EAAC,aAAa;EAAO,UAAU,oBAAoB;CAAgB;AAC5E;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js

@@ -10,7 +10,7 @@ import { streamIn, streamOut, streamOutStringified } from "../../types/streams.j
 import { URLParams } from "../../types/url-params.js";
 import { downstreamSchema } from "./change-streamer.js";
 import { snapshotMessageSchema } from "./snapshot.js";
-import WebSocket from "ws";
+import WebSocket$1 from "ws";
 import websocket from "@fastify/websocket";
 //#region ../zero-cache/src/services/change-streamer/change-streamer-http.ts
 var MIN_SUPPORTED_PROTOCOL_VERSION = 1;
@@ -114,11 +114,11 @@ var ChangeStreamerHttpClient = class {
 		return uri;
 	}
 	async reserveSnapshot(taskID) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(SNAPSHOT_PATH) + `?${new URLSearchParams({ taskID }).toString()}`);
 		return streamIn(this.#lc, ws, snapshotMessageSchema);
 	}
 	async subscribe(ctx) {
-		const ws = new WebSocket(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
+		const ws = new WebSocket$1(await this.#resolveChangeStreamer(CHANGES_PATH) + `?${getParams(ctx).toString()}`);
 		return streamIn(this.#lc, ws, downstreamSchema);
 	}
 };

package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,UAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}
+{"version":3,"file":"change-streamer-http.js","names":["#lc","#opts","#changeStreamer","#backupMonitor","#subscribe","#reserveSnapshot","#receiveWebsocket","#getBackupMonitor","#ensureChangeStreamerStarted","#changeStreamerStarted","#shardID","#changeDB","#changeStreamerURI","#resolveChangeStreamer"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-http.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport websocket from '@fastify/websocket';\nimport type {LogContext} from '@rocicorp/logger';\nimport WebSocket from 'ws';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport type {IncomingMessageSubset} from '../../types/http.ts';\nimport {pgClient, type PostgresDB} from '../../types/pg.ts';\nimport {type Worker} from '../../types/processes.ts';\nimport {type ShardID} from '../../types/shards.ts';\nimport {\n  streamIn,\n  streamOut,\n  streamOutStringified,\n  type Source,\n} from '../../types/streams.ts';\nimport {URLParams} from '../../types/url-params.ts';\nimport {installWebSocketReceiver} from '../../types/websocket-handoff.ts';\nimport {closeWithError, PROTOCOL_ERROR} from '../../types/ws.ts';\nimport {HttpService} from '../http-service.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport {\n  downstreamSchema,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type ChangeStreamerService,\n  type Downstream,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport {discoverChangeStreamerAddress} from './schema/tables.ts';\nimport {snapshotMessageSchema, type SnapshotMessage} from './snapshot.ts';\n\nconst MIN_SUPPORTED_PROTOCOL_VERSION = 1;\n\nconst SNAPSHOT_PATH_PATTERN = '/replication/:version/snapshot';\nconst CHANGES_PATH_PATTERN = '/replication/:version/changes';\nconst PATH_REGEX = /\\/replication\\/v(?<version>\\d+)\\/(changes|snapshot)$/;\n\nconst SNAPSHOT_PATH = `/replication/v${PROTOCOL_VERSION}/snapshot`;\nconst CHANGES_PATH = `/replication/v${PROTOCOL_VERSION}/changes`;\n\ntype Options = {\n  port: number;\n  keepaliveTimeoutMs: number | undefined;\n  startupDelayMs: number;\n};\n\nexport class ChangeStreamerHttpServer extends HttpService {\n  readonly id = 'change-streamer-http-server';\n  readonly #lc: LogContext;\n  readonly #opts: Options;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #backupMonitor: BackupMonitor | null;\n\n  constructor(\n    lc: LogContext,\n    opts: Options,\n    parent: Worker,\n    changeStreamer: ChangeStreamerService,\n    backupMonitor: BackupMonitor | null,\n  ) {\n    super('change-streamer-http-server', lc, opts, async fastify => {\n      await fastify.register(websocket);\n\n      fastify.get(CHANGES_PATH_PATTERN, {websocket: true}, this.#subscribe);\n      fastify.get(\n        SNAPSHOT_PATH_PATTERN,\n        {websocket: true},\n        this.#reserveSnapshot,\n      );\n\n      installWebSocketReceiver<'snapshot' | 'changes'>(\n        lc,\n        fastify.websocketServer,\n        this.#receiveWebsocket,\n        parent,\n      );\n    });\n\n    this.#lc = lc;\n    this.#opts = opts;\n    this.#changeStreamer = changeStreamer;\n    this.#backupMonitor = backupMonitor;\n  }\n\n  #getBackupMonitor() {\n    return must(\n      this.#backupMonitor,\n      'replication-manager is not configured with a ZERO_LITESTREAM_BACKUP_URL',\n    );\n  }\n\n  // Called when receiving a web socket via the main dispatcher handoff.\n  readonly #receiveWebsocket = (\n    ws: WebSocket,\n    action: 'changes' | 'snapshot',\n    msg: IncomingMessageSubset,\n  ) => {\n    switch (action) {\n      case 'snapshot':\n        return this.#reserveSnapshot(ws, msg);\n      case 'changes':\n        return this.#subscribe(ws, msg);\n      default:\n        closeWithError(\n          this._lc,\n          ws,\n          `invalid action \"${action}\" received in handoff`,\n        );\n        return;\n    }\n  };\n\n  readonly #reserveSnapshot = (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const url = new URL(\n        req.url ?? '',\n        req.headers.origin ?? 'http://localhost',\n      );\n      checkProtocolVersion(url.pathname);\n      const taskID = url.searchParams.get('taskID');\n      if (!taskID) {\n        throw new Error('Missing taskID in snapshot request');\n      }\n      const downstream =\n        this.#getBackupMonitor().startSnapshotReservation(taskID);\n      void streamOut(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  readonly #subscribe = async (ws: WebSocket, req: RequestHeaders) => {\n    try {\n      const ctx = getSubscriberContext(req);\n      if (ctx.mode === 'serving') {\n        this.#ensureChangeStreamerStarted('incoming subscription');\n      }\n\n      const downstream = await this.#changeStreamer.subscribe(ctx);\n      if (ctx.initial && ctx.taskID && this.#backupMonitor) {\n        // Now that the change-streamer knows about the subscriber and watermark,\n        // end the reservation to safely resume scheduling cleanup.\n        this.#backupMonitor.endReservation(ctx.taskID);\n      }\n      void streamOutStringified(this._lc, downstream, ws);\n    } catch (err) {\n      closeWithError(this._lc, ws, err, PROTOCOL_ERROR);\n    }\n  };\n\n  #changeStreamerStarted = false;\n\n  #ensureChangeStreamerStarted(reason: string) {\n    if (!this.#changeStreamerStarted && this._state.shouldRun()) {\n      this.#lc.info?.(`starting ChangeStreamerService: ${reason}`);\n      void this.#changeStreamer\n        .run()\n        .catch(e =>\n          this.#lc.warn?.(`ChangeStreamerService ended with error`, e),\n        )\n        .finally(() => this.stop());\n\n      this.#changeStreamerStarted = true;\n    }\n  }\n\n  protected override _onStart(): void {\n    const {startupDelayMs} = this.#opts;\n    this._state.setTimeout(\n      () =>\n        this.#ensureChangeStreamerStarted(\n          `startup delay elapsed (${startupDelayMs} ms)`,\n        ),\n      startupDelayMs,\n    );\n  }\n\n  protected override async _onStop(): Promise<void> {\n    if (this.#changeStreamerStarted) {\n      await this.#changeStreamer.stop();\n    }\n  }\n}\n\nexport class ChangeStreamerHttpClient implements ChangeStreamer {\n  readonly #lc: LogContext;\n  readonly #shardID: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #changeStreamerURI: string | undefined;\n\n  constructor(\n    lc: LogContext,\n    shardID: ShardID,\n    changeDB: string,\n    changeStreamerURI: string | undefined,\n  ) {\n    this.#lc = lc;\n    this.#shardID = shardID;\n    // Create a pg client with a single short-lived connection for the purpose\n    // of change-streamer discovery (i.e. ChangeDB as DNS).\n    this.#changeDB = pgClient(lc, changeDB, 'change-streamer-discovery', {\n      max: 1,\n      ['idle_timeout']: 15,\n    });\n    this.#changeStreamerURI = changeStreamerURI;\n  }\n\n  async #resolveChangeStreamer(path: string) {\n    let baseURL = this.#changeStreamerURI;\n    if (!baseURL) {\n      const address = await discoverChangeStreamerAddress(\n        this.#shardID,\n        this.#changeDB,\n      );\n      if (!address) {\n        throw new Error(`no change-streamer is running`);\n      }\n      baseURL = address.includes('://') ? `${address}/` : `ws://${address}/`;\n    }\n    const uri = new URL(path, baseURL);\n    this.#lc.info?.(`connecting to change-streamer@${uri}`);\n    return uri;\n  }\n\n  async reserveSnapshot(taskID: string): Promise<Source<SnapshotMessage>> {\n    const uri = await this.#resolveChangeStreamer(SNAPSHOT_PATH);\n\n    const params = new URLSearchParams({taskID});\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, snapshotMessageSchema);\n  }\n\n  async subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n    const uri = await this.#resolveChangeStreamer(CHANGES_PATH);\n\n    const params = getParams(ctx);\n    const ws = new WebSocket(uri + `?${params.toString()}`);\n\n    return streamIn(this.#lc, ws, downstreamSchema);\n  }\n}\n\ntype RequestHeaders = Pick<IncomingMessage, 'url' | 'headers'>;\n\nexport function getSubscriberContext(req: RequestHeaders): SubscriberContext {\n  const url = new URL(req.url ?? '', req.headers.origin ?? 'http://localhost');\n  const protocolVersion = checkProtocolVersion(url.pathname);\n  const params = new URLParams(url);\n\n  return {\n    protocolVersion,\n    id: params.get('id', true),\n    taskID: params.get('taskID', false),\n    mode: params.get('mode', false) === 'backup' ? 'backup' : 'serving',\n    replicaVersion: params.get('replicaVersion', true),\n    watermark: params.get('watermark', true),\n    initial: params.getBoolean('initial'),\n  };\n}\n\nfunction checkProtocolVersion(pathname: string): number {\n  const match = PATH_REGEX.exec(pathname);\n  if (!match) {\n    throw new Error(`invalid path: ${pathname}`);\n  }\n  const v = Number(match.groups?.version);\n  if (\n    Number.isNaN(v) ||\n    v > PROTOCOL_VERSION ||\n    v < MIN_SUPPORTED_PROTOCOL_VERSION\n  ) {\n    throw new Error(\n      `Cannot service client at protocol v${v}. ` +\n        `Supported protocols: [v${MIN_SUPPORTED_PROTOCOL_VERSION} ... v${PROTOCOL_VERSION}]`,\n    );\n  }\n  return v;\n}\n\n// This is called from the client-side (i.e. the replicator).\nfunction getParams(ctx: SubscriberContext): URLSearchParams {\n  // The protocolVersion is hard-coded into the CHANGES_PATH.\n  const {protocolVersion, ...stringParams} = ctx;\n  assert(\n    protocolVersion === PROTOCOL_VERSION,\n    `replicator should be setting protocolVersion to ${PROTOCOL_VERSION}`,\n  );\n  return new URLSearchParams({\n    ...stringParams,\n    taskID: ctx.taskID ? ctx.taskID : '',\n    initial: ctx.initial ? 'true' : 'false',\n  });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAgCA,IAAM,iCAAiC;AAEvC,IAAM,wBAAwB;AAC9B,IAAM,uBAAuB;AAC7B,IAAM,aAAa;AAEnB,IAAM,gBAAgB;AACtB,IAAM,eAAe;AAQrB,IAAa,2BAAb,cAA8C,YAAY;CACxD,KAAc;CACd;CACA;CACA;CACA;CAEA,YACE,IACA,MACA,QACA,gBACA,eACA;EACA,MAAM,+BAA+B,IAAI,MAAM,OAAM,YAAW;GAC9D,MAAM,QAAQ,SAAS,SAAS;GAEhC,QAAQ,IAAI,sBAAsB,EAAC,WAAW,KAAI,GAAG,KAAKI,UAAU;GACpE,QAAQ,IACN,uBACA,EAAC,WAAW,KAAI,GAChB,KAAKC,gBACP;GAEA,yBACE,IACA,QAAQ,iBACR,KAAKC,mBACL,MACF;EACF,CAAC;EAED,KAAKN,MAAM;EACX,KAAKC,QAAQ;EACb,KAAKC,kBAAkB;EACvB,KAAKC,iBAAiB;CACxB;CAEA,oBAAoB;EAClB,OAAO,KACL,KAAKA,gBACL,yEACF;CACF;CAGA,qBACE,IACA,QACA,QACG;EACH,QAAQ,QAAR;GACE,KAAK,YACH,OAAO,KAAKE,iBAAiB,IAAI,GAAG;GACtC,KAAK,WACH,OAAO,KAAKD,WAAW,IAAI,GAAG;GAChC;IACE,eACE,KAAK,KACL,IACA,mBAAmB,OAAO,sBAC5B;IACA;EACJ;CACF;CAEA,oBAA6B,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,IAAI,IACd,IAAI,OAAO,IACX,IAAI,QAAQ,UAAU,kBACxB;GACA,qBAAqB,IAAI,QAAQ;GACjC,MAAM,SAAS,IAAI,aAAa,IAAI,QAAQ;GAC5C,IAAI,CAAC,QACH,MAAM,IAAI,MAAM,oCAAoC;GAEtD,MAAM,aACJ,KAAKG,kBAAkB,EAAE,yBAAyB,MAAM;GAC1D,UAAe,KAAK,KAAK,YAAY,EAAE;EACzC,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,aAAsB,OAAO,IAAe,QAAwB;EAClE,IAAI;GACF,MAAM,MAAM,qBAAqB,GAAG;GACpC,IAAI,IAAI,SAAS,WACf,KAAKC,6BAA6B,uBAAuB;GAG3D,MAAM,aAAa,MAAM,KAAKN,gBAAgB,UAAU,GAAG;GAC3D,IAAI,IAAI,WAAW,IAAI,UAAU,KAAKC,gBAGpC,KAAKA,eAAe,eAAe,IAAI,MAAM;GAE/C,qBAA0B,KAAK,KAAK,YAAY,EAAE;EACpD,SAAS,KAAK;GACZ,eAAe,KAAK,KAAK,IAAI,KAAK,cAAc;EAClD;CACF;CAEA,yBAAyB;CAEzB,6BAA6B,QAAgB;EAC3C,IAAI,CAAC,KAAKM,0BAA0B,KAAK,OAAO,UAAU,GAAG;GAC3D,KAAKT,IAAI,OAAO,mCAAmC,QAAQ;GAC3D,KAAUE,gBACP,IAAI,EACJ,OAAM,MACL,KAAKF,IAAI,OAAO,0CAA0C,CAAC,CAC7D,EACC,cAAc,KAAK,KAAK,CAAC;GAE5B,KAAKS,yBAAyB;EAChC;CACF;CAEA,WAAoC;EAClC,MAAM,EAAC,mBAAkB,KAAKR;EAC9B,KAAK,OAAO,iBAER,KAAKO,6BACH,0BAA0B,eAAe,KAC3C,GACF,cACF;CACF;CAEA,MAAyB,UAAyB;EAChD,IAAI,KAAKC,wBACP,MAAM,KAAKP,gBAAgB,KAAK;CAEpC;AACF;AAEA,IAAa,2BAAb,MAAgE;CAC9D;CACA;CACA;CACA;CAEA,YACE,IACA,SACA,UACA,mBACA;EACA,KAAKF,MAAM;EACX,KAAKU,WAAW;EAGhB,KAAKC,YAAY,SAAS,IAAI,UAAU,6BAA6B;GACnE,KAAK;IACJ,iBAAiB;EACpB,CAAC;EACD,KAAKC,qBAAqB;CAC5B;CAEA,MAAMC,uBAAuB,MAAc;EACzC,IAAI,UAAU,KAAKD;EACnB,IAAI,CAAC,SAAS;GACZ,MAAM,UAAU,MAAM,8BACpB,KAAKF,UACL,KAAKC,SACP;GACA,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,+BAA+B;GAEjD,UAAU,QAAQ,SAAS,KAAK,IAAI,GAAG,QAAQ,KAAK,QAAQ,QAAQ;EACtE;EACA,MAAM,MAAM,IAAI,IAAI,MAAM,OAAO;EACjC,KAAKX,IAAI,OAAO,iCAAiC,KAAK;EACtD,OAAO;CACT;CAEA,MAAM,gBAAgB,QAAkD;EAItE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,aAAa,IAG5B,IAAI,IADhB,gBAAgB,EAAC,OAAM,CACP,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,qBAAqB;CACrD;CAEA,MAAM,UAAU,KAAqD;EAInE,MAAM,KAAK,IAAI,YAAU,MAHP,KAAKa,uBAAuB,YAAY,IAG3B,IADhB,UAAU,GACU,EAAO,SAAS,GAAG;EAEtD,OAAO,SAAS,KAAKb,KAAK,IAAI,gBAAgB;CAChD;AACF;AAIA,SAAgB,qBAAqB,KAAwC;CAC3E,MAAM,MAAM,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,QAAQ,UAAU,kBAAkB;CAC3E,MAAM,kBAAkB,qBAAqB,IAAI,QAAQ;CACzD,MAAM,SAAS,IAAI,UAAU,GAAG;CAEhC,OAAO;EACL;EACA,IAAI,OAAO,IAAI,MAAM,IAAI;EACzB,QAAQ,OAAO,IAAI,UAAU,KAAK;EAClC,MAAM,OAAO,IAAI,QAAQ,KAAK,MAAM,WAAW,WAAW;EAC1D,gBAAgB,OAAO,IAAI,kBAAkB,IAAI;EACjD,WAAW,OAAO,IAAI,aAAa,IAAI;EACvC,SAAS,OAAO,WAAW,SAAS;CACtC;AACF;AAEA,SAAS,qBAAqB,UAA0B;CACtD,MAAM,QAAQ,WAAW,KAAK,QAAQ;CACtC,IAAI,CAAC,OACH,MAAM,IAAI,MAAM,iBAAiB,UAAU;CAE7C,MAAM,IAAI,OAAO,MAAM,QAAQ,OAAO;CACtC,IACE,OAAO,MAAM,CAAC,KACd,IAAA,KACA,IAAI,gCAEJ,MAAM,IAAI,MACR,sCAAsC,EAAE,2BACZ,+BAA+B,SAC7D;CAEF,OAAO;AACT;AAGA,SAAS,UAAU,KAAyC;CAE1D,MAAM,EAAC,iBAAiB,GAAG,iBAAgB;CAC3C,OACE,oBAAA,GACA,mDACF;CACA,OAAO,IAAI,gBAAgB;EACzB,GAAG;EACH,QAAQ,IAAI,SAAS,IAAI,SAAS;EAClC,SAAS,IAAI,UAAU,SAAS;CAClC,CAAC;AACH"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAInD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}
+{"version":3,"file":"change-streamer-service.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAGnD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,gBAAgB,EAEtB,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,qCAAqC,CAAC;AAC7C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,2CAA2C,CAAC;AAMjF,OAAO,EACL,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,IAAI,aAAa,EACpC,MAAM,aAAa,CAAC;AAGrB,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG;IAC1C,kCAAkC,EAAE,MAAM,CAAC;CAC5C,CAAC;AAEF;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,QAAQ,EAAE,UAAU,EACpB,YAAY,EAAE,YAAY,EAC1B,0BAA0B,EAAE,0BAA0B,EACtD,iBAAiB,EAAE,iBAAiB,EACpC,SAAS,EAAE,SAAS,GAAG,IAAI,EAC3B,SAAS,EAAE,OAAO,EAClB,IAAI,EAAE,aAAa,EACnB,YAAY,oBAAa,GACxB,OAAO,CAAC,qBAAqB,CAAC,CA4BhC;AAID,MAAM,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM;IACjB,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,MAAM;CACb,CAAC"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js

@@ -8,7 +8,6 @@ import { publishCriticalEvent } from "../../observability/events.js";
 import { AutoResetSignal, ensureReplicationConfig, markResetRequired } from "./schema/tables.js";
 import { Subscription } from "../../types/subscription.js";
 import { publishReplicationError, replicationStatusError } from "../replicator/replication-status.js";
-import { orTimeout } from "../../types/timeout.js";
 import "./change-streamer.js";
 import { Forwarder } from "./forwarder.js";
 import { initChangeStreamerSchema } from "./schema/init.js";
@@ -183,7 +182,6 @@ var ChangeStreamerImpl = class {
 	#changeCounter = getOrCreateCounter("replication", "changes", "Count of replicated changes (DML or DDL statements)");
 	#latestStatus;
 	#purgeLock;
-	#fatalSourceError;
 	#stream;
 	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, changeDB, replicaVersion, source, replicationStatusPublisher, initialPurgeLock, autoReset, opts, setTimeoutFn = setTimeout) {
 		this.id = `change-streamer`;
@@ -219,17 +217,6 @@ var ChangeStreamerImpl = class {
 			try {
 				const { lastWatermark, backfillRequests } = await this.#storer.getStartStreamInitializationParameters();
 				const stream = await this.#source.startStream(lastWatermark, backfillRequests);
-				const waitForFlowControl = async (wait) => {
-					if (!stream.sourceTerminated) {
-						await wait;
-						return;
-					}
-					const sourceError = await Promise.race([wait.then(() => void 0), stream.sourceTerminated]);
-					if (sourceError) {
-						this.#fatalSourceError = sourceError;
-						throw sourceError;
-					}
-				};
 				this.#storer.run().catch((e) => stream.changes.cancel(e));
 				this.#stream = stream;
 				if (this.#state.resetBackoff() > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS) this.#replicationStatusPublisher.publish(this.#lc, "Replicating", `Replicating from ${lastWatermark}`);
@@ -268,12 +255,13 @@ var ChangeStreamerImpl = class {
 					unflushedBytes += json.length;
 					if (unflushedBytes < flushBytesThreshold) this.#forwarder.forward(entry);
 					else {
-						await waitForFlowControl(this.#forwarder.forwardWithFlowControl(entry));
+						const forwarded = this.#forwarder.forwardWithFlowControl(entry);
+						await stream.changes.doneOr(forwarded);
 						unflushedBytes = 0;
 					}
 					if (type === "commit" || type === "rollback") watermark = null;
 					const readyForMore = this.#storer.readyForMore();
-					if (readyForMore) await waitForFlowControl(readyForMore);
+					if (readyForMore) await stream.changes.doneOr(readyForMore);
 				}
 			} catch (e) {
 				err = e;
@@ -281,10 +269,6 @@ var ChangeStreamerImpl = class {
 				this.#stream?.changes.cancel();
 				this.#stream = void 0;
 			}
-			if (this.#fatalSourceError && err === this.#fatalSourceError) {
-				this.#forwarder.stopProgressMonitor();
-				throw err;
-			}
 			if (watermark) {
 				this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);
 				this.#storer.abort();
@@ -383,12 +367,7 @@ var ChangeStreamerImpl = class {
 	async stop(err) {
 		this.#state.stop(this.#lc, err);
 		this.#stream?.changes.cancel();
-		if (this.#fatalSourceError) {
-			await orTimeout(Promise.allSettled([this.#storer.stop(), this.#source.stop()]), 100);
-			return;
-		}
-		await this.#storer.stop();
-		await this.#source.stop();
+		await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);
 	}
 };
 var CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#fatalSourceError","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {orTimeout} from '../../types/timeout.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #fatalSourceError: Error | undefined;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        const waitForFlowControl = async (wait: Promise<void>) => {\n          if (!stream.sourceTerminated) {\n            await wait;\n            return;\n          }\n          const sourceError = await Promise.race([\n            wait.then(() => undefined),\n            stream.sourceTerminated,\n          ]);\n          if (sourceError) {\n            this.#fatalSourceError = sourceError;\n            throw sourceError;\n          }\n        };\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await waitForFlowControl(forwarded);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await waitForFlowControl(readyForMore);\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      if (this.#fatalSourceError && err === this.#fatalSourceError) {\n        // A blocked flow-control wait may not drain; restart from durable state.\n        this.#forwarder.stopProgressMonitor();\n        throw err;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    if (this.#fatalSourceError) {\n      // Cleanup is best-effort before intentional process replacement. A\n      // wedged Storer may be unable to stop without the process exiting.\n      await orTimeout(\n        Promise.allSettled([this.#storer.stop(), this.#source.stop()]),\n        100,\n      );\n      return;\n    }\n    await this.#storer.stop();\n    await this.#source.stop();\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAgEA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,MAAM,qBAAqB,OAAO,SAAwB;KACxD,IAAI,CAAC,OAAO,kBAAkB;MAC5B,MAAM;MACN;KACF;KACA,MAAM,cAAc,MAAM,QAAQ,KAAK,CACrC,KAAK,WAAW,KAAA,CAAS,GACzB,OAAO,gBACT,CAAC;KACD,IAAI,aAAa;MACf,KAAKa,oBAAoB;MACzB,MAAM;KACR;IACF;IACA,KAAKZ,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKE,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKN,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MASL,MAAM,mBADY,KAAKA,WAAW,uBAAuB,KAChC,CAAS;MAClC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,mBAAmB,YAAY;IAEzC;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAEA,IAAI,KAAKG,qBAAqB,QAAQ,KAAKA,mBAAmB;IAE5D,KAAKX,WAAW,oBAAoB;IACpC,MAAM;GACR;GAGA,IAAI,WAAW;IACb,KAAKN,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMkB,sBAAsB,KAA6B;EACvD,KAAKlB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKU,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKd,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMgB,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKT,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKU,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKV,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,IAAI,KAAKG,mBAAmB;GAG1B,MAAM,UACJ,QAAQ,WAAW,CAAC,KAAKZ,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC,GAC7D,GACF;GACA;EACF;EACA,MAAM,KAAKC,QAAQ,KAAK;EACxB,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}
+{"version":3,"file":"change-streamer-service.js","names":["#lc","#shard","#changeDB","#replicaVersion","#source","#storer","#forwarder","#replicationStatusPublisher","#autoReset","#state","#initialWatermarks","#serving","#txCounter","#changeCounter","#stream","#purgeLock","#latestStatus","#handleControlMessage","#purgeOldChanges"],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import {getDefaultHighWaterMark} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {publishCriticalEvent} from '../../observability/events.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n  min,\n  type AtLeastOne,\n  type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../change-source/change-source.ts';\nimport {\n  type ChangeStreamControl,\n  type ChangeStreamData,\n  type Rollback,\n} from '../change-source/protocol/current/downstream.ts';\nimport {\n  publishReplicationError,\n  replicationStatusError,\n  type ReplicationStatusPublisher,\n} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  DEFAULT_MAX_RETRY_DELAY_MS,\n  RunningState,\n  UnrecoverableError,\n} from '../running-state.ts';\nimport {\n  type ChangeStreamerService,\n  type Status,\n  type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n  AutoResetSignal,\n  ensureReplicationConfig,\n  markResetRequired,\n} from './schema/tables.ts';\nimport {\n  Storer,\n  type PurgeLock,\n  type TuningOptions as StorerOptions,\n} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\nexport type TuningOptions = StorerOptions & {\n  flowControlConsensusPaddingSeconds: number;\n};\n\n/**\n * Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n */\nexport async function initializeStreamer(\n  lc: LogContext,\n  shard: ShardID,\n  taskID: string,\n  discoveryAddress: string,\n  discoveryProtocol: string,\n  changeDB: PostgresDB,\n  changeSource: ChangeSource,\n  replicationStatusPublisher: ReplicationStatusPublisher,\n  subscriptionState: SubscriptionState,\n  purgeLock: PurgeLock | null,\n  autoReset: boolean,\n  opts: TuningOptions,\n  setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n  // Make sure the ChangeLog DB is set up.\n  await initChangeStreamerSchema(lc, changeDB, shard);\n  await ensureReplicationConfig(\n    lc,\n    changeDB,\n    subscriptionState,\n    shard,\n    autoReset,\n    setTimeoutFn,\n  );\n\n  const {replicaVersion} = subscriptionState;\n  return new ChangeStreamerImpl(\n    lc,\n    shard,\n    taskID,\n    discoveryAddress,\n    discoveryProtocol,\n    changeDB,\n    replicaVersion,\n    changeSource,\n    replicationStatusPublisher,\n    purgeLock,\n    autoReset,\n    opts,\n    setTimeoutFn,\n  );\n}\n\nconst REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS = 5000;\n\nexport type ChangeTag = ChangeStreamData[1]['tag'];\n\n/**\n * Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n *\n * However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n *\n * Additionally, the ChangeStreamData is eagerly stringified once, after which\n * the string is passed to the changeLog and all subscribers, eliminating\n * redundant stringification and reducing GC churn.\n */\nexport type WatermarkedChange = [\n  watermark: string,\n  tag: ChangeTag,\n  json: string,\n];\n\n/**\n * Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n *\n * ### Subscriber Catchup\n *\n * Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n *\n *\n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * | Historic changes in storage |  Pending (streamed) tx  |   Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n *                                           Replication stream\n *                                           >  >  >  >  >  >  >  >  >\n *           ^  ---> required catchup --->   ^\n * Subscriber watermark               Subscription begins\n * ```\n *\n * Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n *\n * Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n *\n * * If the streamer is in the middle of a pending Transaction, the\n *   Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n *   state, new changes are *not* forwarded to the Subscriber, and catchup\n *   is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n *   by the Storer, it begins catchup on the Subscriber (with a READONLY\n *   snapshot so that it does not block subsequent storage operations).\n *   This catchup is thus guaranteed to load the change log entries of\n *   that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n *   Subscriber from the \"queued\" to the \"active\" set of clients such that\n *   the Subscriber begins receiving new changes, starting from the next\n *   Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n *   is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n *   and the Subscriber henceforth forwards changes as they are received.\n *\n * In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n *\n * ### Watermarks and ordering\n *\n * The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n *\n * https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n *\n * > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n *\n * In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n *\n * A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n *\n * ```\n * LSN:   1     2     3  4    5   6   7     8   9      10\n * tx1: begin  data     data     data     commit\n * tx2:             begin    data    data      data  commit\n * ```\n *\n * will arrive as:\n *\n * ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n *\n * Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n *\n * Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n *\n * * A `commit` message is assigned to a watermark corresponding to its LSN.\n *   These are guaranteed to be in commit order by definition.\n *\n * * `begin` and `data` messages are assigned to the watermark of the\n *   preceding `commit` (the previous transaction, or the replication\n *   slot's starting LSN) plus 1. This guarantees that they will be sorted\n *   after the previously commit transaction even if their LSNs came before it.\n *   This is referred to as the `preCommitWatermark`.\n *\n * * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n *   the position of the message within its transaction, with the `begin` message\n *   starting at `0`. This guarantees that `begin` and `data` messages will be\n *   fetched in the original ChangeSource order during catchup.\n *\n * `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n *\n * ### Cleanup\n *\n * As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n *\n * More concretely:\n *\n * * The `initial`, backup-derived watermark is the earliest to which cleanup\n *   should ever happen.\n *\n * * However, it is possible for the replica backup to be *ahead* of a connected\n *   subscriber; and if a network error causes that subscriber to retry from its\n *   last watermark, the change streamer must support it.\n *\n * Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n */\nclass ChangeStreamerImpl implements ChangeStreamerService {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #shard: ShardID;\n  readonly #changeDB: PostgresDB;\n  readonly #replicaVersion: string;\n  readonly #source: ChangeSource;\n  readonly #storer: Storer;\n  readonly #forwarder: Forwarder;\n  readonly #replicationStatusPublisher: ReplicationStatusPublisher;\n\n  readonly #autoReset: boolean;\n  readonly #state: RunningState;\n  readonly #initialWatermarks = new Set<string>();\n\n  // Starting the (Postgres) ChangeStream results in killing the previous\n  // Postgres subscriber, potentially creating a gap in which the old\n  // change-streamer has shut down and the new change-streamer has not yet\n  // been recognized as \"healthy\" (and thus does not get any requests).\n  //\n  // To minimize this gap, delay starting the ChangeStream until the first\n  // request from a `serving` replicator, indicating that higher level\n  // load-balancing / routing logic has begun routing requests to this task.\n  readonly #serving = resolver();\n\n  readonly #txCounter = getOrCreateCounter(\n    'replication',\n    'transactions',\n    'Count of replicated transactions',\n  );\n  readonly #changeCounter = getOrCreateCounter(\n    'replication',\n    'changes',\n    'Count of replicated changes (DML or DDL statements)',\n  );\n\n  #latestStatus: Status;\n  #purgeLock: PurgeLock | null;\n  #stream: ChangeStream | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    taskID: string,\n    discoveryAddress: string,\n    discoveryProtocol: string,\n    changeDB: PostgresDB,\n    replicaVersion: string,\n    source: ChangeSource,\n    replicationStatusPublisher: ReplicationStatusPublisher,\n    initialPurgeLock: PurgeLock | null,\n    autoReset: boolean,\n    opts: TuningOptions,\n    setTimeoutFn = setTimeout,\n  ) {\n    this.id = `change-streamer`;\n    this.#lc = lc.withContext('component', 'change-streamer');\n    this.#shard = shard;\n    this.#changeDB = changeDB;\n    this.#replicaVersion = replicaVersion;\n    this.#source = source;\n    this.#storer = new Storer(\n      lc,\n      shard,\n      taskID,\n      discoveryAddress,\n      discoveryProtocol,\n      changeDB,\n      replicaVersion,\n      consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n      err => this.stop(err),\n      opts,\n    );\n    this.#forwarder = new Forwarder(lc, {\n      flowControlConsensusPaddingSeconds:\n        opts.flowControlConsensusPaddingSeconds,\n    });\n    this.#replicationStatusPublisher = replicationStatusPublisher;\n    this.#purgeLock = initialPurgeLock;\n    this.#autoReset = autoReset;\n    this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n    this.#latestStatus = {tag: 'status'};\n  }\n\n  async run() {\n    this.#lc.info?.('starting change stream');\n\n    this.#forwarder.startProgressMonitor();\n\n    const lagReport = await this.#source.startLagReporter();\n    if (lagReport) {\n      this.#latestStatus.lagReport = lagReport;\n    }\n\n    // Once this change-streamer acquires \"ownership\" of the change DB,\n    // it is safe to start the storer.\n    await this.#storer.assumeOwnership(this.#purgeLock);\n    this.#purgeLock = null;\n\n    // The threshold in (estimated number of) bytes to send() on subscriber\n    // websockets before `await`-ing the I/O buffers to be ready for more.\n    const flushBytesThreshold = getDefaultHighWaterMark(false);\n\n    while (this.#state.shouldRun()) {\n      let err: unknown;\n      let watermark: string | null = null;\n      let unflushedBytes = 0;\n      try {\n        const {lastWatermark, backfillRequests} =\n          await this.#storer.getStartStreamInitializationParameters();\n        const stream = await this.#source.startStream(\n          lastWatermark,\n          backfillRequests,\n        );\n        this.#storer.run().catch(e => stream.changes.cancel(e));\n\n        this.#stream = stream;\n        if (\n          this.#state.resetBackoff() >\n          REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n        ) {\n          // After recovering from a backoff for which a replication status\n          // error was published, publish an OK status\n          this.#replicationStatusPublisher.publish(\n            this.#lc,\n            'Replicating',\n            `Replicating from ${lastWatermark}`,\n          );\n        }\n        watermark = null;\n\n        for await (const change of stream.changes) {\n          const [type, msg] = change;\n          switch (type) {\n            case 'status':\n              if (msg.ack) {\n                this.#storer.status(change); // storer acks once it gets through its queue\n              }\n              if (msg.lagReport) {\n                // Lag reports are not stored in the cdc change log, but rather\n                // only forwarded on \"live\" connections. When a new subscriber\n                // is catching up, it is initialized with the #latestStatus\n                // from which it can measure lag while catching up.\n                this.#latestStatus.lagReport = msg.lagReport;\n                this.#forwarder.sendStatus(this.#latestStatus);\n              }\n              continue;\n            case 'control':\n              await this.#handleControlMessage(msg);\n              continue; // control messages are not stored/forwarded\n            case 'begin':\n              watermark = change[2].commitWatermark;\n              break;\n            case 'commit':\n              if (watermark !== change[2].watermark) {\n                throw new UnrecoverableError(\n                  `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n                );\n              }\n              this.#txCounter.add(1);\n              break;\n            default:\n              if (type === 'data') {\n                this.#changeCounter.add(1);\n              }\n              if (watermark === null) {\n                throw new UnrecoverableError(\n                  `${type} change (${msg.tag}) received before 'begin' message`,\n                );\n              }\n              break;\n          }\n\n          const json = this.#storer.store(watermark, change);\n          const entry: WatermarkedChange = [watermark, change[1].tag, json];\n          unflushedBytes += json.length;\n          if (unflushedBytes < flushBytesThreshold) {\n            // pipeline changes until flushBytesThreshold\n            this.#forwarder.forward(entry);\n          } else {\n            // Wait for messages to clear socket buffers to ensure that they\n            // make their way to subscribers. Without this `await`, the\n            // messages end up being buffered in this process, which:\n            // (1) results in memory pressure and increased GC activity\n            // (2) prevents subscribers from processing the messages as they\n            //     arrive, instead getting them in a large batch after being\n            //     idle while they were queued (causing further delays).\n            const forwarded = this.#forwarder.forwardWithFlowControl(entry);\n            await stream.changes.doneOr(forwarded);\n            unflushedBytes = 0;\n          }\n\n          if (type === 'commit' || type === 'rollback') {\n            watermark = null;\n          }\n\n          // Allow the storer to exert back pressure.\n          const readyForMore = this.#storer.readyForMore();\n          if (readyForMore) {\n            await stream.changes.doneOr(readyForMore);\n          }\n        }\n      } catch (e) {\n        err = e;\n      } finally {\n        this.#stream?.changes.cancel();\n        this.#stream = undefined;\n      }\n\n      // When the change stream is interrupted, abort any pending transaction.\n      if (watermark) {\n        this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n        this.#storer.abort();\n        this.#forwarder.forward([watermark, 'rollback', ROLLBACK_JSON]);\n      }\n\n      // Backoff and drain any pending entries in the storer before reconnecting.\n      await Promise.all([\n        this.#storer.stop(),\n        this.#state.backoff(this.#lc, err),\n        this.#state.retryDelay > REPLICATION_STATUS_ERROR_DELAY_THRESHOLD_MS\n          ? publishCriticalEvent(\n              this.#lc,\n              replicationStatusError(this.#lc, 'Replicating', err),\n            )\n          : promiseVoid,\n      ]);\n    }\n\n    this.#forwarder.stopProgressMonitor();\n    this.#lc.info?.('ChangeStreamer stopped');\n  }\n\n  async #handleControlMessage(msg: ChangeStreamControl[1]) {\n    this.#lc.info?.('received control message', msg);\n    const {tag} = msg;\n\n    switch (tag) {\n      case 'reset-required':\n        await markResetRequired(this.#changeDB, this.#shard);\n        await publishReplicationError(\n          this.#lc,\n          'Replicating',\n          msg.message ?? 'Resync required',\n          msg.errorDetails,\n        );\n        if (this.#autoReset) {\n          this.#lc.warn?.('shutting down for auto-reset');\n          await this.stop(new AutoResetSignal());\n        }\n        break;\n      default:\n        unreachable(tag);\n    }\n  }\n\n  subscribe(ctx: SubscriberContext): Promise<Source<string>> {\n    const {protocolVersion, id, mode, replicaVersion, watermark} = ctx;\n    if (mode === 'serving') {\n      this.#serving.resolve();\n    }\n    const downstream = Subscription.create<string>({\n      cleanup: () => this.#forwarder.remove(subscriber),\n    });\n    const subscriber = new Subscriber(\n      protocolVersion,\n      id,\n      watermark,\n      downstream,\n      () => this.#latestStatus,\n    );\n    if (replicaVersion !== this.#replicaVersion) {\n      this.#lc.warn?.(\n        `rejecting subscriber at replica version ${replicaVersion}`,\n      );\n      subscriber.close(\n        ErrorType.WrongReplicaVersion,\n        `current replica version is ${\n          this.#replicaVersion\n        } (requested ${replicaVersion})`,\n      );\n    } else {\n      this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n      this.#forwarder.add(subscriber);\n      this.#storer.catchup(subscriber, mode);\n    }\n    return Promise.resolve(downstream);\n  }\n\n  scheduleCleanup(watermark: string) {\n    const origSize = this.#initialWatermarks.size;\n    this.#initialWatermarks.add(watermark);\n\n    if (origSize === 0) {\n      this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n    }\n  }\n\n  async getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }> {\n    const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n    if (!minWatermark) {\n      this.#lc.warn?.(\n        `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n      );\n    }\n    return {\n      replicaVersion: this.#replicaVersion,\n      minWatermark: minWatermark ?? this.#replicaVersion,\n    };\n  }\n\n  /**\n   * Makes a best effort to purge the change log. In the event of a database\n   * error, exceptions will be logged and swallowed, so this method is safe\n   * to run in a timeout.\n   */\n  async #purgeOldChanges(): Promise<void> {\n    const initial = [...this.#initialWatermarks];\n    if (initial.length === 0) {\n      this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n      return;\n    }\n    const current = [...this.#forwarder.getAcks()];\n    if (current.length === 0) {\n      // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n      // Bail to be safe.\n      this.#lc.warn?.('No subscribers to confirm cleanup');\n      return;\n    }\n    try {\n      const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n      const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n      if (earliestCurrent < earliestInitial) {\n        this.#lc.info?.(\n          `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n        );\n      } else {\n        this.#lc.info?.(`Purging changes before ${earliestInitial} ...`);\n        const start = performance.now();\n        const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n        const elapsed = (performance.now() - start).toFixed(2);\n        this.#lc.info?.(\n          `Purged ${deleted} changes before ${earliestInitial} (${elapsed} ms)`,\n        );\n        this.#initialWatermarks.delete(earliestInitial);\n      }\n    } catch (e) {\n      this.#lc.warn?.(`error purging change log`, e);\n    } finally {\n      if (this.#initialWatermarks.size) {\n        // If there are unpurged watermarks to check, schedule the next purge.\n        this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n      }\n    }\n  }\n\n  async stop(err?: unknown) {\n    this.#state.stop(this.#lc, err);\n    this.#stream?.changes.cancel();\n    await Promise.allSettled([this.#storer.stop(), this.#source.stop()]);\n  }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n//    `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS * 3;\n\nconst ROLLBACK_JSON = JSON.stringify([\n  'rollback',\n  {tag: 'rollback'},\n] satisfies Rollback);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA+DA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,4BACA,mBACA,WACA,WACA,MACA,eAAe,YACiB;CAEhC,MAAM,yBAAyB,IAAI,UAAU,KAAK;CAClD,MAAM,wBACJ,IACA,UACA,mBACA,OACA,WACA,YACF;CAEA,MAAM,EAAC,mBAAkB;CACzB,OAAO,IAAI,mBACT,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,cACA,4BACA,WACA,WACA,MACA,YACF;AACF;AAEA,IAAM,8CAA8C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkKpD,IAAM,qBAAN,MAA0D;CACxD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA;CACA;CACA,qCAA8B,IAAI,IAAY;CAU9C,WAAoB,SAAS;CAE7B,aAAsB,mBACpB,eACA,gBACA,kCACF;CACA,iBAA0B,mBACxB,eACA,WACA,qDACF;CAEA;CACA;CACA;CAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,4BACA,kBACA,WACA,MACA,eAAe,YACf;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GAAG,YAAY,aAAa,iBAAiB;EACxD,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,UAAU,IAAI,OACjB,IACA,OACA,QACA,kBACA,mBACA,UACA,iBACA,aAAY,KAAKS,SAAS,KAAK,KAAK;GAAC;GAAU,SAAS;GAAI,SAAS;EAAE,CAAC,IACxE,QAAO,KAAK,KAAK,GAAG,GACpB,IACF;EACA,KAAKR,aAAa,IAAI,UAAU,IAAI,EAClC,oCACE,KAAK,mCACT,CAAC;EACD,KAAKC,8BAA8B;EACnC,KAAKQ,aAAa;EAClB,KAAKP,aAAa;EAClB,KAAKC,SAAS,IAAI,aAAa,KAAK,IAAI,KAAA,GAAW,YAAY;EAC/D,KAAKO,gBAAgB,EAAC,KAAK,SAAQ;CACrC;CAEA,MAAM,MAAM;EACV,KAAKhB,IAAI,OAAO,wBAAwB;EAExC,KAAKM,WAAW,qBAAqB;EAErC,MAAM,YAAY,MAAM,KAAKF,QAAQ,iBAAiB;EACtD,IAAI,WACF,KAAKY,cAAc,YAAY;EAKjC,MAAM,KAAKX,QAAQ,gBAAgB,KAAKU,UAAU;EAClD,KAAKA,aAAa;EAIlB,MAAM,sBAAsB,wBAAwB,KAAK;EAEzD,OAAO,KAAKN,OAAO,UAAU,GAAG;GAC9B,IAAI;GACJ,IAAI,YAA2B;GAC/B,IAAI,iBAAiB;GACrB,IAAI;IACF,MAAM,EAAC,eAAe,qBACpB,MAAM,KAAKJ,QAAQ,uCAAuC;IAC5D,MAAM,SAAS,MAAM,KAAKD,QAAQ,YAChC,eACA,gBACF;IACA,KAAKC,QAAQ,IAAI,EAAE,OAAM,MAAK,OAAO,QAAQ,OAAO,CAAC,CAAC;IAEtD,KAAKS,UAAU;IACf,IACE,KAAKL,OAAO,aAAa,IACzB,6CAIA,KAAKF,4BAA4B,QAC/B,KAAKP,KACL,eACA,oBAAoB,eACtB;IAEF,YAAY;IAEZ,WAAW,MAAM,UAAU,OAAO,SAAS;KACzC,MAAM,CAAC,MAAM,OAAO;KACpB,QAAQ,MAAR;MACE,KAAK;OACH,IAAI,IAAI,KACN,KAAKK,QAAQ,OAAO,MAAM;OAE5B,IAAI,IAAI,WAAW;QAKjB,KAAKW,cAAc,YAAY,IAAI;QACnC,KAAKV,WAAW,WAAW,KAAKU,aAAa;OAC/C;OACA;MACF,KAAK;OACH,MAAM,KAAKC,sBAAsB,GAAG;OACpC;MACF,KAAK;OACH,YAAY,OAAO,GAAG;OACtB;MACF,KAAK;OACH,IAAI,cAAc,OAAO,GAAG,WAC1B,MAAM,IAAI,mBACR,oBAAoB,OAAO,GAAG,UAAU,oCAAoC,WAC9E;OAEF,KAAKL,WAAW,IAAI,CAAC;OACrB;MACF;OACE,IAAI,SAAS,QACX,KAAKC,eAAe,IAAI,CAAC;OAE3B,IAAI,cAAc,MAChB,MAAM,IAAI,mBACR,GAAG,KAAK,WAAW,IAAI,IAAI,kCAC7B;OAEF;KACJ;KAEA,MAAM,OAAO,KAAKR,QAAQ,MAAM,WAAW,MAAM;KACjD,MAAM,QAA2B;MAAC;MAAW,OAAO,GAAG;MAAK;KAAI;KAChE,kBAAkB,KAAK;KACvB,IAAI,iBAAiB,qBAEnB,KAAKC,WAAW,QAAQ,KAAK;UACxB;MAQL,MAAM,YAAY,KAAKA,WAAW,uBAAuB,KAAK;MAC9D,MAAM,OAAO,QAAQ,OAAO,SAAS;MACrC,iBAAiB;KACnB;KAEA,IAAI,SAAS,YAAY,SAAS,YAChC,YAAY;KAId,MAAM,eAAe,KAAKD,QAAQ,aAAa;KAC/C,IAAI,cACF,MAAM,OAAO,QAAQ,OAAO,YAAY;IAE5C;GACF,SAAS,GAAG;IACV,MAAM;GACR,UAAU;IACR,KAAKS,SAAS,QAAQ,OAAO;IAC7B,KAAKA,UAAU,KAAA;GACjB;GAGA,IAAI,WAAW;IACb,KAAKd,IAAI,OAAO,oCAAoC,WAAW;IAC/D,KAAKK,QAAQ,MAAM;IACnB,KAAKC,WAAW,QAAQ;KAAC;KAAW;KAAY;IAAa,CAAC;GAChE;GAGA,MAAM,QAAQ,IAAI;IAChB,KAAKD,QAAQ,KAAK;IAClB,KAAKI,OAAO,QAAQ,KAAKT,KAAK,GAAG;IACjC,KAAKS,OAAO,aAAa,8CACrB,qBACE,KAAKT,KACL,uBAAuB,KAAKA,KAAK,eAAe,GAAG,CACrD,IACA;GACN,CAAC;EACH;EAEA,KAAKM,WAAW,oBAAoB;EACpC,KAAKN,IAAI,OAAO,wBAAwB;CAC1C;CAEA,MAAMiB,sBAAsB,KAA6B;EACvD,KAAKjB,IAAI,OAAO,4BAA4B,GAAG;EAC/C,MAAM,EAAC,QAAO;EAEd,QAAQ,KAAR;GACE,KAAK;IACH,MAAM,kBAAkB,KAAKE,WAAW,KAAKD,MAAM;IACnD,MAAM,wBACJ,KAAKD,KACL,eACA,IAAI,WAAW,mBACf,IAAI,YACN;IACA,IAAI,KAAKQ,YAAY;KACnB,KAAKR,IAAI,OAAO,8BAA8B;KAC9C,MAAM,KAAK,KAAK,IAAI,gBAAgB,CAAC;IACvC;IACA;GACF,SACE,YAAY,GAAG;EACnB;CACF;CAEA,UAAU,KAAiD;EACzD,MAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,cAAa;EAC/D,IAAI,SAAS,WACX,KAAKW,SAAS,QAAQ;EAExB,MAAM,aAAa,aAAa,OAAe,EAC7C,eAAe,KAAKL,WAAW,OAAO,UAAU,EAClD,CAAC;EACD,MAAM,aAAa,IAAI,WACrB,iBACA,IACA,WACA,kBACM,KAAKU,aACb;EACA,IAAI,mBAAmB,KAAKb,iBAAiB;GAC3C,KAAKH,IAAI,OACP,2CAA2C,gBAC7C;GACA,WAAW,MACT,GACA,8BACE,KAAKG,gBACN,cAAc,eAAe,EAChC;EACF,OAAO;GACL,KAAKH,IAAI,QAAQ,qBAAqB,WAAW,IAAI;GAErD,KAAKM,WAAW,IAAI,UAAU;GAC9B,KAAKD,QAAQ,QAAQ,YAAY,IAAI;EACvC;EACA,OAAO,QAAQ,QAAQ,UAAU;CACnC;CAEA,gBAAgB,WAAmB;EACjC,MAAM,WAAW,KAAKK,mBAAmB;EACzC,KAAKA,mBAAmB,IAAI,SAAS;EAErC,IAAI,aAAa,GACf,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;CAE1E;CAEA,MAAM,oBAGH;EACD,MAAM,eAAe,MAAM,KAAKb,QAAQ,0BAA0B;EAClE,IAAI,CAAC,cACH,KAAKL,IAAI,OACP,8EACF;EAEF,OAAO;GACL,gBAAgB,KAAKG;GACrB,cAAc,gBAAgB,KAAKA;EACrC;CACF;;;;;;CAOA,MAAMe,mBAAkC;EACtC,MAAM,UAAU,CAAC,GAAG,KAAKR,kBAAkB;EAC3C,IAAI,QAAQ,WAAW,GAAG;GACxB,KAAKV,IAAI,OAAO,4CAA4C;GAC5D;EACF;EACA,MAAM,UAAU,CAAC,GAAG,KAAKM,WAAW,QAAQ,CAAC;EAC7C,IAAI,QAAQ,WAAW,GAAG;GAGxB,KAAKN,IAAI,OAAO,mCAAmC;GACnD;EACF;EACA,IAAI;GACF,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,MAAM,kBAAkB,IAAI,GAAI,OAAmC;GACnE,IAAI,kBAAkB,iBACpB,KAAKA,IAAI,OACP,yCAAyC,gBAAgB,KAAK,gBAAgB,EAChF;QACK;IACL,KAAKA,IAAI,OAAO,0BAA0B,gBAAgB,KAAK;IAC/D,MAAM,QAAQ,YAAY,IAAI;IAC9B,MAAM,UAAU,MAAM,KAAKK,QAAQ,mBAAmB,eAAe;IACrE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO,QAAQ,CAAC;IACrD,KAAKL,IAAI,OACP,UAAU,QAAQ,kBAAkB,gBAAgB,IAAI,QAAQ,KAClE;IACA,KAAKU,mBAAmB,OAAO,eAAe;GAChD;EACF,SAAS,GAAG;GACV,KAAKV,IAAI,OAAO,4BAA4B,CAAC;EAC/C,UAAU;GACR,IAAI,KAAKU,mBAAmB,MAE1B,KAAKD,OAAO,iBAAiB,KAAKS,iBAAiB,GAAG,gBAAgB;EAE1E;CACF;CAEA,MAAM,KAAK,KAAe;EACxB,KAAKT,OAAO,KAAK,KAAKT,KAAK,GAAG;EAC9B,KAAKc,SAAS,QAAQ,OAAO;EAC7B,MAAM,QAAQ,WAAW,CAAC,KAAKT,QAAQ,KAAK,GAAG,KAAKD,QAAQ,KAAK,CAAC,CAAC;CACrE;AACF;AAiBA,IAAM,mBAAmB,6BAA6B;AAEtD,IAAM,gBAAgB,KAAK,UAAU,CACnC,YACA,EAAC,KAAK,WAAU,CAClB,CAAoB"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -12,6 +12,7 @@ export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
     changeLogBatchSize: number;
+    drainTimeoutMs?: number | undefined;
 };
 /**
  * Handles the storage of changes and the catchup of subscribers
@@ -46,7 +47,7 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
     getStartStreamInitializationParameters(): Promise<{
         lastWatermark: string;
@@ -72,6 +73,11 @@ export declare class Storer implements Service {
      * This is only used in tests.
      */
     allProcessed(): Promise<void>;
+    /**
+     * Stops the storer and waits up to a drain timeout for entries to drain,
+     * throwing an exception if it doesn't drain in time, in order to abort
+     * the server when PG (or the connection to it) appears to be wedged.
+     */
     stop(): Promise<void>;
 }
 /**

package/out/zero-cache/src/services/change-streamer/storer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAmBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,GACnB,EAAE,aAAa;IAgCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAMpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IAyEzC;;;OAGG;IACG,GAAG;IA0eT;;;OAGG;IACG,YAAY;IAQlB,IAAI;CAOL;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}
+{"version":3,"file":"storer.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/storer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,eAAe,EAAC,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAC,KAAK,UAAU,EAA2B,MAAM,mBAAmB,CAAC;AAC5E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,MAAM,EAKZ,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,MAAM,EACZ,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EACV,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,6CAA6C,CAAC;AACrD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAS3C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAuChD,MAAM,MAAM,aAAa,GAAG;IAC1B,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC,CAAC;AAaF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,MAAO,YAAW,OAAO;;IACpC,QAAQ,CAAC,EAAE,YAAY;gBAoBrB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,EACxB,iBAAiB,EAAE,MAAM,EACzB,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,qBAAqB,KAAK,IAAI,EACvD,OAAO,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,EAC7B,EACE,+BAA+B,EAC/B,kBAAkB,EAClB,kBAAkB,EAClB,cAAuB,GACxB,EAAE,aAAa;IAiCZ,eAAe,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI;IA2B5C,sCAAsC,IAAI,OAAO,CAAC;QACtD,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,eAAe,EAAE,CAAC;KACrC,CAAC;IAkCI,yBAAyB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOzD,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAyCtD;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB;IAkB/C,KAAK;IAIL,MAAM,CAAC,CAAC,EAAE,uBAAuB;IAIjC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,cAAc;IAOpD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;IA6FzC;;;OAGG;IACG,GAAG;IA2eT;;;OAGG;IACG,YAAY;IAQlB;;;;OAIG;IACG,IAAI;CAaX;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,iBAAiB,EAAE,MAAM,EACzB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,UAmB/B;AA+BD,qBAAa,SAAS;;IAGpB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAG5B,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,eAAe,EACnB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM;IAUb,OAAO;CAWd;AAED,qBAAa,WAAW;;gBAKV,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU;IAWpD,OAAO;CA4Bd"}

package/out/zero-cache/src/services/change-streamer/storer.js

@@ -14,6 +14,7 @@ import { runTx } from "../../db/run-transaction.js";
 import { AutoResetSignal, markResetRequired } from "./schema/tables.js";
 import { Queue } from "../../../../shared/src/queue.js";
 import { TransactionPool } from "../../db/transaction-pool.js";
+import { orTimeout } from "../../types/timeout.js";
 import { resolver } from "@rocicorp/resolver";
 import "postgres";
 import { getHeapStatistics } from "node:v8";
@@ -64,9 +65,10 @@ var Storer = class {
 	#backPressureThresholdBytes;
 	#statementTimeoutMs;
 	#changeLogBatchSize;
+	#drainTimeoutMs;
 	#approximateQueuedBytes = 0;
 	#running = false;
-	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize }) {
+	constructor(lc, shard, taskID, discoveryAddress, discoveryProtocol, db, replicaVersion, onConsumed, onFatal, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, drainTimeoutMs = 3e4 }) {
 		this.#lc = lc.withContext("component", "change-log");
 		this.#shard = shard;
 		this.#taskID = taskID;
@@ -78,6 +80,7 @@ var Storer = class {
 		this.#onFatal = onFatal;
 		this.#statementTimeoutMs = statementTimeoutMs;
 		this.#changeLogBatchSize = Math.max(1, changeLogBatchSize);
+		this.#drainTimeoutMs = drainTimeoutMs;
 		const heapStats = getHeapStatistics();
 		this.#backPressureThresholdBytes = (heapStats.heap_size_limit - heapStats.used_heap_size) * backPressureLimitHeapProportion;
 		this.#lc.info?.(`Using up to ${(this.#backPressureThresholdBytes / 1024 ** 2).toFixed(2)} MB of --max-old-space-size (~${(heapStats.heap_size_limit / 1024 ** 2).toFixed(2)} MB) to absorb upstream spikes`, { heapStats });
@@ -183,20 +186,30 @@ var Storer = class {
 		}]);
 	}
 	#readyForMore = null;
+	#backpressureTimeout;
 	readyForMore() {
 		if (!this.#running) return;
 		if (this.#readyForMore === null && this.#approximateQueuedBytes > this.#backPressureThresholdBytes) {
 			this.#lc.warn?.(`applying back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)\n\nTo inspect changeLog backlog in your change DB:\n  SELECT\n    (change->'relation'->>'schema') || '.' || (change->'relation'->>'name') AS table_name,\n    change->>'tag' AS operation,\n    COUNT(*) AS count\n  FROM "<app_id>/cdc"."changeLog"\n  GROUP BY 1, 2\n  ORDER BY 3 DESC\n  LIMIT 20;`);
 			this.#readyForMore = resolver();
+			this.#resetBackpressureTimeout(true);
 		}
 		return this.#readyForMore?.promise;
 	}
+	#resetBackpressureTimeout(reschedule) {
+		clearTimeout(this.#backpressureTimeout);
+		this.#backpressureTimeout = reschedule ? setTimeout(() => {
+			this.#readyForMore?.reject(new AbortError(`No statement processed within the last ${this.#statementTimeoutMs}ms. Considering the change-log to be wedged.`));
+			this.#readyForMore = null;
+		}, this.#statementTimeoutMs) : void 0;
+	}
 	#maybeReleaseBackPressure() {
-		if (this.#readyForMore !== null && this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
+		if (this.#readyForMore !== null) if (this.#approximateQueuedBytes < this.#backPressureThresholdBytes * .8) {
 			this.#lc.info?.(`releasing back pressure with ${this.#queue.size()} queued changes (~${(this.#approximateQueuedBytes / 1024 ** 2).toFixed(2)} MB)`);
 			this.#readyForMore.resolve();
 			this.#readyForMore = null;
-		}
+			this.#resetBackpressureTimeout(false);
+		} else this.#resetBackpressureTimeout(true);
 	}
 	/**
 	* Flushes any buffered {@link PendingTransaction.batch} rows to the changeLog.
@@ -246,6 +259,7 @@ var Storer = class {
 				this.#readyForMore.resolve();
 				this.#readyForMore = null;
 			}
+			this.#resetBackpressureTimeout(false);
 			this.#cancelQueueEntries(this.#queue.drain().filter((entry) => entry !== void 0), err);
 			this.#running = false;
 			signalStopped();
@@ -533,12 +547,17 @@ var Storer = class {
 			await promise;
 		}
 	}
-	stop() {
+	/**
+	* Stops the storer and waits up to a drain timeout for entries to drain,
+	* throwing an exception if it doesn't drain in time, in order to abort
+	* the server when PG (or the connection to it) appears to be wedged.
+	*/
+	async stop() {
 		if (this.#running) {
 			this.#lc.info?.(`draining ${this.#queue.size()} changeLog entries`);
 			this.#queue.enqueue("stop");
 		}
-		return this.#stopped;
+		if (await orTimeout(this.#stopped, this.#drainTimeoutMs) === "timed-out") throw new AbortError(`changeLog did not drain within ${this.#drainTimeoutMs}ms`);
 	}
 };
 /**