@rocicorp/zero 1.7.0 → 1.8.0-canary.0

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => parseStreamMessage(lc, buffer, parser),\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,GAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,GACxE;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAEA,MAAM,gBAAgB,kBAAkB;EAEtC,IADY,KAAK,IACb,IAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;CACF,GAAG,yBAAyB,CAAC;CAE7B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU,mBAAmB,IAAI,QAAQ,MAAM;EAItD,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Postgres promises to send a keepalive at roughly wal_sender_timeout / 2.\n  // If we go long enough without receiving anything (data or keepalive), the\n  // inbound half of the connection is silently dead — our outbound acks keep\n  // flowing into the void, neither side errors, and the change-source sits\n  // there forever. Tear down the readable to force a reconnect.\n  //\n  // The threshold is 2x wal_sender_timeout — i.e. two consecutive keepalives\n  // missed — to avoid spurious reconnects from a single delayed keepalive\n  // (PG scheduling jitter under load, our own event-loop stalls, etc.).\n  const inboundTimeoutMs = walSenderTimeoutMs * 2;\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n    const sinceLastReceived = now - lastReceivedTime;\n    if (sinceLastReceived > inboundTimeoutMs && !readable.destroyed) {\n      lc.warn?.(\n        `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n          `(> 2x wal_sender_timeout ${walSenderTimeoutMs}ms). Destroying ` +\n          `replication stream to force reconnect.`,\n      );\n      readable.destroy(\n        new Error(\n          `replication stream inbound timeout after ${sinceLastReceived}ms`,\n        ),\n      );\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,GAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,GACxE;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAWA,MAAM,mBAAmB,qBAAqB;CAC9C,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAAkB;EACtC,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,IAAI,oBAAoB,oBAAoB,CAAC,SAAS,WAAW;GAC/D,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,8BACtC,mBAAmB,uDAEnD;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,yBAAyB,CAAC;CAE7B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts

@@ -0,0 +1,18 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { NormalizedZeroConfig } from '../../config/normalize.ts';
+import type { BackupMonitor } from './backup-monitor.ts';
+import type { ChangeStreamerService } from './change-streamer.ts';
+import { type BackupStateVerifier } from './litestream3-backup-monitor.ts';
+import { type VfsBackupWatermarkSource } from './vfs-backup-monitor.ts';
+export type BackupCleanupMonitorFactoryOptions = {
+    lc: LogContext;
+    config: NormalizedZeroConfig;
+    replicaFile: string;
+    changeStreamer: ChangeStreamerService;
+    initialCleanupDelayMs: number;
+    verifyBackupState?: BackupStateVerifier | undefined;
+    vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;
+    env?: NodeJS.ProcessEnv | undefined;
+};
+export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env, }: BackupCleanupMonitorFactoryOptions): BackupMonitor | null;
+//# sourceMappingURL=backup-cleanup-monitor-factory.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACpD,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,SAAS,CAAC;IAChE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,wBAAwB,EACxB,GAAG,GACJ,EAAE,kCAAkC,GAAG,aAAa,GAAG,IAAI,CAgC3D"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -0,0 +1,17 @@
+import { forkChildWorker } from "../../types/processes.js";
+import { BACKUP_WATERMARK_READER_URL } from "../../server/worker-urls.js";
+import { getLastBackupTime } from "../litestream/commands.js";
+import { VfsBackupWatermarkWorkerSource } from "../litestream/vfs-watermark-worker-source.js";
+import { Litestream3BackupMonitor } from "./litestream3-backup-monitor.js";
+import { VfsBackupMonitor } from "./vfs-backup-monitor.js";
+//#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
+function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env }) {
+	const { backupURL, port: metricsPort } = config.litestream;
+	if (!backupURL) return null;
+	if (config.litestream.backupUsingV5) return new VfsBackupMonitor(lc, backupURL, changeStreamer, initialCleanupDelayMs, config.litestream.vfsProbeIntervalMs, vfsBackupWatermarkSource ?? new VfsBackupWatermarkWorkerSource(lc, () => forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env), config.litestream.vfsProbeTimeoutMs));
+	return new Litestream3BackupMonitor(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, initialCleanupDelayMs, verifyBackupState ?? (() => getLastBackupTime(lc, config)));
+}
+//#endregion
+export { createBackupCleanupMonitor };
+
+//# sourceMappingURL=backup-cleanup-monitor-factory.js.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_READER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {VfsBackupWatermarkWorkerSource} from '../litestream/vfs-watermark-worker-source.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  Litestream3BackupMonitor,\n  type BackupStateVerifier,\n} from './litestream3-backup-monitor.ts';\nimport {\n  VfsBackupMonitor,\n  type VfsBackupWatermarkSource,\n} from './vfs-backup-monitor.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  initialCleanupDelayMs: number;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  initialCleanupDelayMs,\n  verifyBackupState,\n  vfsBackupWatermarkSource,\n  env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor | null {\n  const {backupURL, port: metricsPort} = config.litestream;\n  if (!backupURL) {\n    return null;\n  }\n\n  if (config.litestream.backupUsingV5) {\n    return new VfsBackupMonitor(\n      lc,\n      backupURL,\n      changeStreamer,\n      initialCleanupDelayMs,\n      config.litestream.vfsProbeIntervalMs,\n      vfsBackupWatermarkSource ??\n        new VfsBackupWatermarkWorkerSource(\n          lc,\n          () =>\n            forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env),\n          config.litestream.vfsProbeTimeoutMs,\n        ),\n    );\n  }\n\n  return new Litestream3BackupMonitor(\n    lc,\n    replicaFile,\n    backupURL,\n    `http://localhost:${metricsPort}/metrics`,\n    changeStreamer,\n    initialCleanupDelayMs,\n    verifyBackupState ?? (() => getLastBackupTime(lc, config)),\n  );\n}\n"],"mappings":";;;;;;;AA4BA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,uBACA,mBACA,0BACA,OAC2D;CAC3D,MAAM,EAAC,WAAW,MAAM,gBAAe,OAAO;CAC9C,IAAI,CAAC,WACH,OAAO;CAGT,IAAI,OAAO,WAAW,eACpB,OAAO,IAAI,iBACT,IACA,WACA,gBACA,uBACA,OAAO,WAAW,oBAClB,4BACE,IAAI,+BACF,UAEE,gBAAgB,6BAA6B,OAAO,QAAQ,GAAG,GACjE,OAAO,WAAW,iBACpB,CACJ;CAGF,OAAO,IAAI,yBACT,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,gBACA,uBACA,4BAA4B,kBAAkB,IAAI,MAAM,EAC1D;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -1,93 +1,8 @@
-import type { LogContext } from '@rocicorp/logger';
-import { Subscription } from '../../types/subscription.ts';
-import type { Service } from '../service.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
+import type { Subscription } from '../../types/subscription.ts';
+import type { SingletonService } from '../service.ts';
 import type { SnapshotMessage } from './snapshot.ts';
-export declare const CHECK_INTERVAL_MS = 60000;
-/**
- * Allowance for clock skew between the machine reporting litestream metrics
- * and the timestamps reported by the backup destination (e.g. S3).
- */
-export declare const BACKUP_VERIFICATION_SLACK_MS = 60000;
-/**
- * How long the actual backup state may remain *continuously* behind the
- * backup progress claimed by litestream metrics before the backup is
- * considered genuinely wedged and the process is shut down.
- *
- * A wedged backup is dangerous: litestream believes it is making durable
- * progress when it is not, which can silently corrupt the backup (the WAL
- * format only makes *some* such gaps detectable). Once that is confirmed to
- * be the persistent state, continuing to run is worse than crashing, so the
- * process exits loudly (non-zero, with a logged error) and the wedged backup
- * destination becomes the priority to investigate.
- *
- * The change-log is conservatively *not* purged for the entire time the
- * backup is stale, so the only cost of a generous grace period is unbounded
- * change-log growth. We therefore err well on the side of slack: the backup
- * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear
- * the server down, so that a transient hiccup (a slow or briefly unreachable
- * destination, or litestream restarting) does not trigger a shutdown. The
- * staleness clock is reset the moment a purge is confirmed against a real
- * upload.
- */
-export declare const WEDGED_SHUTDOWN_GRACE_MS: number;
-/**
- * Returns the time of the most recent object actually uploaded to the
- * backup replica destination (e.g. as determined by listing the snapshots
- * and WAL segments in S3). Rejects if the backup state cannot be determined.
- *
- * See `getLastBackupTime()` in `../litestream/commands.ts` for the
- * production implementation.
- */
-export type BackupStateVerifier = () => Promise<Date>;
-/**
- * The BackupMonitor polls the litestream "/metrics" endpoint to track the
- * watermark (label) value of the `litestream_replica_progress` gauge and
- * schedules cleanup of change log entries that can be purged as a result.
- *
- * See: https://github.com/rocicorp/litestream/pull/3
- *
- * Note that change log entries cannot simply be purged as soon as they
- * have been applied and backed up by litestream. Consider the case in which
- * litestream backs up new wal segments every minute, but it takes 5 minutes
- * to restore a replica: if a zero-cache starts restoring a replica at
- * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
- * purging changelog records as soon as those watermarks are replicated would
- * result in the zero-cache not being able to catch up from minute 0 once it
- * has finished restoring the replica.
- *
- * The `/snapshot` reservation protocol is used to prevent premature change
- * log cleanup:
- * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
- *   request open while it restores its snapshot, prepares it, and
- *   starts its subscription to the change stream. During this time, no
- *   cleanups are scheduled.
- * - When the subscription is started, the interval since the beginning of
- *   of the reservation is tracked to increase the background cleanup delay
- *   interval if needed. The reservation is ended (and request closed), and
- *   cleanup scheduling is resumed with the current delay interval.
- *
- * Note that the reservation request is the primary mechanism by which
- * premature change log cleanup is prevented. The cleanup delay interval is
- * a secondary safeguard.
- *
- * Additionally, because the watermarks reported by litestream metrics
- * reflect what litestream *believes* has been backed up (which has been
- * observed to diverge from reality when uploads silently fail), the cleanup
- * watermark is only advanced after verifying it against the actual backup
- * state in the replica destination via a {@link BackupStateVerifier}. If the
- * actual backup state stays behind the claimed progress for longer than
- * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
- * process is shut down rather than risk corrupting the backup.
- */
-export declare class BackupMonitor implements Service {
-    #private;
-    readonly id = "backup-monitor";
-    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, verifyBackupState: BackupStateVerifier);
-    run(): Promise<void>;
+export interface BackupMonitor extends SingletonService {
     startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
-    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
-    stop(): Promise<void>;
+    endReservation(taskID: string, updateCleanupDelay?: boolean | undefined): void;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAGxC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAOtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAsC7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,iBAAiB,EAAE,mBAAmB;IAkBxC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAcpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAqMF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;IACxE,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,GACvC,IAAI,CAAC;CACT"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts

@@ -0,0 +1,93 @@
+import type { LogContext } from '@rocicorp/logger';
+import { Subscription } from '../../types/subscription.ts';
+import type { BackupMonitor } from './backup-monitor.ts';
+import type { ChangeStreamerService } from './change-streamer.ts';
+import type { SnapshotMessage } from './snapshot.ts';
+export declare const CHECK_INTERVAL_MS = 60000;
+/**
+ * Allowance for clock skew between the machine reporting litestream metrics
+ * and the timestamps reported by the backup destination (e.g. S3).
+ */
+export declare const BACKUP_VERIFICATION_SLACK_MS = 60000;
+/**
+ * How long the actual backup state may remain *continuously* behind the
+ * backup progress claimed by litestream metrics before the backup is
+ * considered genuinely wedged and the process is shut down.
+ *
+ * A wedged backup is dangerous: litestream believes it is making durable
+ * progress when it is not, which can silently corrupt the backup (the WAL
+ * format only makes *some* such gaps detectable). Once that is confirmed to
+ * be the persistent state, continuing to run is worse than crashing, so the
+ * process exits loudly (non-zero, with a logged error) and the wedged backup
+ * destination becomes the priority to investigate.
+ *
+ * The change-log is conservatively *not* purged for the entire time the
+ * backup is stale, so the only cost of a generous grace period is unbounded
+ * change-log growth. We therefore err well on the side of slack: the backup
+ * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear
+ * the server down, so that a transient hiccup (a slow or briefly unreachable
+ * destination, or litestream restarting) does not trigger a shutdown. The
+ * staleness clock is reset the moment a purge is confirmed against a real
+ * upload.
+ */
+export declare const WEDGED_SHUTDOWN_GRACE_MS: number;
+/**
+ * Returns the time of the most recent object actually uploaded to the
+ * backup replica destination (e.g. as determined by listing the snapshots
+ * and WAL segments in S3). Rejects if the backup state cannot be determined.
+ *
+ * See `getLastBackupTime()` in `../litestream/commands.ts` for the
+ * production implementation.
+ */
+export type BackupStateVerifier = () => Promise<Date>;
+/**
+ * The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+ * track the watermark (label) value of the `litestream_replica_progress` gauge
+ * and schedules cleanup of change log entries that can be purged as a result.
+ *
+ * See: https://github.com/rocicorp/litestream/pull/3
+ *
+ * Note that change log entries cannot simply be purged as soon as they
+ * have been applied and backed up by litestream. Consider the case in which
+ * litestream backs up new wal segments every minute, but it takes 5 minutes
+ * to restore a replica: if a zero-cache starts restoring a replica at
+ * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+ * purging changelog records as soon as those watermarks are replicated would
+ * result in the zero-cache not being able to catch up from minute 0 once it
+ * has finished restoring the replica.
+ *
+ * The `/snapshot` reservation protocol is used to prevent premature change
+ * log cleanup:
+ * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
+ *   request open while it restores its snapshot, prepares it, and
+ *   starts its subscription to the change stream. During this time, no
+ *   cleanups are scheduled.
+ * - When the subscription is started, the interval since the beginning of
+ *   of the reservation is tracked to increase the background cleanup delay
+ *   interval if needed. The reservation is ended (and request closed), and
+ *   cleanup scheduling is resumed with the current delay interval.
+ *
+ * Note that the reservation request is the primary mechanism by which
+ * premature change log cleanup is prevented. The cleanup delay interval is
+ * a secondary safeguard.
+ *
+ * Additionally, because the watermarks reported by litestream metrics
+ * reflect what litestream *believes* has been backed up (which has been
+ * observed to diverge from reality when uploads silently fail), the cleanup
+ * watermark is only advanced after verifying it against the actual backup
+ * state in the replica destination via a {@link BackupStateVerifier}. If the
+ * actual backup state stays behind the claimed progress for longer than
+ * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
+ * process is shut down rather than risk corrupting the backup.
+ */
+export declare class Litestream3BackupMonitor implements BackupMonitor {
+    #private;
+    readonly id = "backup-monitor";
+    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, verifyBackupState: BackupStateVerifier);
+    run(): Promise<void>;
+    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
+    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
+    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=litestream3-backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream3-backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAGxC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAOtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,wBAAyB,YAAW,aAAa;;IAC5D,QAAQ,CAAC,EAAE,oBAAoB;gBAsC7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,iBAAiB,EAAE,mBAAmB;IAkBxC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAcpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAqMF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}

package/out/zero-cache/src/services/change-streamer/{backup-monitor.js → litestream3-backup-monitor.js}

@@ -6,7 +6,7 @@ import { RunningState, UnrecoverableError } from "../running-state.js";
 import { Subscription } from "../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
 import parsePrometheusTextFormat from "parse-prometheus-text-format";
-//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+//#region ../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts
 var CHECK_INTERVAL_MS = 6e4;
 var MIN_CLEANUP_DELAY_MS = 3e4;
 /**
@@ -37,9 +37,9 @@ var BACKUP_VERIFICATION_SLACK_MS = 6e4;
 */
 var WEDGED_SHUTDOWN_GRACE_MS = 15 * 6e4;
 /**
-* The BackupMonitor polls the litestream "/metrics" endpoint to track the
-* watermark (label) value of the `litestream_replica_progress` gauge and
-* schedules cleanup of change log entries that can be purged as a result.
+* The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+* track the watermark (label) value of the `litestream_replica_progress` gauge
+* and schedules cleanup of change log entries that can be purged as a result.
 *
 * See: https://github.com/rocicorp/litestream/pull/3
 *
@@ -76,7 +76,7 @@ var WEDGED_SHUTDOWN_GRACE_MS = 15 * 6e4;
 * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
 * process is shut down rather than risk corrupting the backup.
 */
-var BackupMonitor = class {
+var Litestream3BackupMonitor = class {
 	id = "backup-monitor";
 	#lc;
 	#replicaFile;
@@ -290,6 +290,6 @@ var BackupMonitor = class {
 	}
 };
 //#endregion
-export { BackupMonitor };
+export { Litestream3BackupMonitor };
 
-//# sourceMappingURL=backup-monitor.js.map
+//# sourceMappingURL=litestream3-backup-monitor.js.map

package/out/zero-cache/src/services/change-streamer/{backup-monitor.js.map → litestream3-backup-monitor.js.map}

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#verifyBackupState","#purgesBlocked","#fatal","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxWatermarkUpTo","#confirmedDurable","#lastVerifiedUploadTime","#backupStaleSince","#shutDownOnWedgedBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The BackupMonitor polls the litestream \"/metrics\" endpoint to track the\n * watermark (label) value of the `litestream_replica_progress` gauge and\n * schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n *\n * Additionally, because the watermarks reported by litestream metrics\n * reflect what litestream *believes* has been backed up (which has been\n * observed to diverge from reality when uploads silently fail), the cleanup\n * watermark is only advanced after verifying it against the actual backup\n * state in the replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class BackupMonitor implements Service {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  // Rejected when the backup is determined to be genuinely wedged, which\n  // surfaces as a rejection of the `run()` promise and shuts the process down.\n  readonly #fatal = resolver<void>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#verifyBackupState = verifyBackupState;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    // Resolves on a normal stop; rejects if the backup is found to be wedged.\n    return Promise.race([this.#state.stopped(), this.#fatal.promise]);\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    const maxWatermark = this.#maxWatermarkUpTo(latestCleanupTime);\n    if (maxWatermark.length === 0) {\n      return;\n    }\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    const claimedTime = must(this.#watermarks.get(maxWatermark));\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupState();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedWatermark = this.#maxWatermarkUpTo(\n      Math.min(\n        latestCleanupTime,\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n      ),\n    );\n    if (verifiedWatermark.length === 0) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    this.#changeStreamer.scheduleCleanup(verifiedWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    clearInterval(this.#checkMetricsTimer);\n    this.#fatal.reject(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxWatermarkUpTo(cutoff: number): string {\n    let max = '';\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;AAgBA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;AAM7B,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyD7C,IAAa,gBAAb,MAA8C;CAC5C,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAID,SAAkB,SAAe;CAEjC,iBAAyB;CACzB,oBAAiC;CACjC,0BAAuC;CAKvC,oBAAmC;CACnC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKC,kBAAkB;EACvB,KAAKI,qBAAqB;EAC1B,KAAKG,kBAAkB,KAAK,IAC1B,uBACA,oBACF;EAEA,KAAKX,IAAI,OACP,0BAA0B,sBAAsB,2BAClD;CACF;CAEA,MAAqB;EACnB,KAAKA,IAAI,OACP,yBAAyB,KAAKG,iBAAiB,QAC1C,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,qBAAqB,YACxB,KAAK,mCACL,iBACF;EACA,KAAKC,qBAAqB;EAE1B,OAAO,QAAQ,KAAK,CAAC,KAAKR,OAAO,QAAQ,GAAG,KAAKK,OAAO,OAAO,CAAC;CAClE;CAEA,yBAAyB,QAA+C;EACtE,KAAKV,IAAI,OAAO,oCAAoC,OAAO,WAAW;EAEtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAGvD,KAAUF,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKF;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKF,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EAEX,IAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;GAC5C,KAAKN,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKW,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKX,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,oCAA6C,YAAY;EACvD,IAAI;GACF,MAAM,KAAKc,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;CACF;CAEA,OAAOgB,mBAIJ;EACD,MAAM,kBAAkB,KAAKb;EAC7B,MAAM,SAAS,KAAKE,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKL,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMW,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKE,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKV,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKP,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKO,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKW,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMH,mBAAmB;EACvB,IAAI,KAAKT,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EACA,MAAM,oBAAoB,KAAK,IAAI,IAAI,KAAKK;EAC5C,MAAM,eAAe,KAAKQ,kBAAkB,iBAAiB;EAC7D,IAAI,aAAa,WAAW,GAC1B;EAWF,MAAM,cAAc,KAAK,KAAKZ,YAAY,IAAI,YAAY,CAAC;EAC3D,IAAI,CAAC,KAAKa,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKb,mBAAmB;EAC/D,SAAS,GAAG;GACV,KAAKC,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKT,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKqB,uBAAuB;EACpD,MAAM,oBAAoB,KAAKF,kBAC7B,KAAK,IACH,mBACA,WAAW,QAAQ,IAAI,4BACzB,CACF;EACA,IAAI,kBAAkB,WAAW,GAAG;GAClC,KAAKV,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKa,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,KAAKlB,gBAAgB,gBAAgB,iBAAiB;EACtD,KAAK,MAAM,aAAa,KAAKG,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKU,iBAAiB;CACxB;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKR,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKP,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,cAAc,KAAKY,kBAAkB;EACrC,KAAKF,OAAO,OAAO,GAAG;CACxB;;;;;CAMA,kBAAkB,QAAwB;EACxC,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKH,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAChD,MAAM;EAGV,OAAO;CACT;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKc;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;CAEA,OAAsB;EACpB,cAAc,KAAKT,kBAAkB;EACrC,KAAK,MAAM,EAAC,SAAQ,KAAKN,cAAc,OAAO,GAK5C,IAAI,OAAO;EAEb,KAAKD,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,KAAKc,iBAAiB;GACjD,IAAI,CAAC,cAAc;IACjB,KAAKd,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKC,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,QAAQ,CAAC;IAClE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKD,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}
+{"version":3,"file":"litestream3-backup-monitor.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#changeStreamer","#state","#reservations","#watermarks","#verifyBackupState","#purgesBlocked","#fatal","#cleanupDelayMs","#checkMetricsTimer","#initBackupLagMetric","#checkWatermarks","#scheduleCleanup","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxWatermarkUpTo","#confirmedDurable","#lastVerifiedUploadTime","#backupStaleSince","#shutDownOnWedgedBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {must} from '../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateGauge,\n} from '../../observability/metrics.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport type {SnapshotMessage} from './snapshot.ts';\n\nexport const CHECK_INTERVAL_MS = 60_000;\nconst MIN_CLEANUP_DELAY_MS = 30_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\ntype Reservation = {\n  start: Date;\n  sub: Subscription<SnapshotMessage>;\n};\n\n/**\n * The Litestream3BackupMonitor polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and schedules cleanup of change log entries that can be purged as a result.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Note that change log entries cannot simply be purged as soon as they\n * have been applied and backed up by litestream. Consider the case in which\n * litestream backs up new wal segments every minute, but it takes 5 minutes\n * to restore a replica: if a zero-cache starts restoring a replica at\n * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,\n * purging changelog records as soon as those watermarks are replicated would\n * result in the zero-cache not being able to catch up from minute 0 once it\n * has finished restoring the replica.\n *\n * The `/snapshot` reservation protocol is used to prevent premature change\n * log cleanup:\n * - Clients restoring a snapshot initiate a `/snapshot` request and hold that\n *   request open while it restores its snapshot, prepares it, and\n *   starts its subscription to the change stream. During this time, no\n *   cleanups are scheduled.\n * - When the subscription is started, the interval since the beginning of\n *   of the reservation is tracked to increase the background cleanup delay\n *   interval if needed. The reservation is ended (and request closed), and\n *   cleanup scheduling is resumed with the current delay interval.\n *\n * Note that the reservation request is the primary mechanism by which\n * premature change log cleanup is prevented. The cleanup delay interval is\n * a secondary safeguard.\n *\n * Additionally, because the watermarks reported by litestream metrics\n * reflect what litestream *believes* has been backed up (which has been\n * observed to diverge from reality when uploads silently fail), the cleanup\n * watermark is only advanced after verifying it against the actual backup\n * state in the replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3BackupMonitor implements BackupMonitor {\n  readonly id = 'backup-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #backupURL: string;\n  readonly #metricsEndpoint: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  readonly #reservations = new Map<string, Reservation>();\n  readonly #watermarks = new Map<string, Date>();\n\n  readonly #verifyBackupState: BackupStateVerifier;\n  readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n    description:\n      'Number of change-log purges blocked because the actual backup state ' +\n      '(as listed from the replica destination) could not be verified, or ' +\n      'is older than the backup progress claimed by litestream metrics. ' +\n      'A steadily increasing value indicates a wedged or failing backup. ' +\n      'The \"backup-wedged\" reason is emitted once just before the process ' +\n      'shuts itself down due to a persistently stale backup.',\n  });\n\n  // Rejected when the backup is determined to be genuinely wedged, which\n  // surfaces as a rejection of the `run()` promise and shuts the process down.\n  readonly #fatal = resolver<void>();\n\n  #lastWatermark: string = '';\n  #latestBackupTime: Date | null = null;\n  #lastVerifiedUploadTime: Date | null = null;\n  // Epoch ms at which the actual backup state was first observed to be\n  // continuously behind the watermark litestream claims to have backed up,\n  // or `null` while the backup is keeping up. Reset whenever a purge is\n  // confirmed against a real upload.\n  #backupStaleSince: number | null = null;\n  #cleanupDelayMs: number;\n  #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    backupURL: string,\n    metricsEndpoint: string,\n    changeStreamer: ChangeStreamerService,\n    initialCleanupDelayMs: number,\n    verifyBackupState: BackupStateVerifier,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#backupURL = backupURL;\n    this.#metricsEndpoint = metricsEndpoint;\n    this.#changeStreamer = changeStreamer;\n    this.#verifyBackupState = verifyBackupState;\n    this.#cleanupDelayMs = Math.max(\n      initialCleanupDelayMs,\n      MIN_CLEANUP_DELAY_MS, // purely for peace of mind\n    );\n\n    this.#lc.info?.(\n      `backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`,\n    );\n  }\n\n  run(): Promise<void> {\n    this.#lc.info?.(\n      `monitoring backups at ${this.#metricsEndpoint} with ` +\n        `${this.#cleanupDelayMs} ms cleanup delay`,\n    );\n    this.#checkMetricsTimer = setInterval(\n      this.checkWatermarksAndScheduleCleanup,\n      CHECK_INTERVAL_MS,\n    );\n    this.#initBackupLagMetric();\n    // Resolves on a normal stop; rejects if the backup is found to be wedged.\n    return Promise.race([this.#state.stopped(), this.#fatal.promise]);\n  }\n\n  startSnapshotReservation(taskID: string): Subscription<SnapshotMessage> {\n    this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);\n    // In the case of retries, only track the last reservation.\n    this.#reservations.get(taskID)?.sub.cancel();\n\n    const sub = Subscription.create<SnapshotMessage>({\n      // If the reservation still exists when the connection closes\n      // (e.g. subscriber crashed), clean it up without updating the\n      // cleanup delay.\n      cleanup: () => this.endReservation(taskID, false),\n    });\n    this.#reservations.set(taskID, {start: new Date(), sub});\n    // Note: the Subscription must be returned immediately so that the\n    //       websocket can begin sending liveness pings.\n    void this.#changeStreamer\n      .getChangeLogState()\n      .then(changeLogState => {\n        sub.push([\n          'status',\n          {tag: 'status', backupURL: this.#backupURL, ...changeLogState},\n        ]);\n      })\n      .catch(e => {\n        this.#lc.warn?.(`failing snapshot reservation`, e);\n        sub.fail(e);\n      });\n    return sub;\n  }\n\n  endReservation(taskID: string, updateCleanupDelay = true) {\n    const res = this.#reservations.get(taskID);\n    if (res === undefined) {\n      return;\n    }\n    this.#reservations.delete(taskID);\n    const {start, sub} = res;\n    sub.cancel(); // closes the connection if still open\n\n    if (updateCleanupDelay) {\n      const duration = Date.now() - start.getTime();\n      this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);\n      if (duration > this.#cleanupDelayMs) {\n        this.#cleanupDelayMs = duration;\n        this.#lc.info?.(`increased cleanup delay to ${duration} ms`);\n      }\n    }\n  }\n\n  // Exported for testing\n  readonly checkWatermarksAndScheduleCleanup = async () => {\n    try {\n      await this.#checkWatermarks();\n    } catch (e) {\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n    }\n    try {\n      await this.#scheduleCleanup();\n    } catch (e) {\n      this.#lc.warn?.(`error scheduling cleanup`, e);\n    }\n  };\n\n  async *#fetchWatermarks(): AsyncGenerator<{\n    watermark: string;\n    time: Date;\n    name?: string | undefined;\n  }> {\n    const metricsEndpoint = this.#metricsEndpoint;\n    const signal = this.#state.signal;\n    let resp;\n    try {\n      resp = await fetch(metricsEndpoint, {signal});\n    } catch (e) {\n      if (signal.aborted) {\n        // not an error.\n        return;\n      }\n      // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n      // log them and skip the watermark check until the next interval.\n      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n      return;\n    }\n    if (!resp.ok) {\n      this.#lc.warn?.(\n        `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n      );\n      return;\n    }\n\n    const families = parsePrometheusTextFormat(await resp.text());\n    for (const family of families) {\n      if (\n        family.type === 'GAUGE' &&\n        family.name === 'litestream_replica_progress'\n      ) {\n        for (const metric of family.metrics) {\n          const watermark = metric.labels?.watermark;\n          const name = metric.labels?.name;\n          const time = new Date(parseFloat(metric.value) * 1000);\n\n          if (watermark) {\n            yield {watermark, time, name};\n          }\n        }\n      }\n    }\n  }\n\n  async #checkWatermarks() {\n    for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n      if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n        this.#lc.info?.(\n          `replicated watermark=${watermark} to ${name}` +\n            ` at ${time.toISOString()}.`,\n        );\n        this.#watermarks.set(watermark, time);\n        this.#latestBackupTime = time;\n      }\n    }\n    return this.#latestBackupTime;\n  }\n\n  async #scheduleCleanup() {\n    if (this.#reservations.size > 0) {\n      this.#lc.info?.(\n        `watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`,\n      );\n      return;\n    }\n    const latestCleanupTime = Date.now() - this.#cleanupDelayMs;\n    const maxWatermark = this.#maxWatermarkUpTo(latestCleanupTime);\n    if (maxWatermark.length === 0) {\n      return;\n    }\n    // Purge guard: the watermarks (and their backup times) come from\n    // litestream metrics, which are exported when litestream *believes*\n    // an upload succeeded, and have been observed to advance even when\n    // nothing was actually written to the backup destination. Purging the\n    // change-log based on a falsely advancing watermark permanently breaks\n    // the ability to restore + catch up. Before advancing the cleanup\n    // watermark, verify it against the actual backup state: a claimed\n    // backup time is only trusted if an object was actually uploaded to\n    // the replica destination at (or after) that time, modulo clock skew.\n    const claimedTime = must(this.#watermarks.get(maxWatermark));\n    if (!this.#confirmedDurable(claimedTime)) {\n      try {\n        this.#lastVerifiedUploadTime = await this.#verifyBackupState();\n      } catch (e) {\n        this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n        // Skipping the purge is safe: the change-log just grows.\n        this.#lc.warn?.(\n          `unable to verify backup state. skipping change-log cleanup ` +\n            `up to watermark ${maxWatermark} ` +\n            `(claimed backup time ${claimedTime.toISOString()})`,\n          e,\n        );\n        return;\n      }\n    }\n    // Watermarks whose backup time isn't yet confirmed durable remain in the\n    // map and are re-evaluated at the next check.\n    const lastUpload = must(this.#lastVerifiedUploadTime);\n    const verifiedWatermark = this.#maxWatermarkUpTo(\n      Math.min(\n        latestCleanupTime,\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n      ),\n    );\n    if (verifiedWatermark.length === 0) {\n      this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n      const now = Date.now();\n      if (this.#backupStaleSince === null) {\n        this.#backupStaleSince = now;\n      }\n      const staleForMs = now - this.#backupStaleSince;\n      this.#lc.warn?.(\n        `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n          `litestream claims it was backed up at ` +\n          `${claimedTime.toISOString()}, but the last object actually ` +\n          `uploaded to ${this.#backupURL} was at ` +\n          `${lastUpload.toISOString()}. ` +\n          `The backup has been stale for ${staleForMs} ms.`,\n      );\n      if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n        this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n      }\n      return;\n    }\n    // The cleanup watermark advanced past a real upload, so the backup is\n    // keeping up: clear the staleness clock.\n    this.#backupStaleSince = null;\n    this.#changeStreamer.scheduleCleanup(verifiedWatermark);\n    for (const watermark of this.#watermarks.keys()) {\n      if (watermark <= verifiedWatermark) {\n        this.#watermarks.delete(watermark);\n      }\n    }\n    this.#lastWatermark = verifiedWatermark;\n  }\n\n  /**\n   * Called when the backup has been continuously stale for longer than\n   * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n   * backup, so the process is shut down by rejecting the {@link run()}\n   * promise. The exit is non-zero and logged at ERROR for alerting; on\n   * restart the monitor re-verifies, so a still-wedged backup keeps the\n   * process down until the destination is fixed.\n   */\n  #shutDownOnWedgedBackup(\n    staleForMs: number,\n    claimedTime: Date,\n    lastUpload: Date,\n  ) {\n    this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n    const err = new UnrecoverableError(\n      `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n        `at ${claimedTime.toISOString()}, but the last object actually ` +\n        `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n        `not advanced for ${staleForMs} ms (grace period ` +\n        `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n        `the backup; investigate the backup destination.`,\n    );\n    this.#lc.error?.(err.message);\n    clearInterval(this.#checkMetricsTimer);\n    this.#fatal.reject(err);\n  }\n\n  /**\n   * Returns the newest watermark whose backup time is at or before `cutoff`\n   * (epoch ms), or `''` if there is none.\n   */\n  #maxWatermarkUpTo(cutoff: number): string {\n    let max = '';\n    for (const [watermark, backupTime] of this.#watermarks) {\n      if (backupTime.getTime() <= cutoff && watermark > max) {\n        max = watermark;\n      }\n    }\n    return max;\n  }\n\n  /**\n   * Returns `true` if the actual backup state, as last verified against the\n   * backup destination, confirms that data claimed to be backed up at\n   * `claimedTime` is durable (i.e. an object was actually uploaded at or\n   * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n   * clock skew).\n   */\n  #confirmedDurable(claimedTime: Date): boolean {\n    const lastUpload = this.#lastVerifiedUploadTime;\n    return (\n      lastUpload !== null &&\n      claimedTime.getTime() <=\n        lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n    );\n  }\n\n  stop(): Promise<void> {\n    clearInterval(this.#checkMetricsTimer);\n    for (const {sub} of this.#reservations.values()) {\n      // Close any pending reservations. This commonly happens when a new\n      // replication-manager makes a `/snapshot` reservation on the existing\n      // replication-manager, and then shuts it down when it takes over the\n      // replication slot.\n      sub.cancel();\n    }\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #initBackupLagMetric() {\n    getOrCreateGauge('replica', 'backup_lag', {\n      description:\n        'Latency from when a change is written to the replica ' +\n        'to when it is backed up to litestream. It is expected to create a saw ' +\n        'pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.',\n      unit: 'millisecond',\n    }).addCallback(async o => {\n      // For legacy litestream, we use the watermark metric (and its associated\n      // backup time) exported by litestream metrics to determine the time of\n      // of the backed up watermark. This is technically imprecise--it would be\n      // more correct to use the committed writeTimeMs--but it is good enough\n      // in that it serves the purpose of detecting a non-functioning backup.\n      // With litestream v5, this can be made more precise by querying the\n      // _zero.replicationState row from the backup directly using an LTX-based\n      // database reader.\n      const latestBackup = await this.#checkWatermarks();\n      if (!latestBackup) {\n        this.#lc.warn?.(\n          `no backed up watermarks. unable to report replica.backup_lag`,\n        );\n        return;\n      }\n      const db = new Database(this.#lc, this.#replicaFile, {readonly: true});\n      try {\n        const {writeTimeMs} = db\n          .prepare(/*sql*/ `SELECT writeTimeMs FROM \"_zero.replicationState\"`)\n          .get<{writeTimeMs: number}>();\n        const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());\n        o.observe(backupLag);\n      } catch (e) {\n        this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);\n      } finally {\n        db.close();\n      }\n    });\n  }\n}\n"],"mappings":";;;;;;;;;AAgBA,IAAa,oBAAoB;AACjC,IAAM,uBAAuB;;;;;AAM7B,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyD7C,IAAa,2BAAb,MAA+D;CAC7D,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,gCAAyB,IAAI,IAAyB;CACtD,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAID,SAAkB,SAAe;CAEjC,iBAAyB;CACzB,oBAAiC;CACjC,0BAAuC;CAKvC,oBAAmC;CACnC;CACA;CAEA,YACE,IACA,aACA,WACA,iBACA,gBACA,uBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKC,kBAAkB;EACvB,KAAKI,qBAAqB;EAC1B,KAAKG,kBAAkB,KAAK,IAC1B,uBACA,oBACF;EAEA,KAAKX,IAAI,OACP,0BAA0B,sBAAsB,2BAClD;CACF;CAEA,MAAqB;EACnB,KAAKA,IAAI,OACP,yBAAyB,KAAKG,iBAAiB,QAC1C,KAAKQ,gBAAgB,kBAC5B;EACA,KAAKC,qBAAqB,YACxB,KAAK,mCACL,iBACF;EACA,KAAKC,qBAAqB;EAE1B,OAAO,QAAQ,KAAK,CAAC,KAAKR,OAAO,QAAQ,GAAG,KAAKK,OAAO,OAAO,CAAC;CAClE;CAEA,yBAAyB,QAA+C;EACtE,KAAKV,IAAI,OAAO,oCAAoC,OAAO,WAAW;EAEtE,KAAKM,cAAc,IAAI,MAAM,GAAG,IAAI,OAAO;EAE3C,MAAM,MAAM,aAAa,OAAwB,EAI/C,eAAe,KAAK,eAAe,QAAQ,KAAK,EAClD,CAAC;EACD,KAAKA,cAAc,IAAI,QAAQ;GAAC,uBAAO,IAAI,KAAK;GAAG;EAAG,CAAC;EAGvD,KAAUF,gBACP,kBAAkB,EAClB,MAAK,mBAAkB;GACtB,IAAI,KAAK,CACP,UACA;IAAC,KAAK;IAAU,WAAW,KAAKF;IAAY,GAAG;GAAc,CAC/D,CAAC;EACH,CAAC,EACA,OAAM,MAAK;GACV,KAAKF,IAAI,OAAO,gCAAgC,CAAC;GACjD,IAAI,KAAK,CAAC;EACZ,CAAC;EACH,OAAO;CACT;CAEA,eAAe,QAAgB,qBAAqB,MAAM;EACxD,MAAM,MAAM,KAAKM,cAAc,IAAI,MAAM;EACzC,IAAI,QAAQ,KAAA,GACV;EAEF,KAAKA,cAAc,OAAO,MAAM;EAChC,MAAM,EAAC,OAAO,QAAO;EACrB,IAAI,OAAO;EAEX,IAAI,oBAAoB;GACtB,MAAM,WAAW,KAAK,IAAI,IAAI,MAAM,QAAQ;GAC5C,KAAKN,IAAI,OAAO,2BAA2B,OAAO,MAAM,SAAS,IAAI;GACrE,IAAI,WAAW,KAAKW,iBAAiB;IACnC,KAAKA,kBAAkB;IACvB,KAAKX,IAAI,OAAO,8BAA8B,SAAS,IAAI;GAC7D;EACF;CACF;CAGA,oCAA6C,YAAY;EACvD,IAAI;GACF,MAAM,KAAKc,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;CACF;CAEA,OAAOgB,mBAIJ;EACD,MAAM,kBAAkB,KAAKb;EAC7B,MAAM,SAAS,KAAKE,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKL,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMW,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKE,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKV,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKP,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKO,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKW,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMH,mBAAmB;EACvB,IAAI,KAAKT,cAAc,OAAO,GAAG;GAC/B,KAAKN,IAAI,OACP,6CAA6C,CAAC,GAAG,KAAKM,cAAc,KAAK,CAAC,GAC5E;GACA;EACF;EACA,MAAM,oBAAoB,KAAK,IAAI,IAAI,KAAKK;EAC5C,MAAM,eAAe,KAAKQ,kBAAkB,iBAAiB;EAC7D,IAAI,aAAa,WAAW,GAC1B;EAWF,MAAM,cAAc,KAAK,KAAKZ,YAAY,IAAI,YAAY,CAAC;EAC3D,IAAI,CAAC,KAAKa,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKb,mBAAmB;EAC/D,SAAS,GAAG;GACV,KAAKC,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKT,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKqB,uBAAuB;EACpD,MAAM,oBAAoB,KAAKF,kBAC7B,KAAK,IACH,mBACA,WAAW,QAAQ,IAAI,4BACzB,CACF;EACA,IAAI,kBAAkB,WAAW,GAAG;GAClC,KAAKV,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKa,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,KAAKlB,gBAAgB,gBAAgB,iBAAiB;EACtD,KAAK,MAAM,aAAa,KAAKG,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKU,iBAAiB;CACxB;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKR,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKP,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,cAAc,KAAKY,kBAAkB;EACrC,KAAKF,OAAO,OAAO,GAAG;CACxB;;;;;CAMA,kBAAkB,QAAwB;EACxC,IAAI,MAAM;EACV,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKH,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAChD,MAAM;EAGV,OAAO;CACT;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKc;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;CAEA,OAAsB;EACpB,cAAc,KAAKT,kBAAkB;EACrC,KAAK,MAAM,EAAC,SAAQ,KAAKN,cAAc,OAAO,GAK5C,IAAI,OAAO;EAEb,KAAKD,OAAO,KAAK,KAAKL,GAAG;EACzB,OAAO;CACT;CAEA,uBAAuB;EACrB,iBAAiB,WAAW,cAAc;GACxC,aACE;GAGF,MAAM;EACR,CAAC,EAAE,YAAY,OAAM,MAAK;GASxB,MAAM,eAAe,MAAM,KAAKc,iBAAiB;GACjD,IAAI,CAAC,cAAc;IACjB,KAAKd,IAAI,OACP,8DACF;IACA;GACF;GACA,MAAM,KAAK,IAAI,SAAS,KAAKA,KAAK,KAAKC,cAAc,EAAC,UAAU,KAAI,CAAC;GACrE,IAAI;IACF,MAAM,EAAC,gBAAe,GACnB,QAAgB,kDAAkD,EAClE,IAA2B;IAC9B,MAAM,YAAY,KAAK,IAAI,GAAG,cAAc,aAAa,QAAQ,CAAC;IAClE,EAAE,QAAQ,SAAS;GACrB,SAAS,GAAG;IACV,KAAKD,IAAI,OAAO,6CAA6C,CAAC;GAChE,UAAU;IACR,GAAG,MAAM;GACX;EACF,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/change-streamer/replica-monitor.d.ts

@@ -2,8 +2,8 @@ import type { LogContext } from '@rocicorp/logger';
 import type { Service } from '../service.ts';
 import type { ChangeStreamerService } from './change-streamer.ts';
 /**
- * The single-node equivalent of the {@link BackupMonitor} polls the replica
- * file every 30 seconds and schedules cleanup when the watermark
+ * The single-node equivalent of a backup monitor polls the replica file every
+ * 30 seconds and schedules cleanup when the watermark
  * (i.e. stateVersion) moves forward.
  */
 export declare class ReplicaMonitor implements Service {

package/out/zero-cache/src/services/change-streamer/replica-monitor.js

@@ -6,8 +6,8 @@ import { RunningState } from "../running-state.js";
 //#region ../zero-cache/src/services/change-streamer/replica-monitor.ts
 var CHECK_INTERVAL_MS = 30 * 1e3;
 /**
-* The single-node equivalent of the {@link BackupMonitor} polls the replica
-* file every 30 seconds and schedules cleanup when the watermark
+* The single-node equivalent of a backup monitor polls the replica file every
+* 30 seconds and schedules cleanup when the watermark
 * (i.e. stateVersion) moves forward.
 */
 var ReplicaMonitor = class {

package/out/zero-cache/src/services/change-streamer/replica-monitor.js.map

@@ -1 +1 @@
-{"version":3,"file":"replica-monitor.js","names":["#lc","#replicaFile","#changeStreamer","#state","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of the {@link BackupMonitor} polls the replica\n * file every 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;AASA,IAAM,oBAAoB,KAAK;;;;;;AAO/B,IAAa,iBAAb,MAA+C;CAC7C,KAAc;CACd;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,iBAAyB;CAEzB,YACE,IACA,aACA,gBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,kBAAkB;CACzB;CAEA,MAAM,MAAM;EACV,KAAKF,IAAI,OAAO,0BAA0B;EAC1C,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAEzC,OAAO,KAAKA,OAAO,UAAU,GAAG;GAC9B,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,KAAKC,YAAY;GACnD,IAAI;IACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;IAClE,IAAI,iBAAiB,KAAKG,gBAAgB;KACxC,KAAKA,iBAAiB;KACtB,KAAKJ,IAAI,QAAQ,8BAA8B,cAAc;KAC7D,KAAKE,gBAAgB,gBAAgB,YAAY;IACnD;GACF,SAAS,GAAG;IACV,KAAKF,IAAI,QAAQ,yCAAyC,CAAC;GAC7D,UAAU;IACR,GAAG,MAAM;GACX;GAEA,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAC3C;EACA,KAAKH,IAAI,OAAO,yBAAyB;CAC3C;CAEA,OAAO;EACL,KAAKG,OAAO,KAAK,KAAKH,GAAG;EACzB,OAAO;CACT;AACF"}
+{"version":3,"file":"replica-monitor.js","names":["#lc","#replicaFile","#changeStreamer","#state","#lastWatermark"],"sources":["../../../../../../zero-cache/src/services/change-streamer/replica-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getReplicationState} from '../replicator/schema/replication-state.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\n\nconst CHECK_INTERVAL_MS = 30 * 1000;\n\n/**\n * The single-node equivalent of a backup monitor polls the replica file every\n * 30 seconds and schedules cleanup when the watermark\n * (i.e. stateVersion) moves forward.\n */\nexport class ReplicaMonitor implements Service {\n  readonly id = 'replica-monitor';\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #changeStreamer: ChangeStreamerService;\n  readonly #state = new RunningState(this.id);\n\n  #lastWatermark: string = '';\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    changeStreamer: ChangeStreamerService,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#replicaFile = replicaFile;\n    this.#changeStreamer = changeStreamer;\n  }\n\n  async run() {\n    this.#lc.info?.(`starting replica monitor`);\n    await this.#state.sleep(CHECK_INTERVAL_MS);\n\n    while (this.#state.shouldRun()) {\n      const db = new Database(this.#lc, this.#replicaFile);\n      try {\n        const {stateVersion} = getReplicationState(new StatementRunner(db));\n        if (stateVersion !== this.#lastWatermark) {\n          this.#lastWatermark = stateVersion;\n          this.#lc.debug?.(`replicated up to watermark ${stateVersion}`);\n          this.#changeStreamer.scheduleCleanup(stateVersion);\n        }\n      } catch (e) {\n        this.#lc.error?.(`Unable to read watermark from replica`, e);\n      } finally {\n        db.close();\n      }\n\n      await this.#state.sleep(CHECK_INTERVAL_MS);\n    }\n    this.#lc.info?.(`replica monitor stopped`);\n  }\n\n  stop() {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n}\n"],"mappings":";;;;;;AASA,IAAM,oBAAoB,KAAK;;;;;;AAO/B,IAAa,iBAAb,MAA+C;CAC7C,KAAc;CACd;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAE1C,iBAAyB;CAEzB,YACE,IACA,aACA,gBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,eAAe;EACpB,KAAKC,kBAAkB;CACzB;CAEA,MAAM,MAAM;EACV,KAAKF,IAAI,OAAO,0BAA0B;EAC1C,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAEzC,OAAO,KAAKA,OAAO,UAAU,GAAG;GAC9B,MAAM,KAAK,IAAI,SAAS,KAAKH,KAAK,KAAKC,YAAY;GACnD,IAAI;IACF,MAAM,EAAC,iBAAgB,oBAAoB,IAAI,gBAAgB,EAAE,CAAC;IAClE,IAAI,iBAAiB,KAAKG,gBAAgB;KACxC,KAAKA,iBAAiB;KACtB,KAAKJ,IAAI,QAAQ,8BAA8B,cAAc;KAC7D,KAAKE,gBAAgB,gBAAgB,YAAY;IACnD;GACF,SAAS,GAAG;IACV,KAAKF,IAAI,QAAQ,yCAAyC,CAAC;GAC7D,UAAU;IACR,GAAG,MAAM;GACX;GAEA,MAAM,KAAKG,OAAO,MAAM,iBAAiB;EAC3C;EACA,KAAKH,IAAI,OAAO,yBAAyB;CAC3C;CAEA,OAAO;EACL,KAAKG,OAAO,KAAK,KAAKH,GAAG;EACzB,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/change-streamer/storer.d.ts

@@ -11,6 +11,7 @@ import type { Subscriber } from './subscriber.ts';
 export type TuningOptions = {
     backPressureLimitHeapProportion: number;
     statementTimeoutMs: number;
+    changeLogBatchSize: number;
 };
 /**
  * Handles the storage of changes and the catchup of subscribers
@@ -45,7 +46,7 @@ export type TuningOptions = {
 export declare class Storer implements Service {
     #private;
     readonly id = "storer";
-    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs }: TuningOptions);
+    constructor(lc: LogContext, shard: ShardID, taskID: string, discoveryAddress: string, discoveryProtocol: string, db: PostgresDB, replicaVersion: string, onConsumed: (c: Commit | UpstreamStatusMessage) => void, onFatal: (err: Error) => void, { backPressureLimitHeapProportion, statementTimeoutMs, changeLogBatchSize, }: TuningOptions);
     assumeOwnership(purgeLock?: PurgeLock | null): Promise<void>;
     getStartStreamInitializationParameters(): Promise<{
         lastWatermark: string;