@rocicorp/zero 1.7.0 → 1.8.0-canary.1

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => parseStreamMessage(lc, buffer, parser),\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,GAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,GACxE;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAEA,MAAM,gBAAgB,kBAAkB;EAEtC,IADY,KAAK,IACb,IAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;CACF,GAAG,yBAAyB,CAAC;CAE7B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU,mBAAmB,IAAI,QAAQ,MAAM;EAItD,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}
+{"version":3,"file":"stream.js","names":[],"sources":["../../../../../../../../zero-cache/src/services/change-source/pg/logical-replication/stream.ts"],"sourcesContent":["import {\n  PG_ADMIN_SHUTDOWN,\n  PG_OBJECT_IN_USE,\n  PG_OBJECT_NOT_IN_PREREQUISITE_STATE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport {sleep} from '../../../../../../shared/src/sleep.ts';\nimport {getTypeParsers} from '../../../../db/pg-type-parser.ts';\nimport {type PostgresDB} from '../../../../types/pg.ts';\nimport {id, lit} from '../../../../types/sql.ts';\nimport {pipe, type Sink, type Source} from '../../../../types/streams.ts';\nimport {Subscription} from '../../../../types/subscription.ts';\nimport {AutoResetSignal} from '../../../change-streamer/schema/tables.ts';\nimport {fromBigInt} from '../lsn.ts';\nimport {PgoutputParser} from './pgoutput-parser.ts';\nimport type {Message} from './pgoutput.types.ts';\n\nconst DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE = 5;\n\nexport type StreamMessage = [\n  lsn: bigint,\n  Message | {tag: 'keepalive'; shouldRespond: boolean},\n];\n\n// Expose the `queued` variable of the Subscription to allow\n// the change-source to determine if there are more messages\n// immediately available.\ntype SourceWithPendingQueue<T> = Source<T> & {\n  queued: number;\n};\n\nexport async function subscribe(\n  lc: LogContext,\n  db: PostgresDB,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  retriesIfReplicationSlotActive = DEFAULT_RETRIES_IF_REPLICATION_SLOT_ACTIVE,\n  applicationName = 'zero-replicator',\n): Promise<{\n  messages: SourceWithPendingQueue<StreamMessage>;\n  acks: Sink<bigint>;\n}> {\n  const session = postgres(\n    defu(\n      {\n        max: 1,\n        ['fetch_types']: false, // Necessary for the streaming protocol\n        ['idle_timeout']: null,\n        ['max_lifetime']: null as unknown as number,\n        connection: {\n          ['application_name']: applicationName,\n          replication: 'database', // https://www.postgresql.org/docs/current/protocol-replication.html\n        },\n      },\n      // ParsedOptions are technically compatible with Options, but happen\n      // to not be typed that way. The postgres.js author does an equivalent\n      // merge of ParsedOptions and Options here:\n      // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/subscribe.js#L13\n      db.options as unknown as Options<Record<string, PostgresType>>,\n    ),\n  );\n\n  // Postgres will send keepalives before timing out a wal_sender. It is possible that\n  // these keepalives are not received if there is back-pressure in the replication\n  // stream. To keep the connection alive, explicitly send keepalives if none have been\n  // sent within the last 75% of the wal_sender_timeout.\n  //\n  // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n  const [{walSenderTimeoutMs}] = await session<\n    {walSenderTimeoutMs: number}[]\n  >`SELECT EXTRACT(EPOCH FROM (setting || unit)::interval) * 1000 \n        AS \"walSenderTimeoutMs\" FROM pg_settings\n        WHERE name = 'wal_sender_timeout'`.simple();\n  const manualKeepaliveTimeout = Math.floor(walSenderTimeoutMs * 0.75);\n  lc.info?.(\n    `wal_sender_timeout: ${walSenderTimeoutMs}ms. ` +\n      `Ensuring manual keepalives at least every ${manualKeepaliveTimeout}ms`,\n  );\n\n  const [readable, writable] = await startReplicationStream(\n    lc,\n    session,\n    slot,\n    publications,\n    lsn,\n    retriesIfReplicationSlotActive + 1,\n  );\n\n  let lastAckTime = Date.now();\n  function sendAck(lsn: bigint) {\n    writable.write(makeAck(lsn));\n    lastAckTime = Date.now();\n  }\n\n  // Postgres promises to send a keepalive at roughly wal_sender_timeout / 2.\n  // If we go long enough without receiving anything (data or keepalive), the\n  // inbound half of the connection is silently dead — our outbound acks keep\n  // flowing into the void, neither side errors, and the change-source sits\n  // there forever. Tear down the readable to force a reconnect.\n  //\n  // The threshold is 2x wal_sender_timeout — i.e. two consecutive keepalives\n  // missed — to avoid spurious reconnects from a single delayed keepalive\n  // (PG scheduling jitter under load, our own event-loop stalls, etc.).\n  const inboundTimeoutMs = walSenderTimeoutMs * 2;\n  let lastReceivedTime = Date.now();\n\n  const livenessTimer = setInterval(() => {\n    const now = Date.now();\n    if (now - lastAckTime > manualKeepaliveTimeout) {\n      sendAck(0n);\n      lc.debug?.(`sent manual keepalive`);\n    }\n    const sinceLastReceived = now - lastReceivedTime;\n    if (sinceLastReceived > inboundTimeoutMs && !readable.destroyed) {\n      lc.warn?.(\n        `no message received from ${db.options.host} in ${sinceLastReceived}ms ` +\n          `(> 2x wal_sender_timeout ${walSenderTimeoutMs}ms). Destroying ` +\n          `replication stream to force reconnect.`,\n      );\n      readable.destroy(\n        new Error(\n          `replication stream inbound timeout after ${sinceLastReceived}ms`,\n        ),\n      );\n    }\n  }, manualKeepaliveTimeout / 5);\n\n  let destroyed = false;\n  const typeParsers = await getTypeParsers(db, {returnJsonAsString: true});\n  const parser = new PgoutputParser(typeParsers);\n  const messages = Subscription.create<StreamMessage>({\n    cleanup: () => {\n      destroyed = true;\n      readable.destroyed || readable.destroy();\n      clearInterval(livenessTimer);\n      return session.end();\n    },\n  });\n\n  readable.once(\n    'close',\n    () =>\n      // Only log a warning if the stream was not manually closed.\n      destroyed || lc.warn?.(`replication stream closed by ${db.options.host}`),\n  );\n  readable.once(\n    'error',\n    e =>\n      // Don't log the shutdown signal. This is the expected way for upstream\n      // to close the connection (and will be logged downstream).\n      (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) ||\n      lc.warn?.(`error from ${db.options.host}`, e),\n  );\n\n  pipe({\n    source: readable,\n    sink: messages,\n    parse: buffer => {\n      lastReceivedTime = Date.now();\n      return parseStreamMessage(lc, buffer, parser);\n    },\n    // Allow a small buffer of messages to be queued in the subscription so\n    // that the change-source loop can check the queue to determine if more\n    // messages are immediately available.\n    bufferMessages: 5,\n  });\n\n  return {\n    messages,\n    acks: {push: sendAck},\n  };\n}\n\n/**\n * Formats publication names for the START_REPLICATION command.\n * The replication protocol expects format: publication_names 'pub1,pub2'\n * Each name is escaped to handle any quotes that may have passed validation.\n */\nfunction formatPublicationNames(publications: string[]): string {\n  // lit() returns 'escaped_name' with surrounding quotes\n  // We strip the quotes since the outer quotes are in the template\n  return publications.map(p => lit(p).slice(1, -1)).join(',');\n}\n\nasync function startReplicationStream(\n  lc: LogContext,\n  session: postgres.Sql,\n  slot: string,\n  publications: string[],\n  lsn: bigint,\n  maxAttempts: number,\n) {\n  for (let i = 0; i < maxAttempts; i++) {\n    try {\n      const stream = session\n        .unsafe(\n          `START_REPLICATION SLOT ${id(slot)} LOGICAL ${fromBigInt(lsn)} (\n        proto_version '1',\n        publication_names '${formatPublicationNames(publications)}',\n        messages 'true'\n      )`,\n        )\n        .execute();\n      return await Promise.all([stream.readable(), stream.writable()]);\n    } catch (e) {\n      if (e instanceof postgres.PostgresError) {\n        // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n        if (e.code === PG_OBJECT_IN_USE) {\n          // The freeing up of the replication slot is not transactional;\n          // sometimes it takes time for Postgres to consider the slot\n          // inactive.\n          lc.warn?.(`attempt ${i + 1}: ${String(e)}`, e);\n          await sleep(10);\n          continue;\n        }\n        // error: This slot has been invalidated because it exceeded the maximum reserved size.\n        // (This is a different manifestation of a slot being invalidated when\n        //  the wal exceeds the max_slot_wal_keep_size)\n        if (e.code === PG_OBJECT_NOT_IN_PREREQUISITE_STATE) {\n          lc.error?.(`error starting replication stream`, e);\n          throw new AutoResetSignal(`unable to start replication stream`, {\n            cause: e,\n          });\n        }\n      }\n      throw e;\n    }\n  }\n  throw new Error(\n    `exceeded max attempts (${maxAttempts}) to start the Postgres stream`,\n  );\n}\n\nfunction parseStreamMessage(\n  lc: LogContext,\n  buffer: Buffer,\n  parser: PgoutputParser,\n): StreamMessage | null {\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-START-REPLICATION\n  if (buffer[0] !== 0x77 && buffer[0] !== 0x6b) {\n    lc.warn?.('Unknown message', buffer[0]);\n    return null;\n  }\n  const lsn = buffer.readBigUInt64BE(1);\n  if (buffer[0] === 0x77) {\n    // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-XLOGDATA\n    // (Byte 25 is where the WAL data begins)\n    return [lsn, parser.parse(buffer.subarray(25))];\n  }\n  // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n  // (Byte 17: shouldRespond)\n  const shouldRespond = buffer.readInt8(17) !== 0;\n  return [lsn, {tag: 'keepalive', shouldRespond}];\n}\n\n// https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE\nfunction makeAck(lsn: bigint): Buffer {\n  const microNow = BigInt(Date.now() - Date.UTC(2000, 0, 1)) * BigInt(1000);\n\n  const x = Buffer.alloc(34);\n  x[0] = 'r'.charCodeAt(0);\n  x.writeBigInt64BE(lsn, 1);\n  x.writeBigInt64BE(lsn, 9);\n  x.writeBigInt64BE(lsn, 17);\n  x.writeBigInt64BE(microNow, 25);\n  return x;\n}\n"],"mappings":";;;;;;;;;;;;;AAmBA,IAAM,6CAA6C;AAcnD,eAAsB,UACpB,IACA,IACA,MACA,cACA,KACA,iCAAiC,4CACjC,kBAAkB,mBAIjB;CACD,MAAM,UAAU,SACd,KACE;EACE,KAAK;GACJ,gBAAgB;GAChB,iBAAiB;GACjB,iBAAiB;EAClB,YAAY;IACT,qBAAqB;GACtB,aAAa;EACf;CACF,GAKA,GAAG,OACL,CACF;CAQA,MAAM,CAAC,EAAC,wBAAuB,MAAM,OAEpC;;2CAEwC,OAAO;CAChD,MAAM,yBAAyB,KAAK,MAAM,qBAAqB,GAAI;CACnE,GAAG,OACD,uBAAuB,mBAAmB,gDACK,uBAAuB,GACxE;CAEA,MAAM,CAAC,UAAU,YAAY,MAAM,uBACjC,IACA,SACA,MACA,cACA,KACA,iCAAiC,CACnC;CAEA,IAAI,cAAc,KAAK,IAAI;CAC3B,SAAS,QAAQ,KAAa;EAC5B,SAAS,MAAM,QAAQ,GAAG,CAAC;EAC3B,cAAc,KAAK,IAAI;CACzB;CAWA,MAAM,mBAAmB,qBAAqB;CAC9C,IAAI,mBAAmB,KAAK,IAAI;CAEhC,MAAM,gBAAgB,kBAAkB;EACtC,MAAM,MAAM,KAAK,IAAI;EACrB,IAAI,MAAM,cAAc,wBAAwB;GAC9C,QAAQ,EAAE;GACV,GAAG,QAAQ,uBAAuB;EACpC;EACA,MAAM,oBAAoB,MAAM;EAChC,IAAI,oBAAoB,oBAAoB,CAAC,SAAS,WAAW;GAC/D,GAAG,OACD,4BAA4B,GAAG,QAAQ,KAAK,MAAM,kBAAkB,8BACtC,mBAAmB,uDAEnD;GACA,SAAS,wBACP,IAAI,MACF,4CAA4C,kBAAkB,GAChE,CACF;EACF;CACF,GAAG,yBAAyB,CAAC;CAE7B,IAAI,YAAY;CAEhB,MAAM,SAAS,IAAI,eAAe,MADR,eAAe,IAAI,EAAC,oBAAoB,KAAI,CAAC,CAC1B;CAC7C,MAAM,WAAW,aAAa,OAAsB,EAClD,eAAe;EACb,YAAY;EACZ,SAAS,aAAa,SAAS,QAAQ;EACvC,cAAc,aAAa;EAC3B,OAAO,QAAQ,IAAI;CACrB,EACF,CAAC;CAED,SAAS,KACP,eAGE,aAAa,GAAG,OAAO,gCAAgC,GAAG,QAAQ,MAAM,CAC5E;CACA,SAAS,KACP,UACA,MAGG,aAAa,SAAS,iBAAiB,EAAE,SAAS,qBACnD,GAAG,OAAO,cAAc,GAAG,QAAQ,QAAQ,CAAC,CAChD;CAEA,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,WAAU;GACf,mBAAmB,KAAK,IAAI;GAC5B,OAAO,mBAAmB,IAAI,QAAQ,MAAM;EAC9C;EAIA,gBAAgB;CAClB,CAAC;CAED,OAAO;EACL;EACA,MAAM,EAAC,MAAM,QAAO;CACtB;AACF;;;;;;AAOA,SAAS,uBAAuB,cAAgC;CAG9D,OAAO,aAAa,KAAI,MAAK,IAAI,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,GAAG;AAC5D;AAEA,eAAe,uBACb,IACA,SACA,MACA,cACA,KACA,aACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAC/B,IAAI;EACF,MAAM,SAAS,QACZ,OACC,0BAA0B,GAAG,IAAI,EAAE,WAAW,WAAW,GAAG,EAAE;;6BAE3C,uBAAuB,YAAY,EAAE;;QAG1D,EACC,QAAQ;EACX,OAAO,MAAM,QAAQ,IAAI,CAAC,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC,CAAC;CACjE,SAAS,GAAG;EACV,IAAI,aAAa,SAAS,eAAe;GAEvC,IAAI,EAAE,SAAS,kBAAkB;IAI/B,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C,MAAM,MAAM,EAAE;IACd;GACF;GAIA,IAAI,EAAE,SAAS,qCAAqC;IAClD,GAAG,QAAQ,qCAAqC,CAAC;IACjD,MAAM,IAAI,gBAAgB,sCAAsC,EAC9D,OAAO,EACT,CAAC;GACH;EACF;EACA,MAAM;CACR;CAEF,MAAM,IAAI,MACR,0BAA0B,YAAY,+BACxC;AACF;AAEA,SAAS,mBACP,IACA,QACA,QACsB;CAEtB,IAAI,OAAO,OAAO,OAAQ,OAAO,OAAO,KAAM;EAC5C,GAAG,OAAO,mBAAmB,OAAO,EAAE;EACtC,OAAO;CACT;CACA,MAAM,MAAM,OAAO,gBAAgB,CAAC;CACpC,IAAI,OAAO,OAAO,KAGhB,OAAO,CAAC,KAAK,OAAO,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CAKhD,OAAO,CAAC,KAAK;EAAC,KAAK;EAAa,eADV,OAAO,SAAS,EAAE,MAAM;CACD,CAAC;AAChD;AAGA,SAAS,QAAQ,KAAqB;CACpC,MAAM,WAAW,OAAO,KAAK,IAAI,IAAI,KAAK,IAAI,KAAM,GAAG,CAAC,CAAC,IAAI,OAAO,GAAI;CAExE,MAAM,IAAI,OAAO,MAAM,EAAE;CACzB,EAAE,KAAK,IAAI,WAAW,CAAC;CACvB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,CAAC;CACxB,EAAE,gBAAgB,KAAK,EAAE;CACzB,EAAE,gBAAgB,UAAU,EAAE;CAC9B,OAAO;AACT"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts

@@ -0,0 +1,18 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { NormalizedZeroConfig } from '../../config/normalize.ts';
+import type { BackupMonitor } from './backup-monitor.ts';
+import type { ChangeStreamerService } from './change-streamer.ts';
+import { type BackupStateVerifier } from './litestream3-backup-monitor.ts';
+import { type VfsBackupWatermarkSource } from './vfs-backup-monitor.ts';
+export type BackupCleanupMonitorFactoryOptions = {
+    lc: LogContext;
+    config: NormalizedZeroConfig;
+    replicaFile: string;
+    changeStreamer: ChangeStreamerService;
+    initialCleanupDelayMs: number;
+    verifyBackupState?: BackupStateVerifier | undefined;
+    vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;
+    env?: NodeJS.ProcessEnv | undefined;
+};
+export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env, }: BackupCleanupMonitorFactoryOptions): BackupMonitor | null;
+//# sourceMappingURL=backup-cleanup-monitor-factory.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEL,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,qBAAqB,CAAC;IACtC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;IACpD,wBAAwB,CAAC,EAAE,wBAAwB,GAAG,SAAS,CAAC;IAChE,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CACrC,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,wBAAwB,EACxB,GAAG,GACJ,EAAE,kCAAkC,GAAG,aAAa,GAAG,IAAI,CAgC3D"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js

@@ -0,0 +1,17 @@
+import { forkChildWorker } from "../../types/processes.js";
+import { BACKUP_WATERMARK_READER_URL } from "../../server/worker-urls.js";
+import { getLastBackupTime } from "../litestream/commands.js";
+import { VfsBackupWatermarkWorkerSource } from "../litestream/vfs-watermark-worker-source.js";
+import { Litestream3BackupMonitor } from "./litestream3-backup-monitor.js";
+import { VfsBackupMonitor } from "./vfs-backup-monitor.js";
+//#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
+function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, initialCleanupDelayMs, verifyBackupState, vfsBackupWatermarkSource, env }) {
+	const { backupURL, port: metricsPort } = config.litestream;
+	if (!backupURL) return null;
+	if (config.litestream.backupUsingV5) return new VfsBackupMonitor(lc, backupURL, changeStreamer, initialCleanupDelayMs, config.litestream.vfsProbeIntervalMs, vfsBackupWatermarkSource ?? new VfsBackupWatermarkWorkerSource(lc, () => forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env), config.litestream.vfsProbeTimeoutMs));
+	return new Litestream3BackupMonitor(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, changeStreamer, initialCleanupDelayMs, verifyBackupState ?? (() => getLastBackupTime(lc, config)));
+}
+//#endregion
+export { createBackupCleanupMonitor };
+
+//# sourceMappingURL=backup-cleanup-monitor-factory.js.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {BACKUP_WATERMARK_READER_URL} from '../../server/worker-urls.ts';\nimport {forkChildWorker} from '../../types/processes.ts';\nimport {getLastBackupTime} from '../litestream/commands.ts';\nimport {VfsBackupWatermarkWorkerSource} from '../litestream/vfs-watermark-worker-source.ts';\nimport type {BackupMonitor} from './backup-monitor.ts';\nimport type {ChangeStreamerService} from './change-streamer.ts';\nimport {\n  Litestream3BackupMonitor,\n  type BackupStateVerifier,\n} from './litestream3-backup-monitor.ts';\nimport {\n  VfsBackupMonitor,\n  type VfsBackupWatermarkSource,\n} from './vfs-backup-monitor.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n  lc: LogContext;\n  config: NormalizedZeroConfig;\n  replicaFile: string;\n  changeStreamer: ChangeStreamerService;\n  initialCleanupDelayMs: number;\n  verifyBackupState?: BackupStateVerifier | undefined;\n  vfsBackupWatermarkSource?: VfsBackupWatermarkSource | undefined;\n  env?: NodeJS.ProcessEnv | undefined;\n};\n\nexport function createBackupCleanupMonitor({\n  lc,\n  config,\n  replicaFile,\n  changeStreamer,\n  initialCleanupDelayMs,\n  verifyBackupState,\n  vfsBackupWatermarkSource,\n  env,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor | null {\n  const {backupURL, port: metricsPort} = config.litestream;\n  if (!backupURL) {\n    return null;\n  }\n\n  if (config.litestream.backupUsingV5) {\n    return new VfsBackupMonitor(\n      lc,\n      backupURL,\n      changeStreamer,\n      initialCleanupDelayMs,\n      config.litestream.vfsProbeIntervalMs,\n      vfsBackupWatermarkSource ??\n        new VfsBackupWatermarkWorkerSource(\n          lc,\n          () =>\n            forkChildWorker(BACKUP_WATERMARK_READER_URL, env ?? process.env),\n          config.litestream.vfsProbeTimeoutMs,\n        ),\n    );\n  }\n\n  return new Litestream3BackupMonitor(\n    lc,\n    replicaFile,\n    backupURL,\n    `http://localhost:${metricsPort}/metrics`,\n    changeStreamer,\n    initialCleanupDelayMs,\n    verifyBackupState ?? (() => getLastBackupTime(lc, config)),\n  );\n}\n"],"mappings":";;;;;;;AA4BA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,gBACA,uBACA,mBACA,0BACA,OAC2D;CAC3D,MAAM,EAAC,WAAW,MAAM,gBAAe,OAAO;CAC9C,IAAI,CAAC,WACH,OAAO;CAGT,IAAI,OAAO,WAAW,eACpB,OAAO,IAAI,iBACT,IACA,WACA,gBACA,uBACA,OAAO,WAAW,oBAClB,4BACE,IAAI,+BACF,UAEE,gBAAgB,6BAA6B,OAAO,QAAQ,GAAG,GACjE,OAAO,WAAW,iBACpB,CACJ;CAGF,OAAO,IAAI,yBACT,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,gBACA,uBACA,4BAA4B,kBAAkB,IAAI,MAAM,EAC1D;AACF"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts

@@ -1,93 +1,8 @@
-import type { LogContext } from '@rocicorp/logger';
-import { Subscription } from '../../types/subscription.ts';
-import type { Service } from '../service.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
+import type { Subscription } from '../../types/subscription.ts';
+import type { SingletonService } from '../service.ts';
 import type { SnapshotMessage } from './snapshot.ts';
-export declare const CHECK_INTERVAL_MS = 60000;
-/**
- * Allowance for clock skew between the machine reporting litestream metrics
- * and the timestamps reported by the backup destination (e.g. S3).
- */
-export declare const BACKUP_VERIFICATION_SLACK_MS = 60000;
-/**
- * How long the actual backup state may remain *continuously* behind the
- * backup progress claimed by litestream metrics before the backup is
- * considered genuinely wedged and the process is shut down.
- *
- * A wedged backup is dangerous: litestream believes it is making durable
- * progress when it is not, which can silently corrupt the backup (the WAL
- * format only makes *some* such gaps detectable). Once that is confirmed to
- * be the persistent state, continuing to run is worse than crashing, so the
- * process exits loudly (non-zero, with a logged error) and the wedged backup
- * destination becomes the priority to investigate.
- *
- * The change-log is conservatively *not* purged for the entire time the
- * backup is stale, so the only cost of a generous grace period is unbounded
- * change-log growth. We therefore err well on the side of slack: the backup
- * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear
- * the server down, so that a transient hiccup (a slow or briefly unreachable
- * destination, or litestream restarting) does not trigger a shutdown. The
- * staleness clock is reset the moment a purge is confirmed against a real
- * upload.
- */
-export declare const WEDGED_SHUTDOWN_GRACE_MS: number;
-/**
- * Returns the time of the most recent object actually uploaded to the
- * backup replica destination (e.g. as determined by listing the snapshots
- * and WAL segments in S3). Rejects if the backup state cannot be determined.
- *
- * See `getLastBackupTime()` in `../litestream/commands.ts` for the
- * production implementation.
- */
-export type BackupStateVerifier = () => Promise<Date>;
-/**
- * The BackupMonitor polls the litestream "/metrics" endpoint to track the
- * watermark (label) value of the `litestream_replica_progress` gauge and
- * schedules cleanup of change log entries that can be purged as a result.
- *
- * See: https://github.com/rocicorp/litestream/pull/3
- *
- * Note that change log entries cannot simply be purged as soon as they
- * have been applied and backed up by litestream. Consider the case in which
- * litestream backs up new wal segments every minute, but it takes 5 minutes
- * to restore a replica: if a zero-cache starts restoring a replica at
- * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
- * purging changelog records as soon as those watermarks are replicated would
- * result in the zero-cache not being able to catch up from minute 0 once it
- * has finished restoring the replica.
- *
- * The `/snapshot` reservation protocol is used to prevent premature change
- * log cleanup:
- * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
- *   request open while it restores its snapshot, prepares it, and
- *   starts its subscription to the change stream. During this time, no
- *   cleanups are scheduled.
- * - When the subscription is started, the interval since the beginning of
- *   of the reservation is tracked to increase the background cleanup delay
- *   interval if needed. The reservation is ended (and request closed), and
- *   cleanup scheduling is resumed with the current delay interval.
- *
- * Note that the reservation request is the primary mechanism by which
- * premature change log cleanup is prevented. The cleanup delay interval is
- * a secondary safeguard.
- *
- * Additionally, because the watermarks reported by litestream metrics
- * reflect what litestream *believes* has been backed up (which has been
- * observed to diverge from reality when uploads silently fail), the cleanup
- * watermark is only advanced after verifying it against the actual backup
- * state in the replica destination via a {@link BackupStateVerifier}. If the
- * actual backup state stays behind the claimed progress for longer than
- * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
- * process is shut down rather than risk corrupting the backup.
- */
-export declare class BackupMonitor implements Service {
-    #private;
-    readonly id = "backup-monitor";
-    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, verifyBackupState: BackupStateVerifier);
-    run(): Promise<void>;
+export interface BackupMonitor extends SingletonService {
     startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
-    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
-    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
-    stop(): Promise<void>;
+    endReservation(taskID: string, updateCleanupDelay?: boolean | undefined): void;
 }
 //# sourceMappingURL=backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAUjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAGxC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAEpD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAOtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAc,YAAW,OAAO;;IAC3C,QAAQ,CAAC,EAAE,oBAAoB;gBAsC7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,iBAAiB,EAAE,mBAAmB;IAkBxC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAcpB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IA6BvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAWxC;IAqMF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}
+{"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,MAAM,WAAW,aAAc,SAAQ,gBAAgB;IACrD,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC,CAAC;IACxE,cAAc,CACZ,MAAM,EAAE,MAAM,EACd,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,GACvC,IAAI,CAAC;CACT"}

package/out/zero-cache/src/services/change-streamer/change-streamer.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the BackupMonitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}
+{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer.ts"],"sourcesContent":["import type {Enum} from '../../../../shared/src/enum.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {changeStreamDataSchema} from '../change-source/protocol/current/downstream.ts';\nimport type {ReplicatorMode} from '../replicator/replicator.ts';\nimport {changeSourceTimingsSchema} from '../replicator/reporter/report-schema.ts';\nimport type {Service} from '../service.ts';\nimport * as ErrorType from './error-type-enum.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The ChangeStreamer is the component between replicators (\"subscribers\")\n * and a canonical upstream source of changes (e.g. a Postgres logical\n * replication slot). It facilitates multiple subscribers without incurring\n * the associated upstream expense (e.g. PG replication slots are resource\n * intensive) with a \"forward-store-ack\" procedure.\n *\n * * Changes from the upstream source are immediately **forwarded** to\n *   connected subscribers to minimize latency.\n *\n * * They are then **stored** in a separate DB to facilitate catchup\n *   of connecting subscribers that are behind.\n *\n * * **Acknowledgements** are sent upstream after they are successfully\n *   stored.\n *\n * Unlike Postgres replication slots, in which the progress of a static\n * subscriber is tracked in the replication slot, the ChangeStreamer\n * supports a dynamic set of subscribers (i.e.. zero-caches) that can\n * can continually change.\n *\n * However, it is not the case that the ChangeStreamer needs to support\n * arbitrarily old subscribers. Because the replica is continually\n * backed up to a global location and used to initialize new subscriber\n * tasks, an initial subscription request from a subscriber constitutes\n * a signal for how \"behind\" a new subscriber task can be. This is\n * reflected in the {@link SubscriberContext}, which indicates whether\n * the watermark corresponds to an \"initial\" watermark derived from the\n * replica at task startup.\n *\n * The ChangeStreamer uses a combination of this signal with ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\n */\nexport interface ChangeStreamer {\n  /**\n   * Subscribes to changes based on the supplied subscriber `ctx`,\n   * which indicates the watermark at which the subscriber is up to\n   * date.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<Downstream>>;\n}\n\n// v1: v0.18\n//   - Client-side support for JSON_FORMAT. Introduced in 0.18.\n// v2: v0.19\n//   - Adds the \"status\" message which is initially used to signal that the\n//     subscription is valid (i.e. starting at the requested watermark).\n// v3: v0.20\n//   - Adds the \"taskID\" to the subscription context, and support for\n//     the backup monitor-mediated \"/snapshot\" request.\n// v4: v0.25\n//   - Adds the \"replicaVersion\" and \"minWatermark\" fields to the \"/snapshot\"\n//     status request so that a subscriber can verify whether its replica,\n//     whether it be restored or existing in a permanent volume, is compatible\n//     with the change-streamer.\n// v5: v0.26\n//   - Moves relation.keyColumns and relation.replicaIdentity to\n//     relation.rowKey: { columns, type }.\n//   - Adds `metadata` to `create-table` message\n//   - Adds `tableMetadata` to `add-column` message\n//   - Adds `table-update-metadata` message\n// v6: v0.26\n//   - Adds support for `backfill` messages\n// v6: v1.0.1  (backwards compatible, no version change)\n//   - Adds lag reporting to status messages\n\nexport const PROTOCOL_VERSION = 6;\n\nexport type SubscriberContext = {\n  /**\n   * The supported change-streamer protocol version.\n   */\n  protocolVersion: number;\n\n  /**\n   * Task ID. This is used to link the request with a preceding snapshot\n   * reservation.\n   */\n  taskID: string | null; // TODO: Make required when v3 is min.\n\n  /**\n   * Subscriber id. This is only used for debugging.\n   */\n  id: string;\n\n  /**\n   * The ReplicatorMode of the subscriber. 'backup' indicates that the\n   * subscriber is local to the `change-streamer` in the `replication-manager`,\n   * while 'serving' indicates that user-facing requests depend on the subscriber.\n   */\n  mode: ReplicatorMode;\n\n  /**\n   * The ChangeStreamer will return an Error if the subscriber is\n   * on a different replica version (i.e. the initial snapshot associated\n   * with the replication slot).\n   */\n  replicaVersion: string;\n\n  /**\n   * The watermark up to which the subscriber is up to date.\n   * Only changes after the watermark will be streamed.\n   */\n  watermark: string;\n\n  /**\n   * Whether this is the first subscription request made by the task,\n   * i.e. indicating that the watermark comes from a restored replica\n   * backup. The ChangeStreamer uses this to determine which changes\n   * are safe to purge from the Storer.\n   */\n  initial: boolean;\n};\n\n/**\n * The StatusMessage payload for now is empty, but can be extended to\n * include meta-level information in the future.\n */\nexport const statusSchema = v.object({\n  tag: v.literal('status'),\n\n  lagReport: v\n    .object({\n      lastTimings: changeSourceTimingsSchema.optional(),\n      nextSendTimeMs: v.number(),\n    })\n    .optional(),\n});\n\nexport type Status = v.Infer<typeof statusSchema>;\n\nexport const statusMessageSchema = v.tuple([v.literal('status'), statusSchema]);\n\n/**\n * A StatusMessage will be immediately sent on a (v2+) subscription to\n * indicate that the subscription is valid (i.e. starting at the requested\n * watermark). Invalid subscriptions will instead result in a\n * SubscriptionError as the first message.\n */\nexport type StatusMessage = v.Infer<typeof statusMessageSchema>;\n\nconst subscriptionErrorSchema = v.object({\n  type: v.number(), // ErrorType\n  message: v.string().optional(),\n});\n\nexport type SubscriptionError = v.Infer<typeof subscriptionErrorSchema>;\n\nconst errorSchema = v.tuple([v.literal('error'), subscriptionErrorSchema]);\n\nexport const downstreamSchema = v.union(\n  statusMessageSchema,\n  changeStreamDataSchema,\n  errorSchema,\n);\n\nexport type Error = v.Infer<typeof errorSchema>;\n\nexport function errorTypeToReadableName(val: ErrorType) {\n  switch (val) {\n    case ErrorType.WrongReplicaVersion:\n      return 'WrongReplicaVersion';\n    case ErrorType.WatermarkTooOld:\n      return 'WatermarkTooOld';\n    case ErrorType.Unknown:\n      return 'Unknown';\n    default:\n      return 'Unknown';\n  }\n}\n\n/**\n * A stream of transactions, each starting with a {@link Begin} message,\n * containing one or more {@link Data} messages, and ending with a\n * {@link Commit} or {@link Rollback} message. The 'commit' tuple\n * includes a `watermark` that should be stored with the committed\n * data and used for resuming a subscription (e.g. in the\n * {@link SubscriberContext}).\n *\n * A {@link SubscriptionError} indicates an unrecoverable error that requires\n * manual intervention (e.g. configuration / operational error).\n */\nexport type Downstream = v.Infer<typeof downstreamSchema>;\n\nexport interface ChangeStreamerService\n  extends Omit<ChangeStreamer, 'subscribe'>, Service {\n  /**\n   * The server-side interface overrides `subscribe()` to return a stream\n   * of already-stringified {@link Downstream} payloads.\n   */\n  subscribe(ctx: SubscriberContext): Promise<Source<string>>;\n\n  /**\n   * Notifies the change streamer of a watermark that has been backed up,\n   * indicating that changes before the watermark can be purged if active\n   * subscribers have progressed beyond the watermark.\n   */\n  scheduleCleanup(watermark: string): void;\n\n  getChangeLogState(): Promise<{\n    replicaVersion: string;\n    minWatermark: string;\n  }>;\n}\n"],"mappings":";;;;;;;;AAkIA,IAAa,eAAe,eAAE,OAAO;CACnC,KAAK,eAAE,QAAQ,QAAQ;CAEvB,WAAW,eACR,OAAO;EACN,aAAa,0BAA0B,SAAS;EAChD,gBAAgB,eAAE,OAAO;CAC3B,CAAC,EACA,SAAS;AACd,CAAC;AAID,IAAa,sBAAsB,eAAE,MAAM,CAAC,eAAE,QAAQ,QAAQ,GAAG,YAAY,CAAC;AAU9E,IAAM,0BAA0B,eAAE,OAAO;CACvC,MAAM,eAAE,OAAO;CACf,SAAS,eAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAID,IAAM,cAAc,eAAE,MAAM,CAAC,eAAE,QAAQ,OAAO,GAAG,uBAAuB,CAAC;AAEzE,IAAa,mBAAmB,eAAE,MAChC,qBACA,wBACA,WACF;AAIA,SAAgB,wBAAwB,KAAgB;CACtD,QAAQ,KAAR;EACE,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,KAAK,GACH,OAAO;EACT,SACE,OAAO;CACX;AACF"}

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts

@@ -0,0 +1,114 @@
+import type { LogContext } from '@rocicorp/logger';
+import { Subscription } from '../../types/subscription.ts';
+import type { BackupMonitor } from './backup-monitor.ts';
+import type { ChangeStreamerService } from './change-streamer.ts';
+import type { SnapshotMessage } from './snapshot.ts';
+export declare const CHECK_INTERVAL_MS = 60000;
+/**
+ * Allowance for clock skew between the machine reporting litestream metrics
+ * and the timestamps reported by the backup destination (e.g. S3).
+ */
+export declare const BACKUP_VERIFICATION_SLACK_MS = 60000;
+/**
+ * How long the actual backup state may remain *continuously* behind the
+ * backup progress claimed by litestream metrics before the backup is
+ * considered genuinely wedged and the process is shut down.
+ *
+ * A wedged backup is dangerous: litestream believes it is making durable
+ * progress when it is not, which can silently corrupt the backup (the WAL
+ * format only makes *some* such gaps detectable). Once that is confirmed to
+ * be the persistent state, continuing to run is worse than crashing, so the
+ * process exits loudly (non-zero, with a logged error) and the wedged backup
+ * destination becomes the priority to investigate.
+ *
+ * The change-log is conservatively *not* purged for the entire time the
+ * backup is stale, so the only cost of a generous grace period is unbounded
+ * change-log growth. We therefore err well on the side of slack: the backup
+ * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear
+ * the server down, so that a transient hiccup (a slow or briefly unreachable
+ * destination, or litestream restarting) does not trigger a shutdown. The
+ * staleness clock is reset the moment a purge is confirmed against a real
+ * upload.
+ */
+export declare const WEDGED_SHUTDOWN_GRACE_MS: number;
+/**
+ * How long the BackupMonitor waits for the *first* restorable backup to appear
+ * before concluding that the backup pipeline is broken and shutting the process
+ * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).
+ *
+ * On a fresh stack the first backup is not restorable until the initial
+ * Postgres->replica sync completes and litestream finishes uploading the
+ * initial snapshot — a multipart upload that is not listable in the destination
+ * until it is committed. The producer cannot observe that sub-snapshot progress,
+ * so this is necessarily a generous timeout. It is scaled by replica size to
+ * mirror the platform's storage-scaled startup-probe allowance (~1 hour per
+ * 50GB) so that it never fires before the platform would restart a view-syncer
+ * that is waiting on the backup.
+ */
+export declare const INITIAL_BACKUP_GRACE_MS_PER_UNIT: number;
+/**
+ * How often the BackupMonitor re-checks whether the first restorable backup has
+ * appeared while a view-syncer holds a snapshot reservation open during a cold
+ * start.
+ */
+export declare const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10000;
+/**
+ * Returns the time of the most recent object actually uploaded to the
+ * backup replica destination (e.g. as determined by listing the snapshots
+ * and WAL segments in S3). Rejects if the backup state cannot be determined.
+ *
+ * See `getLastBackupTime()` in `../litestream/commands.ts` for the
+ * production implementation.
+ */
+export type BackupStateVerifier = () => Promise<Date>;
+/**
+ * The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+ * track the watermark (label) value of the `litestream_replica_progress` gauge
+ * and schedules cleanup of change log entries that can be purged as a result.
+ *
+ * See: https://github.com/rocicorp/litestream/pull/3
+ *
+ * Note that change log entries cannot simply be purged as soon as they
+ * have been applied and backed up by litestream. Consider the case in which
+ * litestream backs up new wal segments every minute, but it takes 5 minutes
+ * to restore a replica: if a zero-cache starts restoring a replica at
+ * minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+ * purging changelog records as soon as those watermarks are replicated would
+ * result in the zero-cache not being able to catch up from minute 0 once it
+ * has finished restoring the replica.
+ *
+ * The `/snapshot` reservation protocol is used to prevent premature change
+ * log cleanup:
+ * - Clients restoring a snapshot initiate a `/snapshot` request and hold that
+ *   request open while it restores its snapshot, prepares it, and
+ *   starts its subscription to the change stream. During this time, no
+ *   cleanups are scheduled.
+ * - When the subscription is started, the interval since the beginning of
+ *   of the reservation is tracked to increase the background cleanup delay
+ *   interval if needed. The reservation is ended (and request closed), and
+ *   cleanup scheduling is resumed with the current delay interval.
+ *
+ * Note that the reservation request is the primary mechanism by which
+ * premature change log cleanup is prevented. The cleanup delay interval is
+ * a secondary safeguard.
+ *
+ * Additionally, because the watermarks reported by litestream metrics
+ * reflect what litestream *believes* has been backed up (which has been
+ * observed to diverge from reality when uploads silently fail), the cleanup
+ * watermark is only advanced after verifying it against the actual backup
+ * state in the replica destination via a {@link BackupStateVerifier}. If the
+ * actual backup state stays behind the claimed progress for longer than
+ * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
+ * process is shut down rather than risk corrupting the backup.
+ */
+export declare class Litestream3BackupMonitor implements BackupMonitor {
+    #private;
+    readonly id = "backup-monitor";
+    constructor(lc: LogContext, replicaFile: string, backupURL: string, metricsEndpoint: string, changeStreamer: ChangeStreamerService, initialCleanupDelayMs: number, verifyBackupState: BackupStateVerifier);
+    run(): Promise<void>;
+    startSnapshotReservation(taskID: string): Subscription<SnapshotMessage>;
+    endReservation(taskID: string, updateCleanupDelay?: boolean): void;
+    readonly checkWatermarksAndScheduleCleanup: () => Promise<void>;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=litestream3-backup-monitor.d.ts.map

package/out/zero-cache/src/services/change-streamer/litestream3-backup-monitor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream3-backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAWjD,OAAO,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAEzD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AAEnD,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAGxC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAOtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,wBAAyB,YAAW,aAAa;;IAC5D,QAAQ,CAAC,EAAE,oBAAoB;gBA8C7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,cAAc,EAAE,qBAAqB,EACrC,qBAAqB,EAAE,MAAM,EAC7B,iBAAiB,EAAE,mBAAmB;IAkBxC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAepB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,eAAe,CAAC;IAgFvE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,kBAAkB,UAAO;IAoBxD,QAAQ,CAAC,iCAAiC,sBAgBxC;IAuQF,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkDtB"}

package/out/zero-cache/src/services/change-streamer/{backup-monitor.js → litestream3-backup-monitor.js}

@@ -1,12 +1,14 @@
 import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { sleep } from "../../../../shared/src/sleep.js";
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { getOrCreateCounter, getOrCreateGauge } from "../../observability/metrics.js";
 import { RunningState, UnrecoverableError } from "../running-state.js";
 import { Subscription } from "../../types/subscription.js";
 import { resolver } from "@rocicorp/resolver";
+import { statSync } from "node:fs";
 import parsePrometheusTextFormat from "parse-prometheus-text-format";
-//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
+//#region ../zero-cache/src/services/change-streamer/litestream3-backup-monitor.ts
 var CHECK_INTERVAL_MS = 6e4;
 var MIN_CLEANUP_DELAY_MS = 3e4;
 /**
@@ -36,10 +38,33 @@ var BACKUP_VERIFICATION_SLACK_MS = 6e4;
 * upload.
 */
 var WEDGED_SHUTDOWN_GRACE_MS = 15 * 6e4;
+var GiB = 1024 ** 3;
 /**
-* The BackupMonitor polls the litestream "/metrics" endpoint to track the
-* watermark (label) value of the `litestream_replica_progress` gauge and
-* schedules cleanup of change log entries that can be purged as a result.
+* How long the BackupMonitor waits for the *first* restorable backup to appear
+* before concluding that the backup pipeline is broken and shutting the process
+* down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).
+*
+* On a fresh stack the first backup is not restorable until the initial
+* Postgres->replica sync completes and litestream finishes uploading the
+* initial snapshot — a multipart upload that is not listable in the destination
+* until it is committed. The producer cannot observe that sub-snapshot progress,
+* so this is necessarily a generous timeout. It is scaled by replica size to
+* mirror the platform's storage-scaled startup-probe allowance (~1 hour per
+* 50GB) so that it never fires before the platform would restart a view-syncer
+* that is waiting on the backup.
+*/
+var INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 6e4;
+var INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB;
+/**
+* How often the BackupMonitor re-checks whether the first restorable backup has
+* appeared while a view-syncer holds a snapshot reservation open during a cold
+* start.
+*/
+var RESTORABLE_BACKUP_POLL_INTERVAL_MS = 1e4;
+/**
+* The Litestream3BackupMonitor polls the litestream "/metrics" endpoint to
+* track the watermark (label) value of the `litestream_replica_progress` gauge
+* and schedules cleanup of change log entries that can be purged as a result.
 *
 * See: https://github.com/rocicorp/litestream/pull/3
 *
@@ -76,7 +101,7 @@ var WEDGED_SHUTDOWN_GRACE_MS = 15 * 6e4;
 * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the
 * process is shut down rather than risk corrupting the backup.
 */
-var BackupMonitor = class {
+var Litestream3BackupMonitor = class {
 	id = "backup-monitor";
 	#lc;
 	#replicaFile;
@@ -93,6 +118,7 @@ var BackupMonitor = class {
 	#latestBackupTime = null;
 	#lastVerifiedUploadTime = null;
 	#backupStaleSince = null;
+	#runStartTime = null;
 	#cleanupDelayMs;
 	#checkMetricsTimer;
 	constructor(lc, replicaFile, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs, verifyBackupState) {
@@ -106,6 +132,7 @@ var BackupMonitor = class {
 		this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
 	}
 	run() {
+		this.#runStartTime = Date.now();
 		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`);
 		this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
 		this.#initBackupLagMetric();
@@ -119,17 +146,53 @@ var BackupMonitor = class {
 			start: /* @__PURE__ */ new Date(),
 			sub
 		});
-		this.#changeStreamer.getChangeLogState().then((changeLogState) => {
-			sub.push(["status", {
+		this.#pushStatusWhenRestorable(taskID, sub);
+		return sub;
+	}
+	/**
+	* Pushes the `status` snapshot signal once a restorable backup is confirmed
+	* to exist in the backup destination. On a warm start this is immediate; on a
+	* cold start (a fresh stack whose first backup is still being produced) the
+	* reservation is held open — kept alive by websocket liveness pings — and the
+	* view-syncer stays unready until the first backup lands, rather than being
+	* told to restore a backup that does not yet exist.
+	*/
+	async #pushStatusWhenRestorable(taskID, sub) {
+		try {
+			while (this.#lastVerifiedUploadTime === null) {
+				if (!sub.active) return;
+				if (await this.#confirmRestorableBackup()) break;
+				this.#lc.info?.(`no restorable backup at ${this.#backupURL} yet; holding ${taskID}'s reservation open until it lands`);
+				await sleep(RESTORABLE_BACKUP_POLL_INTERVAL_MS, this.#state.signal);
+			}
+			const changeLogState = await this.#changeStreamer.getChangeLogState();
+			if (sub.active) sub.push(["status", {
 				tag: "status",
 				backupURL: this.#backupURL,
 				...changeLogState
 			}]);
-		}).catch((e) => {
+		} catch (e) {
+			if (this.#state.signal.aborted) return;
 			this.#lc.warn?.(`failing snapshot reservation`, e);
-			sub.fail(e);
-		});
-		return sub;
+			sub.fail(e instanceof Error ? e : new Error(String(e)));
+		}
+	}
+	/**
+	* Verifies whether a restorable backup actually exists in the destination
+	* (as opposed to what litestream metrics merely claim). On success the
+	* verified upload time is cached — seeding the cleanup-verification fast path
+	* ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.
+	* Returns false if no backup is listable yet (e.g. the initial snapshot is
+	* still uploading).
+	*/
+	async #confirmRestorableBackup() {
+		try {
+			this.#lastVerifiedUploadTime = await this.#verifyBackupState();
+			return true;
+		} catch (e) {
+			this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);
+			return false;
+		}
 	}
 	endReservation(taskID, updateCleanupDelay = true) {
 		const res = this.#reservations.get(taskID);
@@ -157,6 +220,11 @@ var BackupMonitor = class {
 		} catch (e) {
 			this.#lc.warn?.(`error scheduling cleanup`, e);
 		}
+		try {
+			await this.#checkInitialBackupDeadline();
+		} catch (e) {
+			this.#lc.warn?.(`error checking initial backup deadline`, e);
+		}
 	};
 	async *#fetchWatermarks() {
 		const metricsEndpoint = this.#metricsEndpoint;
@@ -241,6 +309,47 @@ var BackupMonitor = class {
 		this.#fatal.reject(err);
 	}
 	/**
+	* On a fresh stack, fails loudly if no restorable backup has appeared within
+	* a generous, replica-size-scaled grace period (see
+	* {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the "give up" decision
+	* onto the replication-manager — the producer responsible for creating the
+	* backup — instead of leaving view-syncers to wait (and the platform to keep
+	* restarting them) while the real fault is here. Does nothing once a
+	* restorable backup has been confirmed.
+	*/
+	async #checkInitialBackupDeadline() {
+		if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) return;
+		const elapsed = Date.now() - this.#runStartTime;
+		if (elapsed < this.#initialBackupGraceMs()) return;
+		if (await this.#confirmRestorableBackup()) return;
+		this.#shutDownOnMissingInitialBackup(elapsed);
+	}
+	/**
+	* The grace period for the first restorable backup, scaled by replica size to
+	* mirror the platform's storage-scaled startup-probe allowance. Falls back to
+	* the minimum (one unit) if the replica file size cannot be determined.
+	*/
+	#initialBackupGraceMs() {
+		let bytes = 0;
+		try {
+			bytes = statSync(this.#replicaFile).size;
+		} catch {}
+		return Math.max(1, Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES)) * INITIAL_BACKUP_GRACE_MS_PER_UNIT;
+	}
+	/**
+	* Called when no restorable backup has appeared within
+	* {@link #initialBackupGraceMs}. Continuing to run is pointless — no
+	* view-syncer can restore — and the fault is the backup pipeline, so the
+	* process exits non-zero with a logged error for alerting, mirroring
+	* {@link #shutDownOnWedgedBackup}.
+	*/
+	#shutDownOnMissingInitialBackup(elapsedMs) {
+		const err = new UnrecoverableError(`no restorable backup has appeared at ${this.#backupURL} within ${elapsedMs} ms of startup (grace period ${this.#initialBackupGraceMs()} ms). The initial backup pipeline appears to be broken; shutting down so that the replication-manager is flagged as the cause rather than the view-syncers waiting on it. Investigate litestream replication to ${this.#backupURL}.`);
+		this.#lc.error?.(err.message);
+		clearInterval(this.#checkMetricsTimer);
+		this.#fatal.reject(err);
+	}
+	/**
 	* Returns the newest watermark whose backup time is at or before `cutoff`
 	* (epoch ms), or `''` if there is none.
 	*/
@@ -290,6 +399,6 @@ var BackupMonitor = class {
 	}
 };
 //#endregion
-export { BackupMonitor };
+export { Litestream3BackupMonitor };
 
-//# sourceMappingURL=backup-monitor.js.map
+//# sourceMappingURL=litestream3-backup-monitor.js.map