@rocicorp/zero 1.7.0-canary.5 → 1.8.0-canary.0

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    try {\n      if (await tryRestore(lc, config, replicaConstraints)) {\n        return;\n      }\n    } catch (e) {\n      if (i === 0) {\n        // A restore will fail if the `replicate` process creates a new\n        // snapshot (and compacts old files) at the same time. Snapshots are\n        // infrequent (e.g. once every 12 hours), and the scenario is\n        // recoverable with a retry.\n        lc.warn?.(`initial restore attempt failed. retrying once`, e);\n        continue;\n      }\n      // If it fails again on the retry, though, bail.\n      throw e;\n    }\n    if (replicaConstraints) {\n      // This can happen if the litestream URL is purposefully changed to\n      // force a resync.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n    lc.info?.(\n      `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n    );\n    await sleep(RETRY_INTERVAL_MS);\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  let output: string;\n  try {\n    output = await promise;\n  } finally {\n    clearTimeout(timeout);\n  }\n\n  return parseBackupCreatedTimes(lc, command, output);\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // (i.e. BackupMonitor) returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,IAAI;GACF,IAAI,MAAM,WAAW,IAAI,QAAQ,kBAAkB,GACjD;EAEJ,SAAS,GAAG;GACV,IAAI,MAAM,GAAG;IAKX,GAAG,OAAO,iDAAiD,CAAC;IAC5D;GACF;GAEA,MAAM;EACR;EACA,IAAI,oBAGF,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;EAE/D,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;EACA,MAAM,MAAM,iBAAiB;CAC/B;CACA,MAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;CACA,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,WAAW;EAClB,OAAO,QAAQ;CACjB,GACA;EAAC;EAAK,OAAO;EAAW,aAAa;CAAI,CAC3C;CACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;CAC5C,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;OACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;OAE5C,QAAQ;CAEZ,CAAC;CACD,MAAM;CACN,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GACjC,OAAO;CAET,IAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB,GAAG;EAChE,GAAG,OAAO,6CAA6C;EACvD,aAAa,OAAO,QAAQ,IAAI;EAChC,OAAO;CACT;CACA,OAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO;CACnC,IAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,GAAG,MAAM;CACX;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EACtC;EACA,OAAO;EACP,aAAa;CACf,CAAC;AACH;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM;CACjB,UAAU;EACR,aAAa,OAAO;CACtB;CAEA,OAAO,wBAAwB,IAAI,SAAS,MAAM;AACpD;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  for (let i = 0; i < MAX_RETRIES; i++) {\n    try {\n      if (await tryRestore(lc, config, replicaConstraints)) {\n        return;\n      }\n    } catch (e) {\n      if (i === 0) {\n        // A restore will fail if the `replicate` process creates a new\n        // snapshot (and compacts old files) at the same time. Snapshots are\n        // infrequent (e.g. once every 12 hours), and the scenario is\n        // recoverable with a retry.\n        lc.warn?.(`initial restore attempt failed. retrying once`, e);\n        continue;\n      }\n      // If it fails again on the retry, though, bail.\n      throw e;\n    }\n    if (replicaConstraints) {\n      // This can happen if the litestream URL is purposefully changed to\n      // force a resync.\n      throw new BackupNotFoundException(config.litestream.backupURL);\n    }\n    lc.info?.(\n      `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n    );\n    await sleep(RETRY_INTERVAL_MS);\n  }\n  throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const {litestream, env} = getLitestream(\n    'restore',\n    config,\n    'debug', // Include all output from `litestream restore`, as it's minimal.\n    snapshotStatus?.backupURL,\n  );\n  const {restoreParallelism: parallelism} = config.litestream;\n  const proc = spawn(\n    litestream,\n    [\n      'restore',\n      '-if-db-not-exists',\n      '-if-replica-exists',\n      '-parallelism',\n      String(parallelism),\n      config.replica.file,\n    ],\n    {env, stdio: 'inherit', windowsHide: true},\n  );\n  const {promise, resolve, reject} = resolver();\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(`litestream killed with ${signal}`);\n    } else if (code !== 0) {\n      reject(`litestream exited with code ${code}`);\n    } else {\n      resolve();\n    }\n  });\n  await promise;\n  if (!existsSync(config.replica.file)) {\n    return false;\n  }\n  if (!replicaIsValid(lc, config.replica.file, replicaConstraints)) {\n    lc.info?.(`Deleting local replica and retrying restore`);\n    deleteLiteDB(config.replica.file);\n    return false;\n  }\n  return true;\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  const db = new Database(lc, replica);\n  try {\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  return spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  let output: string;\n  try {\n    output = await promise;\n  } finally {\n    clearTimeout(timeout);\n  }\n\n  return parseBackupCreatedTimes(lc, command, output);\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,IAAM,cAAc;AACpB,IAAM,oBAAoB;AAO1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,KAAK,IAAI,IAAI,GAAG,IAAI,aAAa,KAAK;EACpC,IAAI;GACF,IAAI,MAAM,WAAW,IAAI,QAAQ,kBAAkB,GACjD;EAEJ,SAAS,GAAG;GACV,IAAI,MAAM,GAAG;IAKX,GAAG,OAAO,iDAAiD,CAAC;IAC5D;GACF;GAEA,MAAM;EACR;EACA,IAAI,oBAGF,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;EAE/D,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;EACA,MAAM,MAAM,iBAAiB;CAC/B;CACA,MAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA;CACA,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;CACA,MAAM,EAAC,oBAAoB,gBAAe,OAAO;CACjD,MAAM,OAAO,MACX,YACA;EACE;EACA;EACA;EACA;EACA,OAAO,WAAW;EAClB,OAAO,QAAQ;CACjB,GACA;EAAC;EAAK,OAAO;EAAW,aAAa;CAAI,CAC3C;CACA,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;CAC5C,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;OACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;OAE5C,QAAQ;CAEZ,CAAC;CACD,MAAM;CACN,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GACjC,OAAO;CAET,IAAI,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB,GAAG;EAChE,GAAG,OAAO,6CAA6C;EACvD,aAAa,OAAO,QAAQ,IAAI;EAChC,OAAO;CACT;CACA,OAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,MAAM,KAAK,IAAI,SAAS,IAAI,OAAO;CACnC,IAAI;EACF,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,GAAG,MAAM;CACX;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EACtC;EACA,OAAO;EACP,aAAa;CACf,CAAC;AACH;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM;CACjB,UAAU;EACR,aAAa,OAAO;CACtB;CAEA,OAAO,wBAAwB,IAAI,SAAS,MAAM;AACpD;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts

@@ -0,0 +1,44 @@
+import type { LogContext, LogLevel } from '@rocicorp/logger';
+type LoadableDatabase = {
+    loadExtension(path: string): unknown;
+    close(): void;
+};
+type QueryDatabase = {
+    prepare(sql: string): {
+        get<T>(): T;
+    };
+    close(): void;
+};
+export type VfsBackupWatermark = {
+    watermark: string;
+    writeTimeMs: number | null;
+    txid: string;
+    lagSeconds: number;
+    observedAtMs: number;
+};
+export type VfsBackupWatermarkReaderOptions = {
+    readonly replicaURL: string;
+    readonly extensionPath: string;
+    readonly logLevel?: LogLevel | undefined;
+    readonly logFile?: string | undefined;
+    readonly databaseURI?: string | undefined;
+    readonly env?: NodeJS.ProcessEnv | undefined;
+};
+type VfsBackupWatermarkReaderDeps = {
+    readonly loaderFactory?: (() => LoadableDatabase) | undefined;
+    readonly databaseFactory?: ((lc: LogContext, uri: string) => QueryDatabase) | undefined;
+};
+export declare function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region, }: {
+    readonly backupURL: string;
+    readonly endpoint?: string | undefined;
+    readonly region?: string | undefined;
+}): string;
+export declare class VfsBackupWatermarkReader implements Disposable {
+    #private;
+    constructor(lc: LogContext, options: VfsBackupWatermarkReaderOptions, deps?: VfsBackupWatermarkReaderDeps);
+    readWatermark(): VfsBackupWatermark;
+    close(): void;
+    [Symbol.dispose](): void;
+}
+export {};
+//# sourceMappingURL=vfs-watermark-reader.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-reader.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,QAAQ,EAAC,MAAM,kBAAkB,CAAC;AAQ3D,KAAK,gBAAgB,GAAG;IACtB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG;QACpB,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;KACb,CAAC;IACF,KAAK,IAAI,IAAI,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,+BAA+B,GAAG;IAC5C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;CAC9C,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAClC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,gBAAgB,CAAC,GAAG,SAAS,CAAC;IAC9D,QAAQ,CAAC,eAAe,CAAC,EACrB,CAAC,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,KAAK,aAAa,CAAC,GAChD,SAAS,CAAC;CACf,CAAC;AASF,wBAAgB,4BAA4B,CAAC,EAC3C,SAAS,EACT,QAAQ,EACR,MAAM,GACP,EAAE;IACD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,MAAM,CAiBT;AAED,qBAAa,wBAAyB,YAAW,UAAU;;gBAQvD,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,+BAA+B,EACxC,IAAI,GAAE,4BAAiC;IAiCzC,aAAa,IAAI,kBAAkB;IAQnC,KAAK,IAAI,IAAI;IAKb,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js

@@ -0,0 +1,65 @@
+import { parse, valita_exports } from "../../../../shared/src/valita.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import SQLite3Database from "@rocicorp/zero-sqlite3";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-reader.ts
+var DEFAULT_VFS_DATABASE_URI = "file:zero-backup.db?vfs=litestream&mode=ro";
+var DEFAULT_VFS_LOG_LEVEL = "info";
+var backupWatermarkRowSchema = valita_exports.object({
+	watermark: valita_exports.string(),
+	writeTimeMs: valita_exports.number().nullable(),
+	txid: valita_exports.string(),
+	lagSeconds: valita_exports.number()
+});
+function buildLitestreamVfsReplicaURL({ backupURL, endpoint, region }) {
+	if (endpoint === void 0 && region === void 0) return backupURL;
+	const url = new URL(backupURL);
+	if (url.protocol !== "s3:") return backupURL;
+	if (endpoint !== void 0 && !url.searchParams.has("endpoint")) url.searchParams.set("endpoint", endpoint);
+	if (region !== void 0 && !url.searchParams.has("region")) url.searchParams.set("region", region);
+	return url.toString();
+}
+var VfsBackupWatermarkReader = class {
+	#loader;
+	#db;
+	#stmt;
+	constructor(lc, options, deps = {}) {
+		const env = options.env ?? process.env;
+		env["LITESTREAM_REPLICA_URL"] = options.replicaURL;
+		env["LITESTREAM_LOG_LEVEL"] = (options.logLevel ?? DEFAULT_VFS_LOG_LEVEL).toUpperCase();
+		setOptionalEnv(env, "LITESTREAM_LOG_FILE", options.logFile);
+		const loaderFactory = deps.loaderFactory ?? (() => new SQLite3Database(":memory:"));
+		const databaseFactory = deps.databaseFactory ?? ((lc, uri) => new Database(lc, uri, { readonly: true }));
+		this.#loader = loaderFactory();
+		this.#loader.loadExtension(options.extensionPath);
+		this.#db = databaseFactory(lc.withContext("component", "vfs-backup-watermark-reader"), options.databaseURI ?? DEFAULT_VFS_DATABASE_URI);
+		this.#stmt = this.#db.prepare(`
+      SELECT
+        stateVersion AS watermark,
+        writeTimeMs,
+        litestream_txid() AS txid,
+        litestream_lag() AS lagSeconds
+      FROM "_zero.replicationState"
+    `);
+	}
+	readWatermark() {
+		return {
+			...parse(this.#stmt.get(), backupWatermarkRowSchema),
+			observedAtMs: Date.now()
+		};
+	}
+	close() {
+		this.#db.close();
+		this.#loader.close();
+	}
+	[Symbol.dispose]() {
+		this.close();
+	}
+};
+function setOptionalEnv(env, name, value) {
+	if (value === void 0) delete env[name];
+	else env[name] = value;
+}
+//#endregion
+export { VfsBackupWatermarkReader, buildLitestreamVfsReplicaURL };
+
+//# sourceMappingURL=vfs-watermark-reader.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-reader.js","names":["#loader","#db","#stmt"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-reader.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport Sqlite3Database from '@rocicorp/zero-sqlite3';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\n\nconst DEFAULT_VFS_DATABASE_URI = 'file:zero-backup.db?vfs=litestream&mode=ro';\nconst DEFAULT_VFS_LOG_LEVEL = 'info';\n\ntype LoadableDatabase = {\n  loadExtension(path: string): unknown;\n  close(): void;\n};\n\ntype QueryDatabase = {\n  prepare(sql: string): {\n    get<T>(): T;\n  };\n  close(): void;\n};\n\nexport type VfsBackupWatermark = {\n  watermark: string;\n  writeTimeMs: number | null;\n  txid: string;\n  lagSeconds: number;\n  observedAtMs: number;\n};\n\nexport type VfsBackupWatermarkReaderOptions = {\n  readonly replicaURL: string;\n  readonly extensionPath: string;\n  readonly logLevel?: LogLevel | undefined;\n  readonly logFile?: string | undefined;\n  readonly databaseURI?: string | undefined;\n  readonly env?: NodeJS.ProcessEnv | undefined;\n};\n\ntype VfsBackupWatermarkReaderDeps = {\n  readonly loaderFactory?: (() => LoadableDatabase) | undefined;\n  readonly databaseFactory?:\n    | ((lc: LogContext, uri: string) => QueryDatabase)\n    | undefined;\n};\n\nconst backupWatermarkRowSchema = v.object({\n  watermark: v.string(),\n  writeTimeMs: v.number().nullable(),\n  txid: v.string(),\n  lagSeconds: v.number(),\n});\n\nexport function buildLitestreamVfsReplicaURL({\n  backupURL,\n  endpoint,\n  region,\n}: {\n  readonly backupURL: string;\n  readonly endpoint?: string | undefined;\n  readonly region?: string | undefined;\n}): string {\n  if (endpoint === undefined && region === undefined) {\n    return backupURL;\n  }\n\n  const url = new URL(backupURL);\n  if (url.protocol !== 's3:') {\n    return backupURL;\n  }\n\n  if (endpoint !== undefined && !url.searchParams.has('endpoint')) {\n    url.searchParams.set('endpoint', endpoint);\n  }\n  if (region !== undefined && !url.searchParams.has('region')) {\n    url.searchParams.set('region', region);\n  }\n  return url.toString();\n}\n\nexport class VfsBackupWatermarkReader implements Disposable {\n  readonly #loader: LoadableDatabase;\n  readonly #db: QueryDatabase;\n  readonly #stmt: {\n    get<T>(): T;\n  };\n\n  constructor(\n    lc: LogContext,\n    options: VfsBackupWatermarkReaderOptions,\n    deps: VfsBackupWatermarkReaderDeps = {},\n  ) {\n    const env = options.env ?? process.env;\n    env['LITESTREAM_REPLICA_URL'] = options.replicaURL;\n    env['LITESTREAM_LOG_LEVEL'] = (\n      options.logLevel ?? DEFAULT_VFS_LOG_LEVEL\n    ).toUpperCase();\n    setOptionalEnv(env, 'LITESTREAM_LOG_FILE', options.logFile);\n\n    const loaderFactory =\n      deps.loaderFactory ?? (() => new Sqlite3Database(':memory:'));\n    const databaseFactory =\n      deps.databaseFactory ??\n      ((lc: LogContext, uri: string) =>\n        new Database(lc, uri, {readonly: true}));\n\n    this.#loader = loaderFactory();\n    this.#loader.loadExtension(options.extensionPath);\n\n    this.#db = databaseFactory(\n      lc.withContext('component', 'vfs-backup-watermark-reader'),\n      options.databaseURI ?? DEFAULT_VFS_DATABASE_URI,\n    );\n    this.#stmt = this.#db.prepare(/*sql*/ `\n      SELECT\n        stateVersion AS watermark,\n        writeTimeMs,\n        litestream_txid() AS txid,\n        litestream_lag() AS lagSeconds\n      FROM \"_zero.replicationState\"\n    `);\n  }\n\n  readWatermark(): VfsBackupWatermark {\n    const row = v.parse(this.#stmt.get(), backupWatermarkRowSchema);\n    return {\n      ...row,\n      observedAtMs: Date.now(),\n    };\n  }\n\n  close(): void {\n    this.#db.close();\n    this.#loader.close();\n  }\n\n  [Symbol.dispose](): void {\n    this.close();\n  }\n}\n\nfunction setOptionalEnv(\n  env: NodeJS.ProcessEnv,\n  name: string,\n  value: string | undefined,\n) {\n  if (value === undefined) {\n    delete env[name];\n  } else {\n    env[name] = value;\n  }\n}\n"],"mappings":";;;;AAKA,IAAM,2BAA2B;AACjC,IAAM,wBAAwB;AAsC9B,IAAM,2BAA2B,eAAE,OAAO;CACxC,WAAW,eAAE,OAAO;CACpB,aAAa,eAAE,OAAO,EAAE,SAAS;CACjC,MAAM,eAAE,OAAO;CACf,YAAY,eAAE,OAAO;AACvB,CAAC;AAED,SAAgB,6BAA6B,EAC3C,WACA,UACA,UAKS;CACT,IAAI,aAAa,KAAA,KAAa,WAAW,KAAA,GACvC,OAAO;CAGT,MAAM,MAAM,IAAI,IAAI,SAAS;CAC7B,IAAI,IAAI,aAAa,OACnB,OAAO;CAGT,IAAI,aAAa,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,UAAU,GAC5D,IAAI,aAAa,IAAI,YAAY,QAAQ;CAE3C,IAAI,WAAW,KAAA,KAAa,CAAC,IAAI,aAAa,IAAI,QAAQ,GACxD,IAAI,aAAa,IAAI,UAAU,MAAM;CAEvC,OAAO,IAAI,SAAS;AACtB;AAEA,IAAa,2BAAb,MAA4D;CAC1D;CACA;CACA;CAIA,YACE,IACA,SACA,OAAqC,CAAC,GACtC;EACA,MAAM,MAAM,QAAQ,OAAO,QAAQ;EACnC,IAAI,4BAA4B,QAAQ;EACxC,IAAI,2BACF,QAAQ,YAAY,uBACpB,YAAY;EACd,eAAe,KAAK,uBAAuB,QAAQ,OAAO;EAE1D,MAAM,gBACJ,KAAK,wBAAwB,IAAI,gBAAgB,UAAU;EAC7D,MAAM,kBACJ,KAAK,qBACH,IAAgB,QAChB,IAAI,SAAS,IAAI,KAAK,EAAC,UAAU,KAAI,CAAC;EAE1C,KAAKA,UAAU,cAAc;EAC7B,KAAKA,QAAQ,cAAc,QAAQ,aAAa;EAEhD,KAAKC,MAAM,gBACT,GAAG,YAAY,aAAa,6BAA6B,GACzD,QAAQ,eAAe,wBACzB;EACA,KAAKC,QAAQ,KAAKD,IAAI,QAAgB;;;;;;;KAOrC;CACH;CAEA,gBAAoC;EAElC,OAAO;GACL,GAFU,MAAQ,KAAKC,MAAM,IAAI,GAAG,wBAEjC;GACH,cAAc,KAAK,IAAI;EACzB;CACF;CAEA,QAAc;EACZ,KAAKD,IAAI,MAAM;EACf,KAAKD,QAAQ,MAAM;CACrB;CAEA,CAAC,OAAO,WAAiB;EACvB,KAAK,MAAM;CACb;AACF;AAEA,SAAS,eACP,KACA,MACA,OACA;CACA,IAAI,UAAU,KAAA,GACZ,OAAO,IAAI;MAEX,IAAI,QAAQ;AAEhB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts

@@ -0,0 +1,11 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Worker } from '../../types/processes.ts';
+import type { VfsBackupWatermark } from './vfs-watermark-reader.ts';
+export type VfsBackupWatermarkWorkerFactory = () => Worker;
+export declare class VfsBackupWatermarkWorkerSource {
+    #private;
+    constructor(lc: LogContext, workerFactory: VfsBackupWatermarkWorkerFactory, timeoutMs: number);
+    readWatermark(): Promise<VfsBackupWatermark>;
+    close(): void;
+}
+//# sourceMappingURL=vfs-watermark-worker-source.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAMlE,MAAM,MAAM,+BAA+B,GAAG,MAAM,MAAM,CAAC;AAE3D,qBAAa,8BAA8B;;gBASvC,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,+BAA+B,EAC9C,SAAS,EAAE,MAAM;IAUb,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAclD,KAAK,IAAI,IAAI;CA4Dd"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js

@@ -0,0 +1,74 @@
+import { BackupWatermarkTimeoutError, requestVfsBackupWatermark } from "./vfs-watermark-worker.js";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts
+var VfsBackupWatermarkWorkerSource = class {
+	#lc;
+	#workerFactory;
+	#timeoutMs;
+	#worker;
+	#ready;
+	constructor(lc, workerFactory, timeoutMs) {
+		this.#lc = lc.withContext("component", "vfs-backup-watermark-worker-source");
+		this.#workerFactory = workerFactory;
+		this.#timeoutMs = timeoutMs;
+	}
+	async readWatermark() {
+		const worker = await this.#getWorker();
+		try {
+			return await requestVfsBackupWatermark(worker, this.#timeoutMs, { killOnTimeout: true });
+		} catch (e) {
+			if (e instanceof BackupWatermarkTimeoutError) this.#clearWorker(worker);
+			throw e;
+		}
+	}
+	close() {
+		this.#worker?.kill("SIGTERM");
+		this.#worker = void 0;
+		this.#ready = void 0;
+	}
+	#getWorker() {
+		if (this.#ready) return this.#ready;
+		const worker = this.#workerFactory();
+		this.#worker = worker;
+		this.#ready = new Promise((resolve, reject) => {
+			const cleanup = () => {
+				worker.off("message", onMessage);
+				worker.off("error", onError);
+				worker.off("close", onClose);
+			};
+			const fail = (e) => {
+				cleanup();
+				this.#clearWorker(worker);
+				reject(e);
+			};
+			const onMessage = (data) => {
+				if (!isReadyMessage(data)) return;
+				cleanup();
+				resolve(worker);
+			};
+			const onError = (e) => fail(e);
+			const onClose = (code, signal) => fail(/* @__PURE__ */ new Error(`backup watermark reader worker exited before ready (code=${String(code)}, signal=${String(signal)})`));
+			worker.on("message", onMessage);
+			worker.once("error", onError);
+			worker.once("close", onClose);
+		});
+		worker.once("close", () => this.#clearWorker(worker));
+		worker.once("error", (e) => {
+			this.#lc.warn?.(`backup watermark reader worker failed`, e);
+			this.#clearWorker(worker);
+		});
+		return this.#ready;
+	}
+	#clearWorker(worker) {
+		if (this.#worker === worker) {
+			this.#worker = void 0;
+			this.#ready = void 0;
+		}
+	}
+};
+function isReadyMessage(data) {
+	return Array.isArray(data) && data.length === 2 && data[0] === "ready" && typeof data[1] === "object" && data[1] !== null && "ready" in data[1] && data[1].ready === true;
+}
+//#endregion
+export { VfsBackupWatermarkWorkerSource };
+
+//# sourceMappingURL=vfs-watermark-worker-source.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker-source.js","names":["#lc","#workerFactory","#timeoutMs","#getWorker","#clearWorker","#worker","#ready"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Worker} from '../../types/processes.ts';\nimport type {VfsBackupWatermark} from './vfs-watermark-reader.ts';\nimport {\n  BackupWatermarkTimeoutError,\n  requestVfsBackupWatermark,\n} from './vfs-watermark-worker.ts';\n\nexport type VfsBackupWatermarkWorkerFactory = () => Worker;\n\nexport class VfsBackupWatermarkWorkerSource {\n  readonly #lc: LogContext;\n  readonly #workerFactory: VfsBackupWatermarkWorkerFactory;\n  readonly #timeoutMs: number;\n\n  #worker: Worker | undefined;\n  #ready: Promise<Worker> | undefined;\n\n  constructor(\n    lc: LogContext,\n    workerFactory: VfsBackupWatermarkWorkerFactory,\n    timeoutMs: number,\n  ) {\n    this.#lc = lc.withContext(\n      'component',\n      'vfs-backup-watermark-worker-source',\n    );\n    this.#workerFactory = workerFactory;\n    this.#timeoutMs = timeoutMs;\n  }\n\n  async readWatermark(): Promise<VfsBackupWatermark> {\n    const worker = await this.#getWorker();\n    try {\n      return await requestVfsBackupWatermark(worker, this.#timeoutMs, {\n        killOnTimeout: true,\n      });\n    } catch (e) {\n      if (e instanceof BackupWatermarkTimeoutError) {\n        this.#clearWorker(worker);\n      }\n      throw e;\n    }\n  }\n\n  close(): void {\n    this.#worker?.kill('SIGTERM');\n    this.#worker = undefined;\n    this.#ready = undefined;\n  }\n\n  #getWorker(): Promise<Worker> {\n    if (this.#ready) {\n      return this.#ready;\n    }\n\n    const worker = this.#workerFactory();\n    this.#worker = worker;\n    this.#ready = new Promise((resolve, reject) => {\n      const cleanup = () => {\n        worker.off('message', onMessage);\n        worker.off('error', onError);\n        worker.off('close', onClose);\n      };\n      const fail = (e: unknown) => {\n        cleanup();\n        this.#clearWorker(worker);\n        reject(e);\n      };\n      const onMessage = (data: unknown) => {\n        if (!isReadyMessage(data)) {\n          return;\n        }\n        cleanup();\n        resolve(worker);\n      };\n      const onError = (e: unknown) => fail(e);\n      const onClose = (code: unknown, signal: unknown) =>\n        fail(\n          new Error(\n            `backup watermark reader worker exited before ready ` +\n              `(code=${String(code)}, signal=${String(signal)})`,\n          ),\n        );\n\n      worker.on('message', onMessage);\n      worker.once('error', onError);\n      worker.once('close', onClose);\n    });\n\n    worker.once('close', () => this.#clearWorker(worker));\n    worker.once('error', e => {\n      this.#lc.warn?.(`backup watermark reader worker failed`, e);\n      this.#clearWorker(worker);\n    });\n\n    return this.#ready;\n  }\n\n  #clearWorker(worker: Worker): void {\n    if (this.#worker === worker) {\n      this.#worker = undefined;\n      this.#ready = undefined;\n    }\n  }\n}\n\nfunction isReadyMessage(data: unknown): boolean {\n  return (\n    Array.isArray(data) &&\n    data.length === 2 &&\n    data[0] === 'ready' &&\n    typeof data[1] === 'object' &&\n    data[1] !== null &&\n    'ready' in data[1] &&\n    data[1].ready === true\n  );\n}\n"],"mappings":";;AAUA,IAAa,iCAAb,MAA4C;CAC1C;CACA;CACA;CAEA;CACA;CAEA,YACE,IACA,eACA,WACA;EACA,KAAKA,MAAM,GAAG,YACZ,aACA,oCACF;EACA,KAAKC,iBAAiB;EACtB,KAAKC,aAAa;CACpB;CAEA,MAAM,gBAA6C;EACjD,MAAM,SAAS,MAAM,KAAKC,WAAW;EACrC,IAAI;GACF,OAAO,MAAM,0BAA0B,QAAQ,KAAKD,YAAY,EAC9D,eAAe,KACjB,CAAC;EACH,SAAS,GAAG;GACV,IAAI,aAAa,6BACf,KAAKE,aAAa,MAAM;GAE1B,MAAM;EACR;CACF;CAEA,QAAc;EACZ,KAAKC,SAAS,KAAK,SAAS;EAC5B,KAAKA,UAAU,KAAA;EACf,KAAKC,SAAS,KAAA;CAChB;CAEA,aAA8B;EAC5B,IAAI,KAAKA,QACP,OAAO,KAAKA;EAGd,MAAM,SAAS,KAAKL,eAAe;EACnC,KAAKI,UAAU;EACf,KAAKC,SAAS,IAAI,SAAS,SAAS,WAAW;GAC7C,MAAM,gBAAgB;IACpB,OAAO,IAAI,WAAW,SAAS;IAC/B,OAAO,IAAI,SAAS,OAAO;IAC3B,OAAO,IAAI,SAAS,OAAO;GAC7B;GACA,MAAM,QAAQ,MAAe;IAC3B,QAAQ;IACR,KAAKF,aAAa,MAAM;IACxB,OAAO,CAAC;GACV;GACA,MAAM,aAAa,SAAkB;IACnC,IAAI,CAAC,eAAe,IAAI,GACtB;IAEF,QAAQ;IACR,QAAQ,MAAM;GAChB;GACA,MAAM,WAAW,MAAe,KAAK,CAAC;GACtC,MAAM,WAAW,MAAe,WAC9B,qBACE,IAAI,MACF,4DACW,OAAO,IAAI,EAAE,WAAW,OAAO,MAAM,EAAE,EACpD,CACF;GAEF,OAAO,GAAG,WAAW,SAAS;GAC9B,OAAO,KAAK,SAAS,OAAO;GAC5B,OAAO,KAAK,SAAS,OAAO;EAC9B,CAAC;EAED,OAAO,KAAK,eAAe,KAAKA,aAAa,MAAM,CAAC;EACpD,OAAO,KAAK,UAAS,MAAK;GACxB,KAAKJ,IAAI,OAAO,yCAAyC,CAAC;GAC1D,KAAKI,aAAa,MAAM;EAC1B,CAAC;EAED,OAAO,KAAKE;CACd;CAEA,aAAa,QAAsB;EACjC,IAAI,KAAKD,YAAY,QAAQ;GAC3B,KAAKA,UAAU,KAAA;GACf,KAAKC,SAAS,KAAA;EAChB;CACF;AACF;AAEA,SAAS,eAAe,MAAwB;CAC9C,OACE,MAAM,QAAQ,IAAI,KAClB,KAAK,WAAW,KAChB,KAAK,OAAO,WACZ,OAAO,KAAK,OAAO,YACnB,KAAK,OAAO,QACZ,WAAW,KAAK,MAChB,KAAK,GAAG,UAAU;AAEtB"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts

@@ -0,0 +1,42 @@
+import type { LogContext } from '@rocicorp/logger';
+import type { Message, Worker } from '../../types/processes.ts';
+import type { Service } from '../service.ts';
+import type { VfsBackupWatermark } from './vfs-watermark-reader.ts';
+export type BackupWatermarkRequest = [
+    'backupWatermarkRequest',
+    {
+        readonly requestID: string;
+    }
+];
+export type BackupWatermarkResponse = [
+    'backupWatermarkResponse',
+    {
+        readonly requestID: string;
+        readonly result?: VfsBackupWatermark | undefined;
+        readonly error?: {
+            readonly name: string;
+            readonly message: string;
+            readonly stack?: string | undefined;
+        } | undefined;
+    }
+];
+export interface BackupWatermarkSource {
+    readWatermark(): VfsBackupWatermark;
+    close?(): void;
+}
+export type BackupWatermarkSourceFactory = () => BackupWatermarkSource;
+export declare class BackupWatermarkTimeoutError extends Error {
+}
+export declare function requestVfsBackupWatermark(worker: Worker, timeoutMs: number, options?: {
+    readonly killOnTimeout?: boolean | undefined;
+    readonly killSignal?: NodeJS.Signals | undefined;
+}): Promise<VfsBackupWatermark>;
+export declare class VfsBackupWatermarkWorkerService implements Service {
+    #private;
+    readonly id = "vfs-backup-watermark-worker";
+    constructor(lc: LogContext, parent: Worker | null, source: BackupWatermarkSource | BackupWatermarkSourceFactory, pollIntervalMs: number);
+    run(): Promise<void>;
+    stop(): Promise<void>;
+}
+export type BackupWatermarkMessage = BackupWatermarkRequest | BackupWatermarkResponse | Message<unknown>;
+//# sourceMappingURL=vfs-watermark-worker.d.ts.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,OAAO,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAElE,MAAM,MAAM,sBAAsB,GAAG;IACnC,wBAAwB;IACxB;QACE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B;CACF,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,yBAAyB;IACzB;QACE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC;QACjD,QAAQ,CAAC,KAAK,CAAC,EACX;YACE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;SACrC,GACD,SAAS,CAAC;KACf;CACF,CAAC;AAEF,MAAM,WAAW,qBAAqB;IACpC,aAAa,IAAI,kBAAkB,CAAC;IACpC,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,MAAM,4BAA4B,GAAG,MAAM,qBAAqB,CAAC;AAIvE,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IACP,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,SAAS,CAAC;CAC7C,GACL,OAAO,CAAC,kBAAkB,CAAC,CAkD7B;AAED,qBAAa,+BAAgC,YAAW,OAAO;;IAC7D,QAAQ,CAAC,EAAE,iCAAiC;gBAS1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,MAAM,EAAE,qBAAqB,GAAG,4BAA4B,EAC5D,cAAc,EAAE,MAAM;IAQlB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CA+BtB;AA6BD,MAAM,MAAM,sBAAsB,GAC9B,sBAAsB,GACtB,uBAAuB,GACvB,OAAO,CAAC,OAAO,CAAC,CAAC"}

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js

@@ -0,0 +1,117 @@
+import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
+import { RunningState } from "../running-state.js";
+//#region ../zero-cache/src/services/litestream/vfs-watermark-worker.ts
+var nextRequestID = 0;
+var BackupWatermarkTimeoutError = class extends Error {};
+function requestVfsBackupWatermark(worker, timeoutMs, options = {}) {
+	const requestID = String(++nextRequestID);
+	return new Promise((resolve, reject) => {
+		const timeout = setTimeout(() => {
+			cleanup();
+			if (options.killOnTimeout) worker.kill(options.killSignal ?? "SIGKILL");
+			reject(new BackupWatermarkTimeoutError(`timed out waiting ${timeoutMs} ms for backup watermark response`));
+		}, timeoutMs);
+		const onMessage = (data) => {
+			const response = asBackupWatermarkResponse(data);
+			if (response === null || response[1].requestID !== requestID) return;
+			cleanup();
+			const { result, error } = response[1];
+			if (error !== void 0) {
+				reject(new Error(error.message, { cause: error }));
+				return;
+			}
+			if (result === void 0) {
+				reject(/* @__PURE__ */ new Error(`backup watermark response missing result`));
+				return;
+			}
+			resolve(result);
+		};
+		const cleanup = () => {
+			clearTimeout(timeout);
+			worker.off("message", onMessage);
+		};
+		worker.on("message", onMessage);
+		worker.send(["backupWatermarkRequest", { requestID }], void 0, (err) => {
+			if (err !== null) {
+				cleanup();
+				reject(err);
+			}
+		});
+	});
+}
+var VfsBackupWatermarkWorkerService = class {
+	id = "vfs-backup-watermark-worker";
+	#lc;
+	#parent;
+	#sourceFactory;
+	#source;
+	#state = new RunningState(this.id);
+	#pollIntervalMs;
+	constructor(lc, parent, source, pollIntervalMs) {
+		this.#lc = lc.withContext("component", this.id);
+		this.#parent = parent;
+		this.#sourceFactory = typeof source === "function" ? source : () => source;
+		this.#pollIntervalMs = pollIntervalMs;
+	}
+	async run() {
+		try {
+			if (this.#parent) {
+				this.#parent.onMessageType("backupWatermarkRequest", (msg) => this.#handleRequest(msg));
+				this.#parent.send(["ready", { ready: true }]);
+				await this.#state.stopped();
+				return;
+			}
+			this.#lc.info?.(`polling backup watermark every ${this.#pollIntervalMs} ms`);
+			while (this.#state.shouldRun()) {
+				try {
+					this.#lc.info?.(`polled backup watermark`, this.#getSource().readWatermark());
+				} catch (e) {
+					this.#lc.warn?.(`error while polling backup watermark`, e);
+				}
+				await this.#state.sleep(this.#pollIntervalMs);
+			}
+		} finally {
+			this.#source?.close?.();
+		}
+	}
+	stop() {
+		this.#state.stop(this.#lc);
+		return promiseVoid;
+	}
+	#handleRequest(msg) {
+		if (!this.#parent) return;
+		try {
+			this.#parent.send(["backupWatermarkResponse", {
+				requestID: msg.requestID,
+				result: this.#getSource().readWatermark()
+			}]);
+		} catch (e) {
+			this.#parent.send(["backupWatermarkResponse", {
+				requestID: msg.requestID,
+				error: serializeError(e)
+			}]);
+		}
+	}
+	#getSource() {
+		return this.#source ??= this.#sourceFactory();
+	}
+};
+function serializeError(e) {
+	if (e instanceof Error) return {
+		name: e.name,
+		message: e.message,
+		stack: e.stack
+	};
+	return {
+		name: "Error",
+		message: String(e)
+	};
+}
+function asBackupWatermarkResponse(data) {
+	if (Array.isArray(data) && data.length === 2 && data[0] === "backupWatermarkResponse") return data;
+	return null;
+}
+//#endregion
+export { BackupWatermarkTimeoutError, VfsBackupWatermarkWorkerService, requestVfsBackupWatermark };
+
+//# sourceMappingURL=vfs-watermark-worker.js.map

package/out/zero-cache/src/services/litestream/vfs-watermark-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-watermark-worker.js","names":["#lc","#parent","#sourceFactory","#state","#pollIntervalMs","#handleRequest","#getSource","#source"],"sources":["../../../../../../zero-cache/src/services/litestream/vfs-watermark-worker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {promiseVoid} from '../../../../shared/src/resolved-promises.ts';\nimport type {Message, Worker} from '../../types/processes.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\nimport type {VfsBackupWatermark} from './vfs-watermark-reader.ts';\n\nexport type BackupWatermarkRequest = [\n  'backupWatermarkRequest',\n  {\n    readonly requestID: string;\n  },\n];\n\nexport type BackupWatermarkResponse = [\n  'backupWatermarkResponse',\n  {\n    readonly requestID: string;\n    readonly result?: VfsBackupWatermark | undefined;\n    readonly error?:\n      | {\n          readonly name: string;\n          readonly message: string;\n          readonly stack?: string | undefined;\n        }\n      | undefined;\n  },\n];\n\nexport interface BackupWatermarkSource {\n  readWatermark(): VfsBackupWatermark;\n  close?(): void;\n}\n\nexport type BackupWatermarkSourceFactory = () => BackupWatermarkSource;\n\nlet nextRequestID = 0;\n\nexport class BackupWatermarkTimeoutError extends Error {}\n\nexport function requestVfsBackupWatermark(\n  worker: Worker,\n  timeoutMs: number,\n  options: {\n    readonly killOnTimeout?: boolean | undefined;\n    readonly killSignal?: NodeJS.Signals | undefined;\n  } = {},\n): Promise<VfsBackupWatermark> {\n  const requestID = String(++nextRequestID);\n  return new Promise((resolve, reject) => {\n    const timeout = setTimeout(() => {\n      cleanup();\n      if (options.killOnTimeout) {\n        worker.kill(options.killSignal ?? 'SIGKILL');\n      }\n      reject(\n        new BackupWatermarkTimeoutError(\n          `timed out waiting ${timeoutMs} ms for backup watermark response`,\n        ),\n      );\n    }, timeoutMs);\n\n    const onMessage = (data: unknown) => {\n      const response = asBackupWatermarkResponse(data);\n      if (response === null || response[1].requestID !== requestID) {\n        return;\n      }\n      cleanup();\n      const {result, error} = response[1];\n      if (error !== undefined) {\n        reject(new Error(error.message, {cause: error}));\n        return;\n      }\n      if (result === undefined) {\n        reject(new Error(`backup watermark response missing result`));\n        return;\n      }\n      resolve(result);\n    };\n\n    const cleanup = () => {\n      clearTimeout(timeout);\n      worker.off('message', onMessage);\n    };\n\n    worker.on('message', onMessage);\n    worker.send<BackupWatermarkRequest>(\n      ['backupWatermarkRequest', {requestID}],\n      undefined,\n      (err: Error | null) => {\n        if (err !== null) {\n          cleanup();\n          reject(err);\n        }\n      },\n    );\n  });\n}\n\nexport class VfsBackupWatermarkWorkerService implements Service {\n  readonly id = 'vfs-backup-watermark-worker';\n  readonly #lc: LogContext;\n  readonly #parent: Worker | null;\n  readonly #sourceFactory: BackupWatermarkSourceFactory;\n  #source: BackupWatermarkSource | undefined;\n  readonly #state = new RunningState(this.id);\n  readonly #pollIntervalMs: number;\n\n  constructor(\n    lc: LogContext,\n    parent: Worker | null,\n    source: BackupWatermarkSource | BackupWatermarkSourceFactory,\n    pollIntervalMs: number,\n  ) {\n    this.#lc = lc.withContext('component', this.id);\n    this.#parent = parent;\n    this.#sourceFactory = typeof source === 'function' ? source : () => source;\n    this.#pollIntervalMs = pollIntervalMs;\n  }\n\n  async run(): Promise<void> {\n    try {\n      if (this.#parent) {\n        this.#parent.onMessageType<BackupWatermarkRequest>(\n          'backupWatermarkRequest',\n          msg => this.#handleRequest(msg),\n        );\n        this.#parent.send(['ready', {ready: true}]);\n        await this.#state.stopped();\n        return;\n      }\n\n      this.#lc.info?.(\n        `polling backup watermark every ${this.#pollIntervalMs} ms`,\n      );\n      while (this.#state.shouldRun()) {\n        try {\n          this.#lc.info?.(\n            `polled backup watermark`,\n            this.#getSource().readWatermark(),\n          );\n        } catch (e) {\n          this.#lc.warn?.(`error while polling backup watermark`, e);\n        }\n        await this.#state.sleep(this.#pollIntervalMs);\n      }\n    } finally {\n      this.#source?.close?.();\n    }\n  }\n\n  stop(): Promise<void> {\n    this.#state.stop(this.#lc);\n    return promiseVoid;\n  }\n\n  #handleRequest(msg: BackupWatermarkRequest[1]) {\n    if (!this.#parent) {\n      return;\n    }\n    try {\n      this.#parent.send<BackupWatermarkResponse>([\n        'backupWatermarkResponse',\n        {\n          requestID: msg.requestID,\n          result: this.#getSource().readWatermark(),\n        },\n      ]);\n    } catch (e) {\n      this.#parent.send<BackupWatermarkResponse>([\n        'backupWatermarkResponse',\n        {\n          requestID: msg.requestID,\n          error: serializeError(e),\n        },\n      ]);\n    }\n  }\n\n  #getSource(): BackupWatermarkSource {\n    return (this.#source ??= this.#sourceFactory());\n  }\n}\n\nfunction serializeError(e: unknown): BackupWatermarkResponse[1]['error'] {\n  if (e instanceof Error) {\n    return {\n      name: e.name,\n      message: e.message,\n      stack: e.stack,\n    };\n  }\n  return {\n    name: 'Error',\n    message: String(e),\n  };\n}\n\nfunction asBackupWatermarkResponse(\n  data: unknown,\n): BackupWatermarkResponse | null {\n  if (\n    Array.isArray(data) &&\n    data.length === 2 &&\n    data[0] === 'backupWatermarkResponse'\n  ) {\n    return data as BackupWatermarkResponse;\n  }\n  return null;\n}\n\nexport type BackupWatermarkMessage =\n  | BackupWatermarkRequest\n  | BackupWatermarkResponse\n  | Message<unknown>;\n"],"mappings":";;;AAoCA,IAAI,gBAAgB;AAEpB,IAAa,8BAAb,cAAiD,MAAM,CAAC;AAExD,SAAgB,0BACd,QACA,WACA,UAGI,CAAC,GACwB;CAC7B,MAAM,YAAY,OAAO,EAAE,aAAa;CACxC,OAAO,IAAI,SAAS,SAAS,WAAW;EACtC,MAAM,UAAU,iBAAiB;GAC/B,QAAQ;GACR,IAAI,QAAQ,eACV,OAAO,KAAK,QAAQ,cAAc,SAAS;GAE7C,OACE,IAAI,4BACF,qBAAqB,UAAU,kCACjC,CACF;EACF,GAAG,SAAS;EAEZ,MAAM,aAAa,SAAkB;GACnC,MAAM,WAAW,0BAA0B,IAAI;GAC/C,IAAI,aAAa,QAAQ,SAAS,GAAG,cAAc,WACjD;GAEF,QAAQ;GACR,MAAM,EAAC,QAAQ,UAAS,SAAS;GACjC,IAAI,UAAU,KAAA,GAAW;IACvB,OAAO,IAAI,MAAM,MAAM,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;IAC/C;GACF;GACA,IAAI,WAAW,KAAA,GAAW;IACxB,uBAAO,IAAI,MAAM,0CAA0C,CAAC;IAC5D;GACF;GACA,QAAQ,MAAM;EAChB;EAEA,MAAM,gBAAgB;GACpB,aAAa,OAAO;GACpB,OAAO,IAAI,WAAW,SAAS;EACjC;EAEA,OAAO,GAAG,WAAW,SAAS;EAC9B,OAAO,KACL,CAAC,0BAA0B,EAAC,UAAS,CAAC,GACtC,KAAA,IACC,QAAsB;GACrB,IAAI,QAAQ,MAAM;IAChB,QAAQ;IACR,OAAO,GAAG;GACZ;EACF,CACF;CACF,CAAC;AACH;AAEA,IAAa,kCAAb,MAAgE;CAC9D,KAAc;CACd;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C;CAEA,YACE,IACA,QACA,QACA,gBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAKC,UAAU;EACf,KAAKC,iBAAiB,OAAO,WAAW,aAAa,eAAe;EACpE,KAAKE,kBAAkB;CACzB;CAEA,MAAM,MAAqB;EACzB,IAAI;GACF,IAAI,KAAKH,SAAS;IAChB,KAAKA,QAAQ,cACX,2BACA,QAAO,KAAKI,eAAe,GAAG,CAChC;IACA,KAAKJ,QAAQ,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;IAC1C,MAAM,KAAKE,OAAO,QAAQ;IAC1B;GACF;GAEA,KAAKH,IAAI,OACP,kCAAkC,KAAKI,gBAAgB,IACzD;GACA,OAAO,KAAKD,OAAO,UAAU,GAAG;IAC9B,IAAI;KACF,KAAKH,IAAI,OACP,2BACA,KAAKM,WAAW,EAAE,cAAc,CAClC;IACF,SAAS,GAAG;KACV,KAAKN,IAAI,OAAO,wCAAwC,CAAC;IAC3D;IACA,MAAM,KAAKG,OAAO,MAAM,KAAKC,eAAe;GAC9C;EACF,UAAU;GACR,KAAKG,SAAS,QAAQ;EACxB;CACF;CAEA,OAAsB;EACpB,KAAKJ,OAAO,KAAK,KAAKH,GAAG;EACzB,OAAO;CACT;CAEA,eAAe,KAAgC;EAC7C,IAAI,CAAC,KAAKC,SACR;EAEF,IAAI;GACF,KAAKA,QAAQ,KAA8B,CACzC,2BACA;IACE,WAAW,IAAI;IACf,QAAQ,KAAKK,WAAW,EAAE,cAAc;GAC1C,CACF,CAAC;EACH,SAAS,GAAG;GACV,KAAKL,QAAQ,KAA8B,CACzC,2BACA;IACE,WAAW,IAAI;IACf,OAAO,eAAe,CAAC;GACzB,CACF,CAAC;EACH;CACF;CAEA,aAAoC;EAClC,OAAQ,KAAKM,YAAY,KAAKL,eAAe;CAC/C;AACF;AAEA,SAAS,eAAe,GAAiD;CACvE,IAAI,aAAa,OACf,OAAO;EACL,MAAM,EAAE;EACR,SAAS,EAAE;EACX,OAAO,EAAE;CACX;CAEF,OAAO;EACL,MAAM;EACN,SAAS,OAAO,CAAC;CACnB;AACF;AAEA,SAAS,0BACP,MACgC;CAChC,IACE,MAAM,QAAQ,IAAI,KAClB,KAAK,WAAW,KAChB,KAAK,OAAO,2BAEZ,OAAO;CAET,OAAO;AACT"}

package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;IAsJT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}
+{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAGnD,OAAO,EAGL,KAAK,cAAc,EAEpB,MAAM,uCAAuC,CAAC;AAK/C,OAAO,KAAK,EAAC,0BAA0B,EAAC,MAAM,yBAAyB,CAAC;AACxE,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAIhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;gBAoB1B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI;IAa9C,GAAG;IA+JT,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC;IAIjC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO;CAGnC"}

package/out/zero-cache/src/services/replicator/incremental-sync.js

@@ -40,7 +40,10 @@ var IncrementalSyncer = class {
 		this.#worker.onError((err) => this.#state.stop(lc, err));
 		lc.info?.(`Starting IncrementalSyncer`);
 		const { watermark: initialWatermark } = await this.#worker.getSubscriptionState();
-		this.#notifier.notifySubscribers();
+		this.#notifier.notifySubscribers({
+			state: "version-ready",
+			watermark: initialWatermark
+		});
 		while (this.#state.shouldRun()) {
 			const { replicaVersion, watermark } = await this.#worker.getSubscriptionState();
 			let downstream;
@@ -120,7 +123,11 @@ var IncrementalSyncer = class {
 			const status = result.completedBackfill;
 			this.#statusPublisher?.publish(lc, "Replicating", `Backfilled ${status.table} table`, 0, () => ({ downloadStatus: [status] }));
 		} else if (result.schemaUpdated) this.#statusPublisher?.publish(lc, "Replicating", "Schema updated");
-		if (result.watermark && result.changeLogUpdated) this.#notifier.notifySubscribers({ state: "version-ready" });
+		if (result.watermark && result.changeLogUpdated) this.#notifier.notifySubscribers({
+			state: "version-ready",
+			watermark: result.watermark,
+			replicaReadyTimeMs: Date.now()
+		});
 	}
 	subscribe() {
 		return this.#notifier.subscribe();

package/out/zero-cache/src/services/replicator/incremental-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    this.#worker.onError(err => this.#state.stop(lc, err));\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    void this.#notifier.notifySubscribers();\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({state: 'version-ready'});\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,KAAKI,QAAQ,SAAQ,QAAO,KAAKK,OAAO,KAAK,IAAI,GAAG,CAAC;EACrD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAG1C,KAAUG,UAAU,kBAAkB;EAEtC,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;IACzB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAEJ,WAAW,MAAM,WAAW,YAAY;KACtC,KAAKI,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,SAAS,MAAM,KAAKF,QAAQ,eAChC,OACF;OAEA,KAAKO,cAAc,IAAI,MAAM;OAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;OAEnB;MACF;KACF;IACF;IACA,KAAKP,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;CACvC;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB,EAAC,OAAO,gBAAe,CAAC;CAElE;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;CAC1B;AACF"}
+{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n  errorTypeToReadableName,\n  PROTOCOL_VERSION,\n  type ChangeStreamer,\n  type Downstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport type {ReplicationStatusPublisher} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n  readonly #lc: LogContext;\n  readonly #taskID: string;\n  readonly #id: string;\n  readonly #changeStreamer: ChangeStreamer;\n  readonly #worker: WriteWorkerClient;\n  readonly #mode: ReplicatorMode;\n  readonly #statusPublisher: ReplicationStatusPublisher | null;\n  readonly #notifier: Notifier;\n  readonly #reporter: ReplicationReportRecorder;\n\n  readonly #state = new RunningState('IncrementalSyncer');\n\n  readonly #replicationEvents = getOrCreateCounter(\n    'replication',\n    'events',\n    'Number of replication events processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    taskID: string,\n    id: string,\n    changeStreamer: ChangeStreamer,\n    worker: WriteWorkerClient,\n    mode: ReplicatorMode,\n    statusPublisher: ReplicationStatusPublisher | null,\n  ) {\n    this.#lc = lc;\n    this.#taskID = taskID;\n    this.#id = id;\n    this.#changeStreamer = changeStreamer;\n    this.#worker = worker;\n    this.#mode = mode;\n    this.#statusPublisher = statusPublisher;\n    this.#notifier = new Notifier();\n    this.#reporter = new ReplicationReportRecorder(lc);\n  }\n\n  async run() {\n    const lc = this.#lc;\n    this.#worker.onError(err => this.#state.stop(lc, err));\n    lc.info?.(`Starting IncrementalSyncer`);\n    const {watermark: initialWatermark} =\n      await this.#worker.getSubscriptionState();\n\n    // Notify any waiting subscribers that the replica is ready to be read.\n    // This initial notification intentionally omits replicaReadyTimeMs because\n    // it represents already-current state, not newly-unserved work.\n    void this.#notifier.notifySubscribers({\n      state: 'version-ready',\n      watermark: initialWatermark,\n    });\n\n    while (this.#state.shouldRun()) {\n      const {replicaVersion, watermark} =\n        await this.#worker.getSubscriptionState();\n\n      let downstream: Source<Downstream> | undefined;\n      let unregister = () => {};\n      let err: unknown | undefined;\n\n      try {\n        downstream = await this.#changeStreamer.subscribe({\n          protocolVersion: PROTOCOL_VERSION,\n          taskID: this.#taskID,\n          id: this.#id,\n          mode: this.#mode,\n          watermark,\n          replicaVersion,\n          initial: watermark === initialWatermark,\n        });\n        this.#state.resetBackoff();\n        unregister = this.#state.cancelOnStop(downstream);\n        this.#statusPublisher?.publish(\n          lc,\n          'Replicating',\n          `Replicating from ${watermark}`,\n        );\n\n        let backfillStatus: DownloadStatus | undefined;\n\n        for await (const message of downstream) {\n          this.#replicationEvents.add(1);\n          switch (message[0]) {\n            case 'status': {\n              const {lagReport} = message[1];\n              if (lagReport) {\n                const report: ReplicationReport = {\n                  nextSendTimeMs: lagReport.nextSendTimeMs,\n                };\n                if (lagReport.lastTimings) {\n                  report.lastTimings = {\n                    ...lagReport.lastTimings,\n                    replicateTimeMs: Date.now(),\n                  };\n                }\n                this.#reporter.record(report);\n              }\n              break;\n            }\n            case 'error': {\n              // Signal from the replication-manager that the view-syncer must\n              // shut down and restore a new backup from litestream.\n              const {type, message: msg} = message[1];\n              this.stop(\n                lc,\n                // Note: The AbortError indicates a clean / intentional shutdown.\n                new AbortError(\n                  `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n                ),\n              );\n              break;\n            }\n            default: {\n              const msg = message[1];\n              if (msg.tag === 'backfill' && msg.status) {\n                const {status} = msg;\n                if (!backfillStatus) {\n                  // Start publishing the status every 3 seconds.\n                  backfillStatus = status;\n                  this.#statusPublisher?.publish(\n                    lc,\n                    'Replicating',\n                    `Backfilling ${msg.relation.name} table`,\n                    3000,\n                    () =>\n                      backfillStatus\n                        ? {\n                            downloadStatus: [\n                              {\n                                ...backfillStatus,\n                                table: msg.relation.name,\n                                columns: [\n                                  ...msg.relation.rowKey.columns,\n                                  ...msg.columns,\n                                ],\n                              },\n                            ],\n                          }\n                        : {},\n                  );\n                }\n                backfillStatus = status; // Update the current status\n              }\n\n              const result = await this.#worker.processMessage(\n                message as ChangeStreamData,\n              );\n\n              this.#handleResult(lc, result);\n              if (result?.completedBackfill) {\n                backfillStatus = undefined;\n              }\n              break;\n            }\n          }\n        }\n        this.#worker.abort();\n      } catch (e) {\n        err = e;\n        this.#worker.abort();\n      } finally {\n        downstream?.cancel();\n        unregister();\n        this.#statusPublisher?.stop();\n      }\n      await this.#state.backoff(lc, err);\n    }\n    lc.info?.('IncrementalSyncer stopped');\n  }\n\n  #handleResult(lc: LogContext, result: CommitResult | null) {\n    if (!result) {\n      return;\n    }\n    if (result.completedBackfill) {\n      // Publish the final status\n      const status = result.completedBackfill;\n      this.#statusPublisher?.publish(\n        lc,\n        'Replicating',\n        `Backfilled ${status.table} table`,\n        0,\n        () => ({downloadStatus: [status]}),\n      );\n    } else if (result.schemaUpdated) {\n      this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n    }\n    if (result.watermark && result.changeLogUpdated) {\n      void this.#notifier.notifySubscribers({\n        state: 'version-ready',\n        watermark: result.watermark,\n        replicaReadyTimeMs: Date.now(),\n      });\n    }\n  }\n\n  subscribe(): Source<ReplicaState> {\n    return this.#notifier.subscribe();\n  }\n\n  stop(lc: LogContext, err?: unknown) {\n    this.#state.stop(lc, err);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,KAAKI,QAAQ,SAAQ,QAAO,KAAKK,OAAO,KAAK,IAAI,GAAG,CAAC;EACrD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;IACzB,CAAC;IACD,KAAKI,OAAO,aAAa;IACzB,aAAa,KAAKA,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAEJ,WAAW,MAAM,WAAW,YAAY;KACtC,KAAKI,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,WAAU;QACjB,IAAI,CAAC,gBAAgB;SAEnB,iBAAiB;SACjB,KAAKF,kBAAkB,QACrB,IACA,eACA,eAAe,IAAI,SAAS,KAAK,SACjC,WAEE,iBACI,EACE,gBAAgB,CACd;UACE,GAAG;UACH,OAAO,IAAI,SAAS;UACpB,SAAS,CACP,GAAG,IAAI,SAAS,OAAO,SACvB,GAAG,IAAI,OACT;SACF,CACF,EACF,IACA,CAAC,CACT;QACF;QACA,iBAAiB;OACnB;OAEA,MAAM,SAAS,MAAM,KAAKF,QAAQ,eAChC,OACF;OAEA,KAAKO,cAAc,IAAI,MAAM;OAC7B,IAAI,QAAQ,mBACV,iBAAiB,KAAA;OAEnB;MACF;KACF;IACF;IACA,KAAKP,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;CACvC;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;CAC1B;AACF"}

package/out/zero-cache/src/services/replicator/notifier.d.ts

@@ -24,6 +24,7 @@ import type { ReplicaState, ReplicaStateNotifier } from './replicator.ts';
  */
 export declare class Notifier implements ReplicaStateNotifier {
     #private;
+    get latestState(): ReplicaState | undefined;
     subscribe(): Subscription<ReplicaState>;
     notifySubscribers(state?: ReplicaState): Promise<Result>[];
 }

package/out/zero-cache/src/services/replicator/notifier.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EACX,YAAY,EACb,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAExE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,QAAS,YAAW,oBAAoB;;IAanD,SAAS,IAAI,YAAY,CAAC,YAAY,CAAC;IAWvC,iBAAiB,CACf,KAAK,GAAE,YAAuC,GAC7C,OAAO,CAAC,MAAM,CAAC,EAAE;CAOrB"}
+{"version":3,"file":"notifier.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/notifier.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,MAAM,EACX,YAAY,EACb,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,iBAAiB,CAAC;AAExE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,QAAS,YAAW,oBAAoB;;IAInD,IAAI,WAAW,IAAI,YAAY,GAAG,SAAS,CAE1C;IAmBD,SAAS,IAAI,YAAY,CAAC,YAAY,CAAC;IAWvC,iBAAiB,CACf,KAAK,GAAE,YAAuC,GAC7C,OAAO,CAAC,MAAM,CAAC,EAAE;CAOrB"}