@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.2

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\ntype RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n): Promise<RestoreAttempt> {\n  const {backupURL} = config;\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const replicaExistedBeforeRestore = existsSync(replicaFile);\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result};\n    }\n    const validationStart = performance.now();\n    const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: valid ? 'success' : 'invalid_replica'},\n    );\n    if (!valid) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      return {restored: false, backupURL, result};\n    }\n    result = 'success';\n    if (!replicaExistedBeforeRestore) {\n      // This restore materialized the replica file, so any change log beside it\n      // was written against a replica that is no longer there. Reconciliation\n      // is the safety net — a restored replica keeps its replicaVersion, so the\n      // rule falls through to truncate-or-reseed on head mismatch — but the log\n      // is a cache and deleting it here is the explicit statement that nothing\n      // in it survives the restore.\n      //\n      // A restore that reuses an existing replica (`-if-db-not-exists` made it\n      // a no-op) deliberately keeps the log: that is the process-restart path,\n      // where the log is still the one written beside this exact replica and\n      // discarding it would cost every reconnecting subscriber a `too-old`.\n      deleteChangeLogDB(replicaFile);\n      litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n        ...attrs,\n        result: 'success',\n      });\n    }\n    return {restored: true, backupURL, result};\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n) {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return false;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return false;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    return true;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${config.backupURL}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;AAgDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAE7C,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\ntype RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    snapshotBackupIntervalHoursV5,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  const snapshotIntervalHours = v5\n    ? snapshotBackupIntervalHoursV5\n    : snapshotBackupIntervalHours;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n): Promise<RestoreAttempt> {\n  const {backupURL} = config;\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const replicaExistedBeforeRestore = existsSync(replicaFile);\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result};\n    }\n    const validationStart = performance.now();\n    const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: valid ? 'success' : 'invalid_replica'},\n    );\n    if (!valid) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      return {restored: false, backupURL, result};\n    }\n    result = 'success';\n    if (!replicaExistedBeforeRestore) {\n      // This restore materialized the replica file, so any change log beside it\n      // was written against a replica that is no longer there. Reconciliation\n      // is the safety net — a restored replica keeps its replicaVersion, so the\n      // rule falls through to truncate-or-reseed on head mismatch — but the log\n      // is a cache and deleting it here is the explicit statement that nothing\n      // in it survives the restore.\n      //\n      // A restore that reuses an existing replica (`-if-db-not-exists` made it\n      // a no-op) deliberately keeps the log: that is the process-restart path,\n      // where the log is still the one written beside this exact replica and\n      // discarding it would cost every reconnecting subscriber a `too-old`.\n      deleteChangeLogDB(replicaFile);\n      litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n        ...attrs,\n        result: 'success',\n      });\n    }\n    return {restored: true, backupURL, result};\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n) {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return false;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return false;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    return true;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${config.backupURL}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,+BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAC7C,MAAM,wBAAwB,KAC1B,gCACA;CAGJ,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,qBACF;IACC,sDAAsD,OACrD,wBAAwB,CAC1B;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/litestream/config-v5.yml

@@ -25,8 +25,13 @@ validation:
 
 dbs:
   - path: ${ZERO_REPLICA_FILE}
-    # The monitor-interval is the interval at which new l0 files are created.
+    # The monitor-interval is the steady-state interval at which the db / wal
+    # is checked to create new l0 files and/or checkpoint the wal. The
+    # application will also explicitly trigger a "/sync" if the wal reaches
+    # ZERO_LITESTREAM_FORCE_CHECKPOINT_THRESHOLD_MB.
     monitor-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}s
+    min-checkpoint-page-count: ${ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT}
+    truncate-page-n: ${ZERO_LITESTREAM_TRUNCATE_PAGE_N}
     replica:
       url: ${ZERO_LITESTREAM_BACKUP_URL}
       endpoint: ${ZERO_LITESTREAM_ENDPOINT}
@@ -40,3 +45,10 @@ logging:
   type: ${ZERO_LOG_FORMAT}
 
 addr: localhost:${LITESTREAM_PORT}
+
+# Control API served as HTTP over a Unix domain socket, used for in-process IPC
+# from zero-cache (e.g. forcing an immediate sync/checkpoint via POST /sync).
+socket:
+  enabled: true
+  path: ${ZERO_LITESTREAM_SOCKET_PATH}
+  permissions: 0600

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js

@@ -0,0 +1,104 @@
+import { AbortError } from "../../../../shared/src/abort-error.js";
+import { RunningState } from "../running-state.js";
+//#region ../zero-cache/src/services/litestream/litestream-checkpointer.ts
+var LITESTREAM_SYNC_TIMEOUT_MS = 1e4;
+var PAUSE_POLL_INTERVAL_MS = 1e3;
+var PAUSE_LOG_INTERVAL_POLLS = 30;
+/**
+* Forces litestream to checkpoint the WAL from the write path, so a sustained
+* high write rate can't grow the WAL until litestream's own blocking TRUNCATE
+* checkpoint locks the db (see ReplicationPauseCheckpointConfig).
+*
+* Two tiers:
+*  - Soft (`checkpointThresholdPages`): after each committed transaction,
+*    once the un-checkpointed WAL backlog reaches the threshold, force an
+*    immediate litestream sync (which seals the WAL to LTX and attempts
+*    litestream's own PASSIVE checkpoint). Non-blocking beyond the sync call
+*    itself. Backs off by a full threshold's worth of WAL growth whenever an
+*    attempt fails to drain the WAL (e.g. litestream is mid-snapshot and its
+*    checkpoints are being skipped), so repeated failures don't reattempt on
+*    every commit.
+*  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after
+*    a soft attempt, pause (block the caller) polling litestream until the
+*    WAL drains below it. This is the fail-safe against unbounded WAL growth
+*    when litestream structurally cannot checkpoint — e.g. an in-progress
+*    snapshot holds litestream's checkpoint lock for its full duration, so
+*    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.
+*/
+var LitestreamCheckpointer = class {
+	#lc;
+	#db;
+	#litestream;
+	#attemptChunk;
+	#maxWalPages;
+	#state = new RunningState("litestream-checkpointer");
+	#nextWalThreshold;
+	constructor(lc, db, litestream, config) {
+		this.#lc = lc.withContext("litestream-checkpointer");
+		this.#db = db;
+		this.#litestream = litestream;
+		this.#attemptChunk = config.checkpointThresholdPages;
+		this.#nextWalThreshold = config.checkpointThresholdPages;
+		this.#maxWalPages = config.maxWalPages;
+	}
+	#getNumWalPages() {
+		const [{ log, checkpointed }] = this.#db.pragma("wal_checkpoint(NOOP)");
+		return log - checkpointed;
+	}
+	/**
+	* Call after a committed transaction. No-ops unless the WAL has grown to
+	* at least the current threshold, in which case it triggers a litestream
+	* sync to perform a checkpoint. Assuming that snapshots are disabled, the
+	* checkpoint always succeeds. In the case where the server is configured to
+	* perform snapshots and checkpoints fail, the "emergency break"
+	* {@link #pauseUntilWalDrained()} option provides the disk use protection
+	* that litestream's (disabled) truncate-page-n would have served.
+	*/
+	async maybeCheckpoint() {
+		const walPages = this.#getNumWalPages();
+		if (walPages < this.#nextWalThreshold) return;
+		const start = performance.now();
+		let result;
+		let err;
+		try {
+			result = await this.#litestream.sync({
+				wait: false,
+				timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS
+			}, this.#state.signal);
+		} catch (e) {
+			err = e;
+		}
+		const elapsed = performance.now() - start;
+		let newWalPages = this.#getNumWalPages();
+		if (newWalPages < this.#nextWalThreshold) this.#lc.info?.(`checkpointed ${walPages} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`, result);
+		else if (result) this.#lc.warn?.(`checkpoint skipped ${walPages} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`, result);
+		else this.#lc.warn?.(`checkpoint failed ${walPages} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`, err);
+		if (this.#maxWalPages !== void 0 && newWalPages > this.#maxWalPages) newWalPages = await this.#pauseUntilWalDrained(this.#maxWalPages);
+		this.#nextWalThreshold = newWalPages + this.#attemptChunk;
+	}
+	async #pauseUntilWalDrained(maxWalPages) {
+		for (let i = 0; this.#state.shouldRun(); i++) {
+			const walPages = this.#getNumWalPages();
+			if (walPages < maxWalPages) {
+				this.#lc.info?.(`wal size (${walPages} pages) is under the max limit of ${maxWalPages}`);
+				return walPages;
+			}
+			if (i % PAUSE_LOG_INTERVAL_POLLS === 0) this.#lc.warn?.(`waiting for litestream to successfully checkpoint ${walPages} wal pages`);
+			await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);
+			await this.#litestream.sync({
+				wait: false,
+				timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS
+			}, this.#state.signal).catch((e) => this.#lc.warn?.(`litestream /sync failed`, e));
+		}
+		throw new AbortError("litestream-controller stopped");
+	}
+	/** Releases the underlying litestream controller's connection. */
+	close() {
+		this.#litestream.close();
+		this.#state.stop(this.#lc);
+	}
+};
+//#endregion
+export { LitestreamCheckpointer };
+
+//# sourceMappingURL=litestream-checkpointer.js.map

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-checkpointer.js","names":["#lc","#db","#litestream","#attemptChunk","#maxWalPages","#state","#nextWalThreshold","#getNumWalPages","#pauseUntilWalDrained"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\nconst LITESTREAM_SYNC_TIMEOUT_MS = 10_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint). Non-blocking beyond the sync call\n *    itself. Backs off by a full threshold's worth of WAL growth whenever an\n *    attempt fails to drain the WAL (e.g. litestream is mid-snapshot and its\n *    checkpoints are being skipped), so repeated failures don't reattempt on\n *    every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    const walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      result = await this.#litestream.sync(\n        {\n          wait: false,\n          timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS,\n        },\n        this.#state.signal,\n      );\n    } catch (e) {\n      err = e;\n    }\n\n    const elapsed = performance.now() - start;\n    let newWalPages = this.#getNumWalPages();\n    if (newWalPages < this.#nextWalThreshold) {\n      this.#lc.info?.(\n        `checkpointed ${walPages} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n        result,\n      );\n    } else if (result) {\n      this.#lc.warn?.(\n        `checkpoint skipped ${walPages} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n        result,\n      );\n    } else {\n      this.#lc.warn?.(\n        `checkpoint failed ${walPages} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n        err,\n      );\n    }\n\n    if (this.#maxWalPages !== undefined && newWalPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      newWalPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = newWalPages + this.#attemptChunk;\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      await this.#litestream\n        .sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        )\n        .catch(e => this.#lc.warn?.(`litestream /sync failed`, e));\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAUA,IAAM,6BAA6B;AACnC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;AAuBjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,yBAAyB;EACnD,KAAKC,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,gBAAgB,OAAO;EAC5B,KAAKG,oBAAoB,OAAO;EAChC,KAAKF,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAKH,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,MAAM,WAAW,KAAKM,gBAAgB;EACtC,IAAI,WAAW,KAAKD,mBAClB;EAGF,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,KAAKJ,YAAY,KAC9B;IACE,MAAM;IACN,WAAW;GACb,GACA,KAAKG,OAAO,MACd;EACF,SAAS,GAAG;GACV,MAAM;EACR;EAEA,MAAM,UAAU,YAAY,IAAI,IAAI;EACpC,IAAI,cAAc,KAAKE,gBAAgB;EACvC,IAAI,cAAc,KAAKD,mBACrB,KAAKN,IAAI,OACP,gBAAgB,SAAS,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAC5E,MACF;OACK,IAAI,QACT,KAAKA,IAAI,OACP,sBAAsB,SAAS,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OACvG,MACF;OAEA,KAAKA,IAAI,OACP,qBAAqB,SAAS,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OACvE,GACF;EAGF,IAAI,KAAKI,iBAAiB,KAAA,KAAa,cAAc,KAAKA,cAOxD,cAAc,MAAM,KAAKI,sBAAsB,KAAKJ,YAAY;EAOlE,KAAKE,oBAAoB,cAAc,KAAKH;CAC9C;CAEA,MAAMK,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAKH,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAKE,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAKP,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAKA,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAKK,OAAO,MAAM,sBAAsB;GAC9C,MAAM,KAAKH,YACR,KACC;IAAC,MAAM;IAAO,WAAW;GAA0B,GACnD,KAAKG,OAAO,MACd,EACC,OAAM,MAAK,KAAKL,IAAI,OAAO,2BAA2B,CAAC,CAAC;EAC7D;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAKE,YAAY,MAAM;EACvB,KAAKG,OAAO,KAAK,KAAKL,GAAG;CAC3B;AACF"}

package/out/zero-cache/src/services/litestream/litestream-controller.d.ts

@@ -0,0 +1,70 @@
+import type { LogContext } from '@rocicorp/logger';
+/**
+ * Path of the litestream control socket, derived from the replica file's
+ * directory. The socket must live on a filesystem the (non-root) runtime user
+ * can write — the replica directory is guaranteed writable since litestream
+ * writes the replica there — so it must NOT default to a root-owned location
+ * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set
+ * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).
+ */
+export declare function litestreamSocketPath(replicaFile: string): string;
+/** Response body from litestream's `POST /sync` control endpoint. */
+export type SyncResponse = {
+    /**
+     * `synced` when `wait` was true and the upload completed, `synced_local`
+     * when the WAL was sealed locally without waiting on the remote replica, or
+     * `no_change` when there was nothing to sync.
+     */
+    status: 'synced' | 'synced_local' | 'no_change';
+    path: string;
+    txid: number;
+    replicated_txid: number;
+};
+export type SyncOptions = {
+    /**
+     * When true, block until the sealed WAL has been uploaded to the remote
+     * replica (litestream's SyncAndWait). When false (the default), litestream
+     * only seals the WAL into local LTX files and runs its checkpoint thresholds,
+     * returning without waiting on the backup store. Use false on the write path
+     * where the goal is to keep the WAL small, not to durably back it up.
+     */
+    wait?: boolean;
+    /** Overall request timeout in milliseconds. Defaults to 30s. */
+    timeoutMs?: number;
+};
+/**
+ * IPC client for the litestream control server. litestream serves an HTTP API
+ * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);
+ * because zero-cache runs litestream as a subprocess in the same container, the
+ * socket is reachable directly on the local filesystem.
+ *
+ * A single keep-alive connection is held open and reused across calls, so a
+ * high-frequency caller (e.g. write-path backpressure) does not pay a
+ * connect/handshake per request.
+ */
+export declare class LitestreamController {
+    #private;
+    /**
+     * @param replicaFile Path of the replica database litestream is managing —
+     *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the
+     *   `path` litestream matches control requests against.
+     * @param socketPath Path of the control socket; defaults to the path derived
+     *   from `replicaFile`, matching config-v5.yml's `socket.path`.
+     */
+    constructor(lc: LogContext, replicaFile: string, socketPath?: string);
+    /**
+     * Forces litestream to sync the WAL into LTX files immediately and run its
+     * checkpoint thresholds, instead of waiting for the next monitor-interval
+     * tick. The checkpoint itself is still litestream's usual (non-blocking)
+     * PASSIVE attempt, so it is skipped if the database is busy — call this once
+     * writes are quiesced.
+     *
+     * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT
+     * while litestream is restarting or before it has created the socket) rather
+     * than blocking application progress on it.
+     */
+    sync(opts?: SyncOptions, signal?: AbortSignal): Promise<SyncResponse>;
+    /** Releases the pooled keep-alive connection. */
+    close(): void;
+}
+//# sourceMappingURL=litestream-controller.d.ts.map

package/out/zero-cache/src/services/litestream/litestream-controller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-controller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEhE;AAID,qEAAqE;AACrE,MAAM,MAAM,YAAY,GAAG;IACzB;;;;OAIG;IACH,MAAM,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;;;;GASG;AACH,qBAAa,oBAAoB;;IAM/B;;;;;;OAMG;IACH,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,UAAU,SAAoC,EAQ/C;IAED;;;;;;;;;;OAUG;IACG,IAAI,CACR,IAAI,GAAE,WAAgB,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,YAAY,CAAC,CAUvB;IAED,iDAAiD;IACjD,KAAK,IAAI,IAAI,CAEZ;CA4DF"}

package/out/zero-cache/src/services/litestream/litestream-controller.js

@@ -0,0 +1,101 @@
+import { Agent, request } from "node:http";
+import { json } from "node:stream/consumers";
+//#region ../zero-cache/src/services/litestream/litestream-controller.ts
+/**
+* Path of the litestream control socket, derived from the replica file's
+* directory. The socket must live on a filesystem the (non-root) runtime user
+* can write — the replica directory is guaranteed writable since litestream
+* writes the replica there — so it must NOT default to a root-owned location
+* like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set
+* from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).
+*/
+function litestreamSocketPath(replicaFile) {
+	return `${replicaFile}.sock`;
+}
+var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
+/**
+* IPC client for the litestream control server. litestream serves an HTTP API
+* over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);
+* because zero-cache runs litestream as a subprocess in the same container, the
+* socket is reachable directly on the local filesystem.
+*
+* A single keep-alive connection is held open and reused across calls, so a
+* high-frequency caller (e.g. write-path backpressure) does not pay a
+* connect/handshake per request.
+*/
+var LitestreamController = class {
+	#lc;
+	#replicaFile;
+	#socketPath;
+	#agent;
+	/**
+	* @param replicaFile Path of the replica database litestream is managing —
+	*   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the
+	*   `path` litestream matches control requests against.
+	* @param socketPath Path of the control socket; defaults to the path derived
+	*   from `replicaFile`, matching config-v5.yml's `socket.path`.
+	*/
+	constructor(lc, replicaFile, socketPath = litestreamSocketPath(replicaFile)) {
+		this.#lc = lc.withContext("component", "litestream-controller");
+		this.#replicaFile = replicaFile;
+		this.#socketPath = socketPath;
+		this.#agent = new Agent({
+			keepAlive: true,
+			maxSockets: 1
+		});
+	}
+	/**
+	* Forces litestream to sync the WAL into LTX files immediately and run its
+	* checkpoint thresholds, instead of waiting for the next monitor-interval
+	* tick. The checkpoint itself is still litestream's usual (non-blocking)
+	* PASSIVE attempt, so it is skipped if the database is busy — call this once
+	* writes are quiesced.
+	*
+	* Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT
+	* while litestream is restarting or before it has created the socket) rather
+	* than blocking application progress on it.
+	*/
+	async sync(opts = {}, signal) {
+		const { wait = false, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS } = opts;
+		const result = await this.#post("/sync", {
+			path: this.#replicaFile,
+			wait
+		}, timeoutMs, signal);
+		this.#lc.debug?.("litestream sync", result);
+		return result;
+	}
+	/** Releases the pooled keep-alive connection. */
+	close() {
+		this.#agent.destroy();
+	}
+	#post(path, body, timeoutMs, signal) {
+		const payload = JSON.stringify(body);
+		return new Promise((resolve, reject) => {
+			const req = request({
+				socketPath: this.#socketPath,
+				method: "POST",
+				path,
+				agent: this.#agent,
+				timeout: timeoutMs,
+				headers: {
+					"content-type": "application/json",
+					"content-length": Buffer.byteLength(payload)
+				},
+				signal
+			}, (res) => {
+				const status = res.statusCode ?? 0;
+				json(res).then((body) => {
+					if (status < 200 || status >= 300) reject(/* @__PURE__ */ new Error(`litestream ${path} failed: ${status} ${JSON.stringify(body)}`));
+					else resolve(body);
+				}, (e) => reject(new Error(`litestream ${path} returned invalid JSON`, { cause: e })));
+			});
+			req.on("timeout", () => req.destroy(/* @__PURE__ */ new Error(`litestream ${path} timed out after ${timeoutMs}ms`)));
+			req.on("error", reject);
+			req.end(payload);
+		});
+	}
+};
+//#endregion
+export { LitestreamController, litestreamSocketPath };
+
+//# sourceMappingURL=litestream-controller.js.map

package/out/zero-cache/src/services/litestream/litestream-controller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"litestream-controller.js","names":["#lc","#replicaFile","#socketPath","#agent","#post"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {wait = false, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS} = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {path: this.#replicaFile, wait},\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAuCnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAKC,eAAe;EACpB,KAAKC,cAAc;EAGnB,KAAKC,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EAAC,OAAO,OAAO,YAAY,+BAA8B;EAC/D,MAAM,SAAS,MAAM,KAAKC,MACxB,SACA;GAAC,MAAM,KAAKH;GAAc;EAAI,GAC9B,WACA,MACF;EACA,KAAKD,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAKG,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAKD;IACjB,QAAQ;IACR;IACA,OAAO,KAAKC;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,EAAE,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}