@rocicorp/zero 1.10.0-canary.0 → 1.10.0-canary.10

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, statSync} from 'node:fs';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n  SnapshotMessage,\n  SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreDuration,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreRuns,\n  litestreamRestoreValidationDuration,\n  litestreamRestoreWaitDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nconst RETRY_INTERVAL_MS = 3000;\n\ntype ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\ntype RestoreResult = 'success' | 'no_backup' | 'invalid_replica' | 'error';\n\ntype RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n};\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * @param replicaConstraints The constraints of the restored backup when\n *        restoring for the change-streamer (replication-manager). For the\n *        view-syncer, this should be unspecified so that the constraints are\n *        retrieved from the replication-manager via the snapshot protocol.\n */\nexport async function restoreReplica(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n) {\n  const start = performance.now();\n  const role: LitestreamRole = replicaConstraints\n    ? 'replication_manager'\n    : 'view_syncer';\n  let backupURL = config.litestream.backupURL;\n  let result: RestoreResult | undefined;\n  // View-syncers (no replicaConstraints) wait indefinitely for the\n  // replication-manager to publish a restorable backup. On a fresh stack the\n  // first backup is not durable until the initial sync completes and litestream\n  // uploads the initial snapshot, which can take many minutes for a large\n  // replica. The platform's startup probe budget (which scales with replica\n  // size) is the backstop, so restoreReplica must not impose its own shorter\n  // cap and self-terminate while the backup is still being produced.\n  //\n  // `consecutiveErrors` distinguishes a transient restore *error* (e.g. the\n  // `replicate` process compacting a snapshot mid-restore) — which is retried\n  // once — from a \"backup not found\" result, which is expected while waiting.\n  let consecutiveErrors = 0;\n  try {\n    for (;;) {\n      try {\n        // `consecutiveErrors` is 0 on a fresh attempt and 1 on the retry, so\n        // `consecutiveErrors >= 1` identifies the final attempt of a streak.\n        // tryRestore surfaces litestream's failure output with a different\n        // message on the final attempt (which the log classifier routes to a\n        // paging alert) than on a retriable earlier attempt (routed to a\n        // non-paging warning). See INC-961.\n        const attempt = await tryRestore(\n          lc,\n          config,\n          replicaConstraints,\n          role,\n          consecutiveErrors >= 1,\n        );\n        backupURL = attempt.backupURL;\n        if (attempt.restored) {\n          result = attempt.result;\n          return;\n        }\n        consecutiveErrors = 0;\n        if (replicaConstraints) {\n          // The replication-manager restores against explicit constraints, so a\n          // missing backup is fatal (e.g. the litestream URL was purposefully\n          // changed to force a resync). Only the view-syncer (no constraints)\n          // waits for a backup to appear.\n          result = attempt.result;\n          throw new BackupNotFoundException(config.litestream.backupURL);\n        }\n      } catch (e) {\n        if (e instanceof BackupNotFoundException) {\n          throw e;\n        }\n        if (++consecutiveErrors <= 1) {\n          // A restore will fail if the `replicate` process creates a new\n          // snapshot (and compacts old files) at the same time. Snapshots are\n          // infrequent (e.g. once every 12 hours), and the scenario is\n          // recoverable with a retry.\n          lc.warn?.(`restore attempt failed. retrying once`, e);\n          continue;\n        }\n        // If it fails again on the retry, though, bail.\n        throw e;\n      }\n      lc.info?.(\n        `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n      );\n      await sleep(RETRY_INTERVAL_MS);\n    }\n  } catch (e) {\n    if (e instanceof BackupNotFoundException && result === undefined) {\n      result = 'no_backup';\n    }\n    throw e;\n  } finally {\n    const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n    const labels = {...attrs, result: result ?? 'error'};\n    litestreamRestoreRuns().add(1, labels);\n    litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: ZeroConfig,\n  logLevelOverride?: LogLevel,\n  backupURLOverride?: string,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    endpoint,\n    region,\n    port = config.port + 2,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config.litestream;\n\n  // Set the snapshot interval to something smaller than x hours so that\n  // the hourly check triggers on the hour, rather than the hour after.\n  const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n  const litestream =\n    // The v0.5.8+ litestream executable can restore from either the new LTX\n    // format or the legacy WAL format, allowing forwards-compatibility /\n    // rollback safety with zero-cache versions that backup to LTX.\n    (mode === 'restore' && restoreUsingV5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: config.replica.file,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n        snapshotBackupIntervalMinutes,\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: config.log.format,\n      ['LITESTREAM_CONFIG']: configPath,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nasync function tryRestore(\n  lc: LogContext,\n  config: ZeroConfig,\n  replicaConstraints: ReplicaConstraints | null,\n  role: LitestreamRole,\n  isFinalAttempt: boolean,\n): Promise<RestoreAttempt> {\n  let snapshotStatus: SnapshotStatus | undefined;\n  if (!replicaConstraints) {\n    // view-syncers fetch replica constraints from the replication-manager\n    // via the snapshot protocol.\n    const waitStart = performance.now();\n    snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n    litestreamRestoreWaitDuration().recordMs(\n      performance.now() - waitStart,\n      litestreamRestoreMetricAttrs(config, role, snapshotStatus.backupURL),\n    );\n    lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n    replicaConstraints = snapshotStatus;\n  }\n\n  const backupURL = snapshotStatus?.backupURL ?? config.litestream.backupURL;\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    const replicaExistedBeforeRestore = existsSync(config.replica.file);\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n      snapshotStatus?.backupURL,\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config.litestream;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided whether the failure is retriable. The captured output is\n    // re-surfaced through `lc` below with a distinct message per attempt, so a\n    // retriable failure is routed to a non-paging warning and only a post-retry\n    // failure pages. See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        config.replica.file,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    proc.stdout.on('data', chunk => (stdout += chunk));\n    proc.stderr.on('data', chunk => (stderr += chunk));\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n      // A successful restore does not emit ERROR-level output; forward\n      // litestream's own (minimal, debug-level) restore logging to the pod's\n      // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n      if (stdout) {\n        process.stdout.write(stdout);\n      }\n      if (stderr) {\n        process.stderr.write(stderr);\n      }\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      // litestream logs its restore failure at ERROR. Re-surface that captured\n      // output through our logger with a distinct message per attempt, so the\n      // cloudzero log classifier can route them differently: the retriable\n      // earlier attempt to a non-paging warning, and the post-retry failure to\n      // a paging critical. Both are emitted at ERROR because the log scraper\n      // only ingests error-level lines, and both carry our `worker` context,\n      // distinguishing them from litestream's untagged raw output. See INC-961.\n      const output = (stdout + stderr).trim();\n      if (isFinalAttempt) {\n        lc.error?.(`litestream restore failed after retry`, output || e);\n      } else {\n        lc.error?.(\n          `litestream restore attempt failed, will retry`,\n          output || e,\n        );\n      }\n      throw e;\n    }\n    if (!existsSync(config.replica.file)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result};\n    }\n    const validationStart = performance.now();\n    const valid = replicaIsValid(lc, config.replica.file, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: valid ? 'success' : 'invalid_replica'},\n    );\n    if (!valid) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(config.replica.file);\n      return {restored: false, backupURL, result};\n    }\n    result = 'success';\n    if (!replicaExistedBeforeRestore) {\n      litestreamRestoredDbBytes().add(statSync(config.replica.file).size, {\n        ...attrs,\n        result: 'success',\n      });\n    }\n    return {restored: true, backupURL, result};\n  } finally {\n    litestreamRestoreAttempts().add(1, {\n      ...attrs,\n      result,\n    });\n  }\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints,\n) {\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (replicaVersion !== constraints.replicaVersion) {\n      lc.warn?.(\n        `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n        constraints,\n      );\n      return false;\n    }\n    if (watermark < constraints.minWatermark) {\n      lc.warn?.(\n        `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n      );\n      return false;\n    }\n    lc.info?.(\n      `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n      constraints,\n    );\n    return true;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: ZeroConfig,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, 'snapshots'),\n    listBackupCreatedTimes(lc, config, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.litestream.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: ZeroConfig,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config);\n  const proc = spawn(litestream, [command, config.replica.file], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n\nfunction reserveAndGetSnapshotStatus(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<SnapshotStatus> {\n  const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n  void (async function () {\n    const abort = new AbortController();\n    process.on('SIGINT', () => abort.abort());\n    process.on('SIGTERM', () => abort.abort());\n\n    for (let i = 0; ; i++) {\n      let err: unknown;\n      try {\n        let resolved = false;\n        const stream = await reserveSnapshot(lc, config);\n        for await (const msg of stream) {\n          // Capture the value of the status message that the change-streamer\n          // backup monitor returns, and hold the connection open to\n          // \"reserve\" the snapshot and prevent change log cleanup.\n          resolve(msg[1]);\n          resolved = true;\n        }\n        // The change-streamer itself closes the connection when the\n        // subscription is started (or the reservation retried).\n        if (resolved) {\n          break;\n        }\n      } catch (e) {\n        err = e;\n      }\n      // Retry in the view-syncer since it cannot proceed until it connects\n      // to a (compatible) replication-manager. In particular, a\n      // replication-manager that does not support the view-syncer's\n      // change-streamer protocol will close the stream with an error; this\n      // retry logic essentially delays the startup of a view-syncer until\n      // a compatible replication-manager has been rolled out, allowing\n      // replication-manager and view-syncer services to be updated in\n      // parallel.\n      lc.warn?.(\n        `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n        String(err),\n      );\n      try {\n        await sleep(5000, abort.signal);\n      } catch (e) {\n        return reject(e);\n      }\n    }\n  })();\n\n  return status;\n}\n\nfunction reserveSnapshot(\n  lc: LogContext,\n  config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n  assertNormalized(config);\n  const {taskID, change, changeStreamer} = config;\n  const shardID = getShardConfig(config);\n\n  const changeStreamerClient = new ChangeStreamerHttpClient(\n    lc,\n    shardID,\n    change.db,\n    changeStreamer.uri,\n  );\n\n  return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAyCA,IAAM,oBAAoB;AAe1B,IAAa,0BAAb,cAA6C,MAAM;CACjD,OAAgB,OAAO;CAEvB,YAAY,WAA+B;EACzC,MAAM,uBAAuB,WAAW;CAC1C;AACF;;;;;;;AAQA,eAAsB,eACpB,IACA,QACA,oBACA;CACA,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAuB,qBACzB,wBACA;CACJ,IAAI,YAAY,OAAO,WAAW;CAClC,IAAI;CAYJ,IAAI,oBAAoB;CACxB,IAAI;EACF,SAAS;GACP,IAAI;IAOF,MAAM,UAAU,MAAM,WACpB,IACA,QACA,oBACA,MACA,qBAAqB,CACvB;IACA,YAAY,QAAQ;IACpB,IAAI,QAAQ,UAAU;KACpB,SAAS,QAAQ;KACjB;IACF;IACA,oBAAoB;IACpB,IAAI,oBAAoB;KAKtB,SAAS,QAAQ;KACjB,MAAM,IAAI,wBAAwB,OAAO,WAAW,SAAS;IAC/D;GACF,SAAS,GAAG;IACV,IAAI,aAAa,yBACf,MAAM;IAER,IAAI,EAAE,qBAAqB,GAAG;KAK5B,GAAG,OAAO,yCAAyC,CAAC;KACpD;IACF;IAEA,MAAM;GACR;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,SAAS,GAAG;EACV,IAAI,aAAa,2BAA2B,WAAW,KAAA,GACrD,SAAS;EAEX,MAAM;CACR,UAAU;EAER,MAAM,SAAS;GAAC,GADF,6BAA6B,QAAQ,MAAM,SACtC;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAEA,SAAS,cACP,MACA,QACA,kBACA,mBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,WACA,UACA,YACA,UACA,QACA,OAAO,OAAO,OAAO,GACrB,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,6BACA,sBACA,kBACE,OAAO;CAIX,MAAM,gCAAgC,8BAA8B,KAAK;CAQzE,OAAO;EACL,aAHC,SAAS,aAAa,iBAAiB,eAAe,eACvD,KAAK,YAAY,iCAAiC;EAGlD,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB,OAAO,QAAQ;IACrC,+BAA+B,KAAK,qBAAqB,SAAS;IAClE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,qDAAqD,OACpD,6BACF;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB,OAAO,IAAI;IAC/B,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAe,WACb,IACA,QACA,oBACA,MACA,gBACyB;CACzB,IAAI;CACJ,IAAI,CAAC,oBAAoB;EAGvB,MAAM,YAAY,YAAY,IAAI;EAClC,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;EAC7D,8BAA8B,EAAE,SAC9B,YAAY,IAAI,IAAI,WACpB,6BAA6B,QAAQ,MAAM,eAAe,SAAS,CACrE;EACA,GAAG,OAAO,yBAAyB,eAAe,WAAW;EAC7D,qBAAqB;CACvB;CAEA,MAAM,YAAY,gBAAgB,aAAa,OAAO,WAAW;CACjE,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,MAAM,8BAA8B,WAAW,OAAO,QAAQ,IAAI;EAClE,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,SACA,gBAAgB,SAClB;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE,OAAO;EACX,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAQD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,OAAO,QAAQ;EACjB,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GAQA,MAAM,UAAU,SAAS,QAAQ,KAAK;GACtC,IAAI,gBACF,GAAG,QAAQ,yCAAyC,UAAU,CAAC;QAE/D,GAAG,QACD,iDACA,UAAU,CACZ;GAEF,MAAM;EACR;EACA,IAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;GACpC,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,OAAO,QAAQ,MAAM,kBAAkB;EACxE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,OAAO,QAAQ,IAAI;GAChC,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BACH,0BAA0B,EAAE,IAAI,SAAS,OAAO,QAAQ,IAAI,EAAE,MAAM;GAClE,GAAG;GACH,QAAQ;EACV,CAAC;EAEH,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,0BAA0B,EAAE,IAAI,GAAG;GACjC,GAAG;GACH;EACF,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,mBAAmB,YAAY,gBAAgB;GACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;GACA,OAAO;EACT;EACA,IAAI,YAAY,YAAY,cAAc;GACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;GACA,OAAO;EACT;EACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW,WAAW;CACxE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,WAAW,GAC9C,uBAAuB,IAAI,QAAQ,KAAK,CAC1C,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WAAW,WAC9D;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,MAAM;CAC3D,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,OAAO,QAAQ,IAAI,GAAG;EAC7D;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT;AAEA,SAAS,4BACP,IACA,QACyB;CACzB,MAAM,EAAC,SAAS,QAAQ,SAAS,WAAU,SAAyB;CAEpE,CAAM,iBAAkB;EACtB,MAAM,QAAQ,IAAI,gBAAgB;EAClC,QAAQ,GAAG,gBAAgB,MAAM,MAAM,CAAC;EACxC,QAAQ,GAAG,iBAAiB,MAAM,MAAM,CAAC;EAEzC,KAAK,IAAI,IAAI,IAAK,KAAK;GACrB,IAAI;GACJ,IAAI;IACF,IAAI,WAAW;IACf,MAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;IAC/C,WAAW,MAAM,OAAO,QAAQ;KAI9B,QAAQ,IAAI,EAAE;KACd,WAAW;IACb;IAGA,IAAI,UACF;GAEJ,SAAS,GAAG;IACV,MAAM;GACR;GASA,GAAG,OACD,uCAAuC,IAAI,EAAE,4BAC7C,OAAO,GAAG,CACZ;GACA,IAAI;IACF,MAAM,MAAM,KAAM,MAAM,MAAM;GAChC,SAAS,GAAG;IACV,OAAO,OAAO,CAAC;GACjB;EACF;CACF,GAAG;CAEH,OAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;CAClC,iBAAiB,MAAM;CACvB,MAAM,EAAC,QAAQ,QAAQ,mBAAkB;CAUzC,OAAO,IAP0B,yBAC/B,IAHc,eAAe,MAI7B,GACA,OAAO,IACP,eAAe,GAGV,EAAqB,gBAAgB,MAAM;AACpD"}
+{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\ntype RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    backupURL,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n): Promise<RestoreAttempt> {\n  const {backupURL} = config;\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const replicaExistedBeforeRestore = existsSync(replicaFile);\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result};\n    }\n    const validationStart = performance.now();\n    const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: valid ? 'success' : 'invalid_replica'},\n    );\n    if (!valid) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      return {restored: false, backupURL, result};\n    }\n    result = 'success';\n    if (!replicaExistedBeforeRestore) {\n      // This restore materialized the replica file, so any change log beside it\n      // was written against a replica that is no longer there. Reconciliation\n      // is the safety net — a restored replica keeps its replicaVersion, so the\n      // rule falls through to truncate-or-reseed on head mismatch — but the log\n      // is a cache and deleting it here is the explicit statement that nothing\n      // in it survives the restore.\n      //\n      // A restore that reuses an existing replica (`-if-db-not-exists` made it\n      // a no-op) deliberately keeps the log: that is the process-restart path,\n      // where the log is still the one written beside this exact replica and\n      // discarding it would cost every reconnecting subscriber a `too-old`.\n      deleteChangeLogDB(replicaFile);\n      litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n        ...attrs,\n        result: 'success',\n      });\n    }\n    return {restored: true, backupURL, result};\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\nfunction replicaIsValid(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n) {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return false;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return false;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    return true;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return false;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${config.backupURL}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${config.backupURL}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;AAgDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAE7C,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/litestream/config-v5.yml

@@ -0,0 +1,42 @@
+snapshot:
+  interval: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS}h
+  retention: ${ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS}h
+
+# Retain raw L0 files well past worst-case restore duration so continuous
+# compaction can't delete a file an in-progress restore still needs.
+l0-retention: 1h
+l0-retention-check-interval: 5m
+
+# Compaction ladder (L1, L2, L3). Defaults are 30s/5m/1h, tuned for the 1s
+# monitor-interval default; at our 15s monitor-interval, the default 30s L1 would
+# only merge ~2 L0 files. These relaxed intervals produce fewer/fatter L1 files
+# and keep the per-run "list all L0" cost flat if l0-retention is extended further
+# (the busy-DB L0 LIST fires at L1 frequency). Coarser layers only reduce
+# timestamp-based historical restore resolution, which we don't use;
+# restore-to-latest is unaffected. Keep L1 <= l0-retention and L1 < L2 < L3.
+levels:
+  - interval: 2m # L1
+  - interval: 10m # L2
+  - interval: 1h # L3
+
+# Metadata-only (S3 ListObjects) TXID-chain check; negligible cost in v5.
+validation:
+  interval: 1h
+
+dbs:
+  - path: ${ZERO_REPLICA_FILE}
+    # The monitor-interval is the interval at which new l0 files are created.
+    monitor-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS}s
+    replica:
+      url: ${ZERO_LITESTREAM_BACKUP_URL}
+      endpoint: ${ZERO_LITESTREAM_ENDPOINT}
+      region: ${ZERO_LITESTREAM_REGION}
+      auto-recover: true
+      concurrency: ${ZERO_LITESTREAM_MULTIPART_CONCURRENCY}
+      part-size: ${ZERO_LITESTREAM_MULTIPART_SIZE}
+
+logging:
+  level: ${ZERO_LITESTREAM_LOG_LEVEL}
+  type: ${ZERO_LOG_FORMAT}
+
+addr: localhost:${LITESTREAM_PORT}

package/out/zero-cache/src/services/litestream/config.yml

@@ -8,8 +8,9 @@ dbs:
       - url: ${ZERO_LITESTREAM_BACKUP_URL}
         endpoint: ${ZERO_LITESTREAM_ENDPOINT}
         region: ${ZERO_LITESTREAM_REGION}
-        retention: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES}m
-        retention-check-interval: 1h
+        snapshot-interval: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS}h
+        retention: ${ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS}h
+        retention-check-interval: 15m
         sync-interval: ${ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES}m
         multipart-concurrency: ${ZERO_LITESTREAM_MULTIPART_CONCURRENCY}
         multipart-size: ${ZERO_LITESTREAM_MULTIPART_SIZE}

package/out/zero-cache/src/services/litestream/metrics.d.ts

@@ -1,4 +1,4 @@
-import type { ZeroConfig } from '../../config/zero-config.ts';
+import type { LitestreamConfig } from '../../config/normalize.ts';
 export type LitestreamRole = 'replication_manager' | 'view_syncer';
 export type LitestreamVersion = 'legacy' | 'v5';
 export type LitestreamMetricAttrs = {
@@ -10,9 +10,9 @@ type LitestreamMultipartMetricAttrs = {
     multipart_concurrency: number;
     multipart_size_mib: number;
 };
-export declare function litestreamRestoreMetricAttrs(config: ZeroConfig, role: LitestreamRole, backupURL?: string | undefined): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
-export declare function litestreamBackupMetricAttrs(config: ZeroConfig): LitestreamMetricAttrs;
-export declare function litestreamBackupProcessMetricAttrs(config: ZeroConfig): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
+export declare function litestreamRestoreMetricAttrs(config: LitestreamConfig, role: LitestreamRole, backupURL?: string | undefined): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
+export declare function litestreamBackupMetricAttrs(config: LitestreamConfig): LitestreamMetricAttrs;
+export declare function litestreamBackupProcessMetricAttrs(config: LitestreamConfig): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs;
 export declare function litestreamMonitorMetricAttrs(backupURL: string, litestream: LitestreamVersion, role: LitestreamRole): LitestreamMetricAttrs;
 export declare function litestreamRestoreRuns(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 export declare function litestreamRestoreAttempts(): import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
@@ -26,5 +26,6 @@ export declare function litestreamBackupProcessDuration(): import("../../observa
 export declare function litestreamBackupListDuration(): import("../../observability/metrics.ts").LatencyHistogram;
 export declare function litestreamBackupVerificationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
 export declare function litestreamSnapshotReservationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
+export declare function litestreamSnapshotReservationConfirmDuration(): import("../../observability/metrics.ts").LatencyHistogram;
 export {};
 //# sourceMappingURL=metrics.d.ts.map

package/out/zero-cache/src/services/litestream/metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAO5D,MAAM,MAAM,cAAc,GAAG,qBAAqB,GAAG,aAAa,CAAC;AACnE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,IAAI,CAAC;AAEhD,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,cAAc,EACpB,SAAS,qBAA8B,GACtC,qBAAqB,GAAG,8BAA8B,CAaxD;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,UAAU,GACjB,qBAAqB,CAUvB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,UAAU,GACjB,qBAAqB,GAAG,8BAA8B,CAKxD;AAED,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,cAAc,GACnB,qBAAqB,CAMvB;AAyBD,wBAAgB,qBAAqB,kFAMpC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,2BAA2B,kFAM1C;AAED,wBAAgB,yBAAyB,8DAKxC;AAED,wBAAgB,6BAA6B,8DAK5C;AAED,wBAAgB,gCAAgC,8DAK/C;AAED,wBAAgB,mCAAmC,8DAKlD;AAED,wBAAgB,+BAA+B,8DAK9C;AAED,wBAAgB,4BAA4B,8DAK3C;AAED,wBAAgB,oCAAoC,8DAKnD;AAED,wBAAgB,qCAAqC,8DAKpD"}
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,MAAM,MAAM,cAAc,GAAG,qBAAqB,GAAG,aAAa,CAAC;AACnE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,IAAI,CAAC;AAEhD,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,cAAc,EACpB,SAAS,qBAAmB,GAC3B,qBAAqB,GAAG,8BAA8B,CAaxD;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,CAUvB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,GAAG,8BAA8B,CAKxD;AAED,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,cAAc,GACnB,qBAAqB,CAMvB;AAuBD,wBAAgB,qBAAqB,kFAMpC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,2BAA2B,kFAM1C;AAED,wBAAgB,yBAAyB,8DAKxC;AAED,wBAAgB,6BAA6B,8DAK5C;AAED,wBAAgB,gCAAgC,8DAK/C;AAED,wBAAgB,mCAAmC,8DAKlD;AAED,wBAAgB,+BAA+B,8DAK9C;AAED,wBAAgB,4BAA4B,8DAK3C;AAED,wBAAgB,oCAAoC,8DAKnD;AAED,wBAAgB,qCAAqC,8DAKpD;AAED,wBAAgB,4CAA4C,8DAM3D"}

package/out/zero-cache/src/services/litestream/metrics.js

@@ -1,7 +1,7 @@
 import { LONG_DURATION_HISTOGRAM_BOUNDARIES_S, getOrCreateCounter, getOrCreateHistogram } from "../../observability/metrics.js";
 //#region ../zero-cache/src/services/litestream/metrics.ts
-function litestreamRestoreMetricAttrs(config, role, backupURL = config.litestream.backupURL) {
-	const { executable, executableV5, restoreUsingV5 } = config.litestream;
+function litestreamRestoreMetricAttrs(config, role, backupURL = config.backupURL) {
+	const { executable, executableV5, restoreUsingV5 } = config;
 	const selectedExecutable = (restoreUsingV5 ? executableV5 : executable) ?? executable;
 	return {
 		role,
@@ -13,8 +13,8 @@ function litestreamRestoreMetricAttrs(config, role, backupURL = config.litestrea
 function litestreamBackupMetricAttrs(config) {
 	return {
 		role: "replication_manager",
-		backup_scheme: litestreamBackupScheme(config.litestream.backupURL),
-		litestream: config.litestream.executableV5 !== void 0 && config.litestream.executable === config.litestream.executableV5 ? "v5" : "legacy"
+		backup_scheme: litestreamBackupScheme(config.backupURL),
+		litestream: config.executableV5 !== void 0 && config.executable === config.executableV5 ? "v5" : "legacy"
 	};
 }
 function litestreamBackupProcessMetricAttrs(config) {
@@ -41,8 +41,8 @@ function litestreamBackupScheme(backupURL) {
 }
 function litestreamMultipartMetricAttrs(config) {
 	return {
-		multipart_concurrency: config.litestream.multipartConcurrency,
-		multipart_size_mib: Math.round(config.litestream.multipartSize / 1024 / 1024)
+		multipart_concurrency: config.multipartConcurrency,
+		multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024)
 	};
 }
 function litestreamRestoreRuns() {
@@ -63,9 +63,6 @@ function litestreamBackupProcessRuns() {
 function litestreamRestoreDuration() {
 	return litestreamDurationHistogram("litestream.restore.duration", "Wall-clock duration of a litestream restore run, labeled by result.");
 }
-function litestreamRestoreWaitDuration() {
-	return litestreamDurationHistogram("litestream.restore.wait_duration", "Time spent waiting for the replication-manager snapshot status before restoring.");
-}
 function litestreamRestoreProcessDuration() {
 	return litestreamDurationHistogram("litestream.restore.process_duration", "Wall-clock duration of the litestream restore subprocess.");
 }
@@ -84,6 +81,9 @@ function litestreamBackupVerificationDuration() {
 function litestreamSnapshotReservationDuration() {
 	return litestreamDurationHistogram("litestream.snapshot.reservation_duration", "Duration of a snapshot reservation while a view-syncer restores and subscribes.");
 }
+function litestreamSnapshotReservationConfirmDuration() {
+	return litestreamDurationHistogram("litestream.snapshot.reservation_confirm_duration", "Time from opening a snapshot reservation to confirming its change-log bounds, by the source those bounds came from.");
+}
 function litestreamDurationHistogram(name, description) {
 	return getOrCreateHistogram("replica", name, {
 		description,
@@ -92,6 +92,6 @@ function litestreamDurationHistogram(name, description) {
 	});
 }
 //#endregion
-export { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoreWaitDuration, litestreamRestoredDbBytes, litestreamSnapshotReservationDuration };
+export { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoredDbBytes, litestreamSnapshotReservationConfirmDuration, litestreamSnapshotReservationDuration };
 
 //# sourceMappingURL=metrics.js.map

package/out/zero-cache/src/services/litestream/metrics.js.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {ZeroConfig} from '../../config/zero-config.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n  role: LitestreamRole;\n  backup_scheme: string;\n  litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n  multipart_concurrency: number;\n  multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n  config: ZeroConfig,\n  role: LitestreamRole,\n  backupURL = config.litestream.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  const {executable, executableV5, restoreUsingV5} = config.litestream;\n  const selectedExecutable =\n    (restoreUsingV5 ? executableV5 : executable) ?? executable;\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream:\n      executableV5 !== undefined && selectedExecutable === executableV5\n        ? 'v5'\n        : 'legacy',\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamBackupMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMetricAttrs {\n  return {\n    role: 'replication_manager',\n    backup_scheme: litestreamBackupScheme(config.litestream.backupURL),\n    litestream:\n      config.litestream.executableV5 !== undefined &&\n      config.litestream.executable === config.litestream.executableV5\n        ? 'v5'\n        : 'legacy',\n  };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  return {\n    ...litestreamBackupMetricAttrs(config),\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamMonitorMetricAttrs(\n  backupURL: string,\n  litestream: LitestreamVersion,\n  role: LitestreamRole,\n): LitestreamMetricAttrs {\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream,\n  };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n  if (!backupURL) {\n    return 'unknown';\n  }\n  try {\n    const protocol = new URL(backupURL).protocol;\n    return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n  } catch {\n    return 'unknown';\n  }\n}\n\nfunction litestreamMultipartMetricAttrs(\n  config: ZeroConfig,\n): LitestreamMultipartMetricAttrs {\n  return {\n    multipart_concurrency: config.litestream.multipartConcurrency,\n    multipart_size_mib: Math.round(\n      config.litestream.multipartSize / 1024 / 1024,\n    ),\n  };\n}\n\nexport function litestreamRestoreRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.runs',\n    'Litestream restore runs, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreAttempts() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.attempts',\n    'Litestream restore subprocess attempts, labeled by result.',\n  );\n}\n\nexport function litestreamRestoredDbBytes() {\n  return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n    description:\n      'SQLite database bytes restored by successful litestream restores.',\n    unit: 'bytes',\n  });\n}\n\nexport function litestreamBackupProcessRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.backup.process_runs',\n    'Litestream backup process exits, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.duration',\n    'Wall-clock duration of a litestream restore run, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreWaitDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.wait_duration',\n    'Time spent waiting for the replication-manager snapshot status before restoring.',\n  );\n}\n\nexport function litestreamRestoreProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.process_duration',\n    'Wall-clock duration of the litestream restore subprocess.',\n  );\n}\n\nexport function litestreamRestoreValidationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.validation_duration',\n    'Time spent validating a restored replica database.',\n  );\n}\n\nexport function litestreamBackupProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.process_duration',\n    'Runtime duration of the litestream backup subprocess before it exits.',\n  );\n}\n\nexport function litestreamBackupListDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.list_duration',\n    'Duration of litestream backup destination listing commands.',\n  );\n}\n\nexport function litestreamBackupVerificationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.verification_duration',\n    'Duration of verifying the actual backup state in the backup destination.',\n  );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_duration',\n    'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n  );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n  return getOrCreateHistogram('replica', name, {\n    description,\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WAAW,WAC0B;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB,OAAO;CAC1D,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,WAAW,SAAS;EACjE,YACE,OAAO,WAAW,iBAAiB,KAAA,KACnC,OAAO,WAAW,eAAe,OAAO,WAAW,eAC/C,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO,WAAW;EACzC,oBAAoB,KAAK,MACvB,OAAO,WAAW,gBAAgB,OAAO,IAC3C;CACF;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AAEA,SAAgB,gCAAgC;CAC9C,OAAO,4BACL,oCACA,kFACF;AACF;AAEA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}
+{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {LitestreamConfig} from '../../config/normalize.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n  role: LitestreamRole;\n  backup_scheme: string;\n  litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n  multipart_concurrency: number;\n  multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n  config: LitestreamConfig,\n  role: LitestreamRole,\n  backupURL = config.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  const {executable, executableV5, restoreUsingV5} = config;\n  const selectedExecutable =\n    (restoreUsingV5 ? executableV5 : executable) ?? executable;\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream:\n      executableV5 !== undefined && selectedExecutable === executableV5\n        ? 'v5'\n        : 'legacy',\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamBackupMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs {\n  return {\n    role: 'replication_manager',\n    backup_scheme: litestreamBackupScheme(config.backupURL),\n    litestream:\n      config.executableV5 !== undefined &&\n      config.executable === config.executableV5\n        ? 'v5'\n        : 'legacy',\n  };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  return {\n    ...litestreamBackupMetricAttrs(config),\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamMonitorMetricAttrs(\n  backupURL: string,\n  litestream: LitestreamVersion,\n  role: LitestreamRole,\n): LitestreamMetricAttrs {\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream,\n  };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n  if (!backupURL) {\n    return 'unknown';\n  }\n  try {\n    const protocol = new URL(backupURL).protocol;\n    return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n  } catch {\n    return 'unknown';\n  }\n}\n\nfunction litestreamMultipartMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMultipartMetricAttrs {\n  return {\n    multipart_concurrency: config.multipartConcurrency,\n    multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024),\n  };\n}\n\nexport function litestreamRestoreRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.runs',\n    'Litestream restore runs, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreAttempts() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.attempts',\n    'Litestream restore subprocess attempts, labeled by result.',\n  );\n}\n\nexport function litestreamRestoredDbBytes() {\n  return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n    description:\n      'SQLite database bytes restored by successful litestream restores.',\n    unit: 'bytes',\n  });\n}\n\nexport function litestreamBackupProcessRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.backup.process_runs',\n    'Litestream backup process exits, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.duration',\n    'Wall-clock duration of a litestream restore run, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreWaitDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.wait_duration',\n    'Time spent waiting for the replication-manager snapshot status before restoring.',\n  );\n}\n\nexport function litestreamRestoreProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.process_duration',\n    'Wall-clock duration of the litestream restore subprocess.',\n  );\n}\n\nexport function litestreamRestoreValidationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.validation_duration',\n    'Time spent validating a restored replica database.',\n  );\n}\n\nexport function litestreamBackupProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.process_duration',\n    'Runtime duration of the litestream backup subprocess before it exits.',\n  );\n}\n\nexport function litestreamBackupListDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.list_duration',\n    'Duration of litestream backup destination listing commands.',\n  );\n}\n\nexport function litestreamBackupVerificationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.verification_duration',\n    'Duration of verifying the actual backup state in the backup destination.',\n  );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_duration',\n    'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n  );\n}\n\nexport function litestreamSnapshotReservationConfirmDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_confirm_duration',\n    'Time from opening a snapshot reservation to confirming its change-log ' +\n      'bounds, by the source those bounds came from.',\n  );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n  return getOrCreateHistogram('replica', name, {\n    description,\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WACqC;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB;CACnD,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,SAAS;EACtD,YACE,OAAO,iBAAiB,KAAA,KACxB,OAAO,eAAe,OAAO,eACzB,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO;EAC9B,oBAAoB,KAAK,MAAM,OAAO,gBAAgB,OAAO,IAAI;CACnE;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AASA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAgB,+CAA+C;CAC7D,OAAO,4BACL,oDACA,qHAEF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}

package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBA8D5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAEvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
+{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBAgE5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAKvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -127,7 +127,7 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
 			return await stmt.execute();
 		} finally {
 			const q = stmt;
-			lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);
+			lc.debug?.(`${q.string} (${q.parameters.length} params)`);
 		}
 	}
 	authorizer.reloadPermissions();
@@ -156,7 +156,10 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
 	await Promise.all(tasks.map((task) => task()));
 }
 function getInsertSQL(tx, create) {
-	return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;
+	return tx`
+    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}
+    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING
+  `;
 }
 function getUpsertSQL(tx, set) {
 	const { tableName, primaryKey, value } = set;

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`INSERT INTO ${tx(create.tableName)} ${tx(create.value)}`;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM;EACX,KAAKC,YAAY;EACjB,KAAKC,SAAS;EACd,KAAKE,WAAW,IAAI,SAAS,KAAKJ,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAKK,mBAAmB,IAAI,oBAC1B,KAAKL,KACL,QACA,KAAKI,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAKE,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAKE,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAKD,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;EACP,IAAI,KAAKA,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAKA,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAKH,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrB,qBACA,qBACF,CAAC;EAEH,KAAKC,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAKP,KACL,UACA,KAAKC,WACL,KAAKC,QACL,KAAK,IACL,UACA,KAAKG,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAKF,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAKK,YACP,OAAO,KAAKL,SAAS;EAEvB,KAAKE,iBAAiB,QAAQ;EAC9B,KAAKG,aAAa;EAClB,KAAKL,SAAS,QAAQ;EACtB,OAAO,KAAKA,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAAS,MAClB,mCACF;CACA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CACrC,GAAG,QAAQ,0BAA0B,QAAQ;CAG7C,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAM,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAAC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GACV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,UAAU,GAAG;EAC3D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE,eAAe,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK;AACjE;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAO,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,EAAE,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQ;CACV,CAAC;AAEL"}
+{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#writeAuthorizer.destroy();\n    this.#isStopped = true;\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  lc.debug?.('Process mutation start', mutation);\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM;EACX,KAAKC,YAAY;EACjB,KAAKC,SAAS;EACd,KAAKE,WAAW,IAAI,SAAS,KAAKJ,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAKK,mBAAmB,IAAI,oBAC1B,KAAKL,KACL,QACA,KAAKI,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAKE,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAKE,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAKD,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;EACP,IAAI,KAAKA,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAKA,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAKH,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrB,qBACA,qBACF,CAAC;EAEH,KAAKC,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAKP,KACL,UACA,KAAKC,WACL,KAAKC,QACL,KAAK,IACL,UACA,KAAKG,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAKF,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAKK,YACP,OAAO,KAAKL,SAAS;EAEvB,KAAKE,iBAAiB,QAAQ;EAC9B,KAAKG,aAAa;EAClB,KAAKL,SAAS,QAAQ;EACtB,OAAO,KAAKA,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAAS,MAClB,mCACF;CACA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CACrC,GAAG,QAAQ,0BAA0B,QAAQ;CAG7C,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAM,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAAC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAO,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,EAAE,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQ;CACV,CAAC;AAEL"}