@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.10

package/out/zero-cache/src/services/litestream/commands.js.map

@@ -1 +1 @@
-{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n  static readonly name = 'BackupNotFoundException';\n\n  constructor(backupURL: string | undefined) {\n    super(`backup not found at ${backupURL}`);\n  }\n}\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Returns the replica version and watermark of the local replica file, or\n * `undefined` if it does not exist or cannot be read.\n */\nexport function readLocalReplica(\n  lc: LogContext,\n  replica: string,\n): {replicaVersion: string; watermark: string} | undefined {\n  if (!existsSync(replica)) {\n    return undefined;\n  }\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica, {readonly: true});\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    return {replicaVersion, watermark};\n  } catch (e) {\n    lc.warn?.(`unable to read the subscription state of ${replica}`, e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAejC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0B,eAAE,OAAO,EACvC,MAAM,eACH,OAAO;CACN,QAAQ,eAAE,QAAQ;CAClB,MAAM,eAAE,OAAO,EAAE,SAAS;AAC5B,CAAC,EACA,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;AAMA,SAAgB,iBACd,IACA,SACyD;CACzD,IAAI,CAAC,WAAW,OAAO,GACrB;CAEF,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,SAAS,EAAC,UAAU,KAAI,CAAC;EAC/C,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,OAAO;GAAC;GAAgB;EAAS;CACnC,SAAS,GAAG;EACV,GAAG,OAAO,4CAA4C,WAAW,CAAC;EAClE;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
+{"version":3,"file":"commands.js","names":["v","v.parse"],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {must} from '../../../../shared/src/must.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n  isSQLiteCorruption,\n  logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n  litestreamBackupListDuration,\n  litestreamBackupMetricAttrs,\n  litestreamBackupProcessDuration,\n  litestreamBackupProcessMetricAttrs,\n  litestreamBackupProcessRuns,\n  litestreamRestoreAttempts,\n  litestreamRestoredDbBytes,\n  litestreamRestoreMetricAttrs,\n  litestreamRestoreProcessDuration,\n  litestreamRestoreValidationDuration,\n  type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n  replicaVersion: string;\n  minWatermark: string;\n};\n\nexport type RestoreResult =\n  | 'success'\n  | 'no_backup'\n  | 'invalid_replica'\n  | 'error';\n\nexport type RestoreAttempt = {\n  restored: boolean;\n  backupURL: string | undefined;\n  result: RestoreResult;\n  /**\n   * Set when the restore forked the backup (see {@link ForkBackup}), which\n   * then continues from the restored replica's WAL. The WAL is kept when the\n   * last connection closes (see `Database.persistWal()`).\n   */\n  forked?: boolean | undefined;\n};\n\n/**\n * Prepares the restored replica to continue from a forked backup\n * instead of starting a new lineage with a full snapshot.\n */\nexport type ForkBackup = {\n  /** The restore also forks its backup into the (new, empty) one at this URL. */\n  url: string;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\n/**\n * The effective backup URL: the `backupURLOverride` if present (i.e. a\n * lineage-specific location computed by the replication-manager), or\n * otherwise the configured `backupURL`.\n */\nexport function backupDestinationURL(\n  config: Pick<LitestreamConfig, 'backupURL' | 'backupURLOverride'>,\n): string | undefined {\n  return config.backupURLOverride ?? config.backupURL;\n}\n\nfunction getLitestream(\n  mode: 'restore' | 'replicate',\n  config: LitestreamConfig,\n  replicaFile: string,\n  logLevelOverride?: LogLevel,\n): {\n  litestream: string;\n  env: NodeJS.ProcessEnv;\n} {\n  const {\n    executable,\n    executableV5,\n    restoreUsingV5,\n    backupUsingV5,\n    logLevel,\n    configPath,\n    configPathV5,\n    endpoint,\n    region,\n    port,\n    checkpointThresholdMB,\n    // Note: This assumes the default page size. If page size configuration\n    // is added, this computation must be adjusted accordingly.\n    minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n    maxCheckpointPageCount = minCheckpointPageCount * 10,\n    forceCheckpointThresholdMB,\n    incrementalBackupIntervalMinutes,\n    incrementalBackupIntervalSeconds,\n    snapshotBackupIntervalHours,\n    multipartConcurrency,\n    multipartSize,\n  } = config;\n\n  const v5 =\n    (mode === 'restore' && restoreUsingV5) ||\n    (mode === 'replicate' && backupUsingV5);\n  const litestream =\n    (v5 ? executableV5 : executable) ??\n    must(executable, `Missing --litestream-executable`);\n  const litestreamConfig = v5 ? configPathV5 : configPath;\n  // Disable truncate-page-n if forced checkpoints are enabled,\n  // and otherwise use litestream's default.\n  const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n  return {\n    litestream,\n    env: {\n      ...process.env,\n      ['ZERO_REPLICA_FILE']: replicaFile,\n      ['ZERO_LITESTREAM_BACKUP_URL']: must(backupDestinationURL(config)),\n      ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n        minCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n        maxCheckpointPageCount,\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n        incrementalBackupIntervalMinutes, // v3 only\n      ),\n      ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n        incrementalBackupIntervalSeconds, // v5 only\n      ),\n      // v5 only\n      ['ZERO_LITESTREAM_MONITOR_INTERVAL_SECONDS']: String(\n        // zero-cache explicitly requests syncs at the incremental backup\n        // interval (see LitestreamSyncRequester), and litestream's\n        // own monitor becomes a backstop that keeps backups and checkpoints\n        // progressing if zero-cache stops requesting them.\n        2 * incrementalBackupIntervalSeconds,\n      ),\n      ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n      ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n      ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours,\n      ),\n      ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n        snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n      ),\n      ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n      ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n      ['ZERO_LOG_FORMAT']: 'json',\n      ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n      ['LITESTREAM_CONFIG']: litestreamConfig,\n      ['LITESTREAM_PORT']: String(port),\n      ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n      ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n      ['LITESTREAM_COMPACTION_SERIALIZE']: 'false',\n    },\n  };\n}\n\nexport async function tryRestore(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal?: AbortSignal,\n  forkBackup?: ForkBackup | undefined,\n): Promise<RestoreAttempt> {\n  const {reusedExisting, ...attempt} = await restoreOnce(\n    lc,\n    config,\n    replicaFile,\n    replicaConstraints,\n    role,\n    signal,\n    forkBackup,\n  );\n  if (attempt.result === 'invalid_replica' && reusedExisting) {\n    // `-if-db-not-exists` made the restore a no-op, and the existing local\n    // replica (e.g. one left behind by a failed initial sync of a different\n    // generation) was invalid. It has since been deleted, so restore from the\n    // backup.\n    const {reusedExisting: _, ...retry} = await restoreOnce(\n      lc,\n      config,\n      replicaFile,\n      replicaConstraints,\n      role,\n      signal,\n      forkBackup,\n    );\n    return retry;\n  }\n  return attempt;\n}\n\nasync function restoreOnce(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  replicaConstraints: ReplicaConstraints | undefined,\n  role: LitestreamRole,\n  signal: AbortSignal | undefined,\n  forkBackup: ForkBackup | undefined,\n): Promise<RestoreAttempt & {reusedExisting: boolean}> {\n  const {backupUsingV5} = config;\n  const backupURL = backupDestinationURL(config);\n  if (forkBackup && !backupUsingV5) {\n    forkBackup = undefined; // Ignore forkBackup if v5 is disabled.\n  }\n  const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n  let result: RestoreResult = 'error';\n  try {\n    logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n    deleteRestoreTempFiles(replicaFile);\n    const reusedExisting = existsSync(replicaFile);\n    if (!reusedExisting) {\n      // The local litestream state (and anything recorded in it) describes a\n      // replica that no longer exists. Discard it before the restore, which\n      // (if it seeds a backup) writes the state that belongs to the new one.\n      deleteLitestreamMetaDir(replicaFile);\n    }\n    const {litestream, env} = getLitestream(\n      'restore',\n      config,\n      replicaFile,\n      'debug', // Include all output from `litestream restore`, as it's minimal.\n    );\n    const {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    } = config;\n    lc.info?.(`starting litestream restore`, {\n      restoreParallelism: parallelism,\n      multipartConcurrency,\n      multipartSize,\n    });\n    // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n    // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n    // pod's stdout — where a log-scraper alert would page on it — before our\n    // code has decided how to handle the failure. The captured output is\n    // included in the thrown error so the caller can log the final outcome.\n    // See INC-961.\n    const proc = spawn(\n      litestream,\n      [\n        'restore',\n        '-if-db-not-exists',\n        '-if-replica-exists',\n        '-parallelism',\n        String(parallelism),\n        ...(forkBackup ? ['-json', '-fork-to-url', forkBackup.url] : []),\n        replicaFile,\n      ],\n      {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n    );\n    let stdout = '';\n    let stderr = '';\n    proc.stdout.setEncoding('utf-8');\n    proc.stderr.setEncoding('utf-8');\n    // Output AND capture the stdout and stderr, the former for observability\n    // and the latter for error handling.\n    proc.stdout.on('data', chunk => {\n      process.stdout.write(chunk);\n      stdout += chunk;\n    });\n    proc.stderr.on('data', chunk => {\n      process.stderr.write(chunk);\n      stderr += chunk;\n    });\n    const {promise, resolve, reject} = resolver();\n    proc.on('error', reject);\n    proc.on('close', (code, signal) => {\n      if (signal) {\n        reject(`litestream killed with ${signal}`);\n      } else if (code !== 0) {\n        reject(`litestream exited with code ${code}`);\n      } else {\n        resolve();\n      }\n    });\n    const processStart = performance.now();\n    try {\n      await promise;\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'success'},\n      );\n    } catch (e) {\n      litestreamRestoreProcessDuration().recordMs(\n        performance.now() - processStart,\n        {...attrs, result: 'error'},\n      );\n      const output = [stdout, stderr]\n        .map(value => value.trim())\n        .filter(Boolean)\n        .join('\\n');\n      throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n        cause: e,\n      });\n    }\n    if (!existsSync(replicaFile)) {\n      result = 'no_backup';\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    const validationStart = performance.now();\n    const db = openValidReplica(lc, replicaFile, replicaConstraints);\n    litestreamRestoreValidationDuration().recordMs(\n      performance.now() - validationStart,\n      {...attrs, result: db ? 'success' : 'invalid_replica'},\n    );\n    if (!db) {\n      result = 'invalid_replica';\n      lc.info?.(`Deleting local replica and retrying restore`);\n      deleteLiteDB(replicaFile);\n      deleteChangeLogDB(replicaFile);\n      // The litestream state (e.g. the anchor of a seeded backup) describes\n      // the deleted replica.\n      deleteLitestreamMetaDir(replicaFile);\n      return {restored: false, backupURL, result, reusedExisting};\n    }\n    try {\n      result = 'success';\n      if (!reusedExisting) {\n        // This restore materialized the replica file, so any change log beside it\n        // was written against a replica that is no longer there. Reconciliation\n        // is the safety net — a restored replica keeps its replicaVersion, so the\n        // rule falls through to truncate-or-reseed on head mismatch — but the log\n        // is a cache and deleting it here is the explicit statement that nothing\n        // in it survives the restore.\n        //\n        // A restore that reuses an existing replica (`-if-db-not-exists` made it\n        // a no-op) deliberately keeps the log: that is the process-restart path,\n        // where the log is still the one written beside this exact replica and\n        // discarding it would cost every reconnecting subscriber a `too-old`.\n        deleteChangeLogDB(replicaFile);\n        litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n          ...attrs,\n          result: 'success',\n        });\n      }\n      const forked =\n        forkBackup !== undefined &&\n        !reusedExisting &&\n        restoreForked(lc, stdout);\n      return {restored: true, backupURL, result, reusedExisting, forked};\n    } finally {\n      db.close();\n    }\n  } finally {\n    try {\n      deleteRestoreTempFiles(replicaFile);\n    } catch (e) {\n      lc.warn?.('Unable to clean up Litestream restore temporary files', {\n        replicaFile,\n        error: String(e),\n      });\n    }\n    logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n    litestreamRestoreAttempts().add(1, {...attrs, result});\n  }\n}\n\n// From litestream/cmd/litestream/restore.go\nconst restoreForkOutputSchema = v.object({\n  fork: v\n    .object({\n      forked: v.boolean(),\n      txid: v.string().optional(),\n    })\n    .optional(),\n});\n\ntype RestoreForkOutput = v.Infer<typeof restoreForkOutputSchema>;\n\n/**\n * Whether the `-json` output of `litestream restore -fork-to-url` reports that\n * the backup was forked. A restore plan that cannot fork the backup is\n * restored without forking.\n */\nfunction restoreForked(lc: LogContext, stdout: string): boolean {\n  const start = stdout.indexOf('{');\n  const end = stdout.lastIndexOf('}');\n  if (start < 0 || end < start) {\n    lc.warn?.('litestream restore did not report a fork result');\n    return false;\n  }\n  let output: RestoreForkOutput;\n  try {\n    output = v.parse(\n      JSON.parse(stdout.slice(start, end + 1)),\n      restoreForkOutputSchema,\n      'passthrough',\n    );\n  } catch (e) {\n    lc.warn?.('unable to parse the litestream restore result', e);\n    return false;\n  }\n  const forked = output.fork?.forked === true;\n  lc.info?.(\n    forked\n      ? `forked the backup at txid ${output.fork?.txid}`\n      : 'restored without forking the backup',\n  );\n  return forked;\n}\n\n/**\n * Deletes litestream's local state for `replicaFile` (its default meta\n * directory, `.<file>-litestream`).\n */\nexport function deleteLitestreamMetaDir(replicaFile: string) {\n  rmSync(join(dirname(replicaFile), `.${basename(replicaFile)}-litestream`), {\n    recursive: true,\n    force: true,\n  });\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n  const temporaryReplicaFile = `${replicaFile}.tmp`;\n  deleteLiteDB(temporaryReplicaFile);\n\n  // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n  // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n  const directory = dirname(temporaryReplicaFile);\n  if (!existsSync(directory)) {\n    return;\n  }\n  const prefix = `${basename(temporaryReplicaFile)}-`;\n  for (const name of readdirSync(directory)) {\n    if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n      continue;\n    }\n    const index = name.slice(prefix.length, -'-wal'.length);\n    if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n      rmSync(join(directory, name), {force: true});\n    }\n  }\n}\n\nfunction logRestoreDirectoryContents(\n  lc: LogContext,\n  replicaFile: string,\n  phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n  const directory = dirname(replicaFile);\n  try {\n    const names = readdirSync(directory).sort();\n    const entries = names\n      .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n      .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n    lc.info?.('Litestream restore directory contents', {\n      phase,\n      directory,\n      replicaFile,\n      entryCount: names.length,\n      omittedEntryCount: Math.max(\n        0,\n        names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n      ),\n      entries,\n    });\n  } catch (e) {\n    lc.warn?.('Unable to inspect Litestream restore directory', {\n      phase,\n      directory,\n      replicaFile,\n      error: String(e),\n    });\n  }\n}\n\n/**\n * Returns the replica version and watermark of the local replica file, or\n * `undefined` if it does not exist or cannot be read.\n */\nexport function readLocalReplica(\n  lc: LogContext,\n  replica: string,\n): {replicaVersion: string; watermark: string} | undefined {\n  if (!existsSync(replica)) {\n    return undefined;\n  }\n  let db: Database | undefined;\n  try {\n    db = new Database(lc, replica, {readonly: true});\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    return {replicaVersion, watermark};\n  } catch (e) {\n    lc.warn?.(`unable to read the subscription state of ${replica}`, e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\n/**\n * Opens and validates the restored replica. Returns the open connection if\n * the replica is valid, which the caller must close; otherwise closes it and\n * returns `undefined`.\n */\nfunction openValidReplica(\n  lc: LogContext,\n  replica: string,\n  constraints: ReplicaConstraints | undefined,\n): Database | undefined {\n  let db: Database | undefined;\n  try {\n    // Note: Open the database and read the subscription state as a\n    // sanity / corruption check, even if there are no constraints.\n    db = new Database(lc, replica);\n    const {replicaVersion, watermark} = getSubscriptionState(\n      new StatementRunner(db),\n    );\n    if (constraints) {\n      if (replicaVersion !== constraints.replicaVersion) {\n        lc.warn?.(\n          `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n          constraints,\n        );\n        return undefined;\n      }\n      if (watermark < constraints.minWatermark) {\n        lc.warn?.(\n          `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n        );\n        return undefined;\n      }\n      lc.info?.(\n        `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n        constraints,\n      );\n    }\n    const valid = db;\n    db = undefined; // owned by the caller\n    return valid;\n  } catch (e) {\n    if (isSQLiteCorruption(e)) {\n      logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n    }\n    lc.error?.('Error while validating restored replica', e);\n    return undefined;\n  } finally {\n    db?.close();\n  }\n}\n\nexport function startReplicaBackupProcess(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): ChildProcess {\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const attrs = litestreamBackupProcessMetricAttrs(config);\n  lc.info?.(`starting litestream backup to ${backupDestinationURL(config)}`);\n  const start = performance.now();\n  const proc = spawn(litestream, ['replicate'], {\n    env,\n    stdio: 'inherit',\n    windowsHide: true,\n  });\n  let recorded = false;\n  const record = (result: 'success' | 'error' | 'stopped') => {\n    if (recorded) {\n      return;\n    }\n    recorded = true;\n    const labels = {...attrs, result};\n    litestreamBackupProcessRuns().add(1, labels);\n    litestreamBackupProcessDuration().recordMs(\n      performance.now() - start,\n      labels,\n    );\n  };\n  proc.on('error', e => {\n    lc.warn?.(`litestream backup process error`, e);\n    record('error');\n  });\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      lc.info?.(`litestream backup process stopped`, {signal});\n      record('stopped');\n    } else if (code === 0) {\n      record('success');\n    } else {\n      lc.warn?.(`litestream backup process exited with code ${code}`);\n      record('error');\n    }\n  });\n  return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n): Promise<Date> {\n  const [snapshots, wal] = await Promise.all([\n    listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n    listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n  ]);\n  const times = [...snapshots, ...wal];\n  if (times.length === 0) {\n    // Note: the litestream CLI exits with code 0 and logs listing errors\n    // (e.g. S3 failures) to stderr, so an empty listing cannot be\n    // distinguished from a failed one. Since a valid backup always contains\n    // at least one snapshot, an empty listing is treated as a failure.\n    throw new Error(\n      `no snapshots or WAL segments listed at ${backupDestinationURL(config)}`,\n    );\n  }\n  return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica  generation        index  size     created\n * s3       1862f44967b3863f  0      4546445  2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n  lc: LogContext,\n  config: LitestreamConfig,\n  replicaFile: string,\n  command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n  const start = performance.now();\n  let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n  const {litestream, env} = getLitestream('replicate', config, replicaFile);\n  const proc = spawn(litestream, [command, replicaFile], {\n    env,\n    stdio: ['ignore', 'pipe', 'inherit'],\n    windowsHide: true,\n  });\n  const {promise, resolve, reject} = resolver<string>();\n  let stdout = '';\n  proc.stdout.setEncoding('utf-8');\n  proc.stdout.on('data', chunk => (stdout += chunk));\n  proc.on('error', reject);\n  proc.on('close', (code, signal) => {\n    if (signal) {\n      reject(new Error(`litestream ${command} killed with ${signal}`));\n    } else if (code !== 0) {\n      reject(new Error(`litestream ${command} exited with code ${code}`));\n    } else {\n      resolve(stdout);\n    }\n  });\n  const timeout = setTimeout(() => {\n    result = 'timeout';\n    reject(new Error(`timed out listing backup state (litestream ${command})`));\n    proc.kill('SIGKILL');\n  }, LIST_BACKUP_TIMEOUT_MS);\n\n  try {\n    const output = await promise;\n    const times = parseBackupCreatedTimes(lc, command, output);\n    result = times.length ? 'success' : 'empty';\n    return times;\n  } finally {\n    clearTimeout(timeout);\n    litestreamBackupListDuration().recordMs(performance.now() - start, {\n      ...litestreamBackupMetricAttrs(config),\n      command,\n      result,\n    });\n  }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n  lc: LogContext,\n  command: 'snapshots' | 'wal',\n  output: string,\n): Date[] {\n  const times: Date[] = [];\n  for (const line of output.split('\\n')) {\n    const cols = line.trim().split(wsRe);\n    const created = cols.at(-1);\n    if (\n      cols.length < 2 ||\n      created === undefined ||\n      created === 'created' /* header row */\n    ) {\n      continue;\n    }\n    const time = new Date(created);\n    if (Number.isNaN(time.getTime())) {\n      lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n      continue;\n    }\n    times.push(time);\n  }\n  return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAiEA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;;;;;;AAOjC,SAAgB,qBACd,QACoB;CACpB,OAAO,OAAO,qBAAqB,OAAO;AAC5C;AAEA,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAG7C,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,qBAAqB,MAAM,CAAC;IAChE,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IAEC,6CAA6C,OAK5C,IAAI,gCACN;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;IACpD,oCAAoC;EACvC;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACA,YACyB;CACzB,MAAM,EAAC,gBAAgB,GAAG,YAAW,MAAM,YACzC,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;CACA,IAAI,QAAQ,WAAW,qBAAqB,gBAAgB;EAK1D,MAAM,EAAC,gBAAgB,GAAG,GAAG,UAAS,MAAM,YAC1C,IACA,QACA,aACA,oBACA,MACA,QACA,UACF;EACA,OAAO;CACT;CACA,OAAO;AACT;AAEA,eAAe,YACb,IACA,QACA,aACA,oBACA,MACA,QACA,YACqD;CACrD,MAAM,EAAC,kBAAiB;CACxB,MAAM,YAAY,qBAAqB,MAAM;CAC7C,IAAI,cAAc,CAAC,eACjB,aAAa,KAAA;CAEf,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,iBAAiB,WAAW,WAAW;EAC7C,IAAI,CAAC,gBAIH,wBAAwB,WAAW;EAErC,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB,GAAI,aAAa;IAAC;IAAS;IAAgB,WAAW;GAAG,IAAI,CAAC;GAC9D;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,CAAC,CAAC,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,CAAC,CAAC,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,CAAC,CAC5B,KAAI,UAAS,MAAM,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,KAAK,iBAAiB,IAAI,aAAa,kBAAkB;EAC/D,oCAAoC,CAAC,CAAC,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,KAAK,YAAY;EAAiB,CACvD;EACA,IAAI,CAAC,IAAI;GACP,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAG7B,wBAAwB,WAAW;GACnC,OAAO;IAAC,UAAU;IAAO;IAAW;IAAQ;GAAc;EAC5D;EACA,IAAI;GACF,SAAS;GACT,IAAI,CAAC,gBAAgB;IAYnB,kBAAkB,WAAW;IAC7B,0BAA0B,CAAC,CAAC,IAAI,SAAS,WAAW,CAAC,CAAC,MAAM;KAC1D,GAAG;KACH,QAAQ;IACV,CAAC;GACH;GACA,MAAM,SACJ,eAAe,KAAA,KACf,CAAC,kBACD,cAAc,IAAI,MAAM;GAC1B,OAAO;IAAC,UAAU;IAAM;IAAW;IAAQ;IAAgB;GAAM;EACnE,UAAU;GACR,GAAG,MAAM;EACX;CACF,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,CAAC,CAAC,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAGA,IAAM,0BAA0BA,eAAE,OAAO,EACvC,MAAMA,eACH,OAAO;CACN,QAAQA,eAAE,QAAQ;CAClB,MAAMA,eAAE,OAAO,CAAC,CAAC,SAAS;AAC5B,CAAC,CAAC,CACD,SAAS,EACd,CAAC;;;;;;AASD,SAAS,cAAc,IAAgB,QAAyB;CAC9D,MAAM,QAAQ,OAAO,QAAQ,GAAG;CAChC,MAAM,MAAM,OAAO,YAAY,GAAG;CAClC,IAAI,QAAQ,KAAK,MAAM,OAAO;EAC5B,GAAG,OAAO,iDAAiD;EAC3D,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAASC,MACP,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,CAAC,CAAC,GACvC,yBACA,aACF;CACF,SAAS,GAAG;EACV,GAAG,OAAO,iDAAiD,CAAC;EAC5D,OAAO;CACT;CACA,MAAM,SAAS,OAAO,MAAM,WAAW;CACvC,GAAG,OACD,SACI,6BAA6B,OAAO,MAAM,SAC1C,qCACN;CACA,OAAO;AACT;;;;;AAMA,SAAgB,wBAAwB,aAAqB;CAC3D,OAAO,KAAK,QAAQ,WAAW,GAAG,IAAI,SAAS,WAAW,EAAE,YAAY,GAAG;EACzE,WAAW;EACX,OAAO;CACT,CAAC;AACH;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,CAAC,CAAC,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,CAAC,CAC9C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;;;;;AAMA,SAAgB,iBACd,IACA,SACyD;CACzD,IAAI,CAAC,WAAW,OAAO,GACrB;CAEF,IAAI;CACJ,IAAI;EACF,KAAK,IAAI,SAAS,IAAI,SAAS,EAAC,UAAU,KAAI,CAAC;EAC/C,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,OAAO;GAAC;GAAgB;EAAS;CACnC,SAAS,GAAG;EACV,GAAG,OAAO,4CAA4C,WAAW,CAAC;EAClE;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;;;;;AAOA,SAAS,iBACP,IACA,SACA,aACsB;CACtB,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA;GACF;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA;GACF;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,MAAM,QAAQ;EACd,KAAK,KAAA;EACL,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD;CACF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,qBAAqB,MAAM,GAAG;CACzE,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,CAAC,CAAC,IAAI,GAAG,MAAM;EAC3C,gCAAgC,CAAC,CAAC,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,qBAAqB,MAAM,GACvE;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js

@@ -2,7 +2,7 @@ import { AbortError } from "../../../../shared/src/abort-error.js";
 import { RunningState } from "../running-state.js";
 //#region ../zero-cache/src/services/litestream/litestream-checkpointer.ts
 var SOFT_CHECKPOINT_WAIT_MS = 1e4;
-var LITESTREAM_SYNC_TIMEOUT_MS = 5 * 6e4;
+var LITESTREAM_SYNC_TIMEOUT_MS = 3e5;
 var PAUSE_POLL_INTERVAL_MS = 1e3;
 var PAUSE_LOG_INTERVAL_POLLS = 30;
 /**
@@ -126,6 +126,6 @@ var LitestreamCheckpointer = class {
 	}
 };
 //#endregion
-export { LitestreamCheckpointer };
+export { LitestreamCheckpointer, SOFT_CHECKPOINT_WAIT_MS };
 
 //# sourceMappingURL=litestream-checkpointer.js.map

package/out/zero-cache/src/services/litestream/litestream-checkpointer.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-checkpointer.js","names":["#lc","#db","#litestream","#attemptChunk","#maxWalPages","#state","#nextWalThreshold","#getNumWalPages","#pendingSync","#sync","#waitFor","#pauseUntilWalDrained"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\n// How long the soft path blocks the write path waiting for an in-flight sync\n// before letting the writer proceed. This is a backpressure knob, *not* a\n// request timeout: letting it elapse leaves the sync running (see #pendingSync)\n// rather than abandoning it. In normal operation litestream seals + runs its\n// PASSIVE checkpoint well within this budget; a sync that regularly exceeds it\n// indicates a litestream-side problem to fix (e.g. unintended snapshot), not a\n// value to raise.\nexport const SOFT_CHECKPOINT_WAIT_MS = 10_000;\n// Liveness bound on the /sync request itself, well above the soft wait budget.\n// It exists only to recover from a genuinely wedged litestream (so #pendingSync\n// can't hang forever).\nconst LITESTREAM_SYNC_TIMEOUT_MS = 5 * 60_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint) and block the write path until it\n *    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the\n *    backpressure: it throttles upstream reads to litestream's WAL-seal rate\n *    and, in the common case, gives litestream a window to checkpoint without\n *    writer interference. A single sync is kept in flight at a time\n *    (`#pendingSync`): if one is still running when the budget elapses, the\n *    writer proceeds and the next commit waits for that sync rather than\n *    issuing (and orphaning) another. Backs off by a full threshold's worth of\n *    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream\n *    is mid-snapshot and its checkpoints are being skipped), so repeated\n *    failures don't reattempt on every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n  #pendingSync: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    let walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    // Single-flight: reuse the outstanding sync if one is already running.\n    this.#pendingSync ??= this.#sync(walPages);\n\n    // Soft backpressure: block the write path until the sync completes, but no\n    // longer than the wait budget. Letting the budget elapse lets the writer\n    // proceed *without* cancelling the sync (it keeps running, tracked by\n    // #pendingSync); unbounded WAL growth is caught by the hard cap below.\n    await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);\n\n    walPages = this.#getNumWalPages();\n    if (this.#maxWalPages !== undefined && walPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = walPages + this.#attemptChunk;\n  }\n\n  async #waitFor(p: Promise<void>, timeoutMs: number): Promise<void> {\n    const result = await Promise.race([\n      p,\n      this.#state.sleep(timeoutMs).then(() => 'timed-out' as const),\n    ]);\n    if (result === 'timed-out') {\n      this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);\n    }\n  }\n\n  /**\n   * Runs a single litestream sync, logs its outcome, and clears\n   * {@link #pendingSync} when it settles. Never rejects: sync failures and\n   * post-sync bookkeeping errors (e.g. the db being closed during shutdown)\n   * are caught and logged, so the shared promise can be awaited from multiple\n   * places without risking an unhandled rejection.\n   */\n  async #sync(walPagesBefore: number): Promise<void> {\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      try {\n        result = await this.#litestream.sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        );\n      } catch (e) {\n        err = e;\n      }\n\n      const elapsed = performance.now() - start;\n      const newWalPages = this.#getNumWalPages();\n      if (newWalPages < this.#nextWalThreshold) {\n        this.#lc.info?.(\n          `checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else if (result) {\n        this.#lc.warn?.(\n          `checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else {\n        this.#lc.warn?.(\n          `checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n          err,\n        );\n      }\n    } catch (e) {\n      this.#lc.warn?.('litestream-checkpointer sync post-processing failed', e);\n    } finally {\n      this.#pendingSync = undefined;\n    }\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      // Single-flight: reuse the outstanding sync if one is already running.\n      this.#pendingSync ??= this.#sync(this.#getNumWalPages());\n      // await the full liveness timeout configured in #sync()\n      await this.#pendingSync;\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAiBA,IAAa,0BAA0B;AAIvC,IAAM,6BAA6B,IAAI;AACvC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CACA;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAKC,MAAM;EACX,KAAKC,cAAc;EACnB,KAAKC,gBAAgB,OAAO;EAC5B,KAAKG,oBAAoB,OAAO;EAChC,KAAKF,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAKH,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,IAAI,WAAW,KAAKM,gBAAgB;EACpC,IAAI,WAAW,KAAKD,mBAClB;EAIF,KAAKE,iBAAiB,KAAKC,MAAM,QAAQ;EAMzC,MAAM,KAAKC,SAAS,KAAKF,cAAc,uBAAuB;EAE9D,WAAW,KAAKD,gBAAgB;EAChC,IAAI,KAAKH,iBAAiB,KAAA,KAAa,WAAW,KAAKA,cAOrD,WAAW,MAAM,KAAKO,sBAAsB,KAAKP,YAAY;EAO/D,KAAKE,oBAAoB,WAAW,KAAKH;CAC3C;CAEA,MAAMO,SAAS,GAAkB,WAAkC;EAKjE,IAAI,MAJiB,QAAQ,KAAK,CAChC,GACA,KAAKL,OAAO,MAAM,SAAS,EAAE,WAAW,WAAoB,CAC9D,CAAC,MACc,aACb,KAAKL,IAAI,OAAO,uCAAuC,UAAU,IAAI;CAEzE;;;;;;;;CASA,MAAMS,MAAM,gBAAuC;EACjD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IACF,SAAS,MAAM,KAAKP,YAAY,KAC9B;KAAC,MAAM;KAAO,WAAW;IAA0B,GACnD,KAAKG,OAAO,MACd;GACF,SAAS,GAAG;IACV,MAAM;GACR;GAEA,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,cAAc,KAAKE,gBAAgB;GACzC,IAAI,cAAc,KAAKD,mBACrB,KAAKN,IAAI,OACP,gBAAgB,eAAe,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAClF,MACF;QACK,IAAI,QACT,KAAKA,IAAI,OACP,sBAAsB,eAAe,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OAC7G,MACF;QAEA,KAAKA,IAAI,OACP,qBAAqB,eAAe,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC7E,GACF;EAEJ,SAAS,GAAG;GACV,KAAKA,IAAI,OAAO,uDAAuD,CAAC;EAC1E,UAAU;GACR,KAAKQ,eAAe,KAAA;EACtB;CACF;CAEA,MAAMG,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAKN,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAKE,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAKP,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAKA,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAKK,OAAO,MAAM,sBAAsB;GAE9C,KAAKG,iBAAiB,KAAKC,MAAM,KAAKF,gBAAgB,CAAC;GAEvD,MAAM,KAAKC;EACb;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAKN,YAAY,MAAM;EACvB,KAAKG,OAAO,KAAK,KAAKL,GAAG;CAC3B;AACF"}
+{"version":3,"file":"litestream-checkpointer.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Database} from '../../../../zqlite/src/db.ts';\nimport type {ForceCheckpointConfig} from '../replicator/write-worker-client.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {\n  LitestreamController,\n  SyncResponse,\n} from './litestream-controller.ts';\n\n// How long the soft path blocks the write path waiting for an in-flight sync\n// before letting the writer proceed. This is a backpressure knob, *not* a\n// request timeout: letting it elapse leaves the sync running (see #pendingSync)\n// rather than abandoning it. In normal operation litestream seals + runs its\n// PASSIVE checkpoint well within this budget; a sync that regularly exceeds it\n// indicates a litestream-side problem to fix (e.g. unintended snapshot), not a\n// value to raise.\nexport const SOFT_CHECKPOINT_WAIT_MS = 10_000;\n// Liveness bound on the /sync request itself, well above the soft wait budget.\n// It exists only to recover from a genuinely wedged litestream (so #pendingSync\n// can't hang forever).\nconst LITESTREAM_SYNC_TIMEOUT_MS = 5 * 60_000;\nconst PAUSE_POLL_INTERVAL_MS = 1_000;\nconst PAUSE_LOG_INTERVAL_POLLS = 30;\n\n/**\n * Forces litestream to checkpoint the WAL from the write path, so a sustained\n * high write rate can't grow the WAL until litestream's own blocking TRUNCATE\n * checkpoint locks the db (see ReplicationPauseCheckpointConfig).\n *\n * Two tiers:\n *  - Soft (`checkpointThresholdPages`): after each committed transaction,\n *    once the un-checkpointed WAL backlog reaches the threshold, force an\n *    immediate litestream sync (which seals the WAL to LTX and attempts\n *    litestream's own PASSIVE checkpoint) and block the write path until it\n *    completes, but only up to {@link SOFT_CHECKPOINT_WAIT_MS}. Waiting is the\n *    backpressure: it throttles upstream reads to litestream's WAL-seal rate\n *    and, in the common case, gives litestream a window to checkpoint without\n *    writer interference. A single sync is kept in flight at a time\n *    (`#pendingSync`): if one is still running when the budget elapses, the\n *    writer proceeds and the next commit waits for that sync rather than\n *    issuing (and orphaning) another. Backs off by a full threshold's worth of\n *    WAL growth whenever an attempt fails to drain the WAL (e.g. litestream\n *    is mid-snapshot and its checkpoints are being skipped), so repeated\n *    failures don't reattempt on every commit.\n *  - Hard (`walMaxPages`, optional): if the WAL still exceeds this cap after\n *    a soft attempt, pause (block the caller) polling litestream until the\n *    WAL drains below it. This is the fail-safe against unbounded WAL growth\n *    when litestream structurally cannot checkpoint — e.g. an in-progress\n *    snapshot holds litestream's checkpoint lock for its full duration, so\n *    neither PASSIVE nor TRUNCATE checkpoints can run until it completes.\n */\nexport class LitestreamCheckpointer {\n  readonly #lc: LogContext;\n  readonly #db: Database;\n  readonly #litestream: LitestreamController;\n  readonly #attemptChunk: number;\n  readonly #maxWalPages: number | undefined;\n  readonly #state = new RunningState('litestream-checkpointer');\n  #nextWalThreshold: number;\n  #pendingSync: Promise<void> | undefined;\n\n  constructor(\n    lc: LogContext,\n    db: Database,\n    litestream: LitestreamController,\n    config: ForceCheckpointConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-checkpointer');\n    this.#db = db;\n    this.#litestream = litestream;\n    this.#attemptChunk = config.checkpointThresholdPages;\n    this.#nextWalThreshold = config.checkpointThresholdPages;\n    this.#maxWalPages = config.maxWalPages;\n  }\n\n  #getNumWalPages(): number {\n    // wal_checkpoint(NOOP) reads WAL status without doing any work: `log` is\n    // the current WAL frame count and `checkpointed` is how many of those\n    // frames are already applied to the main db, so `log - checkpointed` is\n    // the un-checkpointed backlog that a forced sync would actually drain.\n    // (Both reset once litestream restarts the WAL.)\n    const [{log, checkpointed}] = this.#db.pragma<{\n      log: number;\n      checkpointed: number;\n    }>('wal_checkpoint(NOOP)');\n    return log - checkpointed;\n  }\n\n  /**\n   * Call after a committed transaction. No-ops unless the WAL has grown to\n   * at least the current threshold, in which case it triggers a litestream\n   * sync to perform a checkpoint. Assuming that snapshots are disabled, the\n   * checkpoint always succeeds. In the case where the server is configured to\n   * perform snapshots and checkpoints fail, the \"emergency break\"\n   * {@link #pauseUntilWalDrained()} option provides the disk use protection\n   * that litestream's (disabled) truncate-page-n would have served.\n   */\n  async maybeCheckpoint(): Promise<void> {\n    let walPages = this.#getNumWalPages();\n    if (walPages < this.#nextWalThreshold) {\n      return;\n    }\n\n    // Single-flight: reuse the outstanding sync if one is already running.\n    this.#pendingSync ??= this.#sync(walPages);\n\n    // Soft backpressure: block the write path until the sync completes, but no\n    // longer than the wait budget. Letting the budget elapse lets the writer\n    // proceed *without* cancelling the sync (it keeps running, tracked by\n    // #pendingSync); unbounded WAL growth is caught by the hard cap below.\n    await this.#waitFor(this.#pendingSync, SOFT_CHECKPOINT_WAIT_MS);\n\n    walPages = this.#getNumWalPages();\n    if (this.#maxWalPages !== undefined && walPages > this.#maxWalPages) {\n      // Emergency brake when best-effort checkpoints are being skipped\n      // (i.e. during a snapshot) and the WAL is approaching disk capacity.\n      // Pausing replication is favorable to litestream's truncate-page-n\n      // because the latter involves re-encoding the database under the lock\n      // (https://github.com/benbjohnson/litestream/issues/1332), which would\n      // incur the full latency of an additional snapshot.\n      walPages = await this.#pauseUntilWalDrained(this.#maxWalPages);\n    }\n\n    // In the common case where checkpoints are successful, this is always\n    // just `attemptChunk`. When a checkpoint doesn't drain the WAL (e.g.\n    // litestream is performing a snapshot), wait for another chunk of growth\n    // before the next attempt.\n    this.#nextWalThreshold = walPages + this.#attemptChunk;\n  }\n\n  async #waitFor(p: Promise<void>, timeoutMs: number): Promise<void> {\n    const result = await Promise.race([\n      p,\n      this.#state.sleep(timeoutMs).then(() => 'timed-out' as const),\n    ]);\n    if (result === 'timed-out') {\n      this.#lc.warn?.(`timed out waiting for /sync result (${timeoutMs}ms)`);\n    }\n  }\n\n  /**\n   * Runs a single litestream sync, logs its outcome, and clears\n   * {@link #pendingSync} when it settles. Never rejects: sync failures and\n   * post-sync bookkeeping errors (e.g. the db being closed during shutdown)\n   * are caught and logged, so the shared promise can be awaited from multiple\n   * places without risking an unhandled rejection.\n   */\n  async #sync(walPagesBefore: number): Promise<void> {\n    const start = performance.now();\n    let result: SyncResponse | undefined;\n    let err: unknown;\n    try {\n      try {\n        result = await this.#litestream.sync(\n          {wait: false, timeoutMs: LITESTREAM_SYNC_TIMEOUT_MS},\n          this.#state.signal,\n        );\n      } catch (e) {\n        err = e;\n      }\n\n      const elapsed = performance.now() - start;\n      const newWalPages = this.#getNumWalPages();\n      if (newWalPages < this.#nextWalThreshold) {\n        this.#lc.info?.(\n          `checkpointed ${walPagesBefore} -> ${newWalPages} wal pages (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else if (result) {\n        this.#lc.warn?.(\n          `checkpoint skipped ${walPagesBefore} -> ${newWalPages}. a snapshot may be in progress (${elapsed.toFixed(2)} ms)`,\n          result,\n        );\n      } else {\n        this.#lc.warn?.(\n          `checkpoint failed ${walPagesBefore} -> ${newWalPages} (${elapsed.toFixed(2)} ms)`,\n          err,\n        );\n      }\n    } catch (e) {\n      this.#lc.warn?.('litestream-checkpointer sync post-processing failed', e);\n    } finally {\n      this.#pendingSync = undefined;\n    }\n  }\n\n  async #pauseUntilWalDrained(maxWalPages: number): Promise<number> {\n    for (let i = 0; this.#state.shouldRun(); i++) {\n      const walPages = this.#getNumWalPages();\n      if (walPages < maxWalPages) {\n        this.#lc.info?.(\n          `wal size (${walPages} pages) is under the max limit of ${maxWalPages}`,\n        );\n        return walPages;\n      }\n\n      // Log every 30 seconds.\n      if (i % PAUSE_LOG_INTERVAL_POLLS === 0) {\n        this.#lc.warn?.(\n          `waiting for litestream to successfully checkpoint ${walPages} wal pages`,\n        );\n      }\n      await this.#state.sleep(PAUSE_POLL_INTERVAL_MS);\n      // Single-flight: reuse the outstanding sync if one is already running.\n      this.#pendingSync ??= this.#sync(this.#getNumWalPages());\n      // await the full liveness timeout configured in #sync()\n      await this.#pendingSync;\n    }\n    throw new AbortError('litestream-controller stopped');\n  }\n\n  /** Releases the underlying litestream controller's connection. */\n  close(): void {\n    this.#litestream.close();\n    this.#state.stop(this.#lc);\n  }\n}\n"],"mappings":";;;AAiBA,IAAa,0BAA0B;AAIvC,IAAM,6BAA6B;AACnC,IAAM,yBAAyB;AAC/B,IAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BjC,IAAa,yBAAb,MAAoC;CAClC;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,yBAAyB;CAC5D;CACA;CAEA,YACE,IACA,IACA,YACA,QACA;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,yBAAyB;EAChE,KAAK,MAAM;EACX,KAAK,cAAc;EACnB,KAAK,gBAAgB,OAAO;EAC5B,KAAK,oBAAoB,OAAO;EAChC,KAAK,eAAe,OAAO;CAC7B;CAEA,kBAA0B;EAMxB,MAAM,CAAC,EAAC,KAAK,kBAAiB,KAAK,IAAI,OAGpC,sBAAsB;EACzB,OAAO,MAAM;CACf;;;;;;;;;;CAWA,MAAM,kBAAiC;EACrC,IAAI,WAAW,KAAK,gBAAgB;EACpC,IAAI,WAAW,KAAK,mBAClB;EAIF,KAAK,iBAAiB,KAAK,MAAM,QAAQ;EAMzC,MAAM,KAAK,SAAS,KAAK,cAAc,uBAAuB;EAE9D,WAAW,KAAK,gBAAgB;EAChC,IAAI,KAAK,iBAAiB,KAAA,KAAa,WAAW,KAAK,cAOrD,WAAW,MAAM,KAAK,sBAAsB,KAAK,YAAY;EAO/D,KAAK,oBAAoB,WAAW,KAAK;CAC3C;CAEA,MAAM,SAAS,GAAkB,WAAkC;EAKjE,IAAI,MAJiB,QAAQ,KAAK,CAChC,GACA,KAAK,OAAO,MAAM,SAAS,CAAC,CAAC,WAAW,WAAoB,CAC9D,CAAC,MACc,aACb,KAAK,IAAI,OAAO,uCAAuC,UAAU,IAAI;CAEzE;;;;;;;;CASA,MAAM,MAAM,gBAAuC;EACjD,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI;EACJ,IAAI;EACJ,IAAI;GACF,IAAI;IACF,SAAS,MAAM,KAAK,YAAY,KAC9B;KAAC,MAAM;KAAO,WAAW;IAA0B,GACnD,KAAK,OAAO,MACd;GACF,SAAS,GAAG;IACV,MAAM;GACR;GAEA,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,MAAM,cAAc,KAAK,gBAAgB;GACzC,IAAI,cAAc,KAAK,mBACrB,KAAK,IAAI,OACP,gBAAgB,eAAe,MAAM,YAAY,cAAc,QAAQ,QAAQ,CAAC,EAAE,OAClF,MACF;QACK,IAAI,QACT,KAAK,IAAI,OACP,sBAAsB,eAAe,MAAM,YAAY,mCAAmC,QAAQ,QAAQ,CAAC,EAAE,OAC7G,MACF;QAEA,KAAK,IAAI,OACP,qBAAqB,eAAe,MAAM,YAAY,IAAI,QAAQ,QAAQ,CAAC,EAAE,OAC7E,GACF;EAEJ,SAAS,GAAG;GACV,KAAK,IAAI,OAAO,uDAAuD,CAAC;EAC1E,UAAU;GACR,KAAK,eAAe,KAAA;EACtB;CACF;CAEA,MAAM,sBAAsB,aAAsC;EAChE,KAAK,IAAI,IAAI,GAAG,KAAK,OAAO,UAAU,GAAG,KAAK;GAC5C,MAAM,WAAW,KAAK,gBAAgB;GACtC,IAAI,WAAW,aAAa;IAC1B,KAAK,IAAI,OACP,aAAa,SAAS,oCAAoC,aAC5D;IACA,OAAO;GACT;GAGA,IAAI,IAAI,6BAA6B,GACnC,KAAK,IAAI,OACP,qDAAqD,SAAS,WAChE;GAEF,MAAM,KAAK,OAAO,MAAM,sBAAsB;GAE9C,KAAK,iBAAiB,KAAK,MAAM,KAAK,gBAAgB,CAAC;GAEvD,MAAM,KAAK;EACb;EACA,MAAM,IAAI,WAAW,+BAA+B;CACtD;;CAGA,QAAc;EACZ,KAAK,YAAY,MAAM;EACvB,KAAK,OAAO,KAAK,KAAK,GAAG;CAC3B;AACF"}

package/out/zero-cache/src/services/litestream/litestream-controller.js.map

@@ -1 +1 @@
-{"version":3,"file":"litestream-controller.js","names":["#lc","#replicaFile","#socketPath","#agent","#post"],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean | undefined;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number | undefined;\n\n  /**\n   * Server-side deadline in seconds, sent as litestream's `timeout`. When set,\n   * litestream ends the request itself (with an error response) once it\n   * elapses. Set it below {@link timeoutMs} so that litestream, rather than\n   * the client, ends a slow request. (litestream defaults to 30s when `wait`\n   * is set, and to no deadline otherwise.)\n   */\n  serverTimeoutSeconds?: number | undefined;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {\n      wait = false,\n      timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,\n      serverTimeoutSeconds,\n    } = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {\n        path: this.#replicaFile,\n        wait,\n        ...(serverTimeoutSeconds ? {timeout: serverTimeoutSeconds} : {}),\n      },\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAgDnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAKC,eAAe;EACpB,KAAKC,cAAc;EAGnB,KAAKC,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EACJ,OAAO,OACP,YAAY,4BACZ,yBACE;EACJ,MAAM,SAAS,MAAM,KAAKC,MACxB,SACA;GACE,MAAM,KAAKH;GACX;GACA,GAAI,uBAAuB,EAAC,SAAS,qBAAoB,IAAI,CAAC;EAChE,GACA,WACA,MACF;EACA,KAAKD,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAKG,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAKD;IACjB,QAAQ;IACR;IACA,OAAO,KAAKC;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,EAAE,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}
+{"version":3,"file":"litestream-controller.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/litestream-controller.ts"],"sourcesContent":["import type {IncomingMessage} from 'node:http';\nimport {Agent, request} from 'node:http';\nimport {json as readJSON} from 'node:stream/consumers';\nimport type {LogContext} from '@rocicorp/logger';\n\n/**\n * Path of the litestream control socket, derived from the replica file's\n * directory. The socket must live on a filesystem the (non-root) runtime user\n * can write — the replica directory is guaranteed writable since litestream\n * writes the replica there — so it must NOT default to a root-owned location\n * like /var/run. Kept in sync with config-v5.yml's `socket.path`, which is set\n * from `ZERO_LITESTREAM_SOCKET_PATH` (see getLitestream() in commands.ts).\n */\nexport function litestreamSocketPath(replicaFile: string): string {\n  return `${replicaFile}.sock`;\n}\n\nconst DEFAULT_REQUEST_TIMEOUT_MS = 30_000;\n\n/** Response body from litestream's `POST /sync` control endpoint. */\nexport type SyncResponse = {\n  /**\n   * `synced` when `wait` was true and the upload completed, `synced_local`\n   * when the WAL was sealed locally without waiting on the remote replica, or\n   * `no_change` when there was nothing to sync.\n   */\n  status: 'synced' | 'synced_local' | 'no_change';\n  path: string;\n  txid: number;\n  replicated_txid: number;\n};\n\nexport type SyncOptions = {\n  /**\n   * When true, block until the sealed WAL has been uploaded to the remote\n   * replica (litestream's SyncAndWait). When false (the default), litestream\n   * only seals the WAL into local LTX files and runs its checkpoint thresholds,\n   * returning without waiting on the backup store. Use false on the write path\n   * where the goal is to keep the WAL small, not to durably back it up.\n   */\n  wait?: boolean | undefined;\n\n  /** Overall request timeout in milliseconds. Defaults to 30s. */\n  timeoutMs?: number | undefined;\n\n  /**\n   * Server-side deadline in seconds, sent as litestream's `timeout`. When set,\n   * litestream ends the request itself (with an error response) once it\n   * elapses. Set it below {@link timeoutMs} so that litestream, rather than\n   * the client, ends a slow request. (litestream defaults to 30s when `wait`\n   * is set, and to no deadline otherwise.)\n   */\n  serverTimeoutSeconds?: number | undefined;\n};\n\n/**\n * IPC client for the litestream control server. litestream serves an HTTP API\n * over a Unix domain socket (enabled via the `socket:` block in config-v5.yml);\n * because zero-cache runs litestream as a subprocess in the same container, the\n * socket is reachable directly on the local filesystem.\n *\n * A single keep-alive connection is held open and reused across calls, so a\n * high-frequency caller (e.g. write-path backpressure) does not pay a\n * connect/handshake per request.\n */\nexport class LitestreamController {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #socketPath: string;\n  readonly #agent: Agent;\n\n  /**\n   * @param replicaFile Path of the replica database litestream is managing —\n   *   the same value passed to litestream as `ZERO_REPLICA_FILE`. This is the\n   *   `path` litestream matches control requests against.\n   * @param socketPath Path of the control socket; defaults to the path derived\n   *   from `replicaFile`, matching config-v5.yml's `socket.path`.\n   */\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    socketPath = litestreamSocketPath(replicaFile),\n  ) {\n    this.#lc = lc.withContext('component', 'litestream-controller');\n    this.#replicaFile = replicaFile;\n    this.#socketPath = socketPath;\n    // keepAlive with a single socket keeps one warm connection to the control\n    // socket that is reused across calls.\n    this.#agent = new Agent({keepAlive: true, maxSockets: 1});\n  }\n\n  /**\n   * Forces litestream to sync the WAL into LTX files immediately and run its\n   * checkpoint thresholds, instead of waiting for the next monitor-interval\n   * tick. The checkpoint itself is still litestream's usual (non-blocking)\n   * PASSIVE attempt, so it is skipped if the database is busy — call this once\n   * writes are quiesced.\n   *\n   * Best-effort: callers should tolerate rejection (e.g. ECONNREFUSED/ENOENT\n   * while litestream is restarting or before it has created the socket) rather\n   * than blocking application progress on it.\n   */\n  async sync(\n    opts: SyncOptions = {},\n    signal?: AbortSignal,\n  ): Promise<SyncResponse> {\n    const {\n      wait = false,\n      timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS,\n      serverTimeoutSeconds,\n    } = opts;\n    const result = await this.#post<SyncResponse>(\n      '/sync',\n      {\n        path: this.#replicaFile,\n        wait,\n        ...(serverTimeoutSeconds ? {timeout: serverTimeoutSeconds} : {}),\n      },\n      timeoutMs,\n      signal,\n    );\n    this.#lc.debug?.('litestream sync', result);\n    return result;\n  }\n\n  /** Releases the pooled keep-alive connection. */\n  close(): void {\n    this.#agent.destroy();\n  }\n\n  // Note: Uses ye'old node:http for its unix socketPath support.\n  //   fetch() would have less boiler-plate, but it would require importing\n  //   the `unidici` package to provide a fetch Agent that supports sockets.\n  #post<T>(\n    path: string,\n    body: unknown,\n    timeoutMs: number,\n    signal?: AbortSignal,\n  ): Promise<T> {\n    const payload = JSON.stringify(body);\n    return new Promise<T>((resolve, reject) => {\n      const req = request(\n        {\n          socketPath: this.#socketPath,\n          method: 'POST',\n          path,\n          agent: this.#agent,\n          timeout: timeoutMs,\n          headers: {\n            'content-type': 'application/json',\n            'content-length': Buffer.byteLength(payload),\n          },\n          signal,\n        },\n        (res: IncomingMessage) => {\n          const status = res.statusCode ?? 0;\n          // litestream's success and error responses are both always JSON\n          // (writeJSON / writeJSONError in server.go).\n          readJSON(res).then(\n            body => {\n              if (status < 200 || status >= 300) {\n                reject(\n                  new Error(\n                    `litestream ${path} failed: ${status} ${JSON.stringify(body)}`,\n                  ),\n                );\n              } else {\n                resolve(body as T);\n              }\n            },\n            (e: unknown) =>\n              reject(\n                new Error(`litestream ${path} returned invalid JSON`, {\n                  cause: e,\n                }),\n              ),\n          );\n        },\n      );\n      req.on('timeout', () =>\n        req.destroy(\n          new Error(`litestream ${path} timed out after ${timeoutMs}ms`),\n        ),\n      );\n      req.on('error', reject);\n      req.end(payload);\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;AAaA,SAAgB,qBAAqB,aAA6B;CAChE,OAAO,GAAG,YAAY;AACxB;AAEA,IAAM,6BAA6B;;;;;;;;;;;AAgDnC,IAAa,uBAAb,MAAkC;CAChC;CACA;CACA;CACA;;;;;;;;CASA,YACE,IACA,aACA,aAAa,qBAAqB,WAAW,GAC7C;EACA,KAAK,MAAM,GAAG,YAAY,aAAa,uBAAuB;EAC9D,KAAK,eAAe;EACpB,KAAK,cAAc;EAGnB,KAAK,SAAS,IAAI,MAAM;GAAC,WAAW;GAAM,YAAY;EAAC,CAAC;CAC1D;;;;;;;;;;;;CAaA,MAAM,KACJ,OAAoB,CAAC,GACrB,QACuB;EACvB,MAAM,EACJ,OAAO,OACP,YAAY,4BACZ,yBACE;EACJ,MAAM,SAAS,MAAM,KAAK,MACxB,SACA;GACE,MAAM,KAAK;GACX;GACA,GAAI,uBAAuB,EAAC,SAAS,qBAAoB,IAAI,CAAC;EAChE,GACA,WACA,MACF;EACA,KAAK,IAAI,QAAQ,mBAAmB,MAAM;EAC1C,OAAO;CACT;;CAGA,QAAc;EACZ,KAAK,OAAO,QAAQ;CACtB;CAKA,MACE,MACA,MACA,WACA,QACY;EACZ,MAAM,UAAU,KAAK,UAAU,IAAI;EACnC,OAAO,IAAI,SAAY,SAAS,WAAW;GACzC,MAAM,MAAM,QACV;IACE,YAAY,KAAK;IACjB,QAAQ;IACR;IACA,OAAO,KAAK;IACZ,SAAS;IACT,SAAS;KACP,gBAAgB;KAChB,kBAAkB,OAAO,WAAW,OAAO;IAC7C;IACA;GACF,IACC,QAAyB;IACxB,MAAM,SAAS,IAAI,cAAc;IAGjC,KAAS,GAAG,CAAC,CAAC,MACZ,SAAQ;KACN,IAAI,SAAS,OAAO,UAAU,KAC5B,uBACE,IAAI,MACF,cAAc,KAAK,WAAW,OAAO,GAAG,KAAK,UAAU,IAAI,GAC7D,CACF;UAEA,QAAQ,IAAS;IAErB,IACC,MACC,OACE,IAAI,MAAM,cAAc,KAAK,yBAAyB,EACpD,OAAO,EACT,CAAC,CACH,CACJ;GACF,CACF;GACA,IAAI,GAAG,iBACL,IAAI,wBACF,IAAI,MAAM,cAAc,KAAK,mBAAmB,UAAU,GAAG,CAC/D,CACF;GACA,IAAI,GAAG,SAAS,MAAM;GACtB,IAAI,IAAI,OAAO;EACjB,CAAC;CACH;AACF"}

package/out/zero-cache/src/services/litestream/metrics.js.map

@@ -1 +1 @@
-{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {LitestreamConfig} from '../../config/normalize.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n  role: LitestreamRole;\n  backup_scheme: string;\n  litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n  multipart_concurrency: number;\n  multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n  config: LitestreamConfig,\n  role: LitestreamRole,\n  backupURL = config.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  const {executable, executableV5, restoreUsingV5} = config;\n  const selectedExecutable =\n    (restoreUsingV5 ? executableV5 : executable) ?? executable;\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream:\n      executableV5 !== undefined && selectedExecutable === executableV5\n        ? 'v5'\n        : 'legacy',\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamBackupMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs {\n  return {\n    role: 'replication_manager',\n    backup_scheme: litestreamBackupScheme(config.backupURL),\n    litestream:\n      config.executableV5 !== undefined &&\n      config.executable === config.executableV5\n        ? 'v5'\n        : 'legacy',\n  };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  return {\n    ...litestreamBackupMetricAttrs(config),\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamMonitorMetricAttrs(\n  backupURL: string,\n  litestream: LitestreamVersion,\n  role: LitestreamRole,\n): LitestreamMetricAttrs {\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream,\n  };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n  if (!backupURL) {\n    return 'unknown';\n  }\n  try {\n    const protocol = new URL(backupURL).protocol;\n    return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n  } catch {\n    return 'unknown';\n  }\n}\n\nfunction litestreamMultipartMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMultipartMetricAttrs {\n  return {\n    multipart_concurrency: config.multipartConcurrency,\n    multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024),\n  };\n}\n\nexport function litestreamRestoreRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.runs',\n    'Litestream restore runs, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreAttempts() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.attempts',\n    'Litestream restore subprocess attempts, labeled by result.',\n  );\n}\n\nexport function litestreamRestoredDbBytes() {\n  return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n    description:\n      'SQLite database bytes restored by successful litestream restores.',\n    unit: 'bytes',\n  });\n}\n\nexport function litestreamBackupProcessRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.backup.process_runs',\n    'Litestream backup process exits, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.duration',\n    'Wall-clock duration of a litestream restore run, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreWaitDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.wait_duration',\n    'Time spent waiting for the replication-manager snapshot status before restoring.',\n  );\n}\n\nexport function litestreamRestoreProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.process_duration',\n    'Wall-clock duration of the litestream restore subprocess.',\n  );\n}\n\nexport function litestreamRestoreValidationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.validation_duration',\n    'Time spent validating a restored replica database.',\n  );\n}\n\nexport function litestreamBackupProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.process_duration',\n    'Runtime duration of the litestream backup subprocess before it exits.',\n  );\n}\n\nexport function litestreamBackupListDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.list_duration',\n    'Duration of litestream backup destination listing commands.',\n  );\n}\n\nexport function litestreamBackupVerificationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.verification_duration',\n    'Duration of verifying the actual backup state in the backup destination.',\n  );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_duration',\n    'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n  );\n}\n\nexport function litestreamSnapshotReservationConfirmDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_confirm_duration',\n    'Time from opening a snapshot reservation to confirming its change-log ' +\n      'bounds, by the source those bounds came from.',\n  );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n  return getOrCreateHistogram('replica', name, {\n    description,\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WACqC;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB;CACnD,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,SAAS;EACtD,YACE,OAAO,iBAAiB,KAAA,KACxB,OAAO,eAAe,OAAO,eACzB,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO;EAC9B,oBAAoB,KAAK,MAAM,OAAO,gBAAgB,OAAO,IAAI;CACnE;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AASA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAgB,+CAA+C;CAC7D,OAAO,4BACL,oDACA,qHAEF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}
+{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {LitestreamConfig} from '../../config/normalize.ts';\nimport {\n  getOrCreateCounter,\n  getOrCreateHistogram,\n  LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n  role: LitestreamRole;\n  backup_scheme: string;\n  litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n  multipart_concurrency: number;\n  multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n  config: LitestreamConfig,\n  role: LitestreamRole,\n  backupURL = config.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  const {executable, executableV5, restoreUsingV5} = config;\n  const selectedExecutable =\n    (restoreUsingV5 ? executableV5 : executable) ?? executable;\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream:\n      executableV5 !== undefined && selectedExecutable === executableV5\n        ? 'v5'\n        : 'legacy',\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamBackupMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs {\n  return {\n    role: 'replication_manager',\n    backup_scheme: litestreamBackupScheme(config.backupURL),\n    litestream:\n      config.executableV5 !== undefined &&\n      config.executable === config.executableV5\n        ? 'v5'\n        : 'legacy',\n  };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n  return {\n    ...litestreamBackupMetricAttrs(config),\n    ...litestreamMultipartMetricAttrs(config),\n  };\n}\n\nexport function litestreamMonitorMetricAttrs(\n  backupURL: string,\n  litestream: LitestreamVersion,\n  role: LitestreamRole,\n): LitestreamMetricAttrs {\n  return {\n    role,\n    backup_scheme: litestreamBackupScheme(backupURL),\n    litestream,\n  };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n  if (!backupURL) {\n    return 'unknown';\n  }\n  try {\n    const protocol = new URL(backupURL).protocol;\n    return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n  } catch {\n    return 'unknown';\n  }\n}\n\nfunction litestreamMultipartMetricAttrs(\n  config: LitestreamConfig,\n): LitestreamMultipartMetricAttrs {\n  return {\n    multipart_concurrency: config.multipartConcurrency,\n    multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024),\n  };\n}\n\nexport function litestreamRestoreRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.runs',\n    'Litestream restore runs, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreAttempts() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.restore.attempts',\n    'Litestream restore subprocess attempts, labeled by result.',\n  );\n}\n\nexport function litestreamRestoredDbBytes() {\n  return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n    description:\n      'SQLite database bytes restored by successful litestream restores.',\n    unit: 'bytes',\n  });\n}\n\nexport function litestreamBackupProcessRuns() {\n  return getOrCreateCounter(\n    'replica',\n    'litestream.backup.process_runs',\n    'Litestream backup process exits, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.duration',\n    'Wall-clock duration of a litestream restore run, labeled by result.',\n  );\n}\n\nexport function litestreamRestoreProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.process_duration',\n    'Wall-clock duration of the litestream restore subprocess.',\n  );\n}\n\nexport function litestreamRestoreValidationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.restore.validation_duration',\n    'Time spent validating a restored replica database.',\n  );\n}\n\nexport function litestreamBackupProcessDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.process_duration',\n    'Runtime duration of the litestream backup subprocess before it exits.',\n  );\n}\n\nexport function litestreamBackupListDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.list_duration',\n    'Duration of litestream backup destination listing commands.',\n  );\n}\n\nexport function litestreamBackupVerificationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.backup.verification_duration',\n    'Duration of verifying the actual backup state in the backup destination.',\n  );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_duration',\n    'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n  );\n}\n\nexport function litestreamSnapshotReservationConfirmDuration() {\n  return litestreamDurationHistogram(\n    'litestream.snapshot.reservation_confirm_duration',\n    'Time from opening a snapshot reservation to confirming its change-log ' +\n      'bounds, by the source those bounds came from.',\n  );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n  return getOrCreateHistogram('replica', name, {\n    description,\n    unit: 's',\n    bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n  });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WACqC;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB;CACnD,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,SAAS;EACtD,YACE,OAAO,iBAAiB,KAAA,KACxB,OAAO,eAAe,OAAO,eACzB,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,CAAC,CAAC;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO;EAC9B,oBAAoB,KAAK,MAAM,OAAO,gBAAgB,OAAO,IAAI;CACnE;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AAEA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAgB,+CAA+C;CAC7D,OAAO,4BACL,oDACA,qHAEF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}

package/out/zero-cache/src/services/litestream/restore-progress.js.map

@@ -1 +1 @@
-{"version":3,"file":"restore-progress.js","names":["#lc","#replicaFile","#publishFn","#start","#totalBytes","#lastBytes","#sawTempFile","#update","#timer","#check","#publish"],"sources":["../../../../../../zero-cache/src/services/litestream/restore-progress.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {\n  RestoreStatus,\n  ViewSyncerStage,\n  ViewSyncerStatusEvent,\n} from '../../../../zero-events/src/status.ts';\nimport {publishEvent} from '../../observability/events.ts';\n\nconst PUBLISH_INTERVAL_MS = 5000;\n\ntype PublishFn = (lc: LogContext, event: ViewSyncerStatusEvent) => void;\n\n/**\n * Publishes the progress of a view-syncer's litestream restore.\n *\n * `litestream restore` writes the database page by page into\n * `<replica>.tmp` and renames it to the replica when it is done, so the\n * size of the temporary file is the number of bytes restored so far.\n */\nexport class RestoreProgressReporter {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #publishFn: PublishFn;\n  #start = performance.now();\n  #totalBytes: number | undefined;\n  #lastBytes: number | undefined;\n  #sawTempFile = false;\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    publishFn: PublishFn = publishEvent,\n  ) {\n    this.#lc = lc;\n    this.#replicaFile = replicaFile;\n    this.#publishFn = publishFn;\n  }\n\n  /**\n   * Publishes a `Restoring` event now and then every `intervalMs` while the\n   * number of restored bytes changes, until {@link stop} or {@link done}.\n   */\n  start(totalBytes: number | undefined, intervalMs = PUBLISH_INTERVAL_MS) {\n    this.stop();\n    // Measure the restore itself, not the wait for a restorable backup.\n    this.#start = performance.now();\n    if (totalBytes !== this.#totalBytes) {\n      // Publish the new total even if the restored bytes are unchanged.\n      this.#lastBytes = undefined;\n      this.#totalBytes = totalBytes;\n    }\n    // Start from 0 rather than checking now: a `.tmp` left behind by an\n    // interrupted restore is only deleted when the restore starts.\n    this.#sawTempFile = false;\n    this.#update(0);\n    this.#timer = setInterval(() => this.#check(), intervalMs);\n  }\n\n  stop() {\n    clearInterval(this.#timer);\n    this.#timer = undefined;\n  }\n\n  /** Publishes a `Restored` event with the size of the restored replica. */\n  done() {\n    this.stop();\n    this.#publish('Restored', fileSize(this.#replicaFile) ?? 0);\n  }\n\n  #check() {\n    const tempBytes = fileSize(`${this.#replicaFile}.tmp`);\n    if (tempBytes !== undefined) {\n      this.#sawTempFile = true;\n      this.#update(tempBytes);\n    } else if (this.#sawTempFile) {\n      // litestream renames `.tmp` to the replica before its post-restore\n      // integrity check, so once `.tmp` is gone the replica holds the bytes.\n      // Before `.tmp` appears, an existing replica is not restore progress\n      // (it may be invalid and about to be deleted).\n      this.#update(fileSize(this.#replicaFile) ?? 0);\n    }\n  }\n\n  #update(bytes: number) {\n    if (bytes !== this.#lastBytes) {\n      this.#lastBytes = bytes;\n      this.#publish('Restoring', bytes);\n    }\n  }\n\n  #publish(stage: ViewSyncerStage, bytes: number) {\n    const restoreStatus: RestoreStatus = {\n      bytes,\n      totalBytes: this.#totalBytes,\n      elapsedMs: Math.round(performance.now() - this.#start),\n    };\n    const event: ViewSyncerStatusEvent = {\n      type: 'zero/events/status/view-syncer/v1',\n      component: 'view-syncer',\n      status: 'OK',\n      stage,\n      description:\n        stage === 'Restoring'\n          ? 'Restoring replica from backup'\n          : 'Restored replica from backup',\n      time: new Date().toISOString(),\n      state: {restoreStatus},\n    };\n    this.#publishFn(this.#lc, event);\n  }\n}\n\nfunction fileSize(file: string): number | undefined {\n  try {\n    return statSync(file).size;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;AASA,IAAM,sBAAsB;;;;;;;;AAW5B,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA;CACA,SAAS,YAAY,IAAI;CACzB;CACA;CACA,eAAe;CACf;CAEA,YACE,IACA,aACA,YAAuB,cACvB;EACA,KAAKA,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,aAAa;CACpB;;;;;CAMA,MAAM,YAAgC,aAAa,qBAAqB;EACtE,KAAK,KAAK;EAEV,KAAKC,SAAS,YAAY,IAAI;EAC9B,IAAI,eAAe,KAAKC,aAAa;GAEnC,KAAKC,aAAa,KAAA;GAClB,KAAKD,cAAc;EACrB;EAGA,KAAKE,eAAe;EACpB,KAAKC,QAAQ,CAAC;EACd,KAAKC,SAAS,kBAAkB,KAAKC,OAAO,GAAG,UAAU;CAC3D;CAEA,OAAO;EACL,cAAc,KAAKD,MAAM;EACzB,KAAKA,SAAS,KAAA;CAChB;;CAGA,OAAO;EACL,KAAK,KAAK;EACV,KAAKE,SAAS,YAAY,SAAS,KAAKT,YAAY,KAAK,CAAC;CAC5D;CAEA,SAAS;EACP,MAAM,YAAY,SAAS,GAAG,KAAKA,aAAa,KAAK;EACrD,IAAI,cAAc,KAAA,GAAW;GAC3B,KAAKK,eAAe;GACpB,KAAKC,QAAQ,SAAS;EACxB,OAAO,IAAI,KAAKD,cAKd,KAAKC,QAAQ,SAAS,KAAKN,YAAY,KAAK,CAAC;CAEjD;CAEA,QAAQ,OAAe;EACrB,IAAI,UAAU,KAAKI,YAAY;GAC7B,KAAKA,aAAa;GAClB,KAAKK,SAAS,aAAa,KAAK;EAClC;CACF;CAEA,SAAS,OAAwB,OAAe;EAC9C,MAAM,gBAA+B;GACnC;GACA,YAAY,KAAKN;GACjB,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAKD,MAAM;EACvD;EACA,MAAM,QAA+B;GACnC,MAAM;GACN,WAAW;GACX,QAAQ;GACR;GACA,aACE,UAAU,cACN,kCACA;GACN,uBAAM,IAAI,KAAK,GAAE,YAAY;GAC7B,OAAO,EAAC,cAAa;EACvB;EACA,KAAKD,WAAW,KAAKF,KAAK,KAAK;CACjC;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,IAAI;EACF,OAAO,SAAS,IAAI,EAAE;CACxB,QAAQ;EACN;CACF;AACF"}
+{"version":3,"file":"restore-progress.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/restore-progress.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {\n  RestoreStatus,\n  ViewSyncerStage,\n  ViewSyncerStatusEvent,\n} from '../../../../zero-events/src/status.ts';\nimport {publishEvent} from '../../observability/events.ts';\n\nconst PUBLISH_INTERVAL_MS = 5000;\n\ntype PublishFn = (lc: LogContext, event: ViewSyncerStatusEvent) => void;\n\n/**\n * Publishes the progress of a view-syncer's litestream restore.\n *\n * `litestream restore` writes the database page by page into\n * `<replica>.tmp` and renames it to the replica when it is done, so the\n * size of the temporary file is the number of bytes restored so far.\n */\nexport class RestoreProgressReporter {\n  readonly #lc: LogContext;\n  readonly #replicaFile: string;\n  readonly #publishFn: PublishFn;\n  #start = performance.now();\n  #totalBytes: number | undefined;\n  #lastBytes: number | undefined;\n  #sawTempFile = false;\n  #timer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    replicaFile: string,\n    publishFn: PublishFn = publishEvent,\n  ) {\n    this.#lc = lc;\n    this.#replicaFile = replicaFile;\n    this.#publishFn = publishFn;\n  }\n\n  /**\n   * Publishes a `Restoring` event now and then every `intervalMs` while the\n   * number of restored bytes changes, until {@link stop} or {@link done}.\n   */\n  start(totalBytes: number | undefined, intervalMs = PUBLISH_INTERVAL_MS) {\n    this.stop();\n    // Measure the restore itself, not the wait for a restorable backup.\n    this.#start = performance.now();\n    if (totalBytes !== this.#totalBytes) {\n      // Publish the new total even if the restored bytes are unchanged.\n      this.#lastBytes = undefined;\n      this.#totalBytes = totalBytes;\n    }\n    // Start from 0 rather than checking now: a `.tmp` left behind by an\n    // interrupted restore is only deleted when the restore starts.\n    this.#sawTempFile = false;\n    this.#update(0);\n    this.#timer = setInterval(() => this.#check(), intervalMs);\n  }\n\n  stop() {\n    clearInterval(this.#timer);\n    this.#timer = undefined;\n  }\n\n  /** Publishes a `Restored` event with the size of the restored replica. */\n  done() {\n    this.stop();\n    this.#publish('Restored', fileSize(this.#replicaFile) ?? 0);\n  }\n\n  #check() {\n    const tempBytes = fileSize(`${this.#replicaFile}.tmp`);\n    if (tempBytes !== undefined) {\n      this.#sawTempFile = true;\n      this.#update(tempBytes);\n    } else if (this.#sawTempFile) {\n      // litestream renames `.tmp` to the replica before its post-restore\n      // integrity check, so once `.tmp` is gone the replica holds the bytes.\n      // Before `.tmp` appears, an existing replica is not restore progress\n      // (it may be invalid and about to be deleted).\n      this.#update(fileSize(this.#replicaFile) ?? 0);\n    }\n  }\n\n  #update(bytes: number) {\n    if (bytes !== this.#lastBytes) {\n      this.#lastBytes = bytes;\n      this.#publish('Restoring', bytes);\n    }\n  }\n\n  #publish(stage: ViewSyncerStage, bytes: number) {\n    const restoreStatus: RestoreStatus = {\n      bytes,\n      totalBytes: this.#totalBytes,\n      elapsedMs: Math.round(performance.now() - this.#start),\n    };\n    const event: ViewSyncerStatusEvent = {\n      type: 'zero/events/status/view-syncer/v1',\n      component: 'view-syncer',\n      status: 'OK',\n      stage,\n      description:\n        stage === 'Restoring'\n          ? 'Restoring replica from backup'\n          : 'Restored replica from backup',\n      time: new Date().toISOString(),\n      state: {restoreStatus},\n    };\n    this.#publishFn(this.#lc, event);\n  }\n}\n\nfunction fileSize(file: string): number | undefined {\n  try {\n    return statSync(file).size;\n  } catch {\n    return undefined;\n  }\n}\n"],"mappings":";;;AASA,IAAM,sBAAsB;;;;;;;;AAW5B,IAAa,0BAAb,MAAqC;CACnC;CACA;CACA;CACA,SAAS,YAAY,IAAI;CACzB;CACA;CACA,eAAe;CACf;CAEA,YACE,IACA,aACA,YAAuB,cACvB;EACA,KAAK,MAAM;EACX,KAAK,eAAe;EACpB,KAAK,aAAa;CACpB;;;;;CAMA,MAAM,YAAgC,aAAa,qBAAqB;EACtE,KAAK,KAAK;EAEV,KAAK,SAAS,YAAY,IAAI;EAC9B,IAAI,eAAe,KAAK,aAAa;GAEnC,KAAK,aAAa,KAAA;GAClB,KAAK,cAAc;EACrB;EAGA,KAAK,eAAe;EACpB,KAAK,QAAQ,CAAC;EACd,KAAK,SAAS,kBAAkB,KAAK,OAAO,GAAG,UAAU;CAC3D;CAEA,OAAO;EACL,cAAc,KAAK,MAAM;EACzB,KAAK,SAAS,KAAA;CAChB;;CAGA,OAAO;EACL,KAAK,KAAK;EACV,KAAK,SAAS,YAAY,SAAS,KAAK,YAAY,KAAK,CAAC;CAC5D;CAEA,SAAS;EACP,MAAM,YAAY,SAAS,GAAG,KAAK,aAAa,KAAK;EACrD,IAAI,cAAc,KAAA,GAAW;GAC3B,KAAK,eAAe;GACpB,KAAK,QAAQ,SAAS;EACxB,OAAO,IAAI,KAAK,cAKd,KAAK,QAAQ,SAAS,KAAK,YAAY,KAAK,CAAC;CAEjD;CAEA,QAAQ,OAAe;EACrB,IAAI,UAAU,KAAK,YAAY;GAC7B,KAAK,aAAa;GAClB,KAAK,SAAS,aAAa,KAAK;EAClC;CACF;CAEA,SAAS,OAAwB,OAAe;EAC9C,MAAM,gBAA+B;GACnC;GACA,YAAY,KAAK;GACjB,WAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK,MAAM;EACvD;EACA,MAAM,QAA+B;GACnC,MAAM;GACN,WAAW;GACX,QAAQ;GACR;GACA,aACE,UAAU,cACN,kCACA;GACN,uBAAM,IAAI,KAAK,EAAA,CAAE,YAAY;GAC7B,OAAO,EAAC,cAAa;EACvB;EACA,KAAK,WAAW,KAAK,KAAK,KAAK;CACjC;AACF;AAEA,SAAS,SAAS,MAAkC;CAClD,IAAI;EACF,OAAO,SAAS,IAAI,CAAC,CAAC;CACxB,QAAQ;EACN;CACF;AACF"}

package/out/zero-cache/src/services/mutagen/mutagen.js

@@ -1,4 +1,5 @@
 import { assert, unreachable } from "../../../../shared/src/asserts.js";
+import { resolver } from "../../../../shared/src/resolver.js";
 import { parse } from "../../../../shared/src/valita.js";
 import { InvalidPush, MutationFailed, MutationRateLimited } from "../../../../zero-protocol/src/error-kind-enum.js";
 import { ZeroCache } from "../../../../zero-protocol/src/error-origin-enum.js";
@@ -16,7 +17,6 @@ import { runTx } from "../../db/run-transaction.js";
 import { recordMutation } from "../../server/anonymous-otel-start.js";
 import { WriteAuthorizerImpl } from "../../auth/write-authorizer.js";
 import { SlidingWindowLimiter } from "../limiter/sliding-window-limiter.js";
-import { resolver } from "@rocicorp/resolver";
 import postgres from "postgres";
 import { PG_SERIALIZATION_FAILURE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/mutagen/mutagen.ts
@@ -138,8 +138,7 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
 	}
 	authorizer.reloadPermissions();
 	if (!errorMode) {
-		const { ops } = mutation.args[0];
-		authorizer.validateTableNames(ops);
+		const ops = authorizer.validateOps(mutation.args[0].ops);
 		const normalizedOps = authorizer.normalizeOps(ops);
 		const [canPre, canPost] = await Promise.all([authorizer.canPreMutation(authData, normalizedOps), authorizer.canPostMutation(authData, normalizedOps)]);
 		if (canPre && canPost) for (const op of ops) switch (op.op) {
@@ -208,6 +207,6 @@ async function checkSchemaVersionAndIncrementLastMutationID(tx, shard, clientGro
 	});
 }
 //#endregion
-export { MutagenService };
+export { MutagenService, processMutation };
 
 //# sourceMappingURL=mutagen.js.map

package/out/zero-cache/src/services/mutagen/mutagen.js.map

@@ -1 +1 @@
-{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nexport async function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    const {ops} = mutation.args[0];\n\n    // Validate all table names before processing\n    authorizer.validateTableNames(ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nexport function getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM;EACX,KAAKC,YAAY;EACjB,KAAKC,SAAS;EACd,KAAKE,WAAW,IAAI,SAAS,KAAKJ,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAKK,mBAAmB,IAAI,oBAC1B,KAAKL,KACL,QACA,KAAKI,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAKE,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAKE,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAKD,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;EACP,IAAI,KAAKA,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAKA,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAKH,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrB,qBACA,qBACF,CAAC;EAEH,KAAKC,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAKP,KACL,UACA,KAAKC,WACL,KAAKC,QACL,KAAK,IACL,UACA,KAAKG,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAKF,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAKK,YACP,OAAO,KAAKL,SAAS;EAEvB,KAAKK,aAAa;EAClB,IAAI;GACF,KAAKH,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAKL,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAKI,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAKD,SAAS,QAAQ;EACtB,OAAO,KAAKA,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAAS,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAM,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAAC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAO,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,EAAE,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQ;CACV,CAAC;AAEL"}
+{"version":3,"file":"mutagen.js","names":["ErrorKind.MutationRateLimited","MutationType.CRUD","Mode.SERIALIZABLE","ErrorKind.MutationFailed","v.parse","ErrorKind.InvalidPush","ErrorOrigin.ZeroCache"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport {resolver} from '../../../../shared/src/resolver.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n  isProtocolError,\n  ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n  type CRUDMutation,\n  type DeleteOp,\n  type InsertOp,\n  type Mutation,\n  type UpdateOp,\n  type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n  primaryKeyValueSchema,\n  type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n  WriteAuthorizerImpl,\n  type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n  kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n  desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n  /**\n   * Callers must hold a {@link ref} for the duration of a push (i.e. across\n   * all of its mutations). The service stops, closing its replica handle,\n   * when its ref count drops to zero.\n   */\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled: boolean,\n  ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n  readonly id: string;\n  readonly #lc: LogContext;\n  readonly #upstream: PostgresDB;\n  readonly #shard: ShardID;\n  readonly #stopped = resolver();\n  readonly #replica: Database;\n  readonly #writeAuthorizer: WriteAuthorizerImpl;\n  readonly #limiter: SlidingWindowLimiter | undefined;\n  #refCount = 0;\n  #isStopped = false;\n\n  readonly #crudMutations = getOrCreateCounter(\n    'mutation',\n    'crud',\n    'Number of CRUD mutations processed',\n  );\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    clientGroupID: string,\n    upstream: PostgresDB,\n    config: ZeroConfig,\n    writeAuthzStorage: DatabaseStorage,\n  ) {\n    this.id = clientGroupID;\n    this.#lc = lc;\n    this.#upstream = upstream;\n    this.#shard = shard;\n    this.#replica = new Database(this.#lc, config.replica.file, {\n      fileMustExist: true,\n    });\n    this.#writeAuthorizer = new WriteAuthorizerImpl(\n      this.#lc,\n      config,\n      this.#replica,\n      shard.appID,\n      clientGroupID,\n      writeAuthzStorage,\n    );\n\n    if (config.perUserMutationLimit.max !== undefined) {\n      this.#limiter = new SlidingWindowLimiter(\n        config.perUserMutationLimit.windowMs,\n        config.perUserMutationLimit.max,\n      );\n    }\n  }\n\n  ref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    ++this.#refCount;\n  }\n\n  unref() {\n    assert(!this.#isStopped, 'MutagenService is already stopped');\n    --this.#refCount;\n    if (this.#refCount <= 0) {\n      void this.stop();\n    }\n  }\n\n  hasRefs(): boolean {\n    return this.#refCount > 0;\n  }\n\n  processMutation(\n    mutation: Mutation,\n    authData: JWTPayload | undefined,\n    customMutatorsEnabled = false,\n  ): Promise<MutationError | undefined> {\n    if (this.#limiter?.canDo() === false) {\n      return Promise.resolve([\n        ErrorKind.MutationRateLimited,\n        'Rate limit exceeded',\n      ]);\n    }\n    this.#crudMutations.add(1, {\n      clientGroupID: this.id,\n    });\n    return processMutation(\n      this.#lc,\n      authData,\n      this.#upstream,\n      this.#shard,\n      this.id,\n      mutation,\n      this.#writeAuthorizer,\n      undefined,\n      customMutatorsEnabled,\n    );\n  }\n\n  run(): Promise<void> {\n    return this.#stopped.promise;\n  }\n\n  stop(): Promise<void> {\n    if (this.#isStopped) {\n      return this.#stopped.promise;\n    }\n    this.#isStopped = true;\n    try {\n      this.#writeAuthorizer.destroy();\n    } catch (e) {\n      this.#lc.error?.('error destroying write authorizer storage', e);\n    } finally {\n      // The replica's statistics are maintained by the replicator, so skip\n      // the `PRAGMA optimize` that closing a writable handle would otherwise\n      // run on every client group churn.\n      this.#replica.close({optimize: false});\n    }\n    this.#stopped.resolve();\n    return this.#stopped.promise;\n  }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n  lc: LogContext,\n  authData: JWTPayload | undefined,\n  db: PostgresDB,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: Mutation,\n  writeAuthorizer: WriteAuthorizer,\n  onTxStart?: () => void | Promise<void>, // for testing\n  customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n  assert(\n    mutation.type === MutationType.CRUD,\n    'Only CRUD mutations are supported',\n  );\n  // log-leak-ignore -- the mutation id, not row data\n  lc = lc.withContext('mutationID', mutation.id);\n  lc = lc.withContext('processMutation');\n  // The mutationID context set above identifies it; the mutation itself\n  // carries the row values being written.\n  lc.debug?.('Process mutation start');\n\n  // Record mutation processing attempt for telemetry (regardless of success/failure)\n  recordMutation('crud');\n\n  let result: MutationError | undefined;\n\n  const start = Date.now();\n  try {\n    // Mutations can fail for a variety of reasons:\n    //\n    // - application error\n    // - network/db error\n    // - zero bug\n    //\n    // For application errors what we want is to re-run the mutation in\n    // \"error mode\", which skips the actual mutation and just updates the\n    // lastMutationID. Then return the error to the app.\n    //\n    // However, it's hard to tell the difference between application errors\n    // and the other types.\n    //\n    // A reasonable policy ends up being to just retry every mutation once\n    // in error mode. If the error mode mutation succeeds then we assume it\n    // was an application error and return the error to the app. Otherwise,\n    // we know it was something internal and we log it.\n    //\n    // This is not 100% correct - there are theoretical cases where we\n    // return an internal error to the app that shouldn't have been. But it\n    // would have to be a crazy coincidence: we'd have to have a network\n    // error on the first attempt that resolves by the second attempt.\n    //\n    // One might ask why not try/catch just the calls to the mutators and\n    // consider those application errors. That is actually what we do in\n    // Replicache:\n    //\n    // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n    //\n    // We don't do it here because:\n    //\n    // 1. It's still not perfect. It's hard to isolate SQL errors in\n    //    mutators due to app developer mistakes from SQL errors due to\n    //    Zero mistakes.\n    // 2. It's not possible to do this with the pg library we're using in\n    //    Zero anyway: https://github.com/porsager/postgres/issues/455.\n    //\n    // Personally I think this simple retry policy is nice.\n    let errorMode = false;\n    for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n      try {\n        await runTx(\n          db,\n          async tx => {\n            // Simulates a concurrent request for testing. In production this is a noop.\n            const done = onTxStart?.();\n            try {\n              return await processMutationWithTx(\n                lc,\n                tx,\n                authData,\n                shard,\n                clientGroupID,\n                mutation,\n                errorMode,\n                writeAuthorizer,\n              );\n            } finally {\n              await done;\n            }\n          },\n          {mode: Mode.SERIALIZABLE},\n        );\n        if (errorMode) {\n          lc.debug?.('Ran mutation successfully in error mode');\n        }\n        break;\n      } catch (e) {\n        if (e instanceof MutationAlreadyProcessedError) {\n          lc.debug?.(e.message);\n          // Don't double-count already processed mutations, but they were counted above\n          return undefined;\n        }\n        if (\n          isProtocolError(e) &&\n          !errorMode &&\n          e.kind === ErrorKind.InvalidPush &&\n          customMutatorsEnabled &&\n          i < 2\n        ) {\n          // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n          // This can create a lot of OOO mutation errors since we do not know when the API server\n          // has applied a custom mutation before moving on to process CRUD mutations.\n          // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n          // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n          // They should migrate completely to custom mutators.\n          lc.info?.(\n            'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n            e,\n          );\n          await new Promise(resolve => setTimeout(resolve, 100));\n          continue;\n        }\n        if (isProtocolError(e) || errorMode) {\n          lc.error?.('Process mutation error', e);\n          throw e;\n        }\n        if (\n          e instanceof postgres.PostgresError &&\n          e.code === PG_SERIALIZATION_FAILURE\n        ) {\n          lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n          continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n        }\n        result = [ErrorKind.MutationFailed, String(e)];\n        if (errorMode) {\n          break;\n        }\n        lc.error?.('Got error running mutation, re-running in error mode', e);\n        errorMode = true;\n        i--;\n      }\n    }\n  } finally {\n    lc.debug?.('Process mutation complete in', Date.now() - start);\n  }\n  return result;\n}\n\nasync function processMutationWithTx(\n  lc: LogContext,\n  tx: PostgresTransaction,\n  authData: JWTPayload | undefined,\n  shard: ShardID,\n  clientGroupID: string,\n  mutation: CRUDMutation,\n  errorMode: boolean,\n  authorizer: WriteAuthorizer,\n) {\n  const tasks: (() => Promise<unknown>)[] = [];\n\n  async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n    try {\n      return await stmt.execute();\n    } finally {\n      const q = stmt as unknown as Query;\n      // `q.parameters` are the bound row values -- log the parameterized\n      // statement and the parameter count, never the values themselves.\n      lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n    }\n  }\n\n  authorizer.reloadPermissions();\n\n  if (!errorMode) {\n    // Validates table names and keys every op by the replica's primary key,\n    // which is what the authorizer checks.\n    const ops = authorizer.validateOps(mutation.args[0].ops);\n\n    const normalizedOps = authorizer.normalizeOps(ops);\n    const [canPre, canPost] = await Promise.all([\n      authorizer.canPreMutation(authData, normalizedOps),\n      authorizer.canPostMutation(authData, normalizedOps),\n    ]);\n    if (canPre && canPost) {\n      for (const op of ops) {\n        switch (op.op) {\n          case 'insert':\n            tasks.push(() => execute(getInsertSQL(tx, op)));\n            break;\n          case 'upsert':\n            tasks.push(() => execute(getUpsertSQL(tx, op)));\n            break;\n          case 'update':\n            tasks.push(() => execute(getUpdateSQL(tx, op)));\n            break;\n          case 'delete':\n            tasks.push(() => execute(getDeleteSQL(tx, op)));\n            break;\n          default:\n            unreachable(op);\n        }\n      }\n    }\n  }\n\n  // Confirm the mutation even though it may have been blocked by the authorizer.\n  // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n  tasks.unshift(() =>\n    checkSchemaVersionAndIncrementLastMutationID(\n      tx,\n      shard,\n      clientGroupID,\n      mutation.clientID,\n      mutation.id,\n    ),\n  );\n\n  // Note: An error thrown from any Promise aborts the entire transaction.\n  await Promise.all(tasks.map(task => task()));\n}\n\nfunction getInsertSQL(\n  tx: postgres.TransactionSql,\n  create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  return tx`\n    INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n    ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n  `;\n}\n\nfunction getUpsertSQL(\n  tx: postgres.TransactionSql,\n  set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = set;\n  return tx`\n    INSERT INTO ${tx(tableName)} ${tx(value)}\n    ON CONFLICT (${tx(primaryKey)})\n    DO UPDATE SET ${tx(value)}\n  `;\n}\n\nfunction getUpdateSQL(\n  tx: postgres.TransactionSql,\n  update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const table = update.tableName;\n  const {primaryKey, value} = update;\n  const id: Record<string, PrimaryKeyValue> = {};\n  for (const key of primaryKey) {\n    id[key] = v.parse(value[key], primaryKeyValueSchema);\n  }\n  return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n    id,\n  ).flatMap(([key, value], i) =>\n    i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n  )}`;\n}\n\nfunction getDeleteSQL(\n  tx: postgres.TransactionSql,\n  deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n  const {tableName, primaryKey, value} = deleteOp;\n\n  const conditions = [];\n  for (const key of primaryKey) {\n    if (conditions.length > 0) {\n      conditions.push(tx`AND`);\n    }\n    conditions.push(tx`${tx(key)} = ${value[key]}`);\n  }\n\n  return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n  tx: PostgresTransaction,\n  shard: ShardID,\n  clientGroupID: string,\n  clientID: string,\n  receivedMutationID: number,\n) {\n  const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n    INSERT INTO ${tx(upstreamSchema(shard))}.clients \n      as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n          VALUES (${clientGroupID}, ${clientID}, ${1})\n      ON CONFLICT (\"clientGroupID\", \"clientID\")\n      DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n      RETURNING \"lastMutationID\"\n  `;\n\n  // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n  if (receivedMutationID < lastMutationID) {\n    throw new MutationAlreadyProcessedError(\n      clientID,\n      receivedMutationID,\n      lastMutationID,\n    );\n  } else if (receivedMutationID > lastMutationID) {\n    throw new ProtocolError({\n      kind: ErrorKind.InvalidPush,\n      message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n      origin: ErrorOrigin.ZeroCache,\n    });\n  }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAK,MAAM;EACX,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,WAAW,IAAI,SAAS,KAAK,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAK,mBAAmB,IAAI,oBAC1B,KAAK,KACL,QACA,KAAK,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAK,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAK,YAAY,mCAAmC;EAC5D,EAAE,KAAK;EACP,IAAI,KAAK,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAK,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAK,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrBA,qBACA,qBACF,CAAC;EAEH,KAAK,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAK,KACL,UACA,KAAK,WACL,KAAK,QACL,KAAK,IACL,UACA,KAAK,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAK,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAK,YACP,OAAO,KAAK,SAAS;EAEvB,KAAK,aAAa;EAClB,IAAI;GACF,KAAK,iBAAiB,QAAQ;EAChC,SAAS,GAAG;GACV,KAAK,IAAI,QAAQ,6CAA6C,CAAC;EACjE,UAAU;GAIR,KAAK,SAAS,MAAM,EAAC,UAAU,MAAK,CAAC;EACvC;EACA,KAAK,SAAS,QAAQ;EACtB,OAAO,KAAK,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAASC,MAClB,mCACF;CAEA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CAGrC,GAAG,QAAQ,wBAAwB;CAGnC,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAMC,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAACC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAe,sBACb,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EAGd,MAAM,MAAM,WAAW,YAAY,SAAS,KAAK,EAAE,CAAC,GAAG;EAEvD,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAS,aACP,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAS,aACP,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAOC,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAMC;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQC;CACV,CAAC;AAEL"}