@vzn/vx 0.0.18 → 0.0.22

package/src/cache/output-index.ts

@@ -0,0 +1,323 @@
+// The output fingerprint index: per-entry `output_files` rows (size, mode,
+// millisecond mtime) and `output_dirs` rows (directory mtimes), and the two
+// proofs a cache hit runs against the tree before deciding not to restore.
+// Owns its statements over the store's handle; `Cache` delegates, and
+// writes the file rows through `replaceFileRows` inside its own save
+// transaction.
+
+/** `mtime_ms` of a recorded output prefix that did not exist when the snapshot was taken. */
+const ABSENT_DIR_MTIME = -1
+
+import type { Database, SQLQueryBindings } from 'bun:sqlite'
+import { lstatSync, statSync } from 'node:fs'
+import { lstat, readdir } from 'node:fs/promises'
+import path from 'node:path'
+import {
+  OUTPUT_DIRS_CAP,
+  OUTPUT_DIRS_RACY_MS,
+  type OutputDirRow,
+  type OutputFileRow,
+  WORKSPACE_OUTPUT_PREFIX,
+} from './layer.js'
+
+export class OutputIndex {
+  private readonly insertOutputFile: ReturnType<Database['prepare']>
+  private readonly deleteOutputFiles: ReturnType<Database['prepare']>
+  private readonly insertOutputDir: ReturnType<Database['prepare']>
+  private readonly deleteOutputDirs: ReturnType<Database['prepare']>
+  private readonly entryExists: ReturnType<Database['prepare']>
+  private readonly stampOutputFile: ReturnType<Database['prepare']>
+  /**
+   * Snapshots taken and not yet written, the last per hash. A snapshot is
+   * read by the NEXT run's hit check, never by the task that took it, so
+   * the rows land in one transaction at the first read, at prune, at
+   * stats or at close — the way `accessed_at` bumps do — instead of one
+   * commit per task: 1,000 of them were the whole `output dir snapshots`
+   * stage, 47–72 ms at run end (item 622).
+   */
+  private readonly pendingDirs = new Map<string, Array<[string, number]> | null>()
+  /** Stamps taken and not yet written, flushed with the directory snapshots. */
+  private readonly pendingStamps = new Map<string, Array<[string, number, number]>>()
+
+  constructor(private readonly db: Database) {
+    this.insertOutputFile = this.db.prepare(`
+      INSERT INTO output_files(entry_hash, path, size_bytes, mode, mtime_ms)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(entry_hash, path) DO UPDATE SET
+        size_bytes = excluded.size_bytes,
+        mode       = excluded.mode,
+        mtime_ms   = excluded.mtime_ms
+    `)
+    this.deleteOutputFiles = this.db.prepare('DELETE FROM output_files WHERE entry_hash = ?')
+    this.insertOutputDir = this.db.prepare(
+      'INSERT INTO output_dirs(entry_hash, path, mtime_ms) VALUES (?, ?, ?)',
+    )
+    this.deleteOutputDirs = this.db.prepare('DELETE FROM output_dirs WHERE entry_hash = ?')
+    this.entryExists = this.db.prepare('SELECT 1 FROM entries WHERE hash = ?')
+    this.stampOutputFile = this.db.prepare(
+      'UPDATE output_files SET ino = ?, ctime_ms = ? WHERE entry_hash = ? AND path = ?',
+    )
+  }
+
+  /**
+   * Replace an entry's file rows. Called INSIDE the store's save transaction,
+   * so an entry and its fingerprints land together (an UPDATE on the same
+   * hash refreshes rather than appends).
+   */
+  replaceFileRows(hash: string, rows: ReadonlyArray<[string, number, number, number]>): void {
+    this.deleteOutputFiles.run(hash)
+    for (const [rel, size, mode, mtime] of rows) {
+      this.insertOutputFile.run(hash, rel, size, mode, mtime)
+    }
+  }
+
+  loadOutputFilesBatch(hashes: readonly string[]): Map<string, OutputFileRow[]> {
+    const out = new Map<string, OutputFileRow[]>()
+    if (hashes.length === 0) return out
+    // Inline placeholders for an IN-list — bun:sqlite doesn't ship
+    // rarray, but `IN (?, ?, …)` with N≤~999 is fast and avoids per-
+    // hash select.get() overhead. `db.query` (not `db.prepare`) caches the
+    // compiled statement keyed by the SQL text — so the dominant single-hash
+    // warm-hit path (called up to 3× per hit) reuses one statement instead of
+    // recompiling on every call.
+    // A reader in the same process (`vx watch`'s next cycle, a test) sees
+    // the stamps taken, not only the ones flushed.
+    this.flushOutputDirs()
+    const placeholders = hashes.map(() => '?').join(',')
+    const stmt = this.db.query(
+      `SELECT entry_hash, path, size_bytes, mode, mtime_ms, ino, ctime_ms FROM output_files WHERE entry_hash IN (${placeholders})`,
+    )
+    const rows = stmt.all(...(hashes as readonly SQLQueryBindings[])) as Array<{
+      entry_hash: string
+      path: string
+      size_bytes: number
+      mode: number
+      mtime_ms: number
+      ino: number | null
+      ctime_ms: number | null
+    }>
+    for (const r of rows) {
+      let list = out.get(r.entry_hash)
+      if (!list) {
+        list = []
+        out.set(r.entry_hash, list)
+      }
+      const row: OutputFileRow = {
+        path: r.path,
+        size: r.size_bytes,
+        mode: r.mode,
+        mtimeMs: r.mtime_ms,
+      }
+      if (r.ino !== null && r.ctime_ms !== null) {
+        row.ino = r.ino
+        row.ctimeMs = r.ctime_ms
+      }
+      list.push(row)
+    }
+    return out
+  }
+
+  async isOutputsCurrent(projectDir: string, expected: readonly OutputFileRow[]): Promise<boolean> {
+    // Empty manifest case (a task produced no outputs) → trivially
+    // current; the on-disk tree under projectDir is whatever it was,
+    // and nothing was supposed to land there.
+    if (expected.length === 0) return true
+    // statSync, deliberately. The async form measured faster on 2026-09-02,
+    // when a hit walked its whole output tree; since the directory
+    // short-circuit (`outputDirsCurrent`) a warm hit stats a handful of
+    // paths, and each async stat costs a thread-pool round trip the
+    // scheduler cannot overlap away (its concurrency is the worker count).
+    // Re-measured 2026-09-09, 1,000 warm hits, interleaved A/B: the run
+    // graph stage 100 → 54 ms, the whole process 422 → 368 ms.
+    const results = expected.map((e) => {
+      try {
+        const s = statSync(path.join(projectDir, e.path))
+        return (
+          s.size === e.size &&
+          (s.mode & 0o777) === (e.mode & 0o777) &&
+          // MILLISECOND comparison (sub-ms tolerance for the float
+          // round-trip through utimes). Save rows carry stat-ms and
+          // restoreOutputs re-syncs restored files to the row value,
+          // so equality holds exactly in steady state; legacy
+          // second-precision rows converge on their first restore.
+          Math.abs(s.mtimeMs - e.mtimeMs) < 1 &&
+          // And the file is the one THIS machine last saw under this
+          // entry: two entries whose outputs carry one fixed mtime (a
+          // `tar -x`, `cp -p`, SOURCE_DATE_EPOCH) and one size matched
+          // each other's rows, and a hit left the other entry's bytes in
+          // place under a green run (item 886). No task sets a ctime; the
+          // inode is weaker than it looks, since ext4 gives a restore's new
+          // file the inode the clean freed (item 941). A row without a stamp
+          // (an ingest, a changed file at stamping) is never current. The
+          // residual: a same-size rewrite, in place or as a new file on the
+          // same inode, inside the coarse clock tick of the stamp, with the
+          // mtime forged back.
+          e.ino !== undefined &&
+          s.ino === e.ino &&
+          Math.floor(s.ctimeMs) === e.ctimeMs
+        )
+      } catch {
+        return false
+      }
+    })
+    return results.every(Boolean)
+  }
+
+  /**
+   * Snapshot every directory under each of `prefixes` for `hash`. Called
+   * after a save and after a restore, when the tree is known to equal the
+   * entry's set. Symlinked directories are not descended (the output walk
+   * refuses them too). Over `OUTPUT_DIRS_CAP` directories, or on any
+   * error, the rows are cleared and the next hit keeps the walk.
+   */
+  async recordOutputDirs(
+    hash: string,
+    projectDir: string,
+    prefixes: readonly string[],
+  ): Promise<void> {
+    const rows: Array<[string, number]> = []
+    const walk = async (rel: string, isPrefix = false): Promise<boolean> => {
+      const abs = path.join(projectDir, rel)
+      let st
+      try {
+        st = await lstat(abs)
+      } catch {
+        // A declared prefix the task never produced (`build/**` beside
+        // `dist/**`, medusa's `.medusa/**`) is recorded ABSENT: the check
+        // then asks that it still not exist. Refusing the snapshot instead
+        // made every such task re-glob its outputs on every warm hit — all
+        // 83 of medusa's, 210 ms of a 2.5 s no-op (2026-09-11).
+        if (isPrefix) {
+          rows.push([rel, ABSENT_DIR_MTIME])
+          return true
+        }
+        return false
+      }
+      if (!st.isDirectory()) return false
+      rows.push([rel, st.mtimeMs])
+      if (rows.length > OUTPUT_DIRS_CAP) return false
+      let entries
+      try {
+        entries = await readdir(abs, { withFileTypes: true })
+      } catch {
+        return false
+      }
+      for (const e of entries) {
+        if (e.isDirectory() && !e.isSymbolicLink()) {
+          if (!(await walk(`${rel}/${e.name}`))) return false
+        }
+      }
+      return true
+    }
+    let ok = true
+    for (const prefix of prefixes) {
+      if (!(await walk(prefix, true))) {
+        ok = false
+        break
+      }
+    }
+    // All or nothing: a racy directory dropped alone would leave its
+    // parent trusted while an addition inside it bumps only the dropped one.
+    const youngest = Date.now() - OUTPUT_DIRS_RACY_MS
+    if (rows.some(([, mtime]) => mtime > youngest)) ok = false
+    this.pendingDirs.set(hash, ok ? rows : null)
+  }
+
+  /**
+   * Stamp `hash`'s rows with each file's inode and ctime, once a save or
+   * a restore has left the tree equal to the entry. A file whose size,
+   * mode or mtime no longer matches its row is left unstamped, so a
+   * write that landed since is never vouched for; the next hit restores.
+   * Written with the directory snapshots, in one transaction.
+   */
+  recordOutputStamps(hash: string, projectDir: string, workspaceRoot: string): void {
+    const rows = this.loadOutputFilesBatch([hash]).get(hash) ?? []
+    const stamps: Array<[string, number, number]> = []
+    for (const e of rows) {
+      const ws = e.path.startsWith(WORKSPACE_OUTPUT_PREFIX)
+      const abs = ws
+        ? path.join(workspaceRoot, e.path.slice(WORKSPACE_OUTPUT_PREFIX.length))
+        : path.join(projectDir, e.path)
+      try {
+        const s = statSync(abs)
+        if (
+          s.size === e.size &&
+          (s.mode & 0o777) === (e.mode & 0o777) &&
+          Math.abs(s.mtimeMs - e.mtimeMs) < 1
+        ) {
+          stamps.push([e.path, s.ino, Math.floor(s.ctimeMs)])
+        }
+      } catch {
+        // gone: left unstamped
+      }
+    }
+    if (stamps.length > 0) this.pendingStamps.set(hash, stamps)
+  }
+
+  /** Land every pending stamp and snapshot in one transaction each; a null snapshot clears its rows. */
+  flushOutputDirs(): void {
+    if (this.pendingStamps.size > 0) {
+      const stamps = [...this.pendingStamps]
+      this.pendingStamps.clear()
+      this.db.transaction(() => {
+        for (const [hash, rows] of stamps) {
+          for (const [rel, ino, ctime] of rows) this.stampOutputFile.run(ino, ctime, hash, rel)
+        }
+      })()
+    }
+    if (this.pendingDirs.size === 0) return
+    const pending = [...this.pendingDirs]
+    this.pendingDirs.clear()
+    this.db.transaction(() => {
+      for (const [hash, rows] of pending) {
+        this.deleteOutputDirs.run(hash)
+        // The rows reference the entry: one pruned by another process
+        // between the snapshot and this flush has nothing to describe,
+        // and its insert would fail the whole transaction on the FK.
+        if (rows === null || this.entryExists.get(hash) === null) continue
+        for (const [rel, mtime] of rows) this.insertOutputDir.run(hash, rel, mtime)
+      }
+    })()
+  }
+
+  loadOutputDirsBatch(hashes: readonly string[]): Map<string, OutputDirRow[]> {
+    const out = new Map<string, OutputDirRow[]>()
+    if (hashes.length === 0) return out
+    // A reader in the same process (`vx watch`'s next cycle, a test) sees
+    // what was snapshotted, not what was flushed.
+    this.flushOutputDirs()
+    const placeholders = hashes.map(() => '?').join(',')
+    const rows = this.db
+      .query(
+        `SELECT entry_hash, path, mtime_ms FROM output_dirs WHERE entry_hash IN (${placeholders})`,
+      )
+      .all(...(hashes as readonly SQLQueryBindings[])) as Array<{
+      entry_hash: string
+      path: string
+      mtime_ms: number
+    }>
+    for (const r of rows) {
+      const list = out.get(r.entry_hash)
+      const row = { path: r.path, mtimeMs: r.mtime_ms }
+      if (list) list.push(row)
+      else out.set(r.entry_hash, [row])
+    }
+    return out
+  }
+
+  /** True iff every recorded directory exists with its recorded mtime (ms). Same forged-mtime trade as the file check. */
+  async outputDirsCurrent(projectDir: string, rows: readonly OutputDirRow[]): Promise<boolean> {
+    if (rows.length === 0) return false
+    const results = rows.map((r) => {
+      let st
+      try {
+        st = lstatSync(path.join(projectDir, r.path))
+      } catch {
+        return r.mtimeMs === ABSENT_DIR_MTIME
+      }
+      if (r.mtimeMs === ABSENT_DIR_MTIME) return false
+      return st.isDirectory() && Math.abs(st.mtimeMs - r.mtimeMs) < 1
+    })
+    return results.every(Boolean)
+  }
+}

package/src/cache/policy.ts

@@ -0,0 +1,84 @@
+// The run-level cache policy: four axes (local/remote × read/write) and
+// the `--cache=<spec>` grammar that sets them. Pure data + parsing; no I/O.
+
+import { UserError } from '../util/index.js'
+
+/**
+ * Independent read/write control over the two cache layers (local +
+ * remote). Replaces the old single `noCache` boolean: each axis can be
+ * toggled on its own so `--force` (re-execute but still refresh the
+ * cache) is distinct from `--no-cache` (disable everything).
+ *
+ * Only the task-artifact get/save path is gated. `recordRun`, `stats`,
+ * `prune`, key derivation, and prefetch-ingest are never affected — they
+ * are bookkeeping/analytics that a run policy has no business disabling.
+ */
+export interface CachePolicy {
+  localRead: boolean
+  localWrite: boolean
+  remoteRead: boolean
+  remoteWrite: boolean
+}
+
+/** All four axes on — the default when no cache flag is passed. */
+export const FULL_CACHE_POLICY: CachePolicy = {
+  localRead: true,
+  localWrite: true,
+  remoteRead: true,
+  remoteWrite: true,
+}
+
+/**
+ * Parse a `--cache=<spec>` value into a `CachePolicy`, starting from a
+ * base (defaults to {@link FULL_CACHE_POLICY}). The spec is a
+ * comma-separated list of `layer:flags` segments where `layer` is
+ * `local` or `remote` and `flags` is any subset of `r` (read) and `w`
+ * (write), order-independent and possibly empty.
+ *
+ * A mentioned layer is set EXACTLY to its flags (read = includes `r`,
+ * write = includes `w`); an unmentioned layer keeps its base value.
+ * So `local:rw,remote:r` = remote read-only, `remote:` = remote fully
+ * off, `local:r` = local read-only with remote untouched.
+ *
+ * Throws a {@link UserError} on an unknown layer, an unknown flag, a
+ * duplicated flag, a missing colon, or a repeated layer.
+ */
+export function parseCachePolicy(spec: string, base: CachePolicy = FULL_CACHE_POLICY): CachePolicy {
+  const out: CachePolicy = { ...base }
+  const seen = new Set<string>()
+  for (const rawSeg of spec.split(',')) {
+    const seg = rawSeg.trim()
+    if (seg.length === 0) continue
+    const colon = seg.indexOf(':')
+    if (colon < 0) {
+      throw new UserError(`invalid --cache segment '${seg}': expected '<layer>:<flags>'`)
+    }
+    const layer = seg.slice(0, colon)
+    const flags = seg.slice(colon + 1)
+    if (layer !== 'local' && layer !== 'remote') {
+      throw new UserError(`invalid --cache layer '${layer}': expected 'local' or 'remote'`)
+    }
+    if (seen.has(layer)) {
+      throw new UserError(`--cache layer '${layer}' specified twice`)
+    }
+    seen.add(layer)
+    const flagSet = new Set<string>()
+    for (const ch of flags) {
+      if (ch !== 'r' && ch !== 'w') {
+        throw new UserError(`invalid --cache flag '${ch}' for '${layer}': expected 'r' and/or 'w'`)
+      }
+      if (flagSet.has(ch)) {
+        throw new UserError(`--cache flag '${ch}' repeated for '${layer}'`)
+      }
+      flagSet.add(ch)
+    }
+    if (layer === 'local') {
+      out.localRead = flagSet.has('r')
+      out.localWrite = flagSet.has('w')
+    } else {
+      out.remoteRead = flagSet.has('r')
+      out.remoteWrite = flagSet.has('w')
+    }
+  }
+  return out
+}

package/src/cache/run-history.ts

@@ -0,0 +1,152 @@
+// The run history: one `runs` row per task outcome and one `invocations`
+// header per `vx run`, written together, plus their 30-day retention.
+// What `vx why`, `vx last`, `stats` and the history reader read. Owns
+// its statements over the store's handle; `Cache` delegates.
+
+import type { Database, SQLQueryBindings } from 'bun:sqlite'
+import type { InvocationRecord, RunRecord } from './layer.js'
+
+export class RunHistory {
+  private readonly insertRun: ReturnType<Database['prepare']>
+  private readonly insertInvocation: ReturnType<Database['prepare']>
+
+  constructor(private readonly db: Database) {
+    this.insertRun = this.db.prepare(`
+      INSERT INTO runs(
+        hash, project, task, status, exit_code, duration_ms, forward_args,
+        started_at, ended_at,
+        run_id, cpu_ms, peak_rss_bytes, wallclock_start_ns, wallclock_end_ns,
+        cache_hit, attempts, cached,
+        blocked_by, timed_out, sandbox_violations, not_ready
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,  ?, ?, ?, ?, ?,  ?, ?, ?,  ?, ?, ?, ?)
+    `)
+    this.insertInvocation = this.db.prepare(`
+      INSERT INTO invocations(
+        run_id, command, requested_tasks, cache_policy, concurrency, flow,
+        started_at, ended_at, total_duration_ms,
+        task_count, failed_count, hit_count, hit_local_count, hit_remote_count,
+        exit_ok,
+        commit_sha, branch, dirty, ci, ci_provider,
+        host, os, arch, vx_version, tags
+      )
+      VALUES (?, ?, ?, ?, ?, ?,  ?, ?, ?,  ?, ?, ?, ?, ?,  ?,  ?, ?, ?, ?, ?,  ?, ?, ?, ?, ?)
+      ON CONFLICT(run_id) DO NOTHING
+    `)
+  }
+
+  recordRun(run: RunRecord): void {
+    this.insertRun.run(...bindRun(run))
+  }
+
+  recordRuns(runs: readonly RunRecord[]): void {
+    if (runs.length === 0) return
+    if (runs.length === 1) {
+      this.insertRun.run(...bindRun(runs[0]!))
+      return
+    }
+    // `bun:sqlite`'s `transaction()` returns a callable that wraps the
+    // body in BEGIN/COMMIT, fsyncing once at the end. For a 200-task
+    // run that's one fsync instead of 200.
+    const insert = this.insertRun
+    const tx = this.db.transaction((batch: readonly RunRecord[]) => {
+      for (const r of batch) insert.run(...bindRun(r))
+    })
+    tx(runs)
+  }
+
+  recordRunBundle(bundle: { runs: readonly RunRecord[]; invocation: InvocationRecord }): void {
+    // Whole run records atomically — one transaction, one fsync: the
+    // per-task `runs` rows and the one `invocations` header. The
+    // input-fingerprint rows do NOT live here: they're written inside
+    // the entry-save transaction (`save`/`ingest`) so a warm
+    // all-cache-hit run — which writes no `runs`-vs-`entry_inputs`
+    // mismatch — pays nothing for the moat it isn't refreshing.
+    const insertRun = this.insertRun
+    const insertInvocation = this.insertInvocation
+    this.db.transaction(() => {
+      for (const r of bundle.runs) insertRun.run(...bindRun(r))
+      insertInvocation.run(...bindInvocation(bundle.invocation))
+    })()
+  }
+
+  /**
+   * Retention: the runs table grows by one row per executed task per
+   * invocation — a 2000-task repo accretes ~20k rows in days. 30 days
+   * covers `vx stats` (24 h windows) and CI-side consumers. The
+   * `invocations` header is pruned on the SAME window, or a header would
+   * outlive its rows and `vx last` would list a run whose detail is gone.
+   */
+  pruneOlderThan(cutoff: number): void {
+    this.db.prepare('DELETE FROM runs WHERE started_at < ?').run(cutoff)
+    this.db.prepare('DELETE FROM invocations WHERE started_at < ?').run(cutoff)
+  }
+}
+
+/**
+ * Bind a RunRecord to the positional parameters expected by the
+ * `insertRun` prepared statement (21 columns). Shared between the
+ * single and batched record paths.
+ */
+function bindRun(run: RunRecord): SQLQueryBindings[] {
+  return [
+    // The ONE place the no-key sentinel is applied, so the column's
+    // NOT NULL invariant can't be violated from a call site.
+    run.hash ?? '',
+    run.project,
+    run.task,
+    run.status,
+    run.exitCode,
+    run.durationMs,
+    run.forwardArgs ? JSON.stringify(run.forwardArgs) : null,
+    run.startedAt,
+    run.endedAt,
+    run.runId ?? null,
+    run.cpuMs ?? null,
+    run.peakRssBytes ?? null,
+    run.wallclockStartNs !== undefined ? run.wallclockStartNs : null,
+    run.wallclockEndNs !== undefined ? run.wallclockEndNs : null,
+    run.cacheHit === undefined ? null : run.cacheHit ? 1 : 0,
+    run.attempts ?? null,
+    run.cached === undefined ? null : run.cached ? 1 : 0,
+    run.blockedBy ?? null,
+    run.timedOut === true ? 1 : null,
+    run.sandboxViolations ?? null,
+    run.notReady ?? null,
+  ]
+}
+
+/**
+ * Bind an InvocationRecord to the positional parameters of the
+ * `insertInvocation` prepared statement (25 columns). Booleans map to
+ * 0/1; null-or-bool columns (`dirty`) keep null distinct from 0.
+ */
+function bindInvocation(inv: InvocationRecord): SQLQueryBindings[] {
+  return [
+    inv.runId,
+    inv.command,
+    inv.requestedTasks,
+    inv.cachePolicy,
+    inv.concurrency,
+    inv.flow,
+    inv.startedAt,
+    inv.endedAt,
+    inv.totalDurationMs,
+    inv.taskCount,
+    inv.failedCount,
+    inv.hitCount,
+    inv.hitLocalCount,
+    inv.hitRemoteCount,
+    inv.exitOk ? 1 : 0,
+    inv.commitSha,
+    inv.branch,
+    inv.dirty === null ? null : inv.dirty ? 1 : 0,
+    inv.ci ? 1 : 0,
+    inv.ciProvider,
+    inv.host,
+    inv.os,
+    inv.arch,
+    inv.vxVersion,
+    inv.tags,
+  ]
+}