@vzn/vx 0.0.16 → 0.0.21

package/src/orchestrator/history.ts

@@ -2,9 +2,10 @@
 // uses for predictive priority and `vx info --history` surfaces.
 //
 // The data has been in cache.db.runs since schema v11; what's new is
-// surfacing it. A single SQL CTE per run pulls the last N rows per
-// (project, task) pair; the result becomes a read-only snapshot for
-// the run's lifetime. Loaded once at prepareRun; never mutated mid-run.
+// surfacing it. One SQL statement per call pulls the last N executed rows
+// per (project, task) pair out of a rowid-bounded slice of the table; the
+// result becomes a read-only snapshot for the run's lifetime. Loaded once
+// by the `schedule` stage (or `--dry`); never mutated mid-run.
 //
 // Two providers:
 //   LocalHistoryProvider  — reads cache.db directly (zero-config).
@@ -12,6 +13,9 @@
 //                            when such an RPC actually exists.
 
 import type { Database } from 'bun:sqlite'
+import { EXECUTED_RUNS_SQL } from '../cache/index.js'
+import { failureModeOf, mixedOutcomeKeysSql } from './failure-mode.js'
+import type { FailureMode } from './failure-mode.js'
 
 const DEFAULT_RECENT = 50
 
@@ -27,8 +31,22 @@ export interface TaskHistory {
   successRate: number
   /** Cache hit rate over the recent window ([0, 1]). */
   hitRate: number
-  /** Failure mode classification. */
-  failureMode: 'stable' | 'flaky-recoverable' | 'flaky-fatal'
+  /** Failure mode classification over the same window (`failureModeOf`,
+   *  the one rule `vx why`'s all-time query also applies). */
+  failureMode: FailureMode
+  /**
+   * Largest peak RSS (bytes) any successful execution in the window
+   * reported; undefined when none did (hits report nothing, and a
+   * platform may not expose rusage). The floor a memory reservation
+   * learned from history packs on.
+   */
+  maxPeakRssBytes?: number
+  /**
+   * The most CPU parallelism any successful execution in the window
+   * showed: cpu time over wall time, so 2.0 means two cores busy for the
+   * whole task. Undefined when no row carried both numbers.
+   */
+  maxCpuParallelism?: number
 }
 
 /** Map keyed by `project#task`. */
@@ -52,92 +70,141 @@ export class LocalHistoryProvider implements HistoryProvider {
     private readonly recent: number = DEFAULT_RECENT,
   ) {}
 
+  /**
+   * The window is the last `recent` INVOCATIONS, read as one rowid slice.
+   *
+   * `runs` has no (project, task) index: it cost every run a scattered
+   * B-tree write per task (2026-09-09, 1,000 warm hits: the record stage
+   * 57–79 ms with it, 14–19 ms without) and bought this reader nothing —
+   * ranking every pair's rows newest-first (`ROW_NUMBER` over a partition)
+   * sorted the whole table either way, 230 ms at 116k rows. Rows are
+   * appended in time order, one contiguous block per invocation, so the
+   * rows of the newest `recent` invocations are exactly `id >= MIN(id)` of
+   * the `recent`-th newest header — a primary-key range, no sort, plain
+   * aggregates, and at most one row per pair per invocation, so "the last
+   * `recent` invocations" IS the last `recent` runs for a pair that runs
+   * every time and whatever fewer it has for one that runs less often. A
+   * hint for ordering and prediction, not a ledger; `vx why` / `vx last`
+   * read by `run_id` and stay exact.
+   */
   async loadFor(taskIds: readonly string[]): Promise<HistoryTable> {
     const out = new Map<string, TaskHistory>()
     if (taskIds.length === 0) return out
 
-    // One CTE per call: rank rows per (project, task) descending by
-    // started_at, keep the top `recent`, aggregate. Cache-hit rows
-    // (cache_hit = 1) are excluded from the duration percentiles so
-    // p50/p99 reflect work the runner actually did. successRate +
-    // hitRate are computed over ALL recent rows.
-    const sql = `
-      WITH recent AS (
-        SELECT
-          project,
-          task,
-          status,
-          duration_ms,
-          cache_hit,
-          ROW_NUMBER() OVER (PARTITION BY project, task ORDER BY started_at DESC) AS rn
-        FROM runs
-        WHERE (project || '#' || task) IN (${taskIds.map(() => '?').join(',')})
+    const floorRow = this.db
+      .query(
+        `SELECT MIN(id) AS id FROM runs WHERE run_id =
+           (SELECT run_id FROM invocations ORDER BY started_at DESC LIMIT 1 OFFSET ?)`,
       )
+      .get(this.recent - 1) as { id: number | null }
+    // Fewer invocations than the window (or none): the whole table is the window.
+    const floor = floorRow.id ?? 0
+
+    // `skipped` rows are excluded: a skip is a task the run never executed,
+    // so it belongs in no success/hit RATE and no duration. Percentiles are
+    // over the executed-success rows of the slice — work the runner actually
+    // did — while the rates count every executed row.
+    const sql = `
       SELECT
-        project,
-        task,
+        r.project,
+        r.task,
         COUNT(*) AS total,
-        SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes,
-        SUM(CASE WHEN cache_hit = 1 THEN 1 ELSE 0 END) AS hits,
-        SUM(CASE WHEN cache_hit IS NULL OR cache_hit = 0 THEN 1 ELSE 0 END) AS executed,
-        SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures
-      FROM recent
-      WHERE rn <= ${this.recent}
-      GROUP BY project, task
+        SUM(CASE WHEN r.status = 'success' THEN 1 ELSE 0 END) AS successes,
+        SUM(CASE WHEN r.cache_hit = 1 THEN 1 ELSE 0 END) AS hits,
+        SUM(CASE WHEN r.status = 'failed' THEN 1 ELSE 0 END) AS failures,
+        SUM(CASE WHEN r.attempts > 1 THEN 1 ELSE 0 END) AS retried,
+        GROUP_CONCAT(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
+                          THEN r.duration_ms END) AS ds,
+        MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
+                 THEN r.peak_rss_bytes
+                 WHEN r.cache_hit = 1 THEN e.peak_rss_bytes END) AS rss,
+        MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
+                      AND r.cpu_ms IS NOT NULL AND r.duration_ms > 0
+                 THEN r.cpu_ms * 1.0 / r.duration_ms
+                 WHEN r.cache_hit = 1 AND e.cpu_ms IS NOT NULL AND e.duration_ms > 0
+                 THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu
+      FROM runs r
+      -- A hit executed nothing here, but its entry carries what the
+      -- PRODUCING execution used (the artifact's sidecar, indexed at save
+      -- and ingest alike), so a task this machine has only ever restored
+      -- — a fresh runner behind a remote cache — still has a number. The
+      -- join is by primary key and only for hit rows; an entry pruned
+      -- since contributes nothing.
+      LEFT JOIN entries e ON r.cache_hit = 1 AND e.hash = r.hash
+      WHERE r.id >= ? AND ${EXECUTED_RUNS_SQL}
+      GROUP BY r.project, r.task
     `
-
     type Row = {
       project: string
       task: string
       total: number
       successes: number
       hits: number
-      executed: number
       failures: number
+      retried: number
+      ds: string | null
+      rss: number | null
+      cpu: number | null
     }
-    const rows = this.db.query(sql).all(...(taskIds as string[])) as Row[]
+    // Every pair in the slice is grouped (the slice is bounded, so this is
+    // cheap) and the asked-for ones are picked out here — a per-row IN-list
+    // probe in SQL cost more than the grouping it would have saved.
+    const wanted = new Set(taskIds)
+    const rows = (this.db.query(sql).all(floor) as Row[]).filter((r) =>
+      wanted.has(`${r.project}#${r.task}`),
+    )
+
+    // The flakiness signal needs per-key outcomes, which the aggregate above
+    // folded away — one more pass over the slice, only for the pairs whose
+    // verdict actually depends on it (failed, and no retry already proving
+    // nondeterminism); a green history never pays for it.
+    const mixed = this.mixedOutcomeKeys(
+      floor,
+      rows.filter((r) => (r.failures || 0) > 0 && (r.retried || 0) === 0),
+    )
 
     for (const row of rows) {
       const key = `${row.project}#${row.task}`
       const total = row.total || 0
-      const failures = row.failures || 0
-      const failureMode: TaskHistory['failureMode'] =
-        failures === 0 ? 'stable' : failures < total / 5 ? 'flaky-recoverable' : 'flaky-fatal'
-      const { p50, p99 } = this.percentilesFor(row.project, row.task)
+      const counts = { total, failures: row.failures || 0, retried: row.retried || 0 }
+      const durations = row.ds === null ? undefined : row.ds.split(',').map(Number)
+      if (durations !== undefined) durations.sort((a, b) => a - b)
       out.set(key, {
         runs: total,
-        p50DurationMs: p50,
-        p99DurationMs: p99,
+        p50DurationMs: durations ? pickPercentile(durations, 0.5) : undefined,
+        p99DurationMs: durations ? pickPercentile(durations, 0.99) : undefined,
         successRate: total > 0 ? (row.successes || 0) / total : 0,
         hitRate: total > 0 ? (row.hits || 0) / total : 0,
-        failureMode,
+        failureMode: failureModeOf(counts, () => mixed.get(key) ?? 0),
+        ...(row.rss !== null ? { maxPeakRssBytes: row.rss } : {}),
+        ...(row.cpu !== null ? { maxCpuParallelism: row.cpu } : {}),
       })
     }
     return out
   }
 
-  // Percentiles need a row-wise read; do it in JS rather than try to
-  // express the percentile_cont equivalent in SQLite. Tiny rows.
-  private percentilesFor(
-    project: string,
-    task: string,
-  ): { p50: number | undefined; p99: number | undefined } {
+  /** Mixed-outcome key count per `project#task`, over the slice, for `pairs` only. */
+  private mixedOutcomeKeys(
+    floor: number,
+    pairs: readonly { project: string; task: string }[],
+  ): Map<string, number> {
+    const out = new Map<string, number>()
+    if (pairs.length === 0) return out
+    const placeholders = pairs.map(() => '(?,?)').join(',')
     const rows = this.db
       .query(
-        `SELECT duration_ms FROM runs
-         WHERE project = ? AND task = ?
-           AND (cache_hit IS NULL OR cache_hit = 0)
-           AND status = 'success'
-         ORDER BY started_at DESC
-         LIMIT ?`,
+        `SELECT project, task, COUNT(*) AS n FROM (${mixedOutcomeKeysSql(
+          'runs',
+          ` AND id >= ? AND (project, task) IN (VALUES ${placeholders})`,
+        )}) GROUP BY project, task`,
       )
-      .all(project, task, this.recent) as { duration_ms: number }[]
-    if (rows.length === 0) return { p50: undefined, p99: undefined }
-    const durations = rows.map((r) => r.duration_ms).sort((a, b) => a - b)
-    return {
-      p50: pickPercentile(durations, 0.5),
-      p99: pickPercentile(durations, 0.99),
-    }
+      .all(floor, ...pairs.flatMap((p) => [p.project, p.task])) as {
+      project: string
+      task: string
+      n: number
+    }[]
+    for (const r of rows) out.set(`${r.project}#${r.task}`, r.n)
+    return out
   }
 }
 

package/src/orchestrator/hit-restore.ts

@@ -0,0 +1,221 @@
+// What a HIT leaves behind: the outputs of a confirmed cache entry
+// materialised in the project — or proved already current and left alone —
+// the exact changed paths marked so a same-project consumer need not
+// re-spawn git, the stored stdout replayed, and the cache-hit outcome
+// built. There is ONE restore path: `executeCachedTask` and the local
+// short-circuit (a stable hit restored ahead of its deps) both come here,
+// so the two cannot drift. Stale-hit-critical: a line moved here changes
+// which bytes a green run replays. Moved out of execute-task.ts
+// 2026-09-10 as pure code motion, the mirror of miss-save.ts.
+
+import path from 'node:path'
+import type { CacheConfig } from '../config.js'
+import {
+  type CacheEntry,
+  cleanOutputs,
+  cleanWorkspaceOutputs,
+  resolveOutputs,
+  resolveWorkspaceOutputs,
+  WORKSPACE_OUTPUT_PREFIX,
+} from '../cache/index.js'
+import type { TaskOutcome } from '../graph/index.js'
+import { span, wholeSubtreePrefixes } from '../util/index.js'
+import type { ExecuteArgs } from './execute-task.js'
+
+export interface RestoreHitArgs {
+  args: ExecuteArgs
+  hash: string
+  hit: CacheEntry
+  /** `performance.now()` when the cache op (probe) started — the
+   *  user-perceived restore duration is measured from here. */
+  cacheOpStart: number
+  /** ns offset from run start for the outcome's wallclock window. */
+  taskStartNs: bigint
+}
+
+/**
+ * Materialize a confirmed cache hit: decide skip-restore, clean +
+ * restore declared outputs, mark the exact changed paths so downstream
+ * same-project tasks needn't re-spawn git, replay stored stdout, and
+ * build the cache-hit `TaskOutcome`. Extracted from `executeCachedTask`
+ * so the local short-circuit can restore a stable-key hit ahead of the
+ * schedule using the IDENTICAL logic — there is one restore path, not
+ * two that could drift.
+ */
+export async function restoreHit(restore: RestoreHitArgs): Promise<TaskOutcome> {
+  const { args, hash, hit, cacheOpStart, taskStartNs } = restore
+  const { node, log } = args
+  const cacheCfg: CacheConfig | undefined = node.config.cache
+  const outputs = cacheCfg?.outputs.files ?? []
+  const wsOutputs = cacheCfg?.outputs.workspaceFiles ?? []
+  const anyOutputs = outputs.length > 0 || wsOutputs.length > 0
+  const dirPrefixes = wholeSubtreePrefixes(outputs)
+  const cleanArgs = {
+    projectDir: node.projectDir,
+    outputs,
+    nestedProjectDirs: args.nestedProjectDirs,
+  }
+  const wsCleanArgs = { workspaceRoot: args.workspaceRoot, outputs: wsOutputs }
+
+  // "Tree is already current" short-circuit — skip cleanOutputs
+  // + restoreOutputs when the on-disk state matches what this
+  // entry recorded at save time. Integrity-preserving: we
+  // require BOTH that the output-glob walk yields exactly the
+  // expected paths (no strays, no missing) AND that every file's
+  // (size, mode, mtime) matches the stored fingerprint. Any
+  // divergence falls through to a real clean + restore.
+  //
+  // The output-file fingerprints can't be batch-loaded at
+  // prepareRun time — non-leaf task hashes depend on upstream
+  // outputs that haven't been written yet, so hashes are
+  // necessarily computed mid-run. The short-circuit's batched probe
+  // loads the rows with the entry (`hit.outputRows`); only the lazy
+  // path pays one SELECT here. Either beats reading the manifest from
+  // the tar (decompress + parse) at the same point.
+  // A task with no declared outputs has nothing to clean or restore: its
+  // stdout replays from the row (`hit.stdout`), and its artifact holds
+  // only logs. Extracting it anyway cost an `exists` + a tar read per hit
+  // — 1.7 ms each on a four-task real repo whose `link` and `element`
+  // tasks declare `outputs: []` (2026-09-10, VX_TIMING).
+  let skipRestore = !anyOutputs
+  if (anyOutputs) {
+    const endRows = span('output rows')
+    const expected = hit.outputRows ?? args.cache.loadOutputFilesBatch([hash]).get(hash) ?? []
+    endRows()
+    if (expected.length > 0) {
+      // Two namespaces in the rows: bare rels are project outputs,
+      // `workspace-outputs/<rel>` rows anchor at the workspace root.
+      const projExpected = expected.filter((e) => !e.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
+      const wsExpected = expected
+        .filter((e) => e.path.startsWith(WORKSPACE_OUTPUT_PREFIX))
+        .map((e) => ({ ...e, path: e.path.slice(WORKSPACE_OUTPUT_PREFIX.length) }))
+      // The directory short-circuit: for whole-subtree globs, unchanged
+      // mtimes on every directory recorded at the last save/restore prove
+      // the output SET is unchanged (a file added or removed anywhere the
+      // glob could see bumps a recorded directory), so the walk that cost
+      // 0.36 ms per warm hit is replaced by a few stats. The per-file
+      // fingerprint check below still runs; only the enumeration is skipped.
+      let setKnown = false
+      if (dirPrefixes !== null && wsOutputs.length === 0 && args.cache.outputDirsCurrent) {
+        const endDirs = span('output dirs')
+        // Loaded with the entry (batched by getMany, or with the lazy get).
+        const dirRows = hit.outputDirRows ?? []
+        const covers = dirPrefixes.every((pre) => dirRows.some((r) => r.path === pre))
+        setKnown = covers && (await args.cache.outputDirsCurrent!(node.projectDir, dirRows))
+        endDirs()
+      }
+      let actualAbs: string[]
+      let actualWsAbs: string[]
+      if (setKnown) {
+        actualAbs = projExpected.map((e) => path.join(node.projectDir, e.path))
+        actualWsAbs = []
+      } else {
+        const endGlob = span('output glob')
+        actualAbs = await resolveOutputs({
+          projectDir: node.projectDir,
+          outputs,
+          nestedProjectDirs: args.nestedProjectDirs,
+        })
+        actualWsAbs = await resolveWorkspaceOutputs({
+          workspaceRoot: args.workspaceRoot,
+          outputs: wsOutputs,
+        })
+        endGlob()
+      }
+      const setsMatch = (
+        actual: readonly string[],
+        exp: ReadonlyArray<{ path: string }>,
+      ): boolean => {
+        const expSet = new Set(exp.map((e) => e.path))
+        return actual.length === expSet.size && actual.every((r) => expSet.has(r))
+      }
+      const actualRels = actualAbs.map((p) =>
+        path.relative(node.projectDir, p).split(path.sep).join('/'),
+      )
+      const actualWsRels = actualWsAbs.map((p) =>
+        path.relative(args.workspaceRoot, p).split(path.sep).join('/'),
+      )
+      if (setsMatch(actualRels, projExpected) && setsMatch(actualWsRels, wsExpected)) {
+        const endStat = span('output stat')
+        skipRestore =
+          (await args.cache.isOutputsCurrent(node.projectDir, projExpected)) &&
+          (await args.cache.isOutputsCurrent(args.workspaceRoot, wsExpected))
+        endStat()
+        // The walk ran and proved the tree current (rows absent, or a
+        // directory had moved — a benign touch): snapshot the directories
+        // so the next hit skips the walk.
+        if (skipRestore && !setKnown && dirPrefixes !== null) {
+          await args.cache.recordOutputDirs?.(hash, node.projectDir, dirPrefixes)
+        }
+      }
+    }
+  }
+  if (!skipRestore) {
+    let cleanedRels: string[] = []
+    let cleanedWsRels: string[] = []
+    if (outputs.length > 0) cleanedRels = await cleanOutputs(cleanArgs)
+    if (wsOutputs.length > 0) cleanedWsRels = await cleanWorkspaceOutputs(wsCleanArgs)
+    await args.cache.restoreOutputs(hash, node.projectDir, args.workspaceRoot)
+    // The directory snapshot behind the NEXT hit's skip-restore, taken at
+    // run end when the run keeps a list (as the miss path does, see
+    // miss-save.ts): the restore renamed into these directories
+    // microseconds ago, inside the snapshot's racy window, so a snapshot
+    // taken here was refused and the first warm run after every restore
+    // walked its output trees — 41 of payload's 45 tasks, 14,430 files,
+    // ~75 ms of the no-op's CPU (2026-09-11).
+    if (dirPrefixes !== null) {
+      if (args.outputDirSnapshots !== undefined) {
+        args.outputDirSnapshots.push({ hash, projectDir: node.projectDir, prefixes: dirPrefixes })
+      } else {
+        await args.cache.recordOutputDirs?.(hash, node.projectDir, dirPrefixes)
+      }
+    }
+    // Restored outputs changed the project's tree — but on this
+    // path we know the EXACT changed paths (wiped declared
+    // outputs + the artifact's files). Record them instead of
+    // dropping the snapshot; downstream same-project tasks only
+    // re-spawn git when their input globs can actually see one of
+    // these paths. The cache-miss save path keeps the
+    // unconditional drop (an executed task may write undeclared
+    // files only git can see).
+    if (outputs.length > 0) {
+      args.gitFilesCache?.markOutputsChanged(node.projectDir, [
+        ...cleanedRels,
+        ...hit.outputFiles.filter((p) => !p.startsWith(WORKSPACE_OUTPUT_PREFIX)),
+      ])
+    }
+    if (wsOutputs.length > 0) {
+      args.gitFilesCache?.markWorkspaceOutputsChanged(args.workspaceRoot, [
+        ...cleanedWsRels,
+        ...hit.outputFiles
+          .filter((p) => p.startsWith(WORKSPACE_OUTPUT_PREFIX))
+          .map((p) => p.slice(WORKSPACE_OUTPUT_PREFIX.length)),
+      ])
+    }
+  }
+  if (hit.stdout) log.taskStdout(node, hit.stdout)
+  const status =
+    hit.exitCode !== 0 ? 'failed' : hit.source === 'remote' ? 'cache-hit-remote' : 'cache-hit'
+  // `restored` distinguishes "we just wrote files to disk" from
+  // "disk already matched the cached snapshot". Drives the
+  // "up-to-date" vs "local-cache" / "remote-cache" label in the
+  // framed block. Only meaningful when at least one output was
+  // declared — no-outputs tasks never materialize anything, so
+  // they're vacuously up-to-date.
+  const restored = !skipRestore && anyOutputs
+  return {
+    node,
+    status,
+    exitCode: hit.exitCode,
+    // What THIS run spent: probe + restore. NOT the stored exec time — see
+    // `storedDurationMs`, which is what the hit skipped.
+    durationMs: Math.round(performance.now() - cacheOpStart),
+    storedDurationMs: hit.durationMs,
+    ...(hit.cpuMs !== undefined ? { storedCpuMs: hit.cpuMs } : {}),
+    ...(hit.peakRssBytes !== undefined ? { storedPeakRssBytes: hit.peakRssBytes } : {}),
+    hash,
+    restored,
+    wallclockStartNs: taskStartNs,
+    wallclockEndNs: process.hrtime.bigint() - args.runStartHrTimeNs,
+  }
+}