@vzn/vx 0.0.648 → 0.0.650

package/src/orchestrator/run-failures.ts

@@ -1,20 +1,15 @@
 // A failed run's task output and the files it names, so an agent reads why
-// a task failed without the terminal (`vx last --format json`). One file per failed run, `<cacheDir>/failures/<runId>.json`,
-// the newest 50 kept: files, not cache.db, so the index layout (and every
-// shared store under it) stays as it was.
+// a task failed without the terminal (`vx last --format json`). Rows of
+// cache.db's `failures` (cache/run-notes.ts), the newest 50 failed runs kept.
 
 import type { Database } from 'bun:sqlite'
-import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
-import path from 'node:path'
+import { indexDb, type Cache, keepRunNotes, keptFailures } from '../cache/index.js'
 import { isGroupTask, type TaskOutcome } from '../graph/index.js'
 import { decodeOutputLog } from './output-log.js'
 import { PASS_STATUSES } from '../history/index.js'
-import { isFsRefusal, maskedLine, splitTaskId } from '../util/index.js'
+import { maskedLine, splitTaskId } from '../util/index.js'
 import { fileLocations, fileMemo, type OutputLocation } from '../report/index.js'
 
-/** Failed runs whose output is kept; run ids are UUIDv7, so names sort by age. */
-const KEPT = 50
-
 /** A failed task of a run and what it said. */
 export interface TaskFailure {
   taskId: string
@@ -36,17 +31,19 @@ export interface RunFailures {
   tasks: TaskFailure[]
 }
 
-interface StoredFailure extends TaskFailure {
-  project: string
-  task: string
-}
-
-const dirOf = (cacheDir: string): string => path.join(cacheDir, 'failures')
-
-/** Each failed outcome's output as plain text and the files it names. */
-function failuresOf(outcomes: readonly TaskOutcome[]): StoredFailure[] {
+/**
+ * Keep the run's failures, when any outcome carries output. A refused write
+ * (a full disk) costs only this output: `warn` says so and the run's
+ * verdict stands.
+ */
+export function writeRunFailures(
+  cache: Cache,
+  runId: string,
+  outcomes: readonly TaskOutcome[],
+  warn: (message: string) => void,
+): void {
   const isFile = fileMemo()
-  const out: StoredFailure[] = []
+  const failures = []
   for (const o of outcomes) {
     // A task failed by a moved input may have printed nothing: the input
     // is still why, and an agent reads it here.
@@ -60,76 +57,30 @@ function failuresOf(outcomes: readonly TaskOutcome[]): StoredFailure[] {
               .map((c) => c.text)
               .join(''),
           )
-    out.push({
+    failures.push({
       taskId: o.node.id,
-      project: o.node.projectName,
-      task: o.node.taskName,
-      exitCode: o.exitCode,
-      ...(o.timedOut === true ? { timedOut: true as const } : {}),
       ...(o.inputMoved !== undefined ? { inputMoved: o.inputMoved } : {}),
       output,
       locations: fileLocations(output, o.node.projectDir, isFile),
       ...(o.sandboxFix !== undefined ? { sandboxFix: o.sandboxFix } : {}),
     })
   }
-  return out
-}
-
-/**
- * Write the run's failures, when any outcome carries output, and drop all
- * but the newest `KEPT` files. Atomic: a reader sees the whole file or none.
- * A refused write (a full disk, a read-only cache dir) costs only this
- * output: `warn` says so and the run's verdict stands.
- */
-export function writeRunFailures(
-  cacheDir: string,
-  runId: string,
-  outcomes: readonly TaskOutcome[],
-  warn: (message: string) => void,
-): void {
-  const tasks = failuresOf(outcomes)
-  if (tasks.length === 0) return
+  if (failures.length === 0) return
   try {
-    keep(cacheDir, runId, tasks)
+    keepRunNotes(indexDb(cache), runId, { failures })
   } catch (err) {
-    if (!isFsRefusal(err)) throw err
-    warn(`[vx] failed tasks' output not kept: ${err.message} — the verdict above stands`)
+    const message = err instanceof Error ? err.message : String(err)
+    warn(`[vx] failed tasks' output not kept: ${message} — the verdict above stands`)
   }
 }
 
-function keep(cacheDir: string, runId: string, tasks: TaskFailure[]): void {
-  const dir = dirOf(cacheDir)
-  mkdirSync(dir, { recursive: true })
-  const file = path.join(dir, `${runId}.json`)
-  const tmp = `${file}.${process.pid}.tmp`
-  writeFileSync(tmp, JSON.stringify({ runId, tasks }))
-  renameSync(tmp, file)
-  trimKept(dir, KEPT)
-}
-
-/**
- * Keep the newest `kept` runs' files in `dir`, and the temp files of runs
- * no older: a writer that crashed between its write and its rename left
- * `<runId>.json.<pid>.tmp`, which the trim of `.json` names never took. A
- * live writer's temp is of a run newer than every kept one.
- */
-export function trimKept(dir: string, kept: number): void {
-  const all = readdirSync(dir)
-  const names = all.filter((n) => n.endsWith('.json')).sort()
-  for (const n of names.slice(0, -kept)) rmSync(path.join(dir, n), { force: true })
-  if (names.length < kept) return
-  const oldest = names[names.length - kept]!
-  for (const n of all)
-    if (n.endsWith('.tmp') && n.slice(0, n.lastIndexOf('.json.') + 5) < oldest)
-      rmSync(path.join(dir, n), { force: true })
-}
-
 /**
  * The failed tasks of `runId`, or of the latest failed run when omitted;
- * null when there is no such run. A failed task the file lacks (an older
- * run, a write the disk refused) reads `output: ''`, `locations: []`.
+ * null when there is no such run. A failed task with no kept row (a run
+ * past the newest 50, a write the disk refused) reads `output: ''`,
+ * `locations: []`.
  */
-export function runFailures(cacheDir: string, db: Database, runId?: string): RunFailures | null {
+export function runFailures(db: Database, runId?: string): RunFailures | null {
   const id =
     runId ??
     (
@@ -138,7 +89,7 @@ export function runFailures(cacheDir: string, db: Database, runId?: string): Run
         .get() as { run_id: string } | null
     )?.run_id
   if (id === undefined) return null
-  const kept = storedFailures(cacheDir, id)
+  const kept = keptFailures(db, id)
   const rows = db
     .query(
       `SELECT r.project, r.task, r.exit_code AS exitCode, r.timed_out AS timedOut,
@@ -165,41 +116,16 @@ export function runFailures(cacheDir: string, db: Database, runId?: string): Run
         ...(r.timedOut === 1 ? { timedOut: true as const } : {}),
         ...(k?.inputMoved !== undefined ? { inputMoved: k.inputMoved } : {}),
         output: k?.output ?? '',
-        locations: k?.locations ?? [],
-        ...(k?.sandboxFix !== undefined ? { sandboxFix: k.sandboxFix } : {}),
+        locations: (k?.locations ?? []) as OutputLocation[],
+        ...(k?.sandboxFix !== undefined
+          ? { sandboxFix: k.sandboxFix as NonNullable<TaskOutcome['sandboxFix']> }
+          : {}),
         ...(r.fixedIn !== null ? { fixedIn: r.fixedIn } : {}),
       }
     }),
   }
 }
 
-/** What `runId`'s failures file holds, by task id; empty when the run kept none. */
-function storedFailures(cacheDir: string, runId: string): Map<string, StoredFailure> {
-  const kept = new Map<string, StoredFailure>()
-  // A run id the caller typed names a file in failures/ only.
-  if (path.basename(runId) !== runId || runId === '..') return kept
-  let text: string
-  try {
-    text = readFileSync(path.join(dirOf(cacheDir), `${runId}.json`), 'utf8')
-  } catch (err) {
-    // No file: the run kept no output.
-    if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err
-    return kept
-  }
-  // A file the disk left torn (a rename that landed before its data, on
-  // power loss) is output the run did not keep: it threw on every `vx last`
-  // until 50 newer runs rotated it out.
-  let stored: unknown
-  try {
-    stored = JSON.parse(text)
-  } catch {
-    return kept
-  }
-  const tasks = (stored as { tasks?: unknown } | null)?.tasks
-  if (Array.isArray(tasks)) for (const t of tasks as StoredFailure[]) kept.set(t.taskId, t)
-  return kept
-}
-
 /** One task's output in a recorded run, as `vx last --log` and getTaskLog read it. */
 export interface TaskLog {
   runId: string
@@ -219,12 +145,7 @@ export interface TaskLog {
  * `taskId`'s output in `runId`, or in the latest run that recorded the
  * task when omitted; null when no such run recorded it.
  */
-export function taskLog(
-  cacheDir: string,
-  db: Database,
-  taskId: string,
-  runId?: string,
-): TaskLog | null {
+export function taskLog(db: Database, taskId: string, runId?: string): TaskLog | null {
   const [project, task] = splitTaskId(taskId)
   if (project === '' || task === '') return null
   const row = db
@@ -241,9 +162,9 @@ export function taskLog(
   if (row === null) return null
   const base = { runId: row.runId, taskId, status: row.status }
   if (row.status === 'failed') {
-    // Only the file: runFailures reads `output: ''` for a task it lacks, as
-    // if the task printed nothing, when the run is past the newest 50.
-    const kept = storedFailures(cacheDir, row.runId).get(taskId)
+    // Only the kept row: runFailures reads `output: ''` for a task it lacks,
+    // as if the task printed nothing, when the run is past the newest 50.
+    const kept = keptFailures(db, row.runId).get(taskId)
     if (kept !== undefined) return { ...base, source: 'failure', output: kept.output }
   } else if (row.hash !== '') {
     const stored = db.query('SELECT stdout FROM entry_stdout WHERE hash = ?').get(row.hash) as {

package/src/orchestrator/run.ts

@@ -8,11 +8,13 @@ import { EMPTY_LOCK, inputStabilityEnv, LOCK_INVALID, lockEntryBroken } from '..
 import { realpathSync } from 'node:fs'
 import path from 'node:path'
 import {
+  indexDb,
   Cache,
   type CachePolicy,
   cachesNothing,
   formatCachePolicy,
   networkConcurrency,
+  keepRunNotes,
   stopRuntimeProbes,
   sweepStaging,
 } from '../cache/index.js'
@@ -66,7 +68,9 @@ import { settleProbedHit, unkeyedGroupOutcome, type ExecuteArgs } from './execut
 import { executeArgsOf, type RunScope } from './execute-args.js'
 import { excludedTaint } from './excluded-keys.js'
 import { dependedOn, keyUpstream } from './upstream.js'
+import { createLiveUsage } from './live-usage.js'
 import {
+  blocksCounts,
   busLogger,
   createEventBus,
   createForecast,
@@ -93,7 +97,6 @@ import {
 } from './plugin-host.js'
 import { createSteps } from './steps.js'
 import { liveSocketOf, serveLive } from './run-live.js'
-import { writeRunSteps, writeRunWarnings } from './run-steps.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
 import { claimOutput, guardedOutput } from './output-host.js'
 import { assembleRunSummary } from './telemetry.js'
@@ -910,7 +913,15 @@ async function runOnBus(
     const fault = executorFaults(warnOf(log))
     const footers = executors.filter((e) => e.footer !== undefined)
     const placers = executors.filter((e) => e.running !== undefined)
+    // Every row's `blocks`, the live list's and the results': one closure
+    // over the graph, built on the first row that asks.
+    let blocks: ReadonlyMap<string, number> | undefined
+    const blocksNow = (): ReadonlyMap<string, number> => (blocks ??= blocksCounts(nodes))
+    // The live rows' `cpu` and `mem`; it looks only once the live list asks.
+    const liveUsage = createLiveUsage()
     const runContext = {
+      blocks: blocksNow,
+      usage: liveUsage.usage,
       ...(footers.length > 0
         ? { executorRows: () => footers.flatMap((e) => executorFooter(e, fault)) }
         : {}),
@@ -967,17 +978,17 @@ async function runOnBus(
     // history's only if the region is still up a second in, so a warm run
     // never reads history.
     terminal?.forecast(
-      () => createForecast(nodes, new Map(), concurrency),
+      () => createForecast(nodes, new Map(), concurrency, blocksNow()),
       async () => {
         const ids: string[] = []
         for (const n of nodes.values()) if (!isGroupTask(n)) ids.push(n.id)
         const p50s = await new LocalHistoryProvider(
-          prepared.localCache.dbHandle(),
+          indexDb(prepared.localCache),
           undefined,
           prepared.branch(),
           taskDefinitions(nodes.values()),
         ).p50sFor(ids)
-        return createForecast(nodes, p50s, concurrency)
+        return createForecast(nodes, p50s, concurrency, blocksNow())
       },
     )
 
@@ -1112,7 +1123,7 @@ async function runOnBus(
     // Why a task missed, for a sink: a read of the history per miss, so only
     // when a sink exists.
     const explainMiss =
-      telemetry === undefined ? undefined : createMissExplainer(prepared.localCache.dbHandle())
+      telemetry === undefined ? undefined : createMissExplainer(indexDb(prepared.localCache))
     const lateProbes = probesAfterWrites(nodes, workspaceRoot)
     // The scheduler's fail-fast stop, as the retry loop of a task already
     // in flight sees it.
@@ -1143,7 +1154,15 @@ async function runOnBus(
     if (options.retries !== undefined) scope.retries = options.retries
     if (taskTimeoutDefault !== undefined) scope.timeout = taskTimeoutDefault
     if (cleanOutputs !== undefined) scope.cleanOutputs = cleanOutputs
-    if (telemetry?.track !== undefined) scope.track = telemetry.track
+    const sinkTrack = telemetry?.track
+    scope.track = (taskId, pid) => {
+      const untrack = liveUsage.track(taskId, pid)
+      const sunk = sinkTrack?.(taskId, pid)
+      return () => {
+        untrack()
+        return sunk?.()
+      }
+    }
     if (explainMiss !== undefined) scope.explainMiss = explainMiss
     if (keepRun !== undefined) scope.keepRun = keepRun
     if (sandboxArmer !== null) scope.armSandbox = () => sandboxArmer.arm()
@@ -1215,7 +1234,7 @@ async function runOnBus(
     // rows land, so the row that prints next carries the verdict: nothing
     // prints below the footer. A hit or a skip asks nothing; a green miss
     // is one probe of the failed-row index.
-    const historyDb = prepared.localCache.dbHandle()
+    const historyDb = indexDb(prepared.localCache)
     const cacheOff = cachesNothing(policy)
     const judgeFlaky = (o: TaskOutcome): void => {
       const candidates = flakyCandidates([o], taintedRan)
@@ -1261,8 +1280,8 @@ async function runOnBus(
       serverDied,
       onFailFast: () => failFast.abort(),
       signal: stop.signal,
-      onStart: (node) => {
-        log.taskStart?.(node)
+      onStart: (node, queuedMs) => {
+        log.taskStart?.(node, queuedMs === undefined ? undefined : { waitedMs: queuedMs })
       },
       onFinish: (o) => {
         if (cacheOff) o.cacheOff = true
@@ -1492,8 +1511,11 @@ async function runOnBus(
     // A cache directory this user cannot write keeps none of the run's
     // records: the run's one notice said so.
     const keeps = prepared.localCache.readOnlyNotice() === null
-    const recordFailures = (final: readonly TaskOutcome[]): void => {
-      if (keeps) writeRunFailures(prepared.cacheDir, runId, final, warnOf(log))
+    const recordFailures = (
+      final: readonly TaskOutcome[],
+      into: Cache = prepared.localCache,
+    ): void => {
+      if (keeps) writeRunFailures(into, runId, final, warnOf(log))
     }
     if (stop.stoppedBy === undefined && !historyAfterWait) {
       stepSync('vx:history', () => {
@@ -1535,8 +1557,18 @@ async function runOnBus(
       await Promise.all([cache.drainUploads?.(), keeper?.settled()])
     })
     if (keeps) {
-      writeRunSteps(prepared.cacheDir, runId, steps.ended, warnOf(log))
-      writeRunWarnings(prepared.cacheDir, runId, warned, warnOf(log))
+      // Beside the run's rows (cache/run-notes.ts); a refused write costs only them.
+      try {
+        keepRunNotes(indexDb(prepared.localCache), runId, {
+          steps: steps.ended,
+          warnings: warned.map((w) => Bun.stripANSI(w)),
+        })
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err)
+        warnOf(log)(
+          `[vx] background steps and warnings not kept: ${message} — the verdict above stands`,
+        )
+      }
     }
     // An upload that failed after the rows went in: its row says so too.
     if (prepared.localCache.uploads.failed > 0 && stop.stoppedBy === undefined && !historyAfterWait)
@@ -1670,11 +1702,11 @@ async function runOnBus(
           const local = new Cache(prepared.cacheDir, undefined, prepared.workspaceRoot)
           try {
             local.recordRunBundle(recordsOf(final, ok && !crashed))
+            recordFailures(final, local)
           } finally {
             local.close()
           }
         })
-        recordFailures(final)
       }
       return { ok: ok && !crashed, outcomes: final }
     }
@@ -1789,7 +1821,7 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
       // inspection command, so the history read's cost is fine here even
       // though it stays OFF the default run path.
       history: new LocalHistoryProvider(
-        prepared.localCache.dbHandle(),
+        indexDb(prepared.localCache),
         undefined,
         prepared.branch(),
         taskDefinitions(prepared.nodes.values()),

package/src/orchestrator/steps.ts

@@ -6,9 +6,9 @@
 //
 // A run shows only the steps it waits on, before its first task or its end
 // (2026-10-16). One that runs beside the tasks (`vx:prefetch`) is a
-// `background` step: never shown, and kept for `vx last` (run-steps.ts).
+// `background` step: never shown, and kept for `vx last` (cache/run-notes.ts).
 import type { Logger } from '../report/index.js'
-import type { RunStep } from './run-steps.js'
+import type { RunStep } from '../cache/index.js'
 
 export interface Steps {
   /** Time `work` as step `id` when steps show; else run it as it is. */

package/src/report/events.ts

@@ -45,7 +45,7 @@ export interface RunStartInfo {
  */
 export type RunEvent =
   | { kind: 'run:start'; info: RunStartInfo }
-  | { kind: 'task:start'; node: TaskNode }
+  | { kind: 'task:start'; node: TaskNode; waitedMs?: number }
   | { kind: 'task:stdout'; node: TaskNode; chunk: string }
   | { kind: 'task:stderr'; node: TaskNode; chunk: string }
   | { kind: 'task:complete'; node: TaskNode; outcome: TaskOutcome }
@@ -139,7 +139,12 @@ export function busLogger(bus: EventBus): Logger {
     taskStderr: (node, chunk) => bus.emit({ kind: 'task:stderr', node, chunk }),
     taskComplete: (node, outcome) => bus.emit({ kind: 'task:complete', node, outcome }),
     runStart: (info) => bus.emit({ kind: 'run:start', info }),
-    taskStart: (node) => bus.emit({ kind: 'task:start', node }),
+    taskStart: (node, info) =>
+      bus.emit(
+        info?.waitedMs === undefined
+          ? { kind: 'task:start', node }
+          : { kind: 'task:start', node, waitedMs: info.waitedMs },
+      ),
     runEnd: () => {
       if (ended) return
       ended = true
@@ -174,7 +179,10 @@ export function terminalSubscriber(sink: Logger): RunEventSubscriber {
         sink.runStart?.(event.info)
         return
       case 'task:start':
-        sink.taskStart?.(event.node)
+        sink.taskStart?.(
+          event.node,
+          event.waitedMs === undefined ? undefined : { waitedMs: event.waitedMs },
+        )
         return
       case 'task:stdout':
         sink.taskStdout(event.node, event.chunk)

package/src/report/forecast.ts

@@ -21,12 +21,13 @@ export interface LiveForecast {
    */
   blocked?: readonly BlockedTask[]
   /**
-   * Each running task's count of the tasks not started that depend on it
-   * directly (through a group, which runs nothing): its row's `blocked`
-   * cell, why they are not running. A task waiting on two running tasks
-   * counts under both.
+   * Each running task's count of the tasks downstream of it (a group,
+   * which runs nothing, looked through): its row's `blocks` cell, why they
+   * are not running. A task downstream of two running tasks counts under both.
    */
   blockedBy: ReadonlyMap<string, number>
+  /** Each task's usual time (its executed p50), for the results' `±usual`. */
+  usual?: ReadonlyMap<string, number>
 }
 
 /** A task an unfinished dependency holds back, and what holds it: the running task its longest chain waits on, else that dependency. */
@@ -53,6 +54,8 @@ export function createForecast(
   nodes: ReadonlyMap<string, TaskNode>,
   p50s: ReadonlyMap<string, number>,
   concurrency: number,
+  /** Each task's count of the tasks downstream of it (`blocksCounts`); computed here when absent. */
+  downstream: ReadonlyMap<string, number> = blocksCounts(nodes),
 ): Forecast {
   // Dependencies before dependents, once: each call is one pass over it.
   const order: TaskNode[] = []
@@ -88,22 +91,6 @@ export function createForecast(
   // graph order: once per run, as the scheduler computes it.
   const rank = computeReverseDepCount(nodes)
   const position = new Map(order.map((n, i) => [n.id, i]))
-  // The tasks each node waits on directly, a group's own looked through.
-  const direct = new Map<string, readonly string[]>()
-  const through = (id: string, into: Set<string>, seen: Set<string>): void => {
-    if (seen.has(id)) return
-    seen.add(id)
-    const d = nodes.get(id)
-    if (d === undefined || !isGroupTask(d)) into.add(id)
-    else for (const x of d.deps) through(x, into, seen)
-  }
-  for (const n of order) {
-    if (isGroupTask(n) || n.deps.length === 0) continue
-    const into = new Set<string>()
-    const seen = new Set<string>()
-    for (const d of n.deps) through(d, into, seen)
-    direct.set(n.id, [...into])
-  }
   const before = (a: string, b: string): boolean => {
     const d = (rank.get(a) ?? 0) - (rank.get(b) ?? 0)
     return d !== 0 ? d > 0 : position.get(a)! < position.get(b)!
@@ -144,8 +131,6 @@ export function createForecast(
         since === undefined ? (head.get(n.id) ?? n.deps.find((d) => end.has(d))) : undefined
       if (by !== undefined && !isGroupTask(n)) {
         waiting++
-        for (const on of direct.get(n.id) ?? [])
-          if (running.has(on)) blockedBy.set(on, (blockedBy.get(on) ?? 0) + 1)
         let at = held.length
         while (at > 0 && before(n.id, held[at - 1]!.id)) at--
         if (at < workers) {
@@ -167,7 +152,11 @@ export function createForecast(
       }
       work += own
     }
-    if (!known) return { waiting, expected, blocked: held, blockedBy }
+    for (const id of running.keys()) {
+      const d = downstream.get(id)
+      if (d !== undefined && d > 0) blockedBy.set(id, d)
+    }
+    if (!known) return { waiting, expected, blocked: held, blockedBy, usual: cost }
     const critical = chainEnd === undefined ? undefined : head.get(chainEnd)
     return {
       etaMs: Math.max(chain, work / workers),
@@ -176,6 +165,18 @@ export function createForecast(
       expected,
       blocked: held,
       blockedBy,
+      usual: cost,
     }
   }
 }
+
+/**
+ * Each task's `blocks`: how many tasks, a group's own looked through and
+ * none counted, sit downstream of it in the run, each once (2026-10-29:
+ * every task it holds back, not only the next ones).
+ */
+export function blocksCounts(nodes: ReadonlyMap<string, TaskNode>): Map<string, number> {
+  const tasks = new Map<string, true>()
+  for (const n of nodes.values()) if (!isGroupTask(n)) tasks.set(n.id, true)
+  return computeReverseDepCount(nodes, tasks)
+}