@vzn/vx 0.0.16 → 0.0.21

package/src/orchestrator/failure-mode.ts

@@ -0,0 +1,202 @@
+// The flakiness verdict, in ONE place. Two readers classify `runs` rows —
+// the all-time per-task query here and `LocalHistoryProvider`'s bounded
+// window — and they used to encode the rule independently, which let them
+// drift into opposite verdicts on identical data. Both build on
+// `mixedOutcomeKeysSql` + `failureModeOf`, so the rule cannot fork again.
+// The per-run detector (`detectFlaky`) and the doctor's list (`flakyTasks`)
+// read the same per-key projection (`keyOutcomesSql`).
+
+import type { Database } from 'bun:sqlite'
+import { KEYED_RUNS_SQL } from '../cache/index.js'
+import { isCacheHit, TASK_STATUSES } from './telemetry.js'
+
+// SQL hit set derived from the predicate — never a hand-typed list and never
+// a prefix LIKE (a prefix counts any status merely NAMED cache-hit-*). The
+// status-vocabulary tripwire greps both wrong forms.
+const HIT_STATUSES = `(${TASK_STATUSES.filter(isCacheHit)
+  .map((s) => `'${s}'`)
+  .join(', ')})`
+
+export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
+
+/**
+ * Per-key outcome projection: every keyed (project, task, hash) under
+ * `source` (a relation with `project`, `task`, `hash`, `status`, `cache_hit`
+ * columns) with how often it failed and how often it passed — a pass being
+ * an executed success or a cache hit, which replays a success. Every
+ * flakiness reader is a filter over this one projection.
+ */
+function keyOutcomesSql(source: string, where = ''): string {
+  return `SELECT project, task, hash,
+       SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,
+       SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
+           THEN 1 ELSE 0 END) AS passes
+       FROM ${source}
+       WHERE ${KEYED_RUNS_SQL}${where}
+       GROUP BY project, task, hash`
+}
+
+/**
+ * Per-key outcome subquery: the distinct cache keys under `source` that
+ * produced BOTH a failure and a success — the definitional flake: identical
+ * inputs, different outcomes. A failure whose key never succeeded is a
+ * legitimate break (a changed input that fails), which belongs to the
+ * regressions surface, not flakiness. One projection of the rule serves the
+ * all-time query below and the windowed one in `history.ts`.
+ */
+export function mixedOutcomeKeysSql(source: string, where = ''): string {
+  return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
+       WHERE failures > 0 AND passes > 0`
+}
+
+/** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
+export function mixedOutcomeKeyCount(db: Database, project: string, task: string): number {
+  const row = db
+    .query(
+      `SELECT COUNT(*) AS n FROM (${mixedOutcomeKeysSql('runs', ' AND project = ? AND task = ?')})`,
+    )
+    .get(project, task) as { n: number }
+  return row.n
+}
+
+/**
+ * The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
+ * or a cache key that both failed and succeeded (`mixedKeys`). Failures
+ * alone — each on its own key — are legitimate breaks, however many there
+ * are. `mixedKeys` is a thunk so a task that never failed is stable without
+ * the key count being computed at all.
+ */
+export function failureModeOf(
+  counts: { total: number; failures: number; retried: number },
+  mixedKeys: () => number,
+): FailureMode {
+  const flakySignal = counts.retried > 0 || (counts.failures > 0 && mixedKeys() > 0)
+  if (!flakySignal) return 'stable'
+  return counts.failures < counts.total / 5 ? 'flaky-recoverable' : 'flaky-fatal'
+}
+
+/** `failureModeOf` over a task's whole recorded history (the DB is not touched unless it failed). */
+export function classifyFailureMode(
+  db: Database,
+  project: string,
+  task: string,
+  counts: { total: number; failures: number; retried: number },
+): FailureMode {
+  return failureModeOf(counts, () => mixedOutcomeKeyCount(db, project, task))
+}
+
+/** An executed, keyed outcome of the run in progress — what `detectFlaky` judges. */
+export interface FlakyCandidate {
+  project: string
+  task: string
+  hash: string
+  status: 'success' | 'failed'
+  /** Attempts this run took; > 1 is a within-run retry. */
+  attempts: number
+}
+
+/** One task the run just proved nondeterministic. */
+export interface FlakyFinding extends FlakyCandidate {
+  /** `project#task`. */
+  taskId: string
+  /** Outcomes on record for this exact key, this run's included. */
+  passes: number
+  failures: number
+}
+
+// Bound parameters per statement: SQLite's compile-time cap is 32 766 in
+// every build Bun ships, but a run can execute more tasks than that only in
+// theory — chunking keeps the query shape identical at any size.
+const IN_CHUNK = 500
+
+function chunked<T>(items: readonly T[], each: (chunk: readonly T[]) => void): void {
+  for (let i = 0; i < items.length; i += IN_CHUNK) each(items.slice(i, i + IN_CHUNK))
+}
+
+const keyOf = (c: { project: string; task: string; hash: string }): string =>
+  `${c.project}\0${c.task}\0${c.hash}`
+
+/**
+ * Which of this run's executed tasks are flaky, judged BEFORE the run's own
+ * rows are recorded: a pass on a key that has failed before, a failure on
+ * a key that has passed before, or a within-run retry. The counts fold this
+ * run in.
+ *
+ * The cost follows the run's colour. No candidate (every task a hit or a
+ * skip): no query. A green miss: one probe of the `runs_failed` partial
+ * index for the keys that ever failed — failures are the rare rows, so the
+ * probe is microseconds however long the history. Only a key that DID fail
+ * before, or a task failing now, pays the projection scan over its own
+ * keys, and a run that is failing has more to look at than a scan.
+ */
+export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[]): FlakyFinding[] {
+  if (candidates.length === 0) return []
+  const everFailed = new Set<string>()
+  const passedNow = candidates.filter((c) => c.status === 'success' && c.attempts <= 1)
+  chunked(passedNow, (chunk) => {
+    const rows = db
+      .query(
+        `SELECT DISTINCT project, task, hash FROM runs
+         WHERE status = 'failed' AND hash IN (${chunk.map(() => '?').join(', ')})`,
+      )
+      .all(...chunk.map((c) => c.hash)) as { project: string; task: string; hash: string }[]
+    for (const r of rows) everFailed.add(keyOf(r))
+  })
+  const suspects = candidates.filter(
+    (c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
+  )
+  if (suspects.length === 0) return []
+  const counts = new Map<string, { passes: number; failures: number }>()
+  chunked(suspects, (chunk) => {
+    const rows = db
+      .query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
+      .all(...chunk.map((c) => c.hash)) as {
+      project: string
+      task: string
+      hash: string
+      passes: number
+      failures: number
+    }[]
+    for (const r of rows) counts.set(keyOf(r), { passes: r.passes, failures: r.failures })
+  })
+  const out: FlakyFinding[] = []
+  for (const c of suspects) {
+    const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
+    const passes = before.passes + (c.status === 'success' ? 1 : 0)
+    const failures = before.failures + (c.status === 'failed' ? 1 : 0)
+    if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
+    out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
+  }
+  return out
+}
+
+/** One task the recorded history shows both passing and failing on unchanged inputs. */
+export interface FlakyTask {
+  /** `project#task`. */
+  taskId: string
+  project: string
+  task: string
+  /** Distinct cache keys that both passed and failed. */
+  keys: number
+  /** Outcomes over those keys. */
+  passes: number
+  failures: number
+}
+
+/**
+ * Every task with a mixed-outcome key anywhere in the retained history
+ * (30 days, `RunHistory.pruneOlderThan`), most failures first: the
+ * doctor's list. A full scan of `runs`, which an inspection verb affords.
+ */
+export function flakyTasks(db: Database): FlakyTask[] {
+  const rows = db
+    .query(
+      `SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
+       FROM (${keyOutcomesSql('runs')})
+       WHERE failures > 0 AND passes > 0
+       GROUP BY project, task
+       ORDER BY failures DESC, passes DESC, project, task`,
+    )
+    .all() as { project: string; task: string; keys: number; passes: number; failures: number }[]
+  return rows.map((r) => ({ ...r, taskId: `${r.project}#${r.task}` }))
+}

package/src/orchestrator/framed-output.ts

@@ -28,6 +28,7 @@
 import { isGroupTask, type TaskNode, type TaskOutcome } from '../graph/index.js'
 import { paint, type ColorSupport } from './colors.js'
 import { formatDuration } from './summary.js'
+import { outcomeLabel } from './events.js'
 
 const NO_COLOR: ColorSupport = { enabled: false }
 
@@ -60,6 +61,14 @@ export interface TaskBlockBody {
   stdout?: string
   /** stderr chunks. Renders under `├─ stderr`. */
   stderr?: string
+  /**
+   * Characters the capture dropped from the head of each stream. Non-zero
+   * only for a task whose output is bounded (a persistent one — nothing
+   * else can outrun its own exit), and rendered so a capped log can never
+   * read as a complete one.
+   */
+  droppedStdout?: number
+  droppedStderr?: number
 }
 
 export function formatTaskBlock(
@@ -84,16 +93,6 @@ export function formatTaskBlock(
 
   const idPainted = paintTaskId(node, colors, { bold: true })
   const corner = (s: string) => paint('', s, colors, { dim: true })
-  // Section labels are bold + state-colored (owner design): stdout
-  // green, stderr red, sandbox yellow, command plain white — only
-  // the command CONTENT is greyed (context, not signal). A dim rule
-  // trails each label to the frame width, and sections get vertical
-  // margins so content stands clear of the furniture.
-  const section = (title: string, color?: string) => {
-    const label = paint(color ?? '', title, colors, { bold: true })
-    const rule = corner('\u2500'.repeat(Math.max(1, FRAME_WIDTH - 4 - title.length)))
-    return `${corner('\u251c\u2500')} ${label} ${rule}`
-  }
   const header = formatBlockHeader(outcome, colors)
   const lines: string[] = [`${corner('┌─')} ${idPainted} ${corner('>')} ${header}`]
 
@@ -107,32 +106,51 @@ export function formatTaskBlock(
     lines.push('', corner(`$ ${node.config.exec?.command ?? ''}`), '')
   }
 
-  if (stdout.trim().length > 0) {
-    lines.push(section('STDOUT', SUCCESS), '')
-    pushBodyLines(lines, stdout)
-    lines.push('')
-  }
-
-  if (stderr.trim().length > 0) {
-    lines.push(section('STDERR', ERROR), '')
-    pushBodyLines(lines, stderr)
-    lines.push('')
-  }
+  pushStreamSection(lines, stdout, 'STDOUT', SUCCESS, body.droppedStdout ?? 0, colors)
+  pushStreamSection(lines, stderr, 'STDERR', ERROR, body.droppedStderr ?? 0, colors)
 
-  // Sandbox violations get a dedicated section inside the frame so the
-  // user sees them in context with the failing task, not as loose
-  // status output above the box.
-  const vlines = outcome.sandboxViolationLines
-  if (vlines && vlines.length > 0) {
-    lines.push(section(`SANDBOX VIOLATIONS (${vlines.length})`, WARN), '')
-    for (const v of vlines) lines.push(v)
-    lines.push('')
-  }
+  lines.push(...violationSection(outcome, colors))
 
   lines.push(`${corner('└─')} ${idPainted} ${corner('──')}${formatBlockFooter(outcome, colors)}`)
   return lines.join('\n') + '\n'
 }
 
+/**
+ * One `├─ <LABEL>` section of captured output, with the head-dropped notice
+ * when the capture was bounded. Shared by the task frame and the persistent
+ * tail block so the two renderings of the same bytes cannot drift.
+ */
+function pushStreamSection(
+  lines: string[],
+  text: string,
+  title: string,
+  color: string,
+  lost: number,
+  colors: ColorSupport,
+): void {
+  if (text.trim().length === 0) return
+  lines.push(sectionLine(title, color, colors), '')
+  if (lost > 0) {
+    lines.push(paint('', `… ${lost} earlier characters dropped`, colors, { dim: true }))
+  }
+  pushBodyLines(lines, text)
+  lines.push('')
+}
+
+/**
+ * Section labels are bold + state-colored (owner design): stdout green,
+ * stderr red, sandbox yellow, command plain white — only the command
+ * CONTENT is greyed (context, not signal). A dim rule trails each label to
+ * the frame width, and sections get vertical margins so content stands
+ * clear of the furniture.
+ */
+function sectionLine(title: string, color: string, colors: ColorSupport): string {
+  const corner = (s: string) => paint('', s, colors, { dim: true })
+  const label = paint(color, title, colors, { bold: true })
+  const rule = corner('─'.repeat(Math.max(1, FRAME_WIDTH - 4 - title.length)))
+  return `${corner('├─')} ${label} ${rule}`
+}
+
 /**
  * Append each line of `text` raw. Content carries no border/indent so
  * terminal wrapping never collides with frame glyphs and copy/paste
@@ -143,7 +161,7 @@ function pushBodyLines(lines: string[], text: string): void {
 }
 
 /** Stable per-project hue: FNV-style fold of the name into the palette. */
-export function projectColor(projectName: string): string {
+function projectColor(projectName: string): string {
   let h = 0
   for (let i = 0; i < projectName.length; i++) {
     h = (h * 31 + projectName.charCodeAt(i)) >>> 0
@@ -172,7 +190,7 @@ export function paintIdParts(
   )
 }
 
-export function paintId(
+function paintId(
   projectName: string,
   taskName: string,
   colors: ColorSupport,
@@ -181,11 +199,7 @@ export function paintId(
   return paintIdParts(projectName, projectName, taskName, colors, opts)
 }
 
-export function paintTaskId(
-  node: TaskNode,
-  colors: ColorSupport,
-  opts: { bold?: boolean } = {},
-): string {
+function paintTaskId(node: TaskNode, colors: ColorSupport, opts: { bold?: boolean } = {}): string {
   return paintId(node.projectName, node.taskName, colors, opts)
 }
 
@@ -197,8 +211,8 @@ export function paintTaskId(
 // fixed cell so durations line up and a ticking elapsed never shifts
 // the row. All detail (exit code, output) lives in the framed block.
 export const TIME_COL = 7 // "0ms" … "999.99s"
-export const STATUS_COL = 7 // "success" / "running" / "skipped"
-export const CACHE_COL = 6 // "remote" / "local" / "fresh" / "miss"
+const STATUS_COL = 7 // "success" / "running" / "skipped"
+const CACHE_COL = 8 // "no-cache" / "remote" / "local" / "fresh" / "miss"
 
 /** Right-align the duration in a TIME_COL cell (pad on the left). */
 function timeCell(ms: number | null, colors: ColorSupport): string {
@@ -256,7 +270,8 @@ function cacheOf(o: TaskOutcome): { word: string; color: string } {
   switch (o.status) {
     case 'success':
     case 'failed':
-      return { word: 'miss', color: '' } // dim
+      // A task with no `cache` block never consulted the cache: not a miss.
+      return { word: o.node.config.cache === undefined ? 'no-cache' : 'miss', color: '' } // dim
     case 'cache-hit':
       return o.restored === false
         ? { word: 'fresh', color: SUCCESS }
@@ -271,7 +286,7 @@ function cacheOf(o: TaskOutcome): { word: string; color: string } {
 }
 
 /** The painted status glyph (shape = cache axis, color = task axis). */
-export function taskGlyph(o: TaskOutcome, colors: ColorSupport): string {
+function taskGlyph(o: TaskOutcome, colors: ColorSupport): string {
   return paint(statusOf(o).color, glyphShape(o), colors)
 }
 
@@ -363,6 +378,27 @@ export function formatFrameOpen(node: TaskNode, colors: ColorSupport = NO_COLOR)
   return `${corner('┌─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('>')} $ ${cmd}`
 }
 
+/**
+ * Sandbox violations get a dedicated section inside the frame so the user
+ * sees them in context with the failing task, not as loose status output
+ * above the box. Shared by BOTH renderers: the focused/live frame used to
+ * omit it, so a single-task run — exactly how someone debugging one task
+ * invokes vx — showed a failure with no violations at all (owner,
+ * 2026-09-05).
+ */
+function violationSection(outcome: TaskOutcome, colors: ColorSupport): string[] {
+  const vlines = outcome.sandboxViolationLines
+  if (!vlines || vlines.length === 0) return []
+  // Unique lines only — a task's children trip the same denial many times
+  // and the repeats bury everything else. The lines themselves are printed
+  // VERBATIM: a violation is evidence, and evidence that has been rewritten
+  // for display is not the thing the sandbox reported. The outcome keeps
+  // every record for telemetry.
+  const body = [...new Set(vlines)]
+  // Red, not amber: a violation fails the task, so it reads as a failure.
+  return [sectionLine(`SANDBOX VIOLATIONS (${body.length})`, ERROR, colors), '', ...body, '']
+}
+
 export function formatFrameClose(
   node: TaskNode,
   outcome: TaskOutcome,
@@ -378,13 +414,42 @@ export function formatFrameClose(
     persistent && outcome.status === 'success'
       ? ` ${paint('', `(${formatDuration(outcome.durationMs)})`, colors, { dim: true })} ${paint(ACCENT, 'running', colors)}`
       : formatBlockFooter(outcome, colors)
-  return `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
+  const close = `${corner('└─')} ${mark}${paintTaskId(node, colors, { bold: true })} ${corner('──')}${tail}`
+  const violations = violationSection(outcome, colors)
+  return violations.length === 0 ? close : `${violations.join('\n')}\n${close}`
 }
 
 function isPersistentNode(node: TaskNode): boolean {
   return node.config.exec?.persistent !== undefined
 }
 
+/**
+ * Everything a persistent task wrote AFTER it signalled ready, as one
+ * trailing block. Its outcome landed at ready while the child kept
+ * running, so this uses the live frame's `▸ … running` vocabulary rather
+ * than the success frame `formatTaskBlock` renders — the block is a tail,
+ * not a completed task's log. Section titles say `(since ready)` because
+ * the pre-ready output already went out with the outcome, and a
+ * head-dropped tail names how much it lost so a truncated log can never
+ * read as complete. Empty body → '' (the caller writes nothing).
+ */
+export function formatPersistentTailBlock(
+  node: TaskNode,
+  outcome: TaskOutcome,
+  body: TaskBlockBody,
+  dropped: { stdout?: number; stderr?: number } = {},
+  colors: ColorSupport = NO_COLOR,
+): string {
+  const stdout = body.stdout ?? ''
+  const stderr = body.stderr ?? ''
+  if (stdout.trim().length === 0 && stderr.trim().length === 0) return ''
+  const lines: string[] = [formatFrameOpen(node, colors)]
+  pushStreamSection(lines, stdout, 'STDOUT (since ready)', SUCCESS, dropped.stdout ?? 0, colors)
+  pushStreamSection(lines, stderr, 'STDERR (since ready)', ERROR, dropped.stderr ?? 0, colors)
+  lines.push(formatFrameClose(node, outcome, colors))
+  return lines.join('\n') + '\n'
+}
+
 /**
  * Between the requested task's frame and the summary footer, list the
  * persistent tasks (dev servers / watchers) the run is keeping alive in
@@ -405,56 +470,48 @@ export function formatPersistentList(
 function formatBlockHeader(o: TaskOutcome, colors: ColorSupport): string {
   const shortHash = o.hash ? o.hash.slice(0, 8) : ''
   const dim = (s: string) => paint('', s, colors, { dim: true })
+  const word = outcomeLabel(o)
   switch (o.status) {
     case 'cache-hit':
-      if (o.restored === false) {
-        return `${paint(SUCCESS, 'up-to-date', colors)} ${dim(`• ${shortHash}`)}`
-      }
-      return `${paint(SUCCESS, 'restored-local', colors)} ${dim(`• ${shortHash}`)}`
+      return `${paint(SUCCESS, word, colors)} ${dim(`• ${shortHash}`)}`
     case 'cache-hit-remote':
-      if (o.restored === false) {
-        return `${paint(SUCCESS, 'up-to-date', colors)} ${dim(`• ${shortHash}`)}`
-      }
-      return `${paint(ACCENT, 'restored-remote', colors)} ${dim(`• ${shortHash}`)}`
+      // A remote restore is the one hit painted accent: bytes came off the wire.
+      return `${paint(o.restored === false ? SUCCESS : ACCENT, word, colors)} ${dim(`• ${shortHash}`)}`
     case 'failed':
       // The command lives in its own `├─ command` section; the header
       // carries the outcome like every other status.
-      return paint(ERROR, `failed (exit ${o.exitCode})`, colors, { bold: true })
+      return paint(ERROR, word, colors, { bold: true })
     case 'skipped':
       return paint(WARN, 'skipped (upstream failed)', colors)
-    case 'success':
-      return dim('success')
     default:
-      return o.status
+      return dim(word)
   }
 }
 
 function formatBlockFooter(o: TaskOutcome, colors: ColorSupport): string {
-  // Footer pattern: ` (<dur>) <status>`. Duration is always shown.
-  // For cache hits it's the *original* exec time the entry was
-  // stored with (set by execute-task), not the ~0ms replay cost.
-  // Status differs by outcome — see formatStatusTag.
+  // Footer pattern: ` (<dur>) <status>`. Duration is always shown, and it is
+  // always what THIS run spent — for a cache hit, the probe + restore, not
+  // the exec time the entry was stored with. (The comment here used to claim
+  // the opposite; the code never did, and `--report` summed these as "time
+  // saved" on the strength of it. The stored time lives on
+  // `TaskOutcome.storedDurationMs`.) Status differs by outcome — see
+  // formatStatusTag.
   const dur = paint('', `(${formatDuration(o.durationMs)})`, colors, { dim: true })
   const tag = formatStatusTag(o, colors)
   return ` ${dur} ${tag}`
 }
 
 function formatStatusTag(o: TaskOutcome, colors: ColorSupport): string {
+  const word = outcomeLabel(o)
   switch (o.status) {
+    case 'failed':
+      return paint(ERROR, word, colors, { bold: true })
+    case 'skipped':
+      return paint(WARN, word, colors)
     case 'cache-hit':
-      return paint('', o.restored === false ? 'up-to-date' : 'restored-local', colors, {
-        dim: true,
-      })
     case 'cache-hit-remote':
-      return paint('', o.restored === false ? 'up-to-date' : 'restored-remote', colors, {
-        dim: true,
-      })
     case 'success':
-      return paint('', 'success', colors, { dim: true })
-    case 'failed':
-      return paint(ERROR, `failed (exit ${o.exitCode})`, colors, { bold: true })
-    case 'skipped':
-      return paint(WARN, 'skipped', colors)
+      return paint('', word, colors, { dim: true })
     default:
       return o.status
   }