@vzn/vx 0.0.18 → 0.0.22

package/src/orchestrator/failure-mode.ts

@@ -1,9 +1,10 @@
-// The flakiness verdict, in ONE place. Two surfaces classify the same
-// `runs` rows — the run-history readers and `LocalHistoryProvider` (what
-// `--dry` predictions read) — and they used
-// to encode the rule independently, which let them drift into opposite
-// verdicts on identical data. Both now call `classifyFailureMode`, so the
-// rule cannot fork again.
+// The flakiness verdict, in ONE place. Two readers classify `runs` rows —
+// the all-time per-task query here and `LocalHistoryProvider`'s bounded
+// window — and they used to encode the rule independently, which let them
+// drift into opposite verdicts on identical data. Both build on
+// `mixedOutcomeKeysSql` + `failureModeOf`, so the rule cannot fork again.
+// The per-run detector (`detectFlaky`) and the doctor's list (`flakyTasks`)
+// read the same per-key projection (`keyOutcomesSql`).
 
 import type { Database } from 'bun:sqlite'
 import { KEYED_RUNS_SQL } from '../cache/index.js'
@@ -19,43 +20,183 @@ const HIT_STATUSES = `(${TASK_STATUSES.filter(isCacheHit)
 export type FailureMode = 'stable' | 'flaky-recoverable' | 'flaky-fatal'
 
 /**
- * Distinct cache keys that produced BOTH a failure and a success — the
- * definitional flake: identical inputs, different outcomes. A failure whose
- * key never succeeded is a legitimate break (a changed input that fails),
- * which belongs to the regressions surface, not flakiness.
+ * Per-key outcome projection: every keyed (project, task, hash) under
+ * `source` (a relation with `project`, `task`, `hash`, `status`, `cache_hit`
+ * columns) with how often it failed and how often it passed — a pass being
+ * an executed success or a cache hit, which replays a success. Every
+ * flakiness reader is a filter over this one projection.
  */
+function keyOutcomesSql(source: string, where = ''): string {
+  return `SELECT project, task, hash,
+       SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,
+       SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
+           THEN 1 ELSE 0 END) AS passes
+       FROM ${source}
+       WHERE ${KEYED_RUNS_SQL}${where}
+       GROUP BY project, task, hash`
+}
+
+/**
+ * Per-key outcome subquery: the distinct cache keys under `source` that
+ * produced BOTH a failure and a success — the definitional flake: identical
+ * inputs, different outcomes. A failure whose key never succeeded is a
+ * legitimate break (a changed input that fails), which belongs to the
+ * regressions surface, not flakiness. One projection of the rule serves the
+ * all-time query below and the windowed one in `history.ts`.
+ */
+export function mixedOutcomeKeysSql(source: string, where = ''): string {
+  return `SELECT project, task FROM (${keyOutcomesSql(source, where)})
+       WHERE failures > 0 AND passes > 0`
+}
+
+/** Mixed-outcome keys of one (project, task) over its WHOLE recorded history. */
 export function mixedOutcomeKeyCount(db: Database, project: string, task: string): number {
   const row = db
     .query(
-      `SELECT COUNT(*) AS n FROM (
-         SELECT hash FROM runs
-         WHERE project = ? AND task = ? AND ${KEYED_RUNS_SQL}
-         GROUP BY hash
-         HAVING SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) > 0
-            AND SUM(CASE WHEN status = 'success' OR status IN ${HIT_STATUSES} OR cache_hit = 1
-                    THEN 1 ELSE 0 END) > 0
-       )`,
+      `SELECT COUNT(*) AS n FROM (${mixedOutcomeKeysSql('runs', ' AND project = ? AND task = ?')})`,
     )
     .get(project, task) as { n: number }
   return row.n
 }
 
 /**
- * Flaky requires a NONDETERMINISM signal: a within-run retry, or a cache key
- * that both failed and succeeded. Failures alone — each on its own key — are
- * legitimate breaks, however many there are.
- *
- * The `failures > 0` short-circuit keeps the key-count query off the common
- * path: a task that never failed is stable without touching the DB.
+ * The verdict. Flaky requires a NONDETERMINISM signal: a within-run retry,
+ * or a cache key that both failed and succeeded (`mixedKeys`). Failures
+ * alone — each on its own key — are legitimate breaks, however many there
+ * are. `mixedKeys` is a thunk so a task that never failed is stable without
+ * the key count being computed at all.
  */
+export function failureModeOf(
+  counts: { total: number; failures: number; retried: number },
+  mixedKeys: () => number,
+): FailureMode {
+  const flakySignal = counts.retried > 0 || (counts.failures > 0 && mixedKeys() > 0)
+  if (!flakySignal) return 'stable'
+  return counts.failures < counts.total / 5 ? 'flaky-recoverable' : 'flaky-fatal'
+}
+
+/** `failureModeOf` over a task's whole recorded history (the DB is not touched unless it failed). */
 export function classifyFailureMode(
   db: Database,
   project: string,
   task: string,
   counts: { total: number; failures: number; retried: number },
 ): FailureMode {
-  const flakySignal =
-    counts.retried > 0 || (counts.failures > 0 && mixedOutcomeKeyCount(db, project, task) > 0)
-  if (!flakySignal) return 'stable'
-  return counts.failures < counts.total / 5 ? 'flaky-recoverable' : 'flaky-fatal'
+  return failureModeOf(counts, () => mixedOutcomeKeyCount(db, project, task))
+}
+
+/** An executed, keyed outcome of the run in progress — what `detectFlaky` judges. */
+export interface FlakyCandidate {
+  project: string
+  task: string
+  hash: string
+  status: 'success' | 'failed'
+  /** Attempts this run took; > 1 is a within-run retry. */
+  attempts: number
+}
+
+/** One task the run just proved nondeterministic. */
+export interface FlakyFinding extends FlakyCandidate {
+  /** `project#task`. */
+  taskId: string
+  /** Outcomes on record for this exact key, this run's included. */
+  passes: number
+  failures: number
+}
+
+// Bound parameters per statement: SQLite's compile-time cap is 32 766 in
+// every build Bun ships, but a run can execute more tasks than that only in
+// theory — chunking keeps the query shape identical at any size.
+const IN_CHUNK = 500
+
+function chunked<T>(items: readonly T[], each: (chunk: readonly T[]) => void): void {
+  for (let i = 0; i < items.length; i += IN_CHUNK) each(items.slice(i, i + IN_CHUNK))
+}
+
+const keyOf = (c: { project: string; task: string; hash: string }): string =>
+  `${c.project}\0${c.task}\0${c.hash}`
+
+/**
+ * Which of this run's executed tasks are flaky, judged BEFORE the run's own
+ * rows are recorded: a pass on a key that has failed before, a failure on
+ * a key that has passed before, or a within-run retry. The counts fold this
+ * run in.
+ *
+ * The cost follows the run's colour. No candidate (every task a hit or a
+ * skip): no query. A green miss: one probe of the `runs_failed` partial
+ * index for the keys that ever failed — failures are the rare rows, so the
+ * probe is microseconds however long the history. Only a key that DID fail
+ * before, or a task failing now, pays the projection scan over its own
+ * keys, and a run that is failing has more to look at than a scan.
+ */
+export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[]): FlakyFinding[] {
+  if (candidates.length === 0) return []
+  const everFailed = new Set<string>()
+  const passedNow = candidates.filter((c) => c.status === 'success' && c.attempts <= 1)
+  chunked(passedNow, (chunk) => {
+    const rows = db
+      .query(
+        `SELECT DISTINCT project, task, hash FROM runs
+         WHERE status = 'failed' AND hash IN (${chunk.map(() => '?').join(', ')})`,
+      )
+      .all(...chunk.map((c) => c.hash)) as { project: string; task: string; hash: string }[]
+    for (const r of rows) everFailed.add(keyOf(r))
+  })
+  const suspects = candidates.filter(
+    (c) => c.status === 'failed' || c.attempts > 1 || everFailed.has(keyOf(c)),
+  )
+  if (suspects.length === 0) return []
+  const counts = new Map<string, { passes: number; failures: number }>()
+  chunked(suspects, (chunk) => {
+    const rows = db
+      .query(keyOutcomesSql('runs', ` AND hash IN (${chunk.map(() => '?').join(', ')})`))
+      .all(...chunk.map((c) => c.hash)) as {
+      project: string
+      task: string
+      hash: string
+      passes: number
+      failures: number
+    }[]
+    for (const r of rows) counts.set(keyOf(r), { passes: r.passes, failures: r.failures })
+  })
+  const out: FlakyFinding[] = []
+  for (const c of suspects) {
+    const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0 }
+    const passes = before.passes + (c.status === 'success' ? 1 : 0)
+    const failures = before.failures + (c.status === 'failed' ? 1 : 0)
+    if (!(passes > 0 && failures > 0) && c.attempts <= 1) continue
+    out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
+  }
+  return out
+}
+
+/** One task the recorded history shows both passing and failing on unchanged inputs. */
+export interface FlakyTask {
+  /** `project#task`. */
+  taskId: string
+  project: string
+  task: string
+  /** Distinct cache keys that both passed and failed. */
+  keys: number
+  /** Outcomes over those keys. */
+  passes: number
+  failures: number
+}
+
+/**
+ * Every task with a mixed-outcome key anywhere in the retained history
+ * (30 days, `RunHistory.pruneOlderThan`), most failures first: the
+ * doctor's list. A full scan of `runs`, which an inspection verb affords.
+ */
+export function flakyTasks(db: Database): FlakyTask[] {
+  const rows = db
+    .query(
+      `SELECT project, task, COUNT(*) AS keys, SUM(passes) AS passes, SUM(failures) AS failures
+       FROM (${keyOutcomesSql('runs')})
+       WHERE failures > 0 AND passes > 0
+       GROUP BY project, task
+       ORDER BY failures DESC, passes DESC, project, task`,
+    )
+    .all() as { project: string; task: string; keys: number; passes: number; failures: number }[]
+  return rows.map((r) => ({ ...r, taskId: `${r.project}#${r.task}` }))
 }

package/src/orchestrator/failure-recap.ts

@@ -0,0 +1,121 @@
+// The end of each failed task's output, kept for the run's last lines
+// (item 706). A failure's frame prints when the task finishes, which in a
+// long CI log is thousands of lines above the end, and GitHub's API returns
+// only a job log's last 5,000 lines: the summary is the one place a reader
+// reliably looks, so the failure's own words are repeated there.
+//
+// The capture is a ring over the output's END: it keeps at most
+// RECAP_TASK_BYTES characters (and a newline) however much a task prints,
+// and counts what it evicts, so the recap can say how much it is not
+// showing.
+
+/** Lines of each failed task's output the recap repeats. */
+const RECAP_LINES = 30
+/** The byte cap on one task's repeated lines. */
+const RECAP_TASK_BYTES = 8 * 1024
+/**
+ * Tasks that get a tail; the rest are named by id. Five tails at the
+ * per-task cap is 40 KiB, so the recap stays under 64 KiB with no cap
+ * of its own.
+ */
+export const RECAP_TASKS = 5
+
+export interface RecapRing {
+  chunks: string[]
+  chars: number
+  /** Newlines evicted from the head. */
+  cutLines: number
+  /** Bytes evicted from the line the kept text starts with; 0 when it starts a line. */
+  cutPartialBytes: number
+}
+
+export interface RecapTail {
+  /** The last lines, at most RECAP_LINES of them and RECAP_TASK_BYTES bytes. */
+  text: string
+  /** Whole lines above `text`. */
+  earlierLines: number
+  /** Bytes cut from the start of `text`'s first line. */
+  cutBytes: number
+}
+
+export function createRecapRing(): RecapRing {
+  return { chunks: [], chars: 0, cutLines: 0, cutPartialBytes: 0 }
+}
+
+// The ring counts characters, the cap bytes: a UTF-16 unit is at least one
+// UTF-8 byte, so the last RECAP_TASK_BYTES characters always hold the last
+// RECAP_TASK_BYTES bytes. One more for the final newline the tail drops.
+const RING_CHARS = RECAP_TASK_BYTES + 1
+
+export function appendRecapRing(r: RecapRing, chunk: string): void {
+  if (chunk.length === 0) return
+  r.chunks.push(chunk)
+  r.chars += chunk.length
+  // A whole chunk goes only when what follows it still fills the ring:
+  // evicting past the cap would leave the recap fewer lines than it shows.
+  while (r.chunks.length > 1 && r.chars - r.chunks[0]!.length >= RING_CHARS) {
+    const gone = r.chunks.shift()!
+    r.chars -= gone.length
+    evict(r, gone, gone.length)
+  }
+  if (r.chars > RING_CHARS) {
+    const head = r.chunks[0]!
+    const at = r.chars - RING_CHARS
+    evict(r, head, at)
+    r.chunks[0] = head.slice(at)
+    r.chars = RING_CHARS
+  }
+}
+
+function evict(r: RecapRing, s: string, end: number): void {
+  const last = s.lastIndexOf('\n', end - 1)
+  const partial = Buffer.byteLength(s.slice(last + 1, end))
+  if (last < 0) {
+    r.cutPartialBytes += partial
+    return
+  }
+  r.cutLines += countNewlines(s, last + 1)
+  r.cutPartialBytes = partial
+}
+
+function countNewlines(s: string, end: number): number {
+  let n = 0
+  for (let i = s.indexOf('\n'); i !== -1 && i < end; i = s.indexOf('\n', i + 1)) n++
+  return n
+}
+
+/**
+ * The recap's view of what the ring kept: its last RECAP_LINES lines, then
+ * its last RECAP_TASK_BYTES bytes (cut on a character boundary). The text
+ * is a fresh copy, so it holds nothing of the chunks it came from.
+ */
+export function recapTail(r: RecapRing): RecapTail {
+  const t = r.chunks.join('')
+  const end = t.endsWith('\n') ? t.length - 1 : t.length
+  let start = 0
+  for (let pos = end, taken = 1; ; pos = start - 1, taken++) {
+    const nl = pos === 0 ? -1 : t.lastIndexOf('\n', pos - 1)
+    if (nl < 0) {
+      start = 0
+      break
+    }
+    start = nl + 1
+    if (taken === RECAP_LINES) break
+  }
+  let earlierLines = r.cutLines + countNewlines(t, start)
+  let cutBytes = start === 0 ? r.cutPartialBytes : 0
+  const bytes = Buffer.from(t.slice(start, end), 'utf8')
+  if (bytes.length <= RECAP_TASK_BYTES)
+    return { text: bytes.toString('utf8'), earlierLines, cutBytes }
+  let at = bytes.length - RECAP_TASK_BYTES
+  while ((bytes[at]! & 0xc0) === 0x80) at++
+  const lastNl = bytes.lastIndexOf(0x0a, at - 1)
+  if (lastNl < 0) cutBytes += at
+  else {
+    for (let i = bytes.indexOf(0x0a); i !== -1 && i <= lastNl; i = bytes.indexOf(0x0a, i + 1)) {
+      earlierLines++
+    }
+    cutBytes = at - lastNl - 1
+  }
+  return { text: bytes.subarray(at).toString('utf8'), earlierLines, cutBytes }
+}

package/src/orchestrator/fingerprint-watch.ts

@@ -0,0 +1,89 @@
+// The workspace fingerprint is read once per run, before any task, and
+// every key folds it (and a lockfile plugin's per-project parts, taken from
+// the same files). A task in the root project may rewrite one of those
+// files — `pnpm install` without `--frozen-lockfile` — and every key taken
+// before it then names a lockfile the tree no longer holds: a save under
+// one filed bytes built against the new install under the old lockfile's
+// key, replayed whenever the tree went back (item 750). The watch answers
+// "has a fingerprinted file moved since the run read it", re-learning the
+// fact only after a task that may have written one ran (`wrote`), so a run
+// with no such task never touches the files again.
+
+import { readFileSync, statSync } from 'node:fs'
+import path from 'node:path'
+import { FILE_HASH_RACY_MS } from '../cache/index.js'
+import { WORKSPACE_FINGERPRINT_FILES, type WorkspaceFingerprints } from '../workspace/index.js'
+import type { Logger } from './logger.js'
+
+export class FingerprintWatch {
+  private pending = false
+  private movedFiles: string[] | undefined
+  private said = false
+
+  /** `at`: when the run started reading the files (ms since the epoch). */
+  constructor(
+    private readonly workspaceRoot: string,
+    private readonly read: WorkspaceFingerprints,
+    private readonly at: number,
+  ) {}
+
+  /** A task that may rewrite a fingerprinted file (`mayWriteFingerprint`) ran a command. */
+  wrote(): void {
+    this.pending = true
+  }
+
+  /**
+   * The fingerprinted files that no longer hold what the run read, or
+   * undefined. Once one has, the run stays moved: every key already taken
+   * folded the old bytes. One `stat` per table entry after each writer; a
+   * file written since the read (by ctime, with git's racy window) is read
+   * and compared. The stat follows a link as both reads do: a lockfile
+   * that is a symlink, rewritten through it, moves its target's ctime and
+   * never the link's, and an `lstat` here skipped it once the link had
+   * aged past the window — a stale hit on every later run (item 760).
+   */
+  moved(): readonly string[] | undefined {
+    if (this.movedFiles !== undefined || !this.pending) return this.movedFiles
+    this.pending = false
+    const moved: string[] = []
+    for (const f of WORKSPACE_FINGERPRINT_FILES) {
+      const abs = path.join(this.workspaceRoot, f)
+      const before = this.read.files.get(f)
+      let ctimeMs: number
+      try {
+        ctimeMs = statSync(abs).ctimeMs
+      } catch {
+        if (before !== undefined) moved.push(f)
+        continue
+      }
+      if (before !== undefined && ctimeMs < this.at - FILE_HASH_RACY_MS) continue
+      let now: Uint8Array | undefined
+      try {
+        now = readFileSync(abs)
+      } catch {
+        // Not a file (the run-start read skipped it the same way).
+      }
+      if (
+        now === undefined ? before !== undefined : before === undefined || !sameBytes(now, before)
+      ) {
+        moved.push(f)
+      }
+    }
+    if (moved.length > 0) this.movedFiles = moved
+    return this.movedFiles
+  }
+
+  /** The run's one line about it, however many tasks it withholds. */
+  say(log: Logger): void {
+    if (this.said || this.movedFiles === undefined) return
+    this.said = true
+    log.status(
+      `[vx] a task rewrote \`${this.movedFiles.join('`, `')}\` during the run, which every key ` +
+        `folds (the workspace fingerprint): nothing keyed before it is restored or saved from here on`,
+    )
+  }
+}
+
+function sameBytes(a: Uint8Array, b: Uint8Array): boolean {
+  return a.length === b.length && Buffer.compare(a, b) === 0
+}