@vzn/vx 0.0.646 → 0.0.648

package/src/history/failure-mode.ts

@@ -141,7 +141,7 @@ const keyOf = (c: { project: string; task: string; hash: string }): string =>
 /**
  * Which of this run's executed tasks are flaky, judged BEFORE the run's own
  * rows are recorded: a failure on a key that has passed before, a pass on a
- * key that already did so, or a within-run retry. A first pass after only
+ * key that already did so, or a pass that took a within-run retry. A first pass after only
  * failures is a recovery, not a flake. The counts fold this run in.
  *
  * The cost follows the run's colour. No candidate (every task a hit or a
@@ -187,7 +187,10 @@ export function detectFlaky(db: Database, candidates: readonly FlakyCandidate[])
   for (const c of suspects) {
     const before = counts.get(keyOf(c)) ?? { passes: 0, failures: 0, relapses: 0 }
     const relapsed = c.status === 'failed' ? before.passes > 0 : before.relapses > 0
-    if (!relapsed && c.attempts <= 1) continue
+    // A retry proves nothing until one attempt passes: every attempt
+    // failing on a key that never passed is a break, as `flakyTasks` reads it.
+    const retriedPass = c.status === 'success' && c.attempts > 1
+    if (!relapsed && !retriedPass) continue
     const passes = before.passes + (c.status === 'success' ? 1 : 0)
     const failures = before.failures + (c.status === 'failed' ? 1 : 0)
     out.push({ ...c, taskId: `${c.project}#${c.task}`, passes, failures })
@@ -213,7 +216,7 @@ export interface FlakyTask {
  * (30 days, `RunHistory.pruneOlderThan`), most failures first: the
  * doctor's list. A pass that took a retry mixes its key on its own: it is
  * one `success` row whose failed attempts count as failures (`vx info`
- * said `none` over one). A full scan of `runs`, which an inspection verb
+ * said `none` over one). A full scan of `task_outcomes`, which an inspection verb
  * affords.
  */
 export function flakyTasks(db: Database): FlakyTask[] {

package/src/history/history.ts

@@ -1,5 +1,5 @@
 // HistoryTable + HistoryProvider — historical run data the `schedule`
-// plugin (`@vzn/vx-schedule-history`, and its `vx history` verb) and
+// plugin (`@vzn/vx-history`, and its `vx history` verb) and
 // `--dry`'s time prediction read.
 //
 // The data has been in cache.db.runs since schema v11; what's new is
@@ -13,7 +13,7 @@
 
 import type { Database } from 'bun:sqlite'
 import { branchSource, EXECUTED_RUNS_SQL } from '../cache/index.js'
-import { splitTaskId } from '../util/index.js'
+import { isMeasure, splitTaskId } from '../util/index.js'
 import { failureModeOf, mixedOutcomeKeysSql } from './failure-mode.js'
 import type { FailureMode } from './failure-mode.js'
 import { isPassStatus, TASK_STATUSES } from '../graph/index.js'
@@ -33,11 +33,21 @@ export const PASS_STATUSES = `(${TASK_STATUSES.filter(isPassStatus)
 /** The share of a machine a task must use to have filled it. */
 const SATURATED = 0.9
 
+/**
+ * The cores a run of `runs r` kept busy, as need: its CPU over its time,
+ * divided by the share of a core its threads got (`cpu_share`, v49). A run
+ * on a crowded worker used its fair share only: nine test shards on four
+ * cores each used 0.36 of the 1.1 they take alone, and learning 0.36 packed
+ * the next run as tight (main c27c238, 2026-10-23). Never past the machine.
+ */
+const NEED_SQL = `MIN(r.cpu_ms * 1.0 / (r.duration_ms * COALESCE(r.cpu_share, 1)),
+        COALESCE(r.placement_cores, r.cpu_ms * 1.0 / (r.duration_ms * COALESCE(r.cpu_share, 1))))`
+
 /** Per (project#task) — last RECENT runs collapsed into a summary. */
 export interface TaskHistory {
   /** Total runs in the recent window. */
   runs: number
-  /** Wall-clock p50 (ms). Cache-hit rows excluded so this reflects work actually done. */
+  /** Wall-clock p50 (ms), a crowded run's as if alone. Cache-hit rows excluded so this reflects work actually done. */
   p50DurationMs: number | undefined
   /** Wall-clock p99 (ms). Same exclusion. */
   p99DurationMs: number | undefined
@@ -56,17 +66,21 @@ export interface TaskHistory {
    */
   maxPeakRssBytes?: number
   /**
-   * The most CPU parallelism any successful execution in the window
-   * showed: cpu time over wall time, so 2.0 means two cores busy for the
-   * whole task. Undefined when no row carried both numbers.
+   * The CPU parallelism the window's successful executions needed on
+   * average: cpu time over wall time over the share of a core it got, so
+   * 2.0 means two cores busy for the whole task. An average, not the
+   * most: one crowded run's need read 2.84 cores for a build that uses
+   * 1.3 to 1.9, booked it a whole 3-core Mac, and the tasks beside it
+   * waited 45 s (main b5b83bb). Undefined when no row carried both numbers.
    */
   maxCpuParallelism?: number
   /**
    * The most cores, and the most memory, of a machine an executor ran it
    * on (`ExecuteResult.machine`) that it used all of: 90% of the cores
-   * over its whole run, or 90% of the memory. A burst to every core is
+   * over its whole run, averaged over the window's runs on machines that
+   * size, or 90% of the memory in any one. A burst to every core is
    * not one: tasks beside it slow for a moment, and booking a machine for
-   * it left CI agents half idle (M, 2026-10-22). A
+   * it left CI agents half idle (2026-10-22). A
    * task that filled a machine wanted a bigger one. Set by `resourcesFor`.
    */
   saturatedCores?: number
@@ -110,8 +124,37 @@ export class LocalHistoryProvider implements HistoryProvider {
     readonly recent: number = DEFAULT_RECENT,
     /** The run's branch (`currentBranch`); undefined: a forecast reads every branch's timings. */
     private readonly branch?: string | null,
+    /**
+     * Each task's definition (`taskDefinitions`): given, a task reads only
+     * the rows of it as it is now, and a task not in it reads none. A row
+     * of no definition (timings shared before definitions were kept) is
+     * read: dropping them left a PR's run with no task measured (#4150).
+     */
+    private readonly defs?: ReadonlyMap<string, string>,
   ) {}
 
+  /** `r.def` in a SELECT and GROUP BY when rows are matched to definitions. */
+  private get byDef(): string {
+    return this.defs === undefined ? '' : ', r.def'
+  }
+
+  /** Whether a row of `id` under `def` is the task as it is now. */
+  private current(id: string, def: string | null | undefined): boolean {
+    return this.defs === undefined || def == null || this.defs.get(id) === def
+  }
+
+  /** A task id's `[project, task, def?]`, the JSON a statement matches rows by. */
+  private keyOf(id: string): string[] {
+    const [project, task] = splitTaskId(id)
+    if (this.defs === undefined) return [project, task]
+    return [project, task, this.defs.get(id) ?? '']
+  }
+
+  /** The condition on a `task_durations` row's `def`, against `j.value ->> 2`. */
+  private get durationDef(): string {
+    return this.defs === undefined ? '' : 'AND (def IS NULL OR def = j.value ->> 2)'
+  }
+
   /**
    * The window is the last `recent` INVOCATIONS, read as one rowid slice.
    *
@@ -121,8 +164,8 @@ export class LocalHistoryProvider implements HistoryProvider {
    * ranking every pair's rows newest-first (`ROW_NUMBER` over a partition)
    * sorted the whole table either way, 230 ms at 116k rows. Rows are
    * appended in time order, one contiguous block per invocation, so the
-   * rows of the newest `recent` invocations are exactly `id >= MIN(id)` of
-   * the `recent`-th newest header — a primary-key range, no sort, plain
+   * rows of the newest `recent` invocations are exactly `id >=` the first
+   * row of the newest `recent` headers — a primary-key range, no sort, plain
    * aggregates, and at most one row per pair per invocation, so "the last
    * `recent` invocations" IS the last `recent` runs for a pair that runs
    * every time and whatever fewer it has for one that runs less often. A
@@ -150,22 +193,24 @@ export class LocalHistoryProvider implements HistoryProvider {
     const sql = `
       SELECT
         r.project,
-        r.task,
+        r.task${this.defs === undefined ? '' : ', r.def AS def'},
         COUNT(*) AS total,
         SUM(CASE WHEN r.status IN ${PASS_STATUSES} THEN 1 ELSE 0 END) AS successes,
         SUM(CASE WHEN r.cache_hit = 1 THEN 1 ELSE 0 END) AS hits,
         SUM(CASE WHEN r.status = 'failed' THEN 1 ELSE 0 END) AS failures,
-        SUM(CASE WHEN r.attempts > 1 THEN 1 ELSE 0 END) AS retried,
+        SUM(CASE WHEN r.status = 'success' AND r.attempts > 1 THEN 1 ELSE 0 END) AS retried,
         GROUP_CONCAT(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
                           THEN r.duration_ms END) AS ds,
         MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
                  THEN r.peak_rss_bytes
                  WHEN r.cache_hit = 1 THEN e.peak_rss_bytes END) AS rss,
-        MAX(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
-                      AND r.cpu_ms IS NOT NULL AND r.duration_ms > 0
-                 THEN r.cpu_ms * 1.0 / r.duration_ms
-                 WHEN r.cache_hit = 1 AND e.cpu_ms IS NOT NULL AND e.duration_ms > 0
-                 THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu
+        -- A hit's producer use only where no run here measured it (resourcesFor).
+        COALESCE(
+          AVG(CASE WHEN (r.cache_hit IS NULL OR r.cache_hit = 0) AND r.status = 'success'
+                        AND r.cpu_ms IS NOT NULL AND r.duration_ms > 0
+                   THEN ${NEED_SQL} END),
+          MAX(CASE WHEN r.cache_hit = 1 AND e.cpu_ms IS NOT NULL AND e.duration_ms > 0
+                   THEN e.cpu_ms * 1.0 / e.duration_ms END)) AS cpu
       FROM runs r
       -- A hit executed nothing here, but its entry carries what the
       -- PRODUCING execution used (the artifact's sidecar, indexed at save
@@ -175,11 +220,12 @@ export class LocalHistoryProvider implements HistoryProvider {
       -- since contributes nothing.
       LEFT JOIN entries e ON r.cache_hit = 1 AND e.hash = r.hash
       WHERE r.id >= ? AND ${EXECUTED_RUNS_SQL}
-      GROUP BY r.project, r.task
+      GROUP BY r.project, r.task${this.byDef}
     `
     type Row = {
       project: string
       task: string
+      def?: string | null
       total: number
       successes: number
       hits: number
@@ -192,9 +238,10 @@ export class LocalHistoryProvider implements HistoryProvider {
     // Every pair in the slice is grouped (the slice is bounded, so this is
     // cheap) and the asked-for ones are picked out here — a per-row IN-list
     // probe in SQL cost more than the grouping it would have saved.
-    const rows = (this.db.query(sql).all(floor) as Row[]).filter(
-      (r) => wanted === null || wanted.has(`${r.project}#${r.task}`),
-    )
+    const rows = (this.db.query(sql).all(floor) as Row[]).filter((r) => {
+      const id = `${r.project}#${r.task}`
+      return (wanted === null || wanted.has(id)) && this.current(id, r.def)
+    })
 
     // The flakiness signal needs per-key outcomes, which the aggregate above
     // folded away — one more pass over the slice, only for the pairs whose
@@ -264,18 +311,27 @@ export class LocalHistoryProvider implements HistoryProvider {
     sha: string,
     timings: ReadonlyMap<string, number>,
     needs: ReadonlyMap<string, TaskResources> = new Map(),
+    /** Each task's definition when it was timed (`taskDefinition`). */
+    defs: ReadonlyMap<string, string> = new Map(),
   ): void {
     const insert = this.db.query(
       `INSERT OR IGNORE INTO task_durations(project, task, source, sha, duration_ms, recorded_at,
-         cpu_parallelism, peak_rss_bytes, saturated_cores, saturated_memory_bytes)
-       VALUES (?, ?, 'remote', ?, ?, ?, ?, ?, ?, ?)`,
+         cpu_parallelism, peak_rss_bytes, saturated_cores, saturated_memory_bytes, def)
+       VALUES (?, ?, 'remote', ?, ?, ?, ?, ?, ?, ?, ?)`,
     )
     const now = Date.now()
     this.db
       .transaction(() => {
         for (const [id, ms] of timings) {
-          const [project, task] = splitTaskId(id)
           const n = needs.get(id)
+          // A plugin's Infinity or negative, kept, made every later forecast
+          // NaN or below zero: the row is never written again to correct it.
+          if (
+            !isMeasure(ms) ||
+            (n !== undefined && !Object.values(n).every((v) => v === undefined || isMeasure(v)))
+          )
+            continue
+          const [project, task] = splitTaskId(id)
           insert.run(
             project,
             task,
@@ -286,6 +342,7 @@ export class LocalHistoryProvider implements HistoryProvider {
             n?.maxPeakRssBytes ?? null,
             n?.saturatedCores ?? null,
             n?.saturatedMemoryBytes ?? null,
+            defs.get(id) ?? null,
           )
         }
       })
@@ -293,7 +350,7 @@ export class LocalHistoryProvider implements HistoryProvider {
   }
 
   /**
-   * Each task's duration forecast (owner, 2026-10-14): the median of its
+   * Each task's duration forecast (2026-10-14): the median of its
    * last `FORECAST_RUNS` timings on this branch, else the default branch's
    * newest imported timing (`importTimings`), else the median on any
    * branch, else none. The timings are the shared store's `task_durations`, so every
@@ -333,11 +390,11 @@ export class LocalHistoryProvider implements HistoryProvider {
         .query(
           `SELECT j.key AS i, (SELECT group_concat(ms) FROM (
              SELECT duration_ms AS ms FROM task_durations
-             WHERE project = j.value ->> 0 AND task = j.value ->> 1 AND ${tier.where}
+             WHERE project = j.value ->> 0 AND task = j.value ->> 1 AND ${tier.where} ${this.durationDef}
              ORDER BY id DESC LIMIT ${tier.limit})) AS ds
            FROM json_each(?) AS j`,
         )
-        .all(...tier.params, JSON.stringify(pending.map(splitTaskId))) as {
+        .all(...tier.params, JSON.stringify(pending.map((id) => this.keyOf(id)))) as {
         i: number
         ds: string | null
       }[]
@@ -377,70 +434,108 @@ export class LocalHistoryProvider implements HistoryProvider {
     type Executed = {
       project: string
       task: string
+      def?: string | null
       rss: number | null
       cpu: number | null
     }
     const executed = this.db
       .query(
-        `SELECT project, task, MAX(peak_rss_bytes) AS rss,
-           MAX(CASE WHEN cpu_ms IS NOT NULL AND duration_ms > 0
-                    THEN cpu_ms * 1.0 / duration_ms END) AS cpu
-         FROM runs
-         WHERE id >= ? AND status = 'success' AND (cache_hit IS NULL OR cache_hit = 0)
-         GROUP BY project, task`,
+        `SELECT r.project, r.task${this.byDef}, MAX(r.peak_rss_bytes) AS rss,
+           AVG(CASE WHEN r.cpu_ms IS NOT NULL AND r.duration_ms > 0
+                    THEN ${NEED_SQL} END) AS cpu
+         FROM runs r
+         WHERE r.id >= ? AND r.status = 'success' AND (r.cache_hit IS NULL OR r.cache_hit = 0)
+         GROUP BY r.project, r.task${this.byDef}`,
       )
       .all(floor) as Executed[]
     const p50s = this.forecasts(wanted)
     for (const row of executed) {
       const key = `${row.project}#${row.task}`
-      if (!wanted.has(key)) continue
+      if (!wanted.has(key) || !this.current(key, row.def)) continue
       out.set(key, {
         p50DurationMs: p50s.get(key),
         ...(row.rss !== null ? { maxPeakRssBytes: row.rss } : {}),
         ...(row.cpu !== null ? { maxCpuParallelism: row.cpu } : {}),
       })
     }
-    type Hit = { project: string; task: string; rss: number | null; cpu: number | null }
+    type Hit = {
+      project: string
+      task: string
+      def?: string | null
+      rss: number | null
+      cpu: number | null
+      ms: number | null
+    }
     const hits = this.db
       .query(
-        `SELECT h.project, h.task, MAX(e.peak_rss_bytes) AS rss,
+        `SELECT r.project, r.task${this.byDef}, MAX(e.peak_rss_bytes) AS rss,
            MAX(CASE WHEN e.cpu_ms IS NOT NULL AND e.duration_ms > 0
-                    THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu
-         FROM (SELECT DISTINCT project, task, hash FROM runs WHERE id >= ? AND cache_hit = 1) h
-         JOIN entries e ON e.hash = h.hash
-         GROUP BY h.project, h.task`,
+                    THEN e.cpu_ms * 1.0 / e.duration_ms END) AS cpu,
+           MAX(CASE WHEN e.duration_ms > 0 THEN e.duration_ms END) AS ms
+         FROM (SELECT DISTINCT project, task, hash, def FROM runs WHERE id >= ? AND cache_hit = 1) r
+         JOIN entries e ON e.hash = r.hash
+         GROUP BY r.project, r.task${this.byDef}`,
       )
       .all(floor) as Hit[]
+    // A hit's CPU is its producer's use, not need: the entry keeps no
+    // share, so a crowded producer's reads low. Only a task with no need
+    // from a run here or the default branch's import takes it: main's
+    // blob carried each hit shard's crowded use forward, and the next run
+    // packed them as tight (main 357d3aa, 2026-10-23).
+    const hitCpu = new Map<string, number>()
     for (const row of hits) {
       const key = `${row.project}#${row.task}`
-      if (!wanted.has(key) || (row.rss === null && row.cpu === null)) continue
+      if (
+        !wanted.has(key) ||
+        !this.current(key, row.def) ||
+        (row.rss === null && row.cpu === null && row.ms === null)
+      )
+        continue
+      if (row.cpu !== null) hitCpu.set(key, row.cpu)
       const had = out.get(key)
       const rss = Math.max(had?.maxPeakRssBytes ?? -Infinity, row.rss ?? -Infinity)
-      const cpu = Math.max(had?.maxCpuParallelism ?? -Infinity, row.cpu ?? -Infinity)
       out.set(key, {
-        p50DurationMs: p50s.get(key),
+        ...had,
+        // A task this store only restored is timed by its producer: one a
+        // branch's run made, main restored and never timed, and every
+        // later run planned it at a guess (PR #4339's docs chain: 90 s
+        // guessed, 45 s run, 2026-10-24).
+        p50DurationMs: p50s.get(key) ?? row.ms ?? undefined,
         ...(rss !== -Infinity ? { maxPeakRssBytes: rss } : {}),
-        ...(cpu !== -Infinity ? { maxCpuParallelism: cpu } : {}),
       })
     }
     // A failure counts: a task killed for memory filled its machine too.
-    type Saturated = { project: string; task: string; cores: number | null; memory: number | null }
+    type Saturated = {
+      project: string
+      task: string
+      def?: string | null
+      cores: number | null
+      memory: number | null
+    }
     const saturated = this.db
       .query(
-        `SELECT project, task,
-           MAX(CASE WHEN cpu_ms * 1.0 / duration_ms >= ${SATURATED} * placement_cores
-                    THEN placement_cores END) AS cores,
-           MAX(CASE WHEN peak_rss_bytes >= ${SATURATED} * placement_memory
-                    THEN placement_memory END) AS memory
-         FROM runs
-         WHERE id >= ? AND placement_cores IS NOT NULL AND duration_ms > 0
-           AND status IN ('success', 'failed') AND (cache_hit IS NULL OR cache_hit = 0)
-         GROUP BY project, task`,
+        `SELECT r.project, r.task${this.byDef},
+           MAX(CASE WHEN r.fill >= ${SATURATED} THEN r.cores END) AS cores,
+           MAX(r.memory) AS memory
+         FROM (SELECT r.project, r.task${this.byDef}, r.placement_cores AS cores,
+                 AVG(${NEED_SQL} / r.placement_cores) AS fill,
+                 MAX(CASE WHEN r.peak_rss_bytes >= ${SATURATED} * r.placement_memory
+                          THEN r.placement_memory END) AS memory
+               FROM runs r
+               WHERE r.id >= ? AND r.placement_cores IS NOT NULL AND r.duration_ms > 0
+                 AND r.status IN ('success', 'failed') AND (r.cache_hit IS NULL OR r.cache_hit = 0)
+               GROUP BY r.project, r.task${this.byDef}, r.placement_cores) r
+         GROUP BY r.project, r.task${this.byDef}`,
       )
       .all(floor) as Saturated[]
     for (const row of saturated) {
       const key = `${row.project}#${row.task}`
-      if (!wanted.has(key) || (row.cores === null && row.memory === null)) continue
+      if (
+        !wanted.has(key) ||
+        !this.current(key, row.def) ||
+        (row.cores === null && row.memory === null)
+      )
+        continue
       out.set(key, {
         ...(out.get(key) ?? { p50DurationMs: p50s.get(key) }),
         ...(row.cores !== null ? { saturatedCores: row.cores } : {}),
@@ -450,6 +545,10 @@ export class LocalHistoryProvider implements HistoryProvider {
     // A forecast reads past the window: a task with none in it still has one.
     for (const [key, p50] of p50s) if (!out.has(key)) out.set(key, { p50DurationMs: p50 })
     this.remoteNeeds(wanted, out)
+    for (const [key, cpu] of hitCpu) {
+      const had = out.get(key)!
+      if (had.maxCpuParallelism === undefined) out.set(key, { ...had, maxCpuParallelism: cpu })
+    }
     return out
   }
 
@@ -477,10 +576,10 @@ export class LocalHistoryProvider implements HistoryProvider {
          JOIN task_durations d ON d.id = (
            SELECT id FROM task_durations
            WHERE project = j.value ->> 0 AND task = j.value ->> 1 AND source = 'remote'
-             AND cpu_parallelism IS NOT NULL
+             AND cpu_parallelism IS NOT NULL ${this.durationDef}
            ORDER BY id DESC LIMIT 1)`,
       )
-      .all(JSON.stringify(pending.map(splitTaskId))) as {
+      .all(JSON.stringify(pending.map((id) => this.keyOf(id)))) as {
       i: number
       cpu: number
       rss: number | null
@@ -507,14 +606,21 @@ export class LocalHistoryProvider implements HistoryProvider {
 
   /** The first `runs` id of the newest `recent` invocations (see `loadFor`). */
   private floor(): number {
+    // The first row of each invocation in the window, an index probe each:
+    // a run of only a group records a header and no row (X-50), and at the
+    // oldest slot it gave no first row, so the window read the whole table.
     const row = this.db
       .query(
-        `SELECT MIN(id) AS id FROM runs WHERE run_id =
-           (SELECT run_id FROM invocations ORDER BY rowid DESC LIMIT 1 OFFSET ?)`,
+        `SELECT COUNT(*) AS n, MIN(first) AS id,
+           (SELECT COALESCE(MAX(id), 0) + 1 FROM runs) AS past
+         FROM (SELECT (SELECT MIN(id) FROM runs WHERE run_id = i.run_id) AS first
+               FROM (SELECT run_id FROM invocations ORDER BY rowid DESC LIMIT ?) i)`,
       )
-      .get(this.recent - 1) as { id: number | null }
+      .get(this.recent) as { n: number; id: number | null; past: number }
     // Fewer invocations than the window (or none): the whole table is the window.
-    return row.id ?? 0
+    if (row.n < this.recent) return 0
+    // A window of only empty invocations holds no row.
+    return row.id ?? row.past
   }
 
   /** Mixed-outcome key count per `project#task`, over the slice, for `pairs` only. */

package/src/history/index.ts

@@ -24,11 +24,13 @@ export {
   explainCacheKey,
   getInvocation,
   getRun,
+  GROUP_VIA,
   inputWriters,
   latestRunId,
   listInvocations,
   listRuns,
   rootCauses,
+  upstreamTask,
   whyDidThisRerun,
 } from './metrics.js'
 export type {