@kirrosh/zond 0.22.0 → 0.26.0

package/src/core/diagnostics/db-analysis.ts

@@ -1,10 +1,10 @@
 import { getDb } from "../../db/schema.ts";
 import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
 import { join } from "node:path";
-import { statusHint, classifyFailure, envHint, envCategory, schemaHint, computeSharedEnvIssue, recommendedAction, softDeleteHint, type RecommendedAction } from "./failure-hints.ts";
-import { AUTH_PATH_RE } from "../runner/execute-run.ts";
+import { classifyFailure, recommendedActionForGenerated, isGeneratedTest, type RecommendedAction } from "./failure-hints.ts";
+import { buildSuggestedFixes, type SuggestedFix } from "./suggested-fixes.ts";
 
-export function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
+function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
   if (!raw) return undefined;
   if (verbose || raw.length < 500) return raw;
   const lines = raw.split(/\r?\n/);
@@ -24,7 +24,49 @@ export function truncateErrorMessage(raw: string | null | undefined, verbose?: b
   return msgLines.join("\n");
 }
 
-export function parseBodySafe(raw: string | null | undefined): unknown {
+/**
+ * ARV-305: build a short reason string for `by_recommended_action.examples`.
+ *
+ * Preference order:
+ *   1. trimmed `error_message` (assertion- or network-level message)
+ *   2. first failing assertion → `<field> <rule>: got <actual>`
+ *   3. undefined (nothing left to say)
+ *
+ * Without (2) every assertion-only failure leaves examples[].reason as
+ * undefined, so triage agents lose the one signal that lets them route
+ * past method/path/status (regenerate_suite vs tighten_validation
+ * collapse to identical-looking buckets).
+ */
+function buildExampleReason(
+  errorMessage: unknown,
+  assertions: unknown,
+): string | undefined {
+  const trim = (s: string) => (s.length > 120 ? `${s.slice(0, 117)}...` : s);
+  if (typeof errorMessage === "string" && errorMessage.length > 0) {
+    return trim(errorMessage);
+  }
+  if (!Array.isArray(assertions)) return undefined;
+  for (const a of assertions) {
+    if (!a || typeof a !== "object") continue;
+    const row = a as Record<string, unknown>;
+    if (row.passed === false) {
+      const field = typeof row.field === "string" ? row.field : "";
+      const rule = typeof row.rule === "string" ? row.rule : "";
+      const actual = "actual" in row ? row.actual : undefined;
+      const actualStr = actual === undefined || actual === null
+        ? ""
+        : typeof actual === "string"
+          ? actual
+          : JSON.stringify(actual);
+      const parts = [field, rule].filter(Boolean).join(" ");
+      const text = actualStr ? `${parts}: got ${actualStr}` : parts;
+      return text ? trim(text) : undefined;
+    }
+  }
+  return undefined;
+}
+
+function parseBodySafe(raw: string | null | undefined): unknown {
   if (!raw) return undefined;
   const truncated = raw.length > 2000 ? raw.slice(0, 2000) + "\u2026[truncated]" : raw;
   try {
@@ -40,7 +82,33 @@ const USEFUL_HEADERS = new Set([
 ]);
 const USEFUL_PREFIXES = ["x-", "ratelimit"];
 
-export function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
+/** ARV-103 (F8): true when at least one assertion on the failing step is
+ *  a schema-validation kind. `--validate-schema` annotates each violated
+ *  field with `kind: "schema"` (set in src/core/runner/schema-validator.ts).
+ *  The assertions column is stored as JSON in SQLite; parse defensively. */
+function hasSchemaAssertion(raw: string | unknown[] | null | undefined): boolean {
+  if (raw === null || raw === undefined) return false;
+  let arr: unknown[];
+  if (Array.isArray(raw)) {
+    arr = raw;
+  } else if (typeof raw === "string") {
+    try {
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed)) return false;
+      arr = parsed;
+    } catch {
+      return false;
+    }
+  } else {
+    return false;
+  }
+  for (const a of arr) {
+    if (a && typeof a === "object" && (a as { kind?: unknown }).kind === "schema") return true;
+  }
+  return false;
+}
+
+function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
   if (!raw) return undefined;
   try {
     const h = JSON.parse(raw) as Record<string, string>;
@@ -118,7 +186,6 @@ export interface FailureGroup {
   count: number;
   failure_type: string;
   recommended_action: RecommendedAction;
-  hint?: string;
   examples: string[];
   response_status: number | null;
 }
@@ -144,10 +211,11 @@ export interface DiagnoseResult {
     assertion_failures: number;
     network_errors: number;
   };
-  agent_directive?: string;
-  env_issue?: string;
-  auth_hint?: string;
   cascade_skips?: CascadeSkipGroup[];
+  /** TASK-29: actionable suggestions populated from 404 placeholder
+   *  detection + .env.yaml unfilled-key audit. Empty / undefined when
+   *  nothing actionable was found. */
+  suggested_fixes?: SuggestedFix[];
   failures: Array<{
     suite_name: string;
     test_name: string;
@@ -159,14 +227,44 @@ export interface DiagnoseResult {
     request_method: string | null;
     request_url: string | null;
     response_status: number | null;
-    hint?: string;
-    schema_hint?: string;
     response_body?: unknown;
     response_headers?: Record<string, string>;
     assertions: unknown;
     duration_ms: number | null;
+    /** ARV-159: when this entry is the representative of a collapsed group
+     *  (status|failure_type signature), the total size of that group. Lets
+     *  consumers reading `.data.failures[]` see "this signature stands for
+     *  N underlying tests" without cross-referencing `.grouped_failures[]`.
+     *  Omitted when no collapsing occurred (failures ≤ 5 or
+     *  --verbose). */
+    group_count?: number;
   }>;
   grouped_failures?: FailureGroup[];
+  /** ARV-101 (F6): top-level aggregation keyed by `recommended_action`
+   *  enum so triage agents (zond-triage skill) can route on the canonical
+   *  action without re-folding `failures[].recommended_action` through
+   *  `jq | group_by`. Built from the *full* failure set (not the compact
+   *  subset), so counts match `.summary.failed`. Each bucket carries
+   *  total count + a small examples list (`<suite>/<test>`). Empty when
+   *  there are no failures. */
+  by_recommended_action?: Record<string, {
+    count: number;
+    /** ARV-228: each example carries the per-failure context the
+     *  zond-triage skill renders in its output template (`POST
+     *  /v1/projects → 500 (×3) — <reason>`). Previously a bare
+     *  `string[]` of `<suite>/<test>` ids — agents had to cross-join
+     *  with `failures[]` to recover method/path/status, which broke
+     *  triage scripts on large runs. Bounded to 5 entries per bucket
+     *  (same cap as the legacy string form). */
+    examples: Array<{
+      suite: string;
+      test: string;
+      method: string;
+      path: string;
+      status: number;
+      reason?: string;
+    }>;
+  }>;
 }
 
 export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, maxExamples?: number): DiagnoseResult {
@@ -187,12 +285,17 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
     .filter(r => r.status === "fail" || r.status === "error")
     .map(r => {
       const parsedBody = parseBodySafe(r.response_body);
-      const hint = envHint(r.request_url, r.error_message, envFilePath) ??
-        softDeleteHint(r.response_status, r.request_method, parsedBody) ??
-        statusHint(r.response_status);
       const failure_type = classifyFailure(r.status, r.response_status);
-      const rec_action = recommendedAction(failure_type, r.response_status);
-      const sHint = schemaHint(failure_type, r.response_status);
+      // ARV-42: generator-emitted suites should not route to fix_test_logic —
+      // editing the YAML gets clobbered on the next `zond audit`.
+      const generated = isGeneratedTest(r.provenance, r.suite_file);
+      // ARV-103 (F8): walk the assertions array to detect a schema-kind
+      // failure (--validate-schema annotates each assertion with its kind).
+      // When present, propagate the flag so the classifier routes to
+      // report_backend_bug — schema violations are real contract bugs, not
+      // test-logic mistakes.
+      const schema_violation = hasSchemaAssertion(r.assertions);
+      const rec_action = recommendedActionForGenerated(failure_type, r.response_status, generated, schema_violation);
       return {
         suite_name: r.suite_name,
         test_name: r.test_name,
@@ -204,8 +307,6 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
         request_method: r.request_method,
         request_url: r.request_url,
         response_status: r.response_status,
-        ...(hint ? { hint } : {}),
-        ...(sHint ? { schema_hint: sHint } : {}),
         response_body: parsedBody,
         response_headers: filterHeaders(r.response_headers),
         assertions: r.assertions,
@@ -213,28 +314,11 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
       };
     });
 
-  const sharedEnvHint = computeSharedEnvIssue(failures, envFilePath);
-
   let apiErrors = 0, assertionFailures = 0, networkErrors = 0;
-  let authFailureCount = 0;
   for (const f of failures) {
     if (f.failure_type === "api_error") apiErrors++;
     else if (f.failure_type === "assertion_failed") assertionFailures++;
     else if (f.failure_type === "network_error") networkErrors++;
-    if (f.response_status === 401 || f.response_status === 403) authFailureCount++;
-  }
-
-  let agent_directive: string | undefined;
-  if (apiErrors > 0) {
-    const fixable = assertionFailures + networkErrors;
-    agent_directive =
-      `${apiErrors} test${apiErrors === 1 ? "" : "s"} returned 5xx server errors. ` +
-      `Do NOT change test expectations to accept 5xx responses. ` +
-      `These are backend bugs, not test logic errors. ` +
-      `Stop iterating on these tests and report the failures to the API team.` +
-      (fixable > 0
-        ? ` The remaining ${fixable} failure${fixable === 1 ? "" : "s"} may be fixable in test logic.`
-        : "");
   }
 
   // Cascade skips: skipped tests due to missing captures from failed create steps
@@ -257,23 +341,67 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
       }))
     : undefined;
 
-  // Auth hint: when many tests fail with 401/403, suggest auth setup
-  let auth_hint: string | undefined;
-  if (authFailureCount >= 5 && authFailureCount / diagRun.total >= 0.3) {
-    const loginEndpoint = allResults.find(
-      r => r.request_method?.toUpperCase() === "POST" && AUTH_PATH_RE.test(r.request_url ?? "")
-    );
-    if (loginEndpoint) {
-      auth_hint = `${authFailureCount} tests failed with 401/403. Found auth endpoint: POST ${loginEndpoint.request_url} — add \`setup: true\` to your auth suite so its captured token is shared with all other suites, or set auth_token manually in .env.yaml`;
-    } else {
-      auth_hint = `${authFailureCount} tests failed with 401/403 — add \`setup: true\` to your auth suite so its captured token is shared with all other suites, or set auth_token in .env.yaml`;
-    }
-  }
-
   const { grouped_failures, compactFailures } = verbose
     ? { grouped_failures: undefined, compactFailures: failures }
     : groupFailures(failures, maxExamples);
 
+  // TASK-29: surface placeholder path-params + unfilled .env.yaml keys.
+  const suggestedFixes = buildSuggestedFixes({
+    failures: failures.map(f => ({
+      response_status: f.response_status,
+      request_url: f.request_url,
+      suite_name: f.suite_name,
+      test_name: f.test_name,
+    })),
+    envFilePath,
+  });
+
+  // ARV-101 (F6): aggregate failures by recommended_action enum so triage
+  // agents read .data.by_recommended_action.fix_env.count instead of
+  // re-folding failures[].recommended_action through `jq | group_by`. Built
+  // from the full failure set (not compactFailures) so counts match
+  // .summary.failed. Bounded examples list (5) keeps payload small while
+  // still pointing at concrete suites the agent can open.
+  //
+  // ARV-228: each example is now an object carrying method/path/status/
+  // reason so the zond-triage skill can render its output template
+  // ("POST /v1/projects → 500 (×3) — TypeError") without cross-joining
+  // failures[]. Bound preserved at 5/bucket; ordering = insertion order
+  // (matches failures[] traversal, deterministic per run).
+  const by_recommended_action: Record<string, NonNullable<DiagnoseResult["by_recommended_action"]>[string]> = {};
+  for (const f of failures) {
+    const key = f.recommended_action;
+    let bucket = by_recommended_action[key];
+    if (!bucket) {
+      bucket = { count: 0, examples: [] };
+      by_recommended_action[key] = bucket;
+    }
+    bucket.count += 1;
+    if (bucket.examples.length < 5) {
+      // Trim reason to keep the bucket compact — full error_message lives
+      // in failures[].error_message for agents that want it.
+      //
+      // ARV-305: when there is no top-level error_message (typical for
+      // assertion failures — the row carries the failing rule in
+      // .assertions, not a free-form message), build a reason out of
+      // the first failing assertion so the example is not stripped
+      // down to method/path/status. The fallback keeps the field
+      // populated whenever any failure context exists.
+      const reason = buildExampleReason(f.error_message, f.assertions);
+      bucket.examples.push({
+        suite: f.suite_name,
+        test: f.test_name,
+        // DB columns are nullable (early steps may not have a request
+        // recorded yet — e.g. fixture-load failures). Coerce to "" / 0
+        // rather than leaking null into the contract.
+        method: f.request_method ?? "",
+        path: f.request_url ?? "",
+        status: f.response_status ?? 0,
+        ...(reason ? { reason } : {}),
+      });
+    }
+  }
+
   return {
     run: {
       id: diagRun.id,
@@ -289,16 +417,15 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
       assertion_failures: assertionFailures,
       network_errors: networkErrors,
     },
-    ...(agent_directive ? { agent_directive } : {}),
-    ...(sharedEnvHint ? { env_issue: sharedEnvHint } : {}),
-    ...(auth_hint ? { auth_hint } : {}),
     ...(cascade_skips ? { cascade_skips } : {}),
+    ...(suggestedFixes.length > 0 ? { suggested_fixes: suggestedFixes } : {}),
     failures: compactFailures,
     ...(grouped_failures ? { grouped_failures } : {}),
+    ...(failures.length > 0 ? { by_recommended_action } : {}),
   };
 }
 
-type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; hint?: string; response_status: number | null };
+type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; response_status: number | null; group_count?: number };
 
 /** Group similar failures for compact output. Exported for testing. */
 export function groupFailures<T extends FailureItem>(failures: T[], maxExamples = 2): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
@@ -306,7 +433,7 @@ export function groupFailures<T extends FailureItem>(failures: T[], maxExamples
     return { compactFailures: failures };
   }
 
-  const groupMap = new Map<string, { items: T[]; failure_type: string; hint?: string; response_status: number | null }>();
+  const groupMap = new Map<string, { items: T[]; failure_type: string; response_status: number | null }>();
 
   for (const f of failures) {
     const key = `${f.response_status ?? "null"}|${f.failure_type}`;
@@ -317,7 +444,6 @@ export function groupFailures<T extends FailureItem>(failures: T[], maxExamples
       groupMap.set(key, {
         items: [f],
         failure_type: f.failure_type,
-        hint: f.hint,
         response_status: f.response_status,
       });
     }
@@ -345,20 +471,45 @@ export function groupFailures<T extends FailureItem>(failures: T[], maxExamples
       count: group.items.length,
       failure_type: group.failure_type,
       recommended_action: group.items[0]!.recommended_action,
-      hint: group.hint,
       examples: (showAll ? group.items : group.items.slice(0, maxExamples)).map(f => `${f.suite_name}/${f.test_name}`),
       response_status: group.response_status,
     });
     if (isApiError) {
       compactFailures.push(...group.items);
     } else {
-      compactFailures.push(group.items[0]!);
+      // ARV-159: tag the representative with the group size so
+      // `.data.failures[]` carries the multiplier inline.
+      const rep = { ...group.items[0]!, group_count: group.items.length };
+      compactFailures.push(rep as T);
     }
   }
 
   return { grouped_failures, compactFailures };
 }
 
+export interface BodyFieldChange {
+  field: string;
+  change: "added" | "removed" | "type_changed";
+  before?: string;
+  after?: string;
+  /** ARV-352: structural scope of the change, derived deterministically from
+   *  the path. `element` = the path crosses an array boundary (`[]`), i.e.
+   *  it's a field of a *collection item* — on list/log endpoints two samplings
+   *  return DIFFERENT objects, so element-level added/removed/type_changed is
+   *  schema-of-union variance across the sampled set, not a contract move.
+   *  `container` = no `[]` in the path — the response envelope/pagination
+   *  skeleton, where a change IS real drift. NOT a suppression heuristic
+   *  (ARV-337): nothing is dropped or down-ranked; the agent judges using the
+   *  scope tag + endpoint context. */
+  scope: "container" | "element";
+}
+
+export interface BodyDiff {
+  suite: string;
+  test: string;
+  changes: BodyFieldChange[];
+}
+
 export interface CompareResult {
   runA: { id: number; started_at: string };
   runB: { id: number; started_at: string };
@@ -368,12 +519,73 @@ export interface CompareResult {
     unchanged: number;
     newTests: number;
     removedTests: number;
+    bodyChanges: number;
+    /** ARV-352: of `bodyChanges`, how many touch the response envelope
+     *  (`container`) vs collection-item fields (`element`). Element-heavy diffs
+     *  on list/log endpoints are schema-of-union variance across a re-sampled
+     *  set, not contract drift — split so triage doesn't read them as regression. */
+    bodyChangesContainer: number;
+    bodyChangesElement: number;
   };
   regressions: Array<{ suite: string; test: string; before: string; after: string }>;
   fixes: Array<{ suite: string; test: string; before: string; after: string }>;
+  /** ARV-339: field-level response-shape diff for tests present in both runs.
+   *  Status-diff answers "what broke"; this answers "how the contract moved". */
+  body_changes: BodyDiff[];
   hasRegressions: boolean;
 }
 
+/** ARV-339: flatten a parsed JSON body into `path → union of leaf types`.
+ *  Array elements collapse under `[]` so item count/order don't add noise. */
+function bodyShape(value: unknown, path: string, out: Map<string, Set<string>>): void {
+  if (Array.isArray(value)) {
+    if (value.length === 0) addShape(out, path, "array");
+    for (const item of value) bodyShape(item, `${path}[]`, out);
+  } else if (value !== null && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>);
+    if (entries.length === 0) addShape(out, path, "object");
+    for (const [k, v] of entries) bodyShape(v, path ? `${path}.${k}` : k, out);
+  } else {
+    addShape(out, path, value === null ? "null" : typeof value);
+  }
+}
+
+function addShape(out: Map<string, Set<string>>, path: string, type: string): void {
+  const key = path || "$";
+  const set = out.get(key) ?? new Set<string>();
+  set.add(type);
+  out.set(key, set);
+}
+
+const typeLabel = (s: Set<string>): string => [...s].sort().join("|");
+
+/** ARV-339: diff two stored response bodies at field level. Returns [] when
+ *  either side is missing / non-JSON — a shape diff of prose is meaningless. */
+export function diffBodyShapes(rawA: string | null, rawB: string | null): BodyFieldChange[] {
+  if (!rawA || !rawB || rawA === rawB) return [];
+  let a: unknown, b: unknown;
+  try { a = JSON.parse(rawA); b = JSON.parse(rawB); } catch { return []; }
+  if (typeof a !== "object" || a === null || typeof b !== "object" || b === null) return [];
+  const shapeA = new Map<string, Set<string>>();
+  const shapeB = new Map<string, Set<string>>();
+  bodyShape(a, "", shapeA);
+  bodyShape(b, "", shapeB);
+  const scopeOf = (field: string): "container" | "element" =>
+    field.includes("[]") ? "element" : "container";
+  const changes: BodyFieldChange[] = [];
+  for (const [field, typesB] of shapeB) {
+    const typesA = shapeA.get(field);
+    if (!typesA) changes.push({ field, change: "added", after: typeLabel(typesB), scope: scopeOf(field) });
+    else if (typeLabel(typesA) !== typeLabel(typesB)) {
+      changes.push({ field, change: "type_changed", before: typeLabel(typesA), after: typeLabel(typesB), scope: scopeOf(field) });
+    }
+  }
+  for (const [field, typesA] of shapeA) {
+    if (!shapeB.has(field)) changes.push({ field, change: "removed", before: typeLabel(typesA), scope: scopeOf(field) });
+  }
+  return changes.sort((x, y) => x.field.localeCompare(y.field));
+}
+
 export function compareRuns(idA: number, idB: number, dbPath?: string): CompareResult {
   getDb(dbPath);
   const runARecord = getRunById(idA);
@@ -386,8 +598,16 @@ export function compareRuns(idA: number, idB: number, dbPath?: string): CompareR
 
   const mapA = new Map<string, string>();
   const mapB = new Map<string, string>();
-  for (const r of resultsA) mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
-  for (const r of resultsB) mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
+  const bodyA = new Map<string, string | null>();
+  const bodyB = new Map<string, string | null>();
+  for (const r of resultsA) {
+    mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
+    bodyA.set(`${r.suite_name}::${r.test_name}`, r.response_body);
+  }
+  for (const r of resultsB) {
+    mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
+    bodyB.set(`${r.suite_name}::${r.test_name}`, r.response_body);
+  }
 
   const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
   const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
@@ -395,10 +615,14 @@ export function compareRuns(idA: number, idB: number, dbPath?: string): CompareR
   let newTests = 0;
   let removedTests = 0;
 
+  const body_changes: BodyDiff[] = [];
+
   for (const [key, statusB] of mapB) {
     const statusA = mapA.get(key);
     if (statusA === undefined) { newTests++; continue; }
     const [suite, test] = key.split("::") as [string, string];
+    const changes = diffBodyShapes(bodyA.get(key) ?? null, bodyB.get(key) ?? null);
+    if (changes.length > 0) body_changes.push({ suite, test, changes });
     const wasPass = statusA === "pass";
     const isPass = statusB === "pass";
     const wasFail = statusA === "fail" || statusA === "error";
@@ -414,9 +638,19 @@ export function compareRuns(idA: number, idB: number, dbPath?: string): CompareR
   return {
     runA: { id: idA, started_at: runARecord.started_at },
     runB: { id: idB, started_at: runBRecord.started_at },
-    summary: { regressions: regressions.length, fixes: fixes.length, unchanged, newTests, removedTests },
+    summary: {
+      regressions: regressions.length,
+      fixes: fixes.length,
+      unchanged,
+      newTests,
+      removedTests,
+      bodyChanges: body_changes.length,
+      bodyChangesContainer: body_changes.filter(d => d.changes.some(c => c.scope === "container")).length,
+      bodyChangesElement: body_changes.filter(d => d.changes.every(c => c.scope === "element")).length,
+    },
     regressions,
     fixes,
+    body_changes,
     hasRegressions: regressions.length > 0,
   };
 }

package/src/core/diagnostics/failure-class.ts

@@ -0,0 +1,140 @@
+/**
+ * TASK-101: failure classification — definitely_bug / likely_bug / quirk / env_issue.
+ *
+ * Goal: бэкендер за секунду видит «реально баг» vs «quirk зонда / probe
+ * фолс-позитив». Чисто read-only классификация: не меняет статус step,
+ * не влияет на pass/fail. Только tag-овая аналитика.
+ */
+
+import type { StepResult, AssertionResult } from "../runner/types.ts";
+import type { SourceMetadata } from "../parser/types.ts";
+
+export type FailureClass =
+  | "definitely_bug"
+  | "likely_bug"
+  | "quirk"
+  | "env_issue"
+  /** Step was skipped because an upstream step failed to produce a required
+   *  capture (or produced a tainted one). Not a failure on its own — render
+   *  collapsed under the root failure to avoid drowning the user. */
+  | "cascade";
+
+export interface FailureClassification {
+  failure_class: FailureClass;
+  failure_class_reason: string;
+}
+
+function failedAssertionsOf(result: StepResult): AssertionResult[] {
+  return result.assertions.filter((a) => !a.passed);
+}
+
+function ruleStartsWith(a: AssertionResult, prefix: string): boolean {
+  return typeof a.rule === "string" && a.rule.startsWith(prefix);
+}
+
+function expectedStatusList(a: AssertionResult): number[] | null {
+  if (Array.isArray(a.expected)) {
+    const arr = a.expected.filter((v): v is number => typeof v === "number");
+    return arr.length > 0 ? arr : null;
+  }
+  return typeof a.expected === "number" ? [a.expected] : null;
+}
+
+/**
+ * Classify a failed step. Returns `null` for pass/skip/unclassifiable failures —
+ * UI renders those as "unclassified" rather than crashing.
+ */
+export function classifyFailure(result: StepResult): FailureClassification | null {
+  if (result.status === "pass" || result.status === "skip") return null;
+
+  // Network/runtime error before any HTTP response → env-side
+  if (result.status === "error") {
+    return {
+      failure_class: "env_issue",
+      failure_class_reason: result.error ?? "request failed before producing a response",
+    };
+  }
+
+  const respStatus = result.response?.status;
+  const provenance: SourceMetadata | null | undefined = result.provenance;
+  const generator = typeof provenance?.generator === "string" ? provenance.generator : undefined;
+  const failed = failedAssertionsOf(result);
+
+  // 1. Backend 5xx — always a backend bug regardless of test intent.
+  if (typeof respStatus === "number" && respStatus >= 500) {
+    return {
+      failure_class: "definitely_bug",
+      failure_class_reason: `API returned ${respStatus} — server-side error`,
+    };
+  }
+
+  // 2. Response did not match its OpenAPI schema — spec guarantees X, server returned ≠ X.
+  const schemaFail = failed.find((a) => ruleStartsWith(a, "schema."));
+  if (schemaFail) {
+    return {
+      failure_class: "definitely_bug",
+      failure_class_reason: `Response violates OpenAPI schema at ${schemaFail.field}`,
+    };
+  }
+
+  // 3. Mass-assignment probe: extras must not apply. A failed not_equals
+  //    assertion on this generator means the sentinel value leaked through.
+  if (generator === "mass-assignment-probe") {
+    const extrasLeak = failed.find(
+      (a) => ruleStartsWith(a, "not_equals") || ruleStartsWith(a, "set_equals"),
+    );
+    if (extrasLeak) {
+      return {
+        failure_class: "definitely_bug",
+        failure_class_reason: `Mass-assignment: client-supplied extras were accepted (${extrasLeak.field})`,
+      };
+    }
+  }
+
+  // 4. ARV-236: 401/403 on a step that expected success → env_issue
+  //    ("token_scope"). Surfaces narrowly-scoped tokens (Resend free plan,
+  //    GitHub PAT without `repo`, Stripe restricted key, etc.) so coverage
+  //    and triage don't tag them as fail. Only fires when the failing
+  //    assertion is on `status` and the expected status was 2xx — a real
+  //    "expect 403" test deliberately checking auth still classifies normally.
+  if (typeof respStatus === "number" && (respStatus === 401 || respStatus === 403)) {
+    const statusFail = failed.find((a) => a.field === "status");
+    if (statusFail) {
+      const expected = expectedStatusList(statusFail);
+      const allExpected2xx = expected?.every((s) => s >= 200 && s < 300) ?? false;
+      if (allExpected2xx) {
+        return {
+          failure_class: "env_issue",
+          failure_class_reason: `token_scope: API returned ${respStatus} where the test expected ${expected!.join("/")} — token lacks scope or plan for this endpoint`,
+        };
+      }
+    }
+  }
+
+  // 5. Negative-probe family — distinguish "API accepted bad input" (likely_bug)
+  //    from "API rejected with a different 4xx" (quirk).
+  if (generator === "negative-probe" || generator === "method-probe") {
+    const statusFail = failed.find((a) => a.field === "status");
+    if (statusFail && typeof respStatus === "number") {
+      const expected = expectedStatusList(statusFail);
+      const allExpected4xx = expected?.every((s) => s >= 400 && s < 500) ?? false;
+      if (allExpected4xx) {
+        if (respStatus >= 200 && respStatus < 300) {
+          return {
+            failure_class: "likely_bug",
+            failure_class_reason: `Negative probe expected 4xx, got ${respStatus} — API accepts invalid input`,
+          };
+        }
+        if (respStatus >= 400 && respStatus < 500) {
+          return {
+            failure_class: "quirk",
+            failure_class_reason: `Negative probe expected ${expected!.join("/")}, got ${respStatus} — different 4xx code`,
+          };
+        }
+      }
+    }
+  }
+
+  // Default: leave unclassified. UI / CLI render as "unclassified".
+  return null;
+}