@kirrosh/zond 0.22.0 → 0.23.0

package/src/core/coverage/reasons.ts

@@ -0,0 +1,300 @@
+/**
+ * Coverage reasons engine — pure function: spec endpoints + run results +
+ * workspace context → matrix cells with explicit reason codes.
+ *
+ * The matrix has rows per endpoint (METHOD path) and three status-class
+ * columns (`2xx`, `4xx`, `5xx`). Each cell carries:
+ *   - status: covered | partial | uncovered
+ *   - reasons: zero or more codes that explain the cell state
+ *   - results: stored step results that contributed to this cell
+ *
+ * "Default" branch is intentionally omitted — extractEndpoints already
+ * skips OpenAPI's "default" status code, so making a column for it would
+ * be permanently empty and noisy.
+ *
+ * The function is intentionally I/O-free: callers (server, exporter, CLI)
+ * load endpoints/results/fixtures separately and feed them in. This keeps
+ * the engine trivial to unit-test.
+ */
+import type { EndpointInfo } from "../generator/types.ts";
+import type { StoredStepResult } from "../../db/queries.ts";
+
+export type StatusClass = "2xx" | "4xx" | "5xx";
+const STATUS_CLASSES: StatusClass[] = ["2xx", "4xx", "5xx"];
+
+export type ReasonCode =
+  | "covered"
+  | "partial-failed"
+  | "not-generated"
+  | "no-spec"
+  | "deprecated"
+  | "no-fixtures"
+  | "ephemeral-only"
+  | "auth-scope-mismatch"
+  | "tag-filtered";
+
+export interface CellResultRef {
+  resultId: number;
+  runId: number;
+  status: string;
+  responseStatus: number | null;
+  failureClass: string | null;
+  testName: string;
+  suiteFile: string | null;
+}
+
+export interface MatrixCell {
+  status: "covered" | "partial" | "uncovered";
+  reasons: ReasonCode[];
+  results: CellResultRef[];
+}
+
+export interface MatrixRow {
+  endpoint: string;
+  method: string;
+  path: string;
+  tags: string[];
+  deprecated: boolean;
+  security: string[];
+  declaredStatuses: number[];
+  cells: Record<StatusClass, MatrixCell>;
+}
+
+export interface BuildMatrixInput {
+  endpoints: EndpointInfo[];
+  results: StoredStepResult[];
+  fixturesAffected: Map<string, { name: string; required: boolean; source: string }[]>;
+  envVars: Set<string>;
+  ephemeralEndpoints: Set<string>;
+  tagFilter: string[];
+  profile: "safe" | "full";
+}
+
+export interface MatrixTotals {
+  endpoints: number;
+  cells: number;
+  covered: number;
+  partial: number;
+  uncovered: number;
+  byReason: Record<ReasonCode, number>;
+}
+
+export interface CoverageMatrix {
+  rows: MatrixRow[];
+  totals: MatrixTotals;
+}
+
+function endpointKey(method: string, path: string): string {
+  return `${method.toUpperCase()} ${path}`;
+}
+
+function classifyStatus(code: number): StatusClass | null {
+  if (code >= 200 && code < 300) return "2xx";
+  if (code >= 400 && code < 500) return "4xx";
+  if (code >= 500 && code < 600) return "5xx";
+  return null;
+}
+
+/**
+ * Build a regex that matches a concrete URL path against an OpenAPI path
+ * template. `/pets/{id}` becomes `^/pets/[^/]+$`.
+ */
+function specPathToRegex(specPath: string): RegExp {
+  const pattern = specPath.replace(/\{[^}]+\}/g, "[^/]+");
+  return new RegExp(`^${pattern}$`);
+}
+
+function extractPathname(url: string | null): string | null {
+  if (!url) return null;
+  try {
+    const u = new URL(url);
+    return u.pathname;
+  } catch {
+    if (url.startsWith("/")) {
+      const q = url.indexOf("?");
+      return q === -1 ? url : url.slice(0, q);
+    }
+    return null;
+  }
+}
+
+function matchResultToEndpoint(
+  result: StoredStepResult,
+  endpointsByKey: Map<string, EndpointInfo>,
+  endpointsByMethod: Map<string, { ep: EndpointInfo; rx: RegExp }[]>,
+): EndpointInfo | null {
+  const provEp = (result.provenance as Record<string, unknown> | null)?.endpoint;
+  if (typeof provEp === "string") {
+    const sp = provEp.indexOf(" ");
+    const normalised = sp === -1
+      ? provEp
+      : `${provEp.slice(0, sp).toUpperCase()}${provEp.slice(sp)}`;
+    const direct = endpointsByKey.get(normalised);
+    if (direct) return direct;
+  }
+  const method = result.request_method?.toUpperCase();
+  if (!method) return null;
+  const candidates = endpointsByMethod.get(method);
+  if (!candidates) return null;
+  const pathname = extractPathname(result.request_url);
+  if (!pathname) return null;
+  for (const c of candidates) if (c.rx.test(pathname)) return c.ep;
+  return null;
+}
+
+function pathParamNames(path: string): string[] {
+  return [...path.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]!);
+}
+
+function hasMissingPathFixtures(
+  ep: EndpointInfo,
+  fixturesAffected: Map<string, { name: string; required: boolean; source: string }[]>,
+  envVars: Set<string>,
+): boolean {
+  const params = pathParamNames(ep.path);
+  if (params.length === 0) return false;
+  const label = endpointKey(ep.method, ep.path);
+  const declared = fixturesAffected.get(label) ?? [];
+  // Manifest entry per param? Required + missing = no-fixtures.
+  const required = declared.filter((f) => f.source === "path" && f.required);
+  for (const f of required) if (!envVars.has(f.name)) return true;
+  // Manifest may not enumerate every param when it was generated before this
+  // endpoint existed, so also require: every {param} must map to an env var.
+  for (const p of params) {
+    if (envVars.has(p)) continue;
+    const declaredForP = declared.find((f) => f.name === p);
+    if (declaredForP && envVars.has(declaredForP.name)) continue;
+    return true;
+  }
+  return false;
+}
+
+function hasMissingAuthFixtures(ep: EndpointInfo, envVars: Set<string>): boolean {
+  if (ep.security.length === 0) return false;
+  // Convention: a security scheme name maps to one of `<name>`, `<name>_token`,
+  // or the lowercased `<name>_token` env var. The endpoint is satisfied if
+  // *any* of its required schemes has a configured token.
+  for (const scheme of ep.security) {
+    const variants = [scheme, `${scheme}_token`, `${scheme.toLowerCase()}_token`, "auth_token"];
+    if (variants.some((v) => envVars.has(v))) return false;
+  }
+  return true;
+}
+
+function declaredStatusClasses(ep: EndpointInfo): Set<StatusClass> {
+  const out = new Set<StatusClass>();
+  for (const r of ep.responses) {
+    const cls = classifyStatus(r.statusCode);
+    if (cls) out.add(cls);
+  }
+  return out;
+}
+
+export function buildCoverageMatrix(input: BuildMatrixInput): CoverageMatrix {
+  const endpointsByKey = new Map<string, EndpointInfo>();
+  const endpointsByMethod = new Map<string, { ep: EndpointInfo; rx: RegExp }[]>();
+  for (const ep of input.endpoints) {
+    endpointsByKey.set(endpointKey(ep.method, ep.path), ep);
+    const method = ep.method.toUpperCase();
+    const list = endpointsByMethod.get(method) ?? [];
+    list.push({ ep, rx: specPathToRegex(ep.path) });
+    endpointsByMethod.set(method, list);
+  }
+
+  // Bucket results: key = "METHOD path" → statusClass → list of refs.
+  const buckets = new Map<string, Record<StatusClass, CellResultRef[]>>();
+  for (const r of input.results) {
+    const ep = matchResultToEndpoint(r, endpointsByKey, endpointsByMethod);
+    if (!ep) continue;
+    const key = endpointKey(ep.method, ep.path);
+    let bucket = buckets.get(key);
+    if (!bucket) {
+      bucket = { "2xx": [], "4xx": [], "5xx": [] };
+      buckets.set(key, bucket);
+    }
+    if (r.response_status == null) continue;
+    const cls = classifyStatus(r.response_status);
+    if (!cls) continue;
+    bucket[cls].push({
+      resultId: r.id,
+      runId: r.run_id,
+      status: r.status,
+      responseStatus: r.response_status,
+      failureClass: r.failure_class,
+      testName: r.test_name,
+      suiteFile: r.suite_file,
+    });
+  }
+
+  const totals: MatrixTotals = {
+    endpoints: input.endpoints.length,
+    cells: 0,
+    covered: 0,
+    partial: 0,
+    uncovered: 0,
+    byReason: {
+      "covered": 0, "partial-failed": 0, "not-generated": 0, "no-spec": 0,
+      "deprecated": 0, "no-fixtures": 0, "ephemeral-only": 0,
+      "auth-scope-mismatch": 0, "tag-filtered": 0,
+    },
+  };
+
+  const filterTags = new Set(input.tagFilter);
+  const rows: MatrixRow[] = input.endpoints.map((ep) => {
+    const key = endpointKey(ep.method, ep.path);
+    const bucket = buckets.get(key) ?? { "2xx": [], "4xx": [], "5xx": [] };
+    const declared = declaredStatusClasses(ep);
+    const tagFiltered = filterTags.size > 0 && !ep.tags.some((t) => filterTags.has(t));
+    const cells: Record<StatusClass, MatrixCell> = { "2xx": null!, "4xx": null!, "5xx": null!};
+
+    for (const cls of STATUS_CLASSES) {
+      const refs = bucket[cls];
+      const passing = refs.some((r) => r.status === "pass");
+      const failing = refs.some((r) => r.status !== "pass");
+      const reasons: ReasonCode[] = [];
+      let cellStatus: MatrixCell["status"];
+      if (passing && !failing) {
+        cellStatus = "covered";
+        reasons.push("covered");
+      } else if (passing && failing) {
+        cellStatus = "covered";
+        reasons.push("covered", "partial-failed");
+      } else if (failing) {
+        cellStatus = "partial";
+        reasons.push("partial-failed");
+      } else {
+        cellStatus = "uncovered";
+        if (!declared.has(cls)) reasons.push("no-spec");
+        if (ep.deprecated) reasons.push("deprecated");
+        if (input.ephemeralEndpoints.has(key) && input.profile === "safe") reasons.push("ephemeral-only");
+        if (tagFiltered) reasons.push("tag-filtered");
+        if (hasMissingPathFixtures(ep, input.fixturesAffected, input.envVars)) reasons.push("no-fixtures");
+        if (hasMissingAuthFixtures(ep, input.envVars)) reasons.push("auth-scope-mismatch");
+        if (reasons.length === 0) reasons.push("not-generated");
+      }
+      // Always tag deprecated even on covered cells — it's an awareness flag.
+      if (ep.deprecated && !reasons.includes("deprecated")) reasons.push("deprecated");
+
+      cells[cls] = { status: cellStatus, reasons, results: refs };
+
+      totals.cells += 1;
+      if (cellStatus === "covered") totals.covered += 1;
+      else if (cellStatus === "partial") totals.partial += 1;
+      else totals.uncovered += 1;
+      for (const code of reasons) totals.byReason[code] += 1;
+    }
+
+    return {
+      endpoint: key,
+      method: ep.method.toUpperCase(),
+      path: ep.path,
+      tags: ep.tags,
+      deprecated: !!ep.deprecated,
+      security: ep.security,
+      declaredStatuses: ep.responses.map((r) => r.statusCode).sort((a, b) => a - b),
+      cells,
+    };
+  });
+
+  return { rows, totals };
+}

package/src/core/diagnostics/db-analysis.ts

@@ -1,10 +1,11 @@
 import { getDb } from "../../db/schema.ts";
 import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
 import { join } from "node:path";
-import { statusHint, classifyFailure, envHint, envCategory, schemaHint, computeSharedEnvIssue, recommendedAction, softDeleteHint, type RecommendedAction } from "./failure-hints.ts";
-import { AUTH_PATH_RE } from "../runner/execute-run.ts";
+import { statusHint, classifyFailure, envHint, envCategory, schemaHint, computeSharedEnvIssue, clusterEnvIssues, buildEnvIssue, recommendedActionForGenerated, isGeneratedTest, softDeleteHint, type RecommendedAction, type EnvIssue } from "./failure-hints.ts";
+import { buildSuggestedFixes, type SuggestedFix } from "./suggested-fixes.ts";
+import { AUTH_PATH_RE } from "../runner/auth-path.ts";
 
-export function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
+function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
   if (!raw) return undefined;
   if (verbose || raw.length < 500) return raw;
   const lines = raw.split(/\r?\n/);
@@ -24,7 +25,7 @@ export function truncateErrorMessage(raw: string | null | undefined, verbose?: b
   return msgLines.join("\n");
 }
 
-export function parseBodySafe(raw: string | null | undefined): unknown {
+function parseBodySafe(raw: string | null | undefined): unknown {
   if (!raw) return undefined;
   const truncated = raw.length > 2000 ? raw.slice(0, 2000) + "\u2026[truncated]" : raw;
   try {
@@ -40,7 +41,33 @@ const USEFUL_HEADERS = new Set([
 ]);
 const USEFUL_PREFIXES = ["x-", "ratelimit"];
 
-export function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
+/** ARV-103 (F8): true when at least one assertion on the failing step is
+ *  a schema-validation kind. `--validate-schema` annotates each violated
+ *  field with `kind: "schema"` (set in src/core/runner/schema-validator.ts).
+ *  The assertions column is stored as JSON in SQLite; parse defensively. */
+function hasSchemaAssertion(raw: string | unknown[] | null | undefined): boolean {
+  if (raw === null || raw === undefined) return false;
+  let arr: unknown[];
+  if (Array.isArray(raw)) {
+    arr = raw;
+  } else if (typeof raw === "string") {
+    try {
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed)) return false;
+      arr = parsed;
+    } catch {
+      return false;
+    }
+  } else {
+    return false;
+  }
+  for (const a of arr) {
+    if (a && typeof a === "object" && (a as { kind?: unknown }).kind === "schema") return true;
+  }
+  return false;
+}
+
+function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
   if (!raw) return undefined;
   try {
     const h = JSON.parse(raw) as Record<string, string>;
@@ -145,9 +172,13 @@ export interface DiagnoseResult {
     network_errors: number;
   };
   agent_directive?: string;
-  env_issue?: string;
+  env_issue?: EnvIssue;
   auth_hint?: string;
   cascade_skips?: CascadeSkipGroup[];
+  /** TASK-29: actionable suggestions populated from 404 placeholder
+   *  detection + .env.yaml unfilled-key audit. Empty / undefined when
+   *  nothing actionable was found. */
+  suggested_fixes?: SuggestedFix[];
   failures: Array<{
     suite_name: string;
     test_name: string;
@@ -165,8 +196,23 @@ export interface DiagnoseResult {
     response_headers?: Record<string, string>;
     assertions: unknown;
     duration_ms: number | null;
+    /** ARV-159: when this entry is the representative of a collapsed group
+     *  (status|failure_type signature), the total size of that group. Lets
+     *  consumers reading `.data.failures[]` see "this signature stands for
+     *  N underlying tests" without cross-referencing `.grouped_failures[]`.
+     *  Omitted when no collapsing occurred (failures ≤ 5 or
+     *  --verbose). */
+    group_count?: number;
   }>;
   grouped_failures?: FailureGroup[];
+  /** ARV-101 (F6): top-level aggregation keyed by `recommended_action`
+   *  enum so triage agents (zond-triage skill) can route on the canonical
+   *  action without re-folding `failures[].recommended_action` through
+   *  `jq | group_by`. Built from the *full* failure set (not the compact
+   *  subset), so counts match `.summary.failed`. Each bucket carries
+   *  total count + a small examples list (`<suite>/<test>`). Empty when
+   *  there are no failures. */
+  by_recommended_action?: Record<string, { count: number; examples: string[] }>;
 }
 
 export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, maxExamples?: number): DiagnoseResult {
@@ -191,7 +237,16 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
         softDeleteHint(r.response_status, r.request_method, parsedBody) ??
         statusHint(r.response_status);
       const failure_type = classifyFailure(r.status, r.response_status);
-      const rec_action = recommendedAction(failure_type, r.response_status);
+      // ARV-42: generator-emitted suites should not route to fix_test_logic —
+      // editing the YAML gets clobbered on the next `zond audit`.
+      const generated = isGeneratedTest(r.provenance, r.suite_file);
+      // ARV-103 (F8): walk the assertions array to detect a schema-kind
+      // failure (--validate-schema annotates each assertion with its kind).
+      // When present, propagate the flag so the classifier routes to
+      // report_backend_bug — schema violations are real contract bugs, not
+      // test-logic mistakes.
+      const schema_violation = hasSchemaAssertion(r.assertions);
+      const rec_action = recommendedActionForGenerated(failure_type, r.response_status, generated, schema_violation);
       const sHint = schemaHint(failure_type, r.response_status);
       return {
         suite_name: r.suite_name,
@@ -213,7 +268,56 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
       };
     });
 
-  const sharedEnvHint = computeSharedEnvIssue(failures, envFilePath);
+  // TASK-70 + TASK-98 — env_issue detector.
+  //
+  // Two passes:
+  //   1. Run-level: if every non-5xx failure shares a single env-category,
+  //      treat it as a global env_issue (legacy TASK-70 behaviour). This
+  //      catches the most common case — base_url unset, every test broken.
+  //   2. Suite-level clustering: group failures by suite, flag each suite
+  //      whose non-5xx failures are ≥80% env-symptomatic (TASK-98). Catches
+  //      per-suite missing variables, expired auth tokens, dead webhook
+  //      hosts — situations where the run is *mixed* but a specific suite
+  //      is clearly env-broken.
+  //
+  // The fix_env override only applies to failures inside an affected suite.
+  // 5xx (api_error) is excluded everywhere — backend bugs stay
+  // report_backend_bug regardless of env state.
+  let env_issue: EnvIssue | undefined;
+  const legacyEnvHint = computeSharedEnvIssue(failures, envFilePath);
+  const clusters = clusterEnvIssues(failures);
+  const built = buildEnvIssue(clusters, envFilePath);
+
+  let affectedSuites: Set<string>;
+  if (built) {
+    env_issue = built;
+    affectedSuites = new Set(built.affected_suites);
+  } else if (legacyEnvHint) {
+    // Legacy global env_issue (no clustered match — e.g. only one failure,
+    // or every suite has a single failing test). Preserve the original
+    // single-message form but expose it via the new envelope shape so
+    // downstream consumers see one stable contract.
+    const allSuites = [...new Set(failures.filter(f => f.failure_type !== "api_error").map(f => f.suite_name))].sort();
+    env_issue = {
+      message: legacyEnvHint,
+      scope: "run",
+      affected_suites: allSuites,
+      symptoms: {},
+    };
+    affectedSuites = new Set(allSuites);
+  } else {
+    affectedSuites = new Set();
+  }
+
+  if (env_issue) {
+    for (const f of failures) {
+      if (f.failure_type === "api_error") continue; // real backend bug — keep
+      if (!affectedSuites.has(f.suite_name)) continue; // out-of-scope suite
+      f.recommended_action = "fix_env";
+      delete f.hint;
+      delete f.schema_hint;
+    }
+  }
 
   let apiErrors = 0, assertionFailures = 0, networkErrors = 0;
   let authFailureCount = 0;
@@ -274,6 +378,37 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
     ? { grouped_failures: undefined, compactFailures: failures }
     : groupFailures(failures, maxExamples);
 
+  // TASK-29: surface placeholder path-params + unfilled .env.yaml keys.
+  const suggestedFixes = buildSuggestedFixes({
+    failures: failures.map(f => ({
+      response_status: f.response_status,
+      request_url: f.request_url,
+      suite_name: f.suite_name,
+      test_name: f.test_name,
+    })),
+    envFilePath,
+  });
+
+  // ARV-101 (F6): aggregate failures by recommended_action enum so triage
+  // agents read .data.by_recommended_action.fix_env.count instead of
+  // re-folding failures[].recommended_action through `jq | group_by`. Built
+  // from the full failure set (not compactFailures) so counts match
+  // .summary.failed. Bounded examples list (5) keeps payload small while
+  // still pointing at concrete suites the agent can open.
+  const by_recommended_action: Record<string, { count: number; examples: string[] }> = {};
+  for (const f of failures) {
+    const key = f.recommended_action;
+    let bucket = by_recommended_action[key];
+    if (!bucket) {
+      bucket = { count: 0, examples: [] };
+      by_recommended_action[key] = bucket;
+    }
+    bucket.count += 1;
+    if (bucket.examples.length < 5) {
+      bucket.examples.push(`${f.suite_name}/${f.test_name}`);
+    }
+  }
+
   return {
     run: {
       id: diagRun.id,
@@ -290,15 +425,17 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, m
       network_errors: networkErrors,
     },
     ...(agent_directive ? { agent_directive } : {}),
-    ...(sharedEnvHint ? { env_issue: sharedEnvHint } : {}),
+    ...(env_issue ? { env_issue } : {}),
     ...(auth_hint ? { auth_hint } : {}),
     ...(cascade_skips ? { cascade_skips } : {}),
+    ...(suggestedFixes.length > 0 ? { suggested_fixes: suggestedFixes } : {}),
     failures: compactFailures,
     ...(grouped_failures ? { grouped_failures } : {}),
+    ...(failures.length > 0 ? { by_recommended_action } : {}),
   };
 }
 
-type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; hint?: string; response_status: number | null };
+type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; hint?: string; response_status: number | null; group_count?: number };
 
 /** Group similar failures for compact output. Exported for testing. */
 export function groupFailures<T extends FailureItem>(failures: T[], maxExamples = 2): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
@@ -352,7 +489,10 @@ export function groupFailures<T extends FailureItem>(failures: T[], maxExamples
     if (isApiError) {
       compactFailures.push(...group.items);
     } else {
-      compactFailures.push(group.items[0]!);
+      // ARV-159: tag the representative with the group size so
+      // `.data.failures[]` carries the multiplier inline.
+      const rep = { ...group.items[0]!, group_count: group.items.length };
+      compactFailures.push(rep as T);
     }
   }
 

package/src/core/diagnostics/failure-class.ts

@@ -0,0 +1,120 @@
+/**
+ * TASK-101: failure classification — definitely_bug / likely_bug / quirk / env_issue.
+ *
+ * Goal: бэкендер за секунду видит «реально баг» vs «quirk зонда / probe
+ * фолс-позитив». Чисто read-only классификация: не меняет статус step,
+ * не влияет на pass/fail. Только tag-овая аналитика.
+ */
+
+import type { StepResult, AssertionResult } from "../runner/types.ts";
+import type { SourceMetadata } from "../parser/types.ts";
+
+export type FailureClass =
+  | "definitely_bug"
+  | "likely_bug"
+  | "quirk"
+  | "env_issue"
+  /** Step was skipped because an upstream step failed to produce a required
+   *  capture (or produced a tainted one). Not a failure on its own — render
+   *  collapsed under the root failure to avoid drowning the user. */
+  | "cascade";
+
+export interface FailureClassification {
+  failure_class: FailureClass;
+  failure_class_reason: string;
+}
+
+function failedAssertionsOf(result: StepResult): AssertionResult[] {
+  return result.assertions.filter((a) => !a.passed);
+}
+
+function ruleStartsWith(a: AssertionResult, prefix: string): boolean {
+  return typeof a.rule === "string" && a.rule.startsWith(prefix);
+}
+
+function expectedStatusList(a: AssertionResult): number[] | null {
+  if (Array.isArray(a.expected)) {
+    const arr = a.expected.filter((v): v is number => typeof v === "number");
+    return arr.length > 0 ? arr : null;
+  }
+  return typeof a.expected === "number" ? [a.expected] : null;
+}
+
+/**
+ * Classify a failed step. Returns `null` for pass/skip/unclassifiable failures —
+ * UI renders those as "unclassified" rather than crashing.
+ */
+export function classifyFailure(result: StepResult): FailureClassification | null {
+  if (result.status === "pass" || result.status === "skip") return null;
+
+  // Network/runtime error before any HTTP response → env-side
+  if (result.status === "error") {
+    return {
+      failure_class: "env_issue",
+      failure_class_reason: result.error ?? "request failed before producing a response",
+    };
+  }
+
+  const respStatus = result.response?.status;
+  const provenance: SourceMetadata | null | undefined = result.provenance;
+  const generator = typeof provenance?.generator === "string" ? provenance.generator : undefined;
+  const failed = failedAssertionsOf(result);
+
+  // 1. Backend 5xx — always a backend bug regardless of test intent.
+  if (typeof respStatus === "number" && respStatus >= 500) {
+    return {
+      failure_class: "definitely_bug",
+      failure_class_reason: `API returned ${respStatus} — server-side error`,
+    };
+  }
+
+  // 2. Response did not match its OpenAPI schema — spec guarantees X, server returned ≠ X.
+  const schemaFail = failed.find((a) => ruleStartsWith(a, "schema."));
+  if (schemaFail) {
+    return {
+      failure_class: "definitely_bug",
+      failure_class_reason: `Response violates OpenAPI schema at ${schemaFail.field}`,
+    };
+  }
+
+  // 3. Mass-assignment probe: extras must not apply. A failed not_equals
+  //    assertion on this generator means the sentinel value leaked through.
+  if (generator === "mass-assignment-probe") {
+    const extrasLeak = failed.find(
+      (a) => ruleStartsWith(a, "not_equals") || ruleStartsWith(a, "set_equals"),
+    );
+    if (extrasLeak) {
+      return {
+        failure_class: "definitely_bug",
+        failure_class_reason: `Mass-assignment: client-supplied extras were accepted (${extrasLeak.field})`,
+      };
+    }
+  }
+
+  // 4. Negative-probe family — distinguish "API accepted bad input" (likely_bug)
+  //    from "API rejected with a different 4xx" (quirk).
+  if (generator === "negative-probe" || generator === "method-probe") {
+    const statusFail = failed.find((a) => a.field === "status");
+    if (statusFail && typeof respStatus === "number") {
+      const expected = expectedStatusList(statusFail);
+      const allExpected4xx = expected?.every((s) => s >= 400 && s < 500) ?? false;
+      if (allExpected4xx) {
+        if (respStatus >= 200 && respStatus < 300) {
+          return {
+            failure_class: "likely_bug",
+            failure_class_reason: `Negative probe expected 4xx, got ${respStatus} — API accepts invalid input`,
+          };
+        }
+        if (respStatus >= 400 && respStatus < 500) {
+          return {
+            failure_class: "quirk",
+            failure_class_reason: `Negative probe expected ${expected!.join("/")}, got ${respStatus} — different 4xx code`,
+          };
+        }
+      }
+    }
+  }
+
+  // Default: leave unclassified. UI / CLI render as "unclassified".
+  return null;
+}