@kirrosh/zond 0.21.0 → 0.23.0

package/src/core/diagnostics/failure-hints.ts

@@ -3,6 +3,8 @@
  * Extracted from query-db.ts for reuse in Web UI.
  */
 
+import { classify } from "../classifier/recommended-action.ts";
+
 export function statusHint(status: number | null | undefined): string | null {
   if (!status) return null;
   if (status >= 500) return "Server-side error — inspect response_body for errorMessage/errorDetail; likely a backend bug";
@@ -41,20 +43,98 @@ export type RecommendedAction =
   | "report_backend_bug"
   | "fix_auth_config"
   | "fix_test_logic"
-  | "fix_network_config";
+  | "fix_network_config"
+  | "fix_env"
+  /** Fix the OpenAPI spec — emitted by lint-spec.Issue (TASK-294) and
+   *  by `status_code_conformance` / `*_conformance` checks (ARV-11). */
+  | "fix_spec"
+  /** Add or correct a fixture in .env.yaml — emitted by discover for
+   *  miss-* states (TASK-294). */
+  | "fix_fixture"
+  /** ARV-42 — generator-emitted suite produced a body the API rejected
+   *  (4xx with validation hint). Editing the YAML is wrong: the next
+   *  `zond generate` would clobber it. Re-run generate (or refine the
+   *  spec/.api-resources hints) instead. */
+  | "regenerate_suite"
+  /** ARV-11 — server accepted an invalid request body. Backend should
+   *  reject earlier; the test isn't wrong. */
+  | "tighten_validation"
+  /** ARV-11 — server didn't enforce a header marked `required: true`
+   *  in the spec. Either enforce it, or drop `required` in the spec. */
+  | "add_required_header"
+  /** ARV-11 — known limitation that the team has accepted. Agents
+   *  should not retry, file a bug, or include in dashboards. */
+  | "wontfix_known_limitation";
 
 export function recommendedAction(
   failureType: "api_error" | "assertion_failed" | "network_error",
   responseStatus: number | null,
 ): RecommendedAction {
-  if (failureType === "api_error") return "report_backend_bug";
-  if (failureType === "network_error") {
-    if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
-    return "fix_network_config";
-  }
-  // assertion_failed
-  if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
-  return "fix_test_logic";
+  // ARV-56: delegate to the single classifier.
+  const action = classify({
+    finding_class: failureType === "api_error" ? "test:api_error" :
+      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
+    status: responseStatus,
+  });
+  // The three failure_type classes are total in the classifier — a missing
+  // branch means a future refactor stripped one; surface loudly.
+  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
+  return action;
+}
+
+/**
+ * ARV-42: extended recommender that knows whether the failing test was
+ * emitted by `zond generate`. For generated suites, "fix_test_logic" is
+ * actively misleading — the generated YAML carries the header
+ * "⚠️ Edits will be overwritten on regenerate" and the next `zond audit`
+ * really does clobber manual edits. Branch into the actually-actionable
+ * remediation instead.
+ *
+ *  - 4xx (400/422) → regenerate_suite: the body the generator emitted
+ *    didn't pass validation; either re-run generate (so newer heuristics
+ *    apply, e.g. ARV-38 default-string) or tighten .api-resources hints.
+ *  - 404 → fix_fixture: a path-param resolved to an empty / stale id
+ *    in .env.yaml; `prepare-fixtures --seed` is the correct remedy.
+ *  - everything else → falls back to recommendedAction (auth → 401/403,
+ *    api_error → 5xx, etc.).
+ *
+ * `isGenerated` is the heuristic from db-analysis: provenance.type
+ * "openapi-generated" OR suite_file under apis/<api>/tests/.
+ */
+export function recommendedActionForGenerated(
+  failureType: "api_error" | "assertion_failed" | "network_error",
+  responseStatus: number | null,
+  isGenerated: boolean,
+  schemaViolation = false,
+): RecommendedAction {
+  // ARV-56: delegate to classifier. `isGenerated` is encoded via a
+  // synthetic suite_path so the same logic flows through classify().
+  // ARV-103 (F8): `schemaViolation` propagates the assertion-kind flag —
+  // when true, the classifier's "treat schema bugs like 5xx" branch wins
+  // over the generator's regenerate_suite default.
+  const action = classify({
+    finding_class: failureType === "api_error" ? "test:api_error" :
+      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
+    status: responseStatus,
+    ...(isGenerated ? { suite_path: "apis/_/tests/_.yaml" } : {}),
+    ...(schemaViolation ? { schema_violation: true } : {}),
+  });
+  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
+  return action;
+}
+
+/** ARV-42: classify a failing result row as generator-emitted. The two
+ *  signals are independent — provenance is missing on older runs, while
+ *  suite_file disambiguates against ad-hoc YAMLs the user dropped into
+ *  apis/<api>/tests/ themselves (rare but supported). */
+export function isGeneratedTest(
+  provenance: { type?: string; generator?: string } | null | undefined,
+  suite_file: string | null | undefined,
+): boolean {
+  if (provenance?.type === "openapi-generated") return true;
+  if (provenance?.generator && provenance.generator.toLowerCase().includes("zond")) return true;
+  if (typeof suite_file === "string" && /(^|\/)apis\/[^/]+\/tests\//.test(suite_file)) return true;
+  return false;
 }
 
 export function envCategory(hint: string | undefined): string | null {
@@ -111,3 +191,126 @@ export function computeSharedEnvIssue(
   // url_malformed
   return [...failures.map(f => f.hint).filter(Boolean)][0] ?? null;
 }
+
+// ── TASK-98: per-suite env clustering ──────────────────────────────────────
+//
+// Round-3 review showed that the all-or-nothing run-level detector misses
+// real env_issue scenarios: a single suite needs `{{stripe_key}}`, a webhook
+// host is unreachable for one suite only, an auth token expires part-way
+// through. Cluster classification — group by suite, flag a suite when ≥80%
+// of its non-5xx failures share an env-symptom — closes that gap without
+// laundering 5xx (real backend bugs) into env_issue.
+export type EnvSymptom = "missing_var" | "base_url" | "url_malformed" | "auth_expired";
+
+function envSymptomOf(failure: {
+  hint?: string;
+  failure_type: string;
+  response_status: number | null;
+}): EnvSymptom | null {
+  if (failure.failure_type === "api_error") return null; // 5xx never counted
+  const cat = envCategory(failure.hint);
+  if (cat === "unresolved_variable") return "missing_var";
+  if (cat === "base_url_missing") return "base_url";
+  if (cat === "url_malformed") return "url_malformed";
+  if (failure.response_status === 401 || failure.response_status === 403) return "auth_expired";
+  return null;
+}
+
+export interface EnvIssue {
+  /** Human-readable summary; used by reporters and shown to the user. */
+  message: string;
+  /** "run" when the issue spans most/all suites; "suite:<name>" when localized. */
+  scope: "run" | `suite:${string}`;
+  /** Suites the env_issue covers — one entry for suite scope, ≥2 for run scope. */
+  affected_suites: string[];
+  /** Histogram of root-cause symptoms across affected failures. */
+  symptoms: Partial<Record<EnvSymptom, number>>;
+}
+
+/**
+ * Cluster non-5xx failures by suite and return per-suite env clusters that
+ * meet the env-symptom threshold (default ≥80% AND ≥2 failures). 5xx are
+ * excluded so backend bugs cannot be reclassified as env issues.
+ */
+export function clusterEnvIssues(
+  failures: Array<{
+    suite_name: string;
+    hint?: string;
+    failure_type: string;
+    response_status: number | null;
+  }>,
+  threshold = 0.8,
+): Array<{ suite: string; symptoms: Partial<Record<EnvSymptom, number>>; total: number }> {
+  const bySuite = new Map<string, typeof failures>();
+  for (const f of failures) {
+    if (f.failure_type === "api_error") continue;
+    const list = bySuite.get(f.suite_name) ?? [];
+    list.push(f);
+    bySuite.set(f.suite_name, list);
+  }
+  const clusters: Array<{ suite: string; symptoms: Partial<Record<EnvSymptom, number>>; total: number }> = [];
+  for (const [suite, items] of bySuite) {
+    if (items.length === 0) continue;
+    const symptoms: Partial<Record<EnvSymptom, number>> = {};
+    let envCount = 0;
+    for (const f of items) {
+      const s = envSymptomOf(f);
+      if (s) {
+        symptoms[s] = (symptoms[s] ?? 0) + 1;
+        envCount++;
+      }
+    }
+    if (envCount / items.length >= threshold && envCount >= 1) {
+      clusters.push({ suite, symptoms, total: items.length });
+    }
+  }
+  return clusters;
+}
+
+function formatSymptoms(symptoms: Partial<Record<EnvSymptom, number>>): string {
+  const parts: string[] = [];
+  for (const k of ["missing_var", "base_url", "url_malformed", "auth_expired"] as EnvSymptom[]) {
+    const n = symptoms[k];
+    if (n) parts.push(`${k}=${n}`);
+  }
+  return parts.join(", ");
+}
+
+/**
+ * Build an EnvIssue envelope from clustered failures. Returns null when no
+ * cluster exceeded the threshold. When exactly one suite is affected, scope
+ * is `suite:<name>`; ≥2 suites collapse into a `run` scope aggregator.
+ */
+export function buildEnvIssue(
+  clusters: Array<{ suite: string; symptoms: Partial<Record<EnvSymptom, number>>; total: number }>,
+  envFilePath?: string,
+): EnvIssue | null {
+  if (clusters.length === 0) return null;
+  const envFile = envFilePath ?? ".env.yaml";
+
+  const merged: Partial<Record<EnvSymptom, number>> = {};
+  for (const c of clusters) {
+    for (const [k, v] of Object.entries(c.symptoms)) {
+      merged[k as EnvSymptom] = (merged[k as EnvSymptom] ?? 0) + (v ?? 0);
+    }
+  }
+  const affected_suites = clusters.map(c => c.suite).sort();
+
+  if (clusters.length === 1) {
+    const c = clusters[0]!;
+    const breakdown = formatSymptoms(c.symptoms);
+    return {
+      message: `Suite "${c.suite}" looks env-broken (${breakdown}) — check ${envFile}`,
+      scope: `suite:${c.suite}`,
+      affected_suites,
+      symptoms: merged,
+    };
+  }
+  const breakdown = formatSymptoms(merged);
+  return {
+    message: `${clusters.length} suites look env-broken (${breakdown}) — check ${envFile}`,
+    scope: "run",
+    affected_suites,
+    symptoms: merged,
+  };
+}

package/src/core/diagnostics/spec-pointer.ts

@@ -0,0 +1,99 @@
+/**
+ * TASK-102: build a JSON Pointer (RFC 6901) into the OpenAPI document for the
+ * response branch a step exercises, plus a small excerpt of the schema at that
+ * pointer. The excerpt is captured at run time and frozen into the DB so that
+ * later spec edits can't rewrite history.
+ *
+ * Inputs come from {@link SourceMetadata} populated by TASK-100:
+ *   - `endpoint`         e.g. "POST /webhooks"
+ *   - `response_branch`  e.g. "422" or "400|422" (first wins for the pointer)
+ */
+
+import type { SourceMetadata } from "../parser/types.ts";
+
+export interface SpecPointer {
+  pointer: string;
+  excerpt: string;
+}
+
+const EXCERPT_MAX_BYTES = 500;
+
+function escapeJsonPointerSegment(s: string): string {
+  return s.replace(/~/g, "~0").replace(/\//g, "~1");
+}
+
+function parseEndpoint(endpoint: string): { method: string; path: string } | null {
+  const m = endpoint.match(/^([A-Z]+)\s+(\/.*)$/);
+  if (!m) return null;
+  return { method: m[1]!.toLowerCase(), path: m[2]! };
+}
+
+function pickPrimaryStatus(responseBranch: string | undefined): string | null {
+  if (!responseBranch) return null;
+  const first = responseBranch.split(/[|,\s]/).find((s) => /^\d{3}$/.test(s));
+  return first ?? null;
+}
+
+function trimExcerpt(json: string): string {
+  if (json.length <= EXCERPT_MAX_BYTES) return json;
+  return json.slice(0, EXCERPT_MAX_BYTES) + "\n…[truncated]";
+}
+
+/**
+ * Resolve the response operation in the OpenAPI document and produce a pointer
+ * + frozen excerpt. Returns `null` if any link in the chain is missing — caller
+ * persists `null` rather than crashing.
+ */
+export function buildSpecPointer(
+  source: SourceMetadata | null | undefined,
+  openApiDoc: unknown,
+): SpecPointer | null {
+  if (!source || !openApiDoc || typeof openApiDoc !== "object") return null;
+  if (typeof source.endpoint !== "string") return null;
+
+  const parsed = parseEndpoint(source.endpoint);
+  if (!parsed) return null;
+  const status = pickPrimaryStatus(source.response_branch ?? undefined);
+  if (!status) return null;
+
+  const doc = openApiDoc as Record<string, unknown>;
+  const paths = doc.paths as Record<string, unknown> | undefined;
+  if (!paths || typeof paths !== "object") return null;
+
+  const pathItem = paths[parsed.path] as Record<string, unknown> | undefined;
+  if (!pathItem || typeof pathItem !== "object") return null;
+
+  const operation = pathItem[parsed.method] as Record<string, unknown> | undefined;
+  if (!operation || typeof operation !== "object") return null;
+
+  const responses = operation.responses as Record<string, unknown> | undefined;
+  if (!responses || typeof responses !== "object") return null;
+
+  const response = (responses[status] ?? responses.default) as Record<string, unknown> | undefined;
+  if (!response || typeof response !== "object") return null;
+
+  const escapedPath = escapeJsonPointerSegment(parsed.path);
+  let pointer = `#/paths/${escapedPath}/${parsed.method}/responses/${status}`;
+  let excerptValue: unknown = response;
+
+  // Drill into application/json schema when available — that's the most useful
+  // surface for UI rendering ("backend promised X, returned Y").
+  const content = response.content as Record<string, unknown> | undefined;
+  if (content && typeof content === "object") {
+    const jsonMedia = (content["application/json"] ?? content["application/json; charset=utf-8"]) as
+      | Record<string, unknown>
+      | undefined;
+    if (jsonMedia && typeof jsonMedia === "object" && "schema" in jsonMedia) {
+      pointer += "/content/application~1json/schema";
+      excerptValue = jsonMedia.schema;
+    }
+  }
+
+  let excerpt: string;
+  try {
+    excerpt = JSON.stringify(excerptValue, null, 2);
+  } catch {
+    excerpt = String(excerptValue);
+  }
+  return { pointer, excerpt: trimExcerpt(excerpt) };
+}

package/src/core/diagnostics/suggested-fixes.ts

@@ -0,0 +1,156 @@
+/**
+ * TASK-29: actionable "Suggested fixes" surfaces for `zond db diagnose`.
+ *
+ * The base diagnose envelope already classifies failures and wires
+ * agent_directive / recommended_action / env_issue. This layer adds two
+ * concrete, fixable signals that the LLM agent can act on without a second
+ * round-trip:
+ *
+ *   1. Placeholder path-params on 404s — when a 404 hits a URL that still
+ *      contains literal example/placeholder values (`example`, all-zeros
+ *      UUID, `your-…-here`, `00000000-…`, …), surface the exact path
+ *      segment that's broken plus the variable name we'd expect (best-effort
+ *      from the segment shape).
+ *
+ *   2. Untilled .env.yaml keys — read the API's .env.yaml and flag values
+ *      that are empty, `<TODO>`, `example`, `null`, or look like
+ *      placeholders (`replace-me`, `your-...`). These block CRUD sweeps
+ *      silently — without them, the agent can't tell apart "value missing"
+ *      from "value present and wrong".
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { parse as parseYaml } from "yaml";
+
+export type SuggestedFixKind =
+  | "placeholder_path_param"
+  | "unfilled_env_key";
+
+export interface SuggestedFix {
+  kind: SuggestedFixKind;
+  /** When `kind=unfilled_env_key`: the key name. When `kind=placeholder_path_param`:
+   *  the path segment that looks like a placeholder. */
+  key: string;
+  message: string;
+  /** File path the user should edit (usually the API's .env.yaml). */
+  source?: string;
+  /** Optional one-line example of what to put there. */
+  example?: string;
+}
+
+const PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
+  { re: /^example$/i, reason: "literal `example`" },
+  { re: /^placeholder$/i, reason: "literal `placeholder`" },
+  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
+  { re: /^00000000-0000-0000-0000-0+(?:beef|dead|cafe|f00d|0+)$/i, reason: "all-zero / sentinel UUID" },
+  { re: /^0+$/, reason: "all-zero numeric id" },
+  { re: /^replace-me$/i, reason: "`replace-me` placeholder" },
+];
+
+const ENV_PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
+  { re: /^<.*>$/, reason: "TODO / angle-bracket placeholder" },
+  { re: /^example$/i, reason: "literal `example`" },
+  { re: /^placeholder$/i, reason: "literal `placeholder`" },
+  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
+  { re: /^replace-?me$/i, reason: "`replace-me` placeholder" },
+  { re: /^<TODO>$/i, reason: "explicit TODO" },
+];
+
+/** Identify path segments that look like placeholders. Used on 404 URLs. */
+export function detectPlaceholderSegments(url: string | null): Array<{ segment: string; reason: string }> {
+  if (!url) return [];
+  let pathname: string;
+  try {
+    pathname = url.startsWith("http") ? new URL(url).pathname : url.split("?")[0]!;
+  } catch {
+    pathname = url;
+  }
+  const out: Array<{ segment: string; reason: string }> = [];
+  for (const seg of pathname.split("/").filter(Boolean)) {
+    for (const { re, reason } of PLACEHOLDER_PATTERNS) {
+      if (re.test(seg)) {
+        out.push({ segment: seg, reason });
+        break;
+      }
+    }
+  }
+  return out;
+}
+
+/** Read .env.yaml and return keys whose value is empty/null/placeholder-shaped. */
+export function findUnfilledEnvKeys(envFilePath: string | undefined): SuggestedFix[] {
+  if (!envFilePath || !existsSync(envFilePath)) return [];
+  let parsed: unknown;
+  try {
+    parsed = parseYaml(readFileSync(envFilePath, "utf-8"));
+  } catch {
+    return [];
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
+
+  const out: SuggestedFix[] = [];
+  for (const [key, val] of Object.entries(parsed as Record<string, unknown>)) {
+    const reason = classifyEnvValue(val);
+    if (reason) {
+      out.push({
+        kind: "unfilled_env_key",
+        key,
+        message: `\`${key}\` in ${envFilePath} is unfilled (${reason}). Set it to a real value before re-running.`,
+        source: envFilePath,
+      });
+    }
+  }
+  return out;
+}
+
+function classifyEnvValue(val: unknown): string | null {
+  if (val === null || val === undefined) return "null / missing";
+  if (typeof val === "string") {
+    const trimmed = val.trim();
+    if (trimmed === "") return "empty string";
+    for (const { re, reason } of ENV_PLACEHOLDER_PATTERNS) {
+      if (re.test(trimmed)) return reason;
+    }
+  }
+  return null;
+}
+
+export interface BuildSuggestedFixesInput {
+  failures: Array<{
+    response_status: number | null;
+    request_url: string | null;
+    suite_name: string;
+    test_name: string;
+  }>;
+  envFilePath?: string;
+}
+
+/**
+ * Combine placeholder-detection across all 404 failures + env-yaml audit
+ * into a single deduplicated list of fixes the agent should apply before
+ * re-running.
+ */
+export function buildSuggestedFixes(input: BuildSuggestedFixesInput): SuggestedFix[] {
+  const fixes: SuggestedFix[] = [];
+  const seenSegments = new Set<string>();
+
+  for (const f of input.failures) {
+    if (f.response_status !== 404) continue;
+    for (const ph of detectPlaceholderSegments(f.request_url)) {
+      const dedupeKey = `seg:${ph.segment}`;
+      if (seenSegments.has(dedupeKey)) continue;
+      seenSegments.add(dedupeKey);
+      fixes.push({
+        kind: "placeholder_path_param",
+        key: ph.segment,
+        message:
+          `404 on \`${f.request_url}\` — path segment \`${ph.segment}\` is a ${ph.reason}. ` +
+          `Replace with a real id from the live API (e.g. \`zond prepare-fixtures --apply\`) ` +
+          `or set the corresponding fixture in ${input.envFilePath ?? ".env.yaml"}.`,
+        source: input.envFilePath,
+      });
+    }
+  }
+
+  fixes.push(...findUnfilledEnvKeys(input.envFilePath));
+  return fixes;
+}