@kirrosh/zond 0.22.0 → 0.26.0

package/src/core/diagnostics/failure-hints.ts

@@ -1,17 +1,11 @@
 /**
- * Failure classification and diagnostic hints.
- * Extracted from query-db.ts for reuse in Web UI.
+ * Failure classification → closed-enum `recommended_action`.
+ * ARV-338: prose hints (statusHint/envHint/softDeleteHint/schemaHint,
+ * env_issue clustering, auth_hint, agent_directive) removed — the agent
+ * triages by enum + raw evidence, not by heuristic guess-text.
  */
 
-export function statusHint(status: number | null | undefined): string | null {
-  if (!status) return null;
-  if (status >= 500) return "Server-side error — inspect response_body for errorMessage/errorDetail; likely a backend bug";
-  if (status === 401 || status === 403) return "Auth failure — check auth_token/api_key in .env.yaml";
-  if (status === 404) return "Resource not found — verify the path and ID";
-  if (status === 400 || status === 422) return "Validation error — check request body fields match the schema";
-  if (status === 429) return "Rate limited — too many requests. Consider consolidating auth/login steps or adding delays between suites";
-  return null;
-}
+import { classify } from "../classifier/recommended-action.ts";
 
 export function classifyFailure(status: string, responseStatus: number | null): "api_error" | "assertion_failed" | "network_error" {
   if (status === "error" && (responseStatus === null || responseStatus < 500)) return "network_error";
@@ -19,95 +13,100 @@ export function classifyFailure(status: string, responseStatus: number | null):
   return "assertion_failed";
 }
 
-export function envHint(url: string | null, errorMessage: string | null, envFilePath?: string): string | null {
-  const envFile = envFilePath ? envFilePath : ".env.yaml in your API directory";
-
-  if (url && /\{\{[^}]+\}\}/.test(url)) {
-    return `URL contains unresolved variable: "${url}" — variable name may not match the key in ${envFile}`;
-  }
-  if (url && !url.startsWith("http://") && !url.startsWith("https://")) {
-    return `base_url is not set or empty — URL resolved to "${url}". Add base_url to ${envFile}`;
-  }
-  if (errorMessage?.includes("base_url is not configured")) {
-    return `base_url is missing or empty. Add base_url: https://your-api.com to ${envFile}`;
-  }
-  if (errorMessage?.includes("URL is invalid") || errorMessage?.includes("Failed to parse URL")) {
-    return `URL is malformed — likely base_url is empty or invalid. Check base_url in ${envFile}`;
-  }
-  return null;
-}
-
 export type RecommendedAction =
   | "report_backend_bug"
   | "fix_auth_config"
   | "fix_test_logic"
-  | "fix_network_config";
+  | "fix_network_config"
+  | "fix_env"
+  /** Fix the OpenAPI spec — emitted by lint-spec.Issue (TASK-294) and
+   *  by `status_code_conformance` / `*_conformance` checks (ARV-11). */
+  | "fix_spec"
+  /** Add or correct a fixture in .env.yaml — emitted by discover for
+   *  miss-* states (TASK-294). */
+  | "fix_fixture"
+  /** ARV-42 — generator-emitted suite produced a body the API rejected
+   *  (4xx with validation hint). Editing the YAML is wrong: the next
+   *  `zond generate` would clobber it. Re-run generate (or refine the
+   *  spec/.api-resources hints) instead. */
+  | "regenerate_suite"
+  /** ARV-11 — server accepted an invalid request body. Backend should
+   *  reject earlier; the test isn't wrong. */
+  | "tighten_validation"
+  /** ARV-11 — server didn't enforce a header marked `required: true`
+   *  in the spec. Either enforce it, or drop `required` in the spec. */
+  | "add_required_header"
+  /** ARV-11 — known limitation that the team has accepted. Agents
+   *  should not retry, file a bug, or include in dashboards. */
+  | "wontfix_known_limitation";
 
 export function recommendedAction(
   failureType: "api_error" | "assertion_failed" | "network_error",
   responseStatus: number | null,
 ): RecommendedAction {
-  if (failureType === "api_error") return "report_backend_bug";
-  if (failureType === "network_error") {
-    if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
-    return "fix_network_config";
-  }
-  // assertion_failed
-  if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
-  return "fix_test_logic";
-}
-
-export function envCategory(hint: string | undefined): string | null {
-  if (!hint) return null;
-  if (hint.includes("base_url is not set") || hint.includes("base_url is missing") || hint.includes("base_url is not configured")) return "base_url_missing";
-  if (hint.includes("unresolved variable")) return "unresolved_variable";
-  if (hint.includes("URL is malformed")) return "url_malformed";
-  return null;
-}
-
-export function schemaHint(
-  failureType: string,
-  responseStatus: number | null | undefined,
-): string | null {
-  if (failureType === "assertion_failed" || responseStatus === 400 || responseStatus === 422) {
-    return "Use describe_endpoint(specPath, method, path) to verify expected request/response schema";
-  }
-  return null;
+  // ARV-56: delegate to the single classifier.
+  const action = classify({
+    finding_class: failureType === "api_error" ? "test:api_error" :
+      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
+    status: responseStatus,
+  });
+  // The three failure_type classes are total in the classifier — a missing
+  // branch means a future refactor stripped one; surface loudly.
+  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
+  return action;
 }
 
-export function softDeleteHint(
-  actualStatus: number | null | undefined,
-  requestMethod: string | null | undefined,
-  responseBody: unknown,
-): string | null {
-  if (actualStatus !== 200 || requestMethod?.toUpperCase() !== "GET") return null;
-  if (responseBody && typeof responseBody === "object") {
-    const hasStatusField =
-      "status" in (responseBody as object) ||
-      "state" in (responseBody as object) ||
-      "deleted" in (responseBody as object) ||
-      "is_deleted" in (responseBody as object);
-    if (hasStatusField) {
-      return 'GET returned 200 with a status/state field after DELETE — likely soft delete. Update the test: remove the "Verify deleted → 404" step and instead assert the status field value (e.g. status: "cancelled")';
-    }
-  }
-  return null;
+/**
+ * ARV-42: extended recommender that knows whether the failing test was
+ * emitted by `zond generate`. For generated suites, "fix_test_logic" is
+ * actively misleading — the generated YAML carries the header
+ * "⚠️ Edits will be overwritten on regenerate" and the next `zond audit`
+ * really does clobber manual edits. Branch into the actually-actionable
+ * remediation instead.
+ *
+ *  - 4xx (400/422) → regenerate_suite: the body the generator emitted
+ *    didn't pass validation; either re-run generate (so newer heuristics
+ *    apply, e.g. ARV-38 default-string) or tighten .api-resources hints.
+ *  - 404 → fix_fixture: a path-param resolved to an empty / stale id
+ *    in .env.yaml; `prepare-fixtures --seed` is the correct remedy.
+ *  - everything else → falls back to recommendedAction (auth → 401/403,
+ *    api_error → 5xx, etc.).
+ *
+ * `isGenerated` is the heuristic from db-analysis: provenance.type
+ * "openapi-generated" OR suite_file under apis/<api>/tests/.
+ */
+export function recommendedActionForGenerated(
+  failureType: "api_error" | "assertion_failed" | "network_error",
+  responseStatus: number | null,
+  isGenerated: boolean,
+  schemaViolation = false,
+): RecommendedAction {
+  // ARV-56: delegate to classifier. `isGenerated` is encoded via a
+  // synthetic suite_path so the same logic flows through classify().
+  // ARV-103 (F8): `schemaViolation` propagates the assertion-kind flag —
+  // when true, the classifier's "treat schema bugs like 5xx" branch wins
+  // over the generator's regenerate_suite default.
+  const action = classify({
+    finding_class: failureType === "api_error" ? "test:api_error" :
+      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
+    status: responseStatus,
+    ...(isGenerated ? { suite_path: "apis/_/tests/_.yaml" } : {}),
+    ...(schemaViolation ? { schema_violation: true } : {}),
+  });
+  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
+  return action;
 }
 
-export function computeSharedEnvIssue(
-  failures: Array<{ hint?: string }>,
-  envFilePath?: string,
-): string | null {
-  const categories = new Set(failures.map(f => envCategory(f.hint)).filter(Boolean));
-  if (categories.size !== 1) return null;
-
-  const envFile = envFilePath ?? ".env.yaml";
-  if (categories.has("base_url_missing")) {
-    return `All failures: base_url is not set — add base_url to ${envFile}`;
-  }
-  if (categories.has("unresolved_variable")) {
-    return `All failures: some variables are not substituted — check variable names in ${envFile}`;
-  }
-  // url_malformed
-  return [...failures.map(f => f.hint).filter(Boolean)][0] ?? null;
+/** ARV-42: classify a failing result row as generator-emitted. The two
+ *  signals are independent — provenance is missing on older runs, while
+ *  suite_file disambiguates against ad-hoc YAMLs the user dropped into
+ *  apis/<api>/tests/ themselves (rare but supported). */
+export function isGeneratedTest(
+  provenance: { type?: string; generator?: string } | null | undefined,
+  suite_file: string | null | undefined,
+): boolean {
+  if (provenance?.type === "openapi-generated") return true;
+  if (provenance?.generator && provenance.generator.toLowerCase().includes("zond")) return true;
+  if (typeof suite_file === "string" && /(^|\/)apis\/[^/]+\/tests\//.test(suite_file)) return true;
+  return false;
 }

package/src/core/diagnostics/spec-pointer.ts

@@ -0,0 +1,99 @@
+/**
+ * TASK-102: build a JSON Pointer (RFC 6901) into the OpenAPI document for the
+ * response branch a step exercises, plus a small excerpt of the schema at that
+ * pointer. The excerpt is captured at run time and frozen into the DB so that
+ * later spec edits can't rewrite history.
+ *
+ * Inputs come from {@link SourceMetadata} populated by TASK-100:
+ *   - `endpoint`         e.g. "POST /webhooks"
+ *   - `response_branch`  e.g. "422" or "400|422" (first wins for the pointer)
+ */
+
+import type { SourceMetadata } from "../parser/types.ts";
+
+export interface SpecPointer {
+  pointer: string;
+  excerpt: string;
+}
+
+const EXCERPT_MAX_BYTES = 500;
+
+function escapeJsonPointerSegment(s: string): string {
+  return s.replace(/~/g, "~0").replace(/\//g, "~1");
+}
+
+function parseEndpoint(endpoint: string): { method: string; path: string } | null {
+  const m = endpoint.match(/^([A-Z]+)\s+(\/.*)$/);
+  if (!m) return null;
+  return { method: m[1]!.toLowerCase(), path: m[2]! };
+}
+
+function pickPrimaryStatus(responseBranch: string | undefined): string | null {
+  if (!responseBranch) return null;
+  const first = responseBranch.split(/[|,\s]/).find((s) => /^\d{3}$/.test(s));
+  return first ?? null;
+}
+
+function trimExcerpt(json: string): string {
+  if (json.length <= EXCERPT_MAX_BYTES) return json;
+  return json.slice(0, EXCERPT_MAX_BYTES) + "\n…[truncated]";
+}
+
+/**
+ * Resolve the response operation in the OpenAPI document and produce a pointer
+ * + frozen excerpt. Returns `null` if any link in the chain is missing — caller
+ * persists `null` rather than crashing.
+ */
+export function buildSpecPointer(
+  source: SourceMetadata | null | undefined,
+  openApiDoc: unknown,
+): SpecPointer | null {
+  if (!source || !openApiDoc || typeof openApiDoc !== "object") return null;
+  if (typeof source.endpoint !== "string") return null;
+
+  const parsed = parseEndpoint(source.endpoint);
+  if (!parsed) return null;
+  const status = pickPrimaryStatus(source.response_branch ?? undefined);
+  if (!status) return null;
+
+  const doc = openApiDoc as Record<string, unknown>;
+  const paths = doc.paths as Record<string, unknown> | undefined;
+  if (!paths || typeof paths !== "object") return null;
+
+  const pathItem = paths[parsed.path] as Record<string, unknown> | undefined;
+  if (!pathItem || typeof pathItem !== "object") return null;
+
+  const operation = pathItem[parsed.method] as Record<string, unknown> | undefined;
+  if (!operation || typeof operation !== "object") return null;
+
+  const responses = operation.responses as Record<string, unknown> | undefined;
+  if (!responses || typeof responses !== "object") return null;
+
+  const response = (responses[status] ?? responses.default) as Record<string, unknown> | undefined;
+  if (!response || typeof response !== "object") return null;
+
+  const escapedPath = escapeJsonPointerSegment(parsed.path);
+  let pointer = `#/paths/${escapedPath}/${parsed.method}/responses/${status}`;
+  let excerptValue: unknown = response;
+
+  // Drill into application/json schema when available — that's the most useful
+  // surface for UI rendering ("backend promised X, returned Y").
+  const content = response.content as Record<string, unknown> | undefined;
+  if (content && typeof content === "object") {
+    const jsonMedia = (content["application/json"] ?? content["application/json; charset=utf-8"]) as
+      | Record<string, unknown>
+      | undefined;
+    if (jsonMedia && typeof jsonMedia === "object" && "schema" in jsonMedia) {
+      pointer += "/content/application~1json/schema";
+      excerptValue = jsonMedia.schema;
+    }
+  }
+
+  let excerpt: string;
+  try {
+    excerpt = JSON.stringify(excerptValue, null, 2);
+  } catch {
+    excerpt = String(excerptValue);
+  }
+  return { pointer, excerpt: trimExcerpt(excerpt) };
+}

package/src/core/diagnostics/suggested-fixes.ts

@@ -0,0 +1,155 @@
+/**
+ * TASK-29: actionable "Suggested fixes" surfaces for `zond db diagnose`.
+ *
+ * The base diagnose envelope already classifies failures and wires
+ * recommended_action. This layer adds two concrete, fixable signals that
+ * the LLM agent can act on without a second round-trip:
+ *
+ *   1. Placeholder path-params on 404s — when a 404 hits a URL that still
+ *      contains literal example/placeholder values (`example`, all-zeros
+ *      UUID, `your-…-here`, `00000000-…`, …), surface the exact path
+ *      segment that's broken plus the variable name we'd expect (best-effort
+ *      from the segment shape).
+ *
+ *   2. Untilled .env.yaml keys — read the API's .env.yaml and flag values
+ *      that are empty, `<TODO>`, `example`, `null`, or look like
+ *      placeholders (`replace-me`, `your-...`). These block CRUD sweeps
+ *      silently — without them, the agent can't tell apart "value missing"
+ *      from "value present and wrong".
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { parse as parseYaml } from "yaml";
+
+export type SuggestedFixKind =
+  | "placeholder_path_param"
+  | "unfilled_env_key";
+
+export interface SuggestedFix {
+  kind: SuggestedFixKind;
+  /** When `kind=unfilled_env_key`: the key name. When `kind=placeholder_path_param`:
+   *  the path segment that looks like a placeholder. */
+  key: string;
+  message: string;
+  /** File path the user should edit (usually the API's .env.yaml). */
+  source?: string;
+  /** Optional one-line example of what to put there. */
+  example?: string;
+}
+
+const PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
+  { re: /^example$/i, reason: "literal `example`" },
+  { re: /^placeholder$/i, reason: "literal `placeholder`" },
+  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
+  { re: /^00000000-0000-0000-0000-0+(?:beef|dead|cafe|f00d|0+)$/i, reason: "all-zero / sentinel UUID" },
+  { re: /^0+$/, reason: "all-zero numeric id" },
+  { re: /^replace-me$/i, reason: "`replace-me` placeholder" },
+];
+
+const ENV_PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
+  { re: /^<.*>$/, reason: "TODO / angle-bracket placeholder" },
+  { re: /^example$/i, reason: "literal `example`" },
+  { re: /^placeholder$/i, reason: "literal `placeholder`" },
+  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
+  { re: /^replace-?me$/i, reason: "`replace-me` placeholder" },
+  { re: /^<TODO>$/i, reason: "explicit TODO" },
+];
+
+/** Identify path segments that look like placeholders. Used on 404 URLs. */
+export function detectPlaceholderSegments(url: string | null): Array<{ segment: string; reason: string }> {
+  if (!url) return [];
+  let pathname: string;
+  try {
+    pathname = url.startsWith("http") ? new URL(url).pathname : url.split("?")[0]!;
+  } catch {
+    pathname = url;
+  }
+  const out: Array<{ segment: string; reason: string }> = [];
+  for (const seg of pathname.split("/").filter(Boolean)) {
+    for (const { re, reason } of PLACEHOLDER_PATTERNS) {
+      if (re.test(seg)) {
+        out.push({ segment: seg, reason });
+        break;
+      }
+    }
+  }
+  return out;
+}
+
+/** Read .env.yaml and return keys whose value is empty/null/placeholder-shaped. */
+export function findUnfilledEnvKeys(envFilePath: string | undefined): SuggestedFix[] {
+  if (!envFilePath || !existsSync(envFilePath)) return [];
+  let parsed: unknown;
+  try {
+    parsed = parseYaml(readFileSync(envFilePath, "utf-8"));
+  } catch {
+    return [];
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
+
+  const out: SuggestedFix[] = [];
+  for (const [key, val] of Object.entries(parsed as Record<string, unknown>)) {
+    const reason = classifyEnvValue(val);
+    if (reason) {
+      out.push({
+        kind: "unfilled_env_key",
+        key,
+        message: `\`${key}\` in ${envFilePath} is unfilled (${reason}). Set it to a real value before re-running.`,
+        source: envFilePath,
+      });
+    }
+  }
+  return out;
+}
+
+function classifyEnvValue(val: unknown): string | null {
+  if (val === null || val === undefined) return "null / missing";
+  if (typeof val === "string") {
+    const trimmed = val.trim();
+    if (trimmed === "") return "empty string";
+    for (const { re, reason } of ENV_PLACEHOLDER_PATTERNS) {
+      if (re.test(trimmed)) return reason;
+    }
+  }
+  return null;
+}
+
+export interface BuildSuggestedFixesInput {
+  failures: Array<{
+    response_status: number | null;
+    request_url: string | null;
+    suite_name: string;
+    test_name: string;
+  }>;
+  envFilePath?: string;
+}
+
+/**
+ * Combine placeholder-detection across all 404 failures + env-yaml audit
+ * into a single deduplicated list of fixes the agent should apply before
+ * re-running.
+ */
+export function buildSuggestedFixes(input: BuildSuggestedFixesInput): SuggestedFix[] {
+  const fixes: SuggestedFix[] = [];
+  const seenSegments = new Set<string>();
+
+  for (const f of input.failures) {
+    if (f.response_status !== 404) continue;
+    for (const ph of detectPlaceholderSegments(f.request_url)) {
+      const dedupeKey = `seg:${ph.segment}`;
+      if (seenSegments.has(dedupeKey)) continue;
+      seenSegments.add(dedupeKey);
+      fixes.push({
+        kind: "placeholder_path_param",
+        key: ph.segment,
+        message:
+          `404 on \`${f.request_url}\` — path segment \`${ph.segment}\` is a ${ph.reason}. ` +
+          `Replace with a real id from the live API (e.g. \`zond prepare-fixtures --apply\`) ` +
+          `or set the corresponding fixture in ${input.envFilePath ?? ".env.yaml"}.`,
+        source: input.envFilePath,
+      });
+    }
+  }
+
+  fixes.push(...findUnfilledEnvKeys(input.envFilePath));
+  return fixes;
+}