@kirrosh/zond 0.26.0 → 0.27.0

package/src/core/diagnostics/failure-class.ts

@@ -1,140 +0,0 @@
-/**
- * TASK-101: failure classification — definitely_bug / likely_bug / quirk / env_issue.
- *
- * Goal: бэкендер за секунду видит «реально баг» vs «quirk зонда / probe
- * фолс-позитив». Чисто read-only классификация: не меняет статус step,
- * не влияет на pass/fail. Только tag-овая аналитика.
- */
-
-import type { StepResult, AssertionResult } from "../runner/types.ts";
-import type { SourceMetadata } from "../parser/types.ts";
-
-export type FailureClass =
-  | "definitely_bug"
-  | "likely_bug"
-  | "quirk"
-  | "env_issue"
-  /** Step was skipped because an upstream step failed to produce a required
-   *  capture (or produced a tainted one). Not a failure on its own — render
-   *  collapsed under the root failure to avoid drowning the user. */
-  | "cascade";
-
-export interface FailureClassification {
-  failure_class: FailureClass;
-  failure_class_reason: string;
-}
-
-function failedAssertionsOf(result: StepResult): AssertionResult[] {
-  return result.assertions.filter((a) => !a.passed);
-}
-
-function ruleStartsWith(a: AssertionResult, prefix: string): boolean {
-  return typeof a.rule === "string" && a.rule.startsWith(prefix);
-}
-
-function expectedStatusList(a: AssertionResult): number[] | null {
-  if (Array.isArray(a.expected)) {
-    const arr = a.expected.filter((v): v is number => typeof v === "number");
-    return arr.length > 0 ? arr : null;
-  }
-  return typeof a.expected === "number" ? [a.expected] : null;
-}
-
-/**
- * Classify a failed step. Returns `null` for pass/skip/unclassifiable failures —
- * UI renders those as "unclassified" rather than crashing.
- */
-export function classifyFailure(result: StepResult): FailureClassification | null {
-  if (result.status === "pass" || result.status === "skip") return null;
-
-  // Network/runtime error before any HTTP response → env-side
-  if (result.status === "error") {
-    return {
-      failure_class: "env_issue",
-      failure_class_reason: result.error ?? "request failed before producing a response",
-    };
-  }
-
-  const respStatus = result.response?.status;
-  const provenance: SourceMetadata | null | undefined = result.provenance;
-  const generator = typeof provenance?.generator === "string" ? provenance.generator : undefined;
-  const failed = failedAssertionsOf(result);
-
-  // 1. Backend 5xx — always a backend bug regardless of test intent.
-  if (typeof respStatus === "number" && respStatus >= 500) {
-    return {
-      failure_class: "definitely_bug",
-      failure_class_reason: `API returned ${respStatus} — server-side error`,
-    };
-  }
-
-  // 2. Response did not match its OpenAPI schema — spec guarantees X, server returned ≠ X.
-  const schemaFail = failed.find((a) => ruleStartsWith(a, "schema."));
-  if (schemaFail) {
-    return {
-      failure_class: "definitely_bug",
-      failure_class_reason: `Response violates OpenAPI schema at ${schemaFail.field}`,
-    };
-  }
-
-  // 3. Mass-assignment probe: extras must not apply. A failed not_equals
-  //    assertion on this generator means the sentinel value leaked through.
-  if (generator === "mass-assignment-probe") {
-    const extrasLeak = failed.find(
-      (a) => ruleStartsWith(a, "not_equals") || ruleStartsWith(a, "set_equals"),
-    );
-    if (extrasLeak) {
-      return {
-        failure_class: "definitely_bug",
-        failure_class_reason: `Mass-assignment: client-supplied extras were accepted (${extrasLeak.field})`,
-      };
-    }
-  }
-
-  // 4. ARV-236: 401/403 on a step that expected success → env_issue
-  //    ("token_scope"). Surfaces narrowly-scoped tokens (Resend free plan,
-  //    GitHub PAT without `repo`, Stripe restricted key, etc.) so coverage
-  //    and triage don't tag them as fail. Only fires when the failing
-  //    assertion is on `status` and the expected status was 2xx — a real
-  //    "expect 403" test deliberately checking auth still classifies normally.
-  if (typeof respStatus === "number" && (respStatus === 401 || respStatus === 403)) {
-    const statusFail = failed.find((a) => a.field === "status");
-    if (statusFail) {
-      const expected = expectedStatusList(statusFail);
-      const allExpected2xx = expected?.every((s) => s >= 200 && s < 300) ?? false;
-      if (allExpected2xx) {
-        return {
-          failure_class: "env_issue",
-          failure_class_reason: `token_scope: API returned ${respStatus} where the test expected ${expected!.join("/")} — token lacks scope or plan for this endpoint`,
-        };
-      }
-    }
-  }
-
-  // 5. Negative-probe family — distinguish "API accepted bad input" (likely_bug)
-  //    from "API rejected with a different 4xx" (quirk).
-  if (generator === "negative-probe" || generator === "method-probe") {
-    const statusFail = failed.find((a) => a.field === "status");
-    if (statusFail && typeof respStatus === "number") {
-      const expected = expectedStatusList(statusFail);
-      const allExpected4xx = expected?.every((s) => s >= 400 && s < 500) ?? false;
-      if (allExpected4xx) {
-        if (respStatus >= 200 && respStatus < 300) {
-          return {
-            failure_class: "likely_bug",
-            failure_class_reason: `Negative probe expected 4xx, got ${respStatus} — API accepts invalid input`,
-          };
-        }
-        if (respStatus >= 400 && respStatus < 500) {
-          return {
-            failure_class: "quirk",
-            failure_class_reason: `Negative probe expected ${expected!.join("/")}, got ${respStatus} — different 4xx code`,
-          };
-        }
-      }
-    }
-  }
-
-  // Default: leave unclassified. UI / CLI render as "unclassified".
-  return null;
-}

package/src/core/diagnostics/failure-hints.ts

@@ -1,112 +0,0 @@
-/**
- * Failure classification → closed-enum `recommended_action`.
- * ARV-338: prose hints (statusHint/envHint/softDeleteHint/schemaHint,
- * env_issue clustering, auth_hint, agent_directive) removed — the agent
- * triages by enum + raw evidence, not by heuristic guess-text.
- */
-
-import { classify } from "../classifier/recommended-action.ts";
-
-export function classifyFailure(status: string, responseStatus: number | null): "api_error" | "assertion_failed" | "network_error" {
-  if (status === "error" && (responseStatus === null || responseStatus < 500)) return "network_error";
-  if (responseStatus !== null && responseStatus >= 500) return "api_error";
-  return "assertion_failed";
-}
-
-export type RecommendedAction =
-  | "report_backend_bug"
-  | "fix_auth_config"
-  | "fix_test_logic"
-  | "fix_network_config"
-  | "fix_env"
-  /** Fix the OpenAPI spec — emitted by lint-spec.Issue (TASK-294) and
-   *  by `status_code_conformance` / `*_conformance` checks (ARV-11). */
-  | "fix_spec"
-  /** Add or correct a fixture in .env.yaml — emitted by discover for
-   *  miss-* states (TASK-294). */
-  | "fix_fixture"
-  /** ARV-42 — generator-emitted suite produced a body the API rejected
-   *  (4xx with validation hint). Editing the YAML is wrong: the next
-   *  `zond generate` would clobber it. Re-run generate (or refine the
-   *  spec/.api-resources hints) instead. */
-  | "regenerate_suite"
-  /** ARV-11 — server accepted an invalid request body. Backend should
-   *  reject earlier; the test isn't wrong. */
-  | "tighten_validation"
-  /** ARV-11 — server didn't enforce a header marked `required: true`
-   *  in the spec. Either enforce it, or drop `required` in the spec. */
-  | "add_required_header"
-  /** ARV-11 — known limitation that the team has accepted. Agents
-   *  should not retry, file a bug, or include in dashboards. */
-  | "wontfix_known_limitation";
-
-export function recommendedAction(
-  failureType: "api_error" | "assertion_failed" | "network_error",
-  responseStatus: number | null,
-): RecommendedAction {
-  // ARV-56: delegate to the single classifier.
-  const action = classify({
-    finding_class: failureType === "api_error" ? "test:api_error" :
-      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
-    status: responseStatus,
-  });
-  // The three failure_type classes are total in the classifier — a missing
-  // branch means a future refactor stripped one; surface loudly.
-  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
-  return action;
-}
-
-/**
- * ARV-42: extended recommender that knows whether the failing test was
- * emitted by `zond generate`. For generated suites, "fix_test_logic" is
- * actively misleading — the generated YAML carries the header
- * "⚠️ Edits will be overwritten on regenerate" and the next `zond audit`
- * really does clobber manual edits. Branch into the actually-actionable
- * remediation instead.
- *
- *  - 4xx (400/422) → regenerate_suite: the body the generator emitted
- *    didn't pass validation; either re-run generate (so newer heuristics
- *    apply, e.g. ARV-38 default-string) or tighten .api-resources hints.
- *  - 404 → fix_fixture: a path-param resolved to an empty / stale id
- *    in .env.yaml; `prepare-fixtures --seed` is the correct remedy.
- *  - everything else → falls back to recommendedAction (auth → 401/403,
- *    api_error → 5xx, etc.).
- *
- * `isGenerated` is the heuristic from db-analysis: provenance.type
- * "openapi-generated" OR suite_file under apis/<api>/tests/.
- */
-export function recommendedActionForGenerated(
-  failureType: "api_error" | "assertion_failed" | "network_error",
-  responseStatus: number | null,
-  isGenerated: boolean,
-  schemaViolation = false,
-): RecommendedAction {
-  // ARV-56: delegate to classifier. `isGenerated` is encoded via a
-  // synthetic suite_path so the same logic flows through classify().
-  // ARV-103 (F8): `schemaViolation` propagates the assertion-kind flag —
-  // when true, the classifier's "treat schema bugs like 5xx" branch wins
-  // over the generator's regenerate_suite default.
-  const action = classify({
-    finding_class: failureType === "api_error" ? "test:api_error" :
-      failureType === "network_error" ? "test:network_error" : "test:assertion_failed",
-    status: responseStatus,
-    ...(isGenerated ? { suite_path: "apis/_/tests/_.yaml" } : {}),
-    ...(schemaViolation ? { schema_violation: true } : {}),
-  });
-  if (!action) throw new Error(`classifier returned no action for failure_type=${failureType} status=${responseStatus}`);
-  return action;
-}
-
-/** ARV-42: classify a failing result row as generator-emitted. The two
- *  signals are independent — provenance is missing on older runs, while
- *  suite_file disambiguates against ad-hoc YAMLs the user dropped into
- *  apis/<api>/tests/ themselves (rare but supported). */
-export function isGeneratedTest(
-  provenance: { type?: string; generator?: string } | null | undefined,
-  suite_file: string | null | undefined,
-): boolean {
-  if (provenance?.type === "openapi-generated") return true;
-  if (provenance?.generator && provenance.generator.toLowerCase().includes("zond")) return true;
-  if (typeof suite_file === "string" && /(^|\/)apis\/[^/]+\/tests\//.test(suite_file)) return true;
-  return false;
-}

package/src/core/diagnostics/spec-pointer.ts

@@ -1,99 +0,0 @@
-/**
- * TASK-102: build a JSON Pointer (RFC 6901) into the OpenAPI document for the
- * response branch a step exercises, plus a small excerpt of the schema at that
- * pointer. The excerpt is captured at run time and frozen into the DB so that
- * later spec edits can't rewrite history.
- *
- * Inputs come from {@link SourceMetadata} populated by TASK-100:
- *   - `endpoint`         e.g. "POST /webhooks"
- *   - `response_branch`  e.g. "422" or "400|422" (first wins for the pointer)
- */
-
-import type { SourceMetadata } from "../parser/types.ts";
-
-export interface SpecPointer {
-  pointer: string;
-  excerpt: string;
-}
-
-const EXCERPT_MAX_BYTES = 500;
-
-function escapeJsonPointerSegment(s: string): string {
-  return s.replace(/~/g, "~0").replace(/\//g, "~1");
-}
-
-function parseEndpoint(endpoint: string): { method: string; path: string } | null {
-  const m = endpoint.match(/^([A-Z]+)\s+(\/.*)$/);
-  if (!m) return null;
-  return { method: m[1]!.toLowerCase(), path: m[2]! };
-}
-
-function pickPrimaryStatus(responseBranch: string | undefined): string | null {
-  if (!responseBranch) return null;
-  const first = responseBranch.split(/[|,\s]/).find((s) => /^\d{3}$/.test(s));
-  return first ?? null;
-}
-
-function trimExcerpt(json: string): string {
-  if (json.length <= EXCERPT_MAX_BYTES) return json;
-  return json.slice(0, EXCERPT_MAX_BYTES) + "\n…[truncated]";
-}
-
-/**
- * Resolve the response operation in the OpenAPI document and produce a pointer
- * + frozen excerpt. Returns `null` if any link in the chain is missing — caller
- * persists `null` rather than crashing.
- */
-export function buildSpecPointer(
-  source: SourceMetadata | null | undefined,
-  openApiDoc: unknown,
-): SpecPointer | null {
-  if (!source || !openApiDoc || typeof openApiDoc !== "object") return null;
-  if (typeof source.endpoint !== "string") return null;
-
-  const parsed = parseEndpoint(source.endpoint);
-  if (!parsed) return null;
-  const status = pickPrimaryStatus(source.response_branch ?? undefined);
-  if (!status) return null;
-
-  const doc = openApiDoc as Record<string, unknown>;
-  const paths = doc.paths as Record<string, unknown> | undefined;
-  if (!paths || typeof paths !== "object") return null;
-
-  const pathItem = paths[parsed.path] as Record<string, unknown> | undefined;
-  if (!pathItem || typeof pathItem !== "object") return null;
-
-  const operation = pathItem[parsed.method] as Record<string, unknown> | undefined;
-  if (!operation || typeof operation !== "object") return null;
-
-  const responses = operation.responses as Record<string, unknown> | undefined;
-  if (!responses || typeof responses !== "object") return null;
-
-  const response = (responses[status] ?? responses.default) as Record<string, unknown> | undefined;
-  if (!response || typeof response !== "object") return null;
-
-  const escapedPath = escapeJsonPointerSegment(parsed.path);
-  let pointer = `#/paths/${escapedPath}/${parsed.method}/responses/${status}`;
-  let excerptValue: unknown = response;
-
-  // Drill into application/json schema when available — that's the most useful
-  // surface for UI rendering ("backend promised X, returned Y").
-  const content = response.content as Record<string, unknown> | undefined;
-  if (content && typeof content === "object") {
-    const jsonMedia = (content["application/json"] ?? content["application/json; charset=utf-8"]) as
-      | Record<string, unknown>
-      | undefined;
-    if (jsonMedia && typeof jsonMedia === "object" && "schema" in jsonMedia) {
-      pointer += "/content/application~1json/schema";
-      excerptValue = jsonMedia.schema;
-    }
-  }
-
-  let excerpt: string;
-  try {
-    excerpt = JSON.stringify(excerptValue, null, 2);
-  } catch {
-    excerpt = String(excerptValue);
-  }
-  return { pointer, excerpt: trimExcerpt(excerpt) };
-}

package/src/core/diagnostics/suggested-fixes.ts

@@ -1,155 +0,0 @@
-/**
- * TASK-29: actionable "Suggested fixes" surfaces for `zond db diagnose`.
- *
- * The base diagnose envelope already classifies failures and wires
- * recommended_action. This layer adds two concrete, fixable signals that
- * the LLM agent can act on without a second round-trip:
- *
- *   1. Placeholder path-params on 404s — when a 404 hits a URL that still
- *      contains literal example/placeholder values (`example`, all-zeros
- *      UUID, `your-…-here`, `00000000-…`, …), surface the exact path
- *      segment that's broken plus the variable name we'd expect (best-effort
- *      from the segment shape).
- *
- *   2. Untilled .env.yaml keys — read the API's .env.yaml and flag values
- *      that are empty, `<TODO>`, `example`, `null`, or look like
- *      placeholders (`replace-me`, `your-...`). These block CRUD sweeps
- *      silently — without them, the agent can't tell apart "value missing"
- *      from "value present and wrong".
- */
-import { existsSync, readFileSync } from "node:fs";
-import { parse as parseYaml } from "yaml";
-
-export type SuggestedFixKind =
-  | "placeholder_path_param"
-  | "unfilled_env_key";
-
-export interface SuggestedFix {
-  kind: SuggestedFixKind;
-  /** When `kind=unfilled_env_key`: the key name. When `kind=placeholder_path_param`:
-   *  the path segment that looks like a placeholder. */
-  key: string;
-  message: string;
-  /** File path the user should edit (usually the API's .env.yaml). */
-  source?: string;
-  /** Optional one-line example of what to put there. */
-  example?: string;
-}
-
-const PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
-  { re: /^example$/i, reason: "literal `example`" },
-  { re: /^placeholder$/i, reason: "literal `placeholder`" },
-  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
-  { re: /^00000000-0000-0000-0000-0+(?:beef|dead|cafe|f00d|0+)$/i, reason: "all-zero / sentinel UUID" },
-  { re: /^0+$/, reason: "all-zero numeric id" },
-  { re: /^replace-me$/i, reason: "`replace-me` placeholder" },
-];
-
-const ENV_PLACEHOLDER_PATTERNS: Array<{ re: RegExp; reason: string }> = [
-  { re: /^<.*>$/, reason: "TODO / angle-bracket placeholder" },
-  { re: /^example$/i, reason: "literal `example`" },
-  { re: /^placeholder$/i, reason: "literal `placeholder`" },
-  { re: /^your-[\w-]+-here$/i, reason: "`your-…-here` placeholder" },
-  { re: /^replace-?me$/i, reason: "`replace-me` placeholder" },
-  { re: /^<TODO>$/i, reason: "explicit TODO" },
-];
-
-/** Identify path segments that look like placeholders. Used on 404 URLs. */
-export function detectPlaceholderSegments(url: string | null): Array<{ segment: string; reason: string }> {
-  if (!url) return [];
-  let pathname: string;
-  try {
-    pathname = url.startsWith("http") ? new URL(url).pathname : url.split("?")[0]!;
-  } catch {
-    pathname = url;
-  }
-  const out: Array<{ segment: string; reason: string }> = [];
-  for (const seg of pathname.split("/").filter(Boolean)) {
-    for (const { re, reason } of PLACEHOLDER_PATTERNS) {
-      if (re.test(seg)) {
-        out.push({ segment: seg, reason });
-        break;
-      }
-    }
-  }
-  return out;
-}
-
-/** Read .env.yaml and return keys whose value is empty/null/placeholder-shaped. */
-export function findUnfilledEnvKeys(envFilePath: string | undefined): SuggestedFix[] {
-  if (!envFilePath || !existsSync(envFilePath)) return [];
-  let parsed: unknown;
-  try {
-    parsed = parseYaml(readFileSync(envFilePath, "utf-8"));
-  } catch {
-    return [];
-  }
-  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
-
-  const out: SuggestedFix[] = [];
-  for (const [key, val] of Object.entries(parsed as Record<string, unknown>)) {
-    const reason = classifyEnvValue(val);
-    if (reason) {
-      out.push({
-        kind: "unfilled_env_key",
-        key,
-        message: `\`${key}\` in ${envFilePath} is unfilled (${reason}). Set it to a real value before re-running.`,
-        source: envFilePath,
-      });
-    }
-  }
-  return out;
-}
-
-function classifyEnvValue(val: unknown): string | null {
-  if (val === null || val === undefined) return "null / missing";
-  if (typeof val === "string") {
-    const trimmed = val.trim();
-    if (trimmed === "") return "empty string";
-    for (const { re, reason } of ENV_PLACEHOLDER_PATTERNS) {
-      if (re.test(trimmed)) return reason;
-    }
-  }
-  return null;
-}
-
-export interface BuildSuggestedFixesInput {
-  failures: Array<{
-    response_status: number | null;
-    request_url: string | null;
-    suite_name: string;
-    test_name: string;
-  }>;
-  envFilePath?: string;
-}
-
-/**
- * Combine placeholder-detection across all 404 failures + env-yaml audit
- * into a single deduplicated list of fixes the agent should apply before
- * re-running.
- */
-export function buildSuggestedFixes(input: BuildSuggestedFixesInput): SuggestedFix[] {
-  const fixes: SuggestedFix[] = [];
-  const seenSegments = new Set<string>();
-
-  for (const f of input.failures) {
-    if (f.response_status !== 404) continue;
-    for (const ph of detectPlaceholderSegments(f.request_url)) {
-      const dedupeKey = `seg:${ph.segment}`;
-      if (seenSegments.has(dedupeKey)) continue;
-      seenSegments.add(dedupeKey);
-      fixes.push({
-        kind: "placeholder_path_param",
-        key: ph.segment,
-        message:
-          `404 on \`${f.request_url}\` — path segment \`${ph.segment}\` is a ${ph.reason}. ` +
-          `Replace with a real id from the live API (e.g. \`zond prepare-fixtures --apply\`) ` +
-          `or set the corresponding fixture in ${input.envFilePath ?? ".env.yaml"}.`,
-        source: input.envFilePath,
-      });
-    }
-  }
-
-  fixes.push(...findUnfilledEnvKeys(input.envFilePath));
-  return fixes;
-}