@kirrosh/zond 0.22.0 → 0.26.0

package/src/core/runner/learn-drift.ts

@@ -0,0 +1,293 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { TestRunResult, AssertionResult, StepResult } from "./types.ts";
+
+export interface DriftCase {
+  suite_name: string;
+  suite_file?: string;
+  step_name: string;
+  method?: string;
+  path?: string;
+  expected: number;
+  observed: number;
+  schema_validated: boolean;
+}
+
+/**
+ * Detect status-code drift cases: step would have passed if `expect.status`
+ * matched the observed status, every body/header assertion is green, and the
+ * response body matches the OpenAPI schema (no `kind: schema` failures).
+ *
+ * Skipped on purpose:
+ *  - `error` steps (network/transport — not a drift signal)
+ *  - `expect.status` arrays (`one of [...]`) — drift only triggers on a single
+ *    expected value; arrays already encode tolerance
+ *  - steps without `kind: schema` evidence — treated as drift_without_schema
+ *    and surfaced separately to the caller via `schema_validated: false`
+ */
+export interface DetectOptions {
+  /** True when the run was launched with a schema validator attached. The
+   *  validator produces no assertions on success, so step.assertions alone
+   *  can't distinguish "schema ok" from "no validator". */
+  schemaValidatorAttached: boolean;
+}
+
+export function detectStatusDrifts(results: TestRunResult[], opts: DetectOptions = { schemaValidatorAttached: false }): DriftCase[] {
+  const cases: DriftCase[] = [];
+  for (const r of results) {
+    for (const step of r.steps) {
+      const drift = classifyStep(step, opts);
+      if (!drift) continue;
+      cases.push({
+        suite_name: r.suite_name,
+        suite_file: r.suite_file,
+        step_name: step.name,
+        method: step.request?.method,
+        path: extractPathFromUrl(step.request?.url),
+        expected: drift.expected,
+        observed: drift.observed,
+        schema_validated: drift.schemaValidated,
+      });
+    }
+  }
+  return cases;
+}
+
+interface StepDrift {
+  expected: number;
+  observed: number;
+  schemaValidated: boolean;
+}
+
+function classifyStep(step: StepResult, opts: DetectOptions): StepDrift | null {
+  if (step.status !== "fail") return null;
+  if (!step.response) return null;
+
+  const statusFails = step.assertions.filter(
+    a => a.field === "status" && !a.passed && typeof a.rule === "string" && a.rule.startsWith("equals "),
+  );
+  if (statusFails.length !== 1) return null;
+
+  const otherFails = step.assertions.filter(a => !a.passed && a !== statusFails[0]);
+  if (otherFails.length > 0) return null;
+
+  const expected = parseExpected(statusFails[0]!);
+  if (expected === null) return null;
+
+  const observed = step.response.status;
+  if (expected === observed) return null;
+
+  // Schema validation evidence — when the validator was attached, assertions
+  // contain `kind: "schema"` entries on failure and nothing on success. So
+  // "validator attached AND no failing schema assertion" is the success case.
+  const schemaFails = step.assertions.filter(a => a.kind === "schema" && !a.passed);
+  if (schemaFails.length > 0) return null; // body diverges from spec — not a drift
+  const schemaValidated = opts.schemaValidatorAttached;
+
+  return { expected, observed, schemaValidated };
+}
+
+function parseExpected(a: AssertionResult): number | null {
+  if (typeof a.expected === "number") return a.expected;
+  if (typeof a.expected === "string") {
+    const n = Number(a.expected);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+function extractPathFromUrl(url: string | undefined): string | undefined {
+  if (!url) return undefined;
+  try {
+    return new URL(url).pathname;
+  } catch {
+    return url;
+  }
+}
+
+export function formatDriftPlan(cases: DriftCase[]): string {
+  if (cases.length === 0) return "No status-code drift detected.\n";
+  const lines: string[] = [];
+  lines.push(`Drift detected (${cases.length} case${cases.length === 1 ? "" : "s"}):`);
+  for (const c of cases) {
+    const ep = `${c.method ?? "?"} ${c.path ?? "?"}`.padEnd(40);
+    const schema = c.schema_validated ? "body-schema=ok" : "body-schema=unverified";
+    lines.push(`  ${ep} spec=${c.expected}  observed=${c.observed}  ${schema}  → suggest: update test, or add to drifts`);
+  }
+  lines.push("");
+  lines.push("Run with --learn-apply --learn-target=test     to rewrite expect.status in YAML");
+  lines.push("Run with --learn-apply --learn-target=drifts   to record in apis/<name>/tolerated-drifts.yaml");
+  return lines.join("\n") + "\n";
+}
+
+export interface ApplyResult {
+  updated: number;
+  errors: { suite_file: string; step_name: string; reason: string }[];
+}
+
+/**
+ * Rewrite `expect.status: <expected>` → `<observed>` in each suite file.
+ *
+ * Implementation is line-based: locate the step block by `name:`, then scan
+ * forward until the next sibling step or dedent looking for the first
+ * `status:` line at a deeper indent. We don't reparse the YAML — preserves
+ * comments, key order, and trailing whitespace so the diff is minimal.
+ */
+export async function applyDriftsToTests(cases: DriftCase[]): Promise<ApplyResult> {
+  const result: ApplyResult = { updated: 0, errors: [] };
+
+  // Group by file — read once, write once per file.
+  const byFile = new Map<string, DriftCase[]>();
+  for (const c of cases) {
+    if (!c.suite_file) {
+      result.errors.push({ suite_file: "<unknown>", step_name: c.step_name, reason: "suite_file missing" });
+      continue;
+    }
+    const list = byFile.get(c.suite_file) ?? [];
+    list.push(c);
+    byFile.set(c.suite_file, list);
+  }
+
+  for (const [file, fileCases] of byFile) {
+    let content: string;
+    try {
+      content = await readFile(file, "utf-8");
+    } catch (err) {
+      for (const c of fileCases) {
+        result.errors.push({ suite_file: file, step_name: c.step_name, reason: `read failed: ${(err as Error).message}` });
+      }
+      continue;
+    }
+
+    let lines = content.split("\n");
+    let touched = false;
+    for (const c of fileCases) {
+      const edited = rewriteExpectStatus(lines, c.step_name, c.expected, c.observed);
+      if (edited.ok) {
+        lines = edited.lines;
+        touched = true;
+        result.updated++;
+      } else {
+        result.errors.push({ suite_file: file, step_name: c.step_name, reason: edited.reason });
+      }
+    }
+
+    if (touched) {
+      try {
+        await writeFile(file, lines.join("\n"), "utf-8");
+      } catch (err) {
+        for (const c of fileCases) {
+          result.errors.push({ suite_file: file, step_name: c.step_name, reason: `write failed: ${(err as Error).message}` });
+        }
+      }
+    }
+  }
+
+  return result;
+}
+
+function rewriteExpectStatus(
+  lines: string[],
+  stepName: string,
+  expected: number,
+  observed: number,
+): { ok: true; lines: string[] } | { ok: false; reason: string } {
+  // Match `- name: foo` or `- name: "foo"` — capture indent of the dash so we
+  // know where the step block ends.
+  const nameRe = new RegExp(`^(\\s*)-\\s+name:\\s+["']?${escapeRegExp(stepName)}["']?\\s*$`);
+  let stepStart = -1;
+  let stepIndent = 0;
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i]!.match(nameRe);
+    if (m) {
+      stepStart = i;
+      stepIndent = m[1]!.length;
+      break;
+    }
+  }
+  if (stepStart < 0) return { ok: false, reason: `step "${stepName}" not found in YAML` };
+
+  // Step block ends at next sibling (`- name:` at the same indent) or at the
+  // first line that dedents past the step's column.
+  let stepEnd = lines.length;
+  for (let i = stepStart + 1; i < lines.length; i++) {
+    const line = lines[i]!;
+    if (line.trim() === "") continue;
+    const indent = line.match(/^(\s*)/)![1]!.length;
+    if (indent <= stepIndent) {
+      stepEnd = i;
+      break;
+    }
+  }
+
+  const statusRe = new RegExp(`^(\\s*)status:\\s*${expected}\\s*(#.*)?$`);
+  for (let i = stepStart + 1; i < stepEnd; i++) {
+    const m = lines[i]!.match(statusRe);
+    if (m) {
+      const tail = m[2] ? ` ${m[2]}` : "";
+      lines[i] = `${m[1]}status: ${observed}${tail}`;
+      return { ok: true, lines };
+    }
+  }
+  return { ok: false, reason: `expect.status: ${expected} not found within step "${stepName}"` };
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * Append drift cases to `<apiDir>/tolerated-drifts.yaml`. The file is a flat
+ * `drifts:` list; we de-duplicate on (method, path, expected, observed).
+ *
+ * Format kept intentionally minimal — runner-side enforcement (skip the
+ * status assertion when a tolerated drift matches) is a follow-up task; this
+ * call just records the data so a human can review.
+ */
+export async function appendToleratedDrifts(apiDir: string, cases: DriftCase[]): Promise<{ written: number; file: string }> {
+  const file = `${apiDir}/tolerated-drifts.yaml`;
+  let existing = "";
+  try {
+    existing = await readFile(file, "utf-8");
+  } catch {
+    // new file — keep empty
+  }
+
+  const seen = new Set<string>();
+  const driftRe = /^\s*-\s*method:\s*(\w+)\s*$\n\s*path:\s*(\S+)\s*$\n\s*expected:\s*(\d+)\s*$\n\s*observed:\s*(\d+)/gm;
+  let m: RegExpExecArray | null;
+  while ((m = driftRe.exec(existing)) !== null) {
+    seen.add(`${m[1]!.toUpperCase()} ${m[2]} ${m[3]}->${m[4]}`);
+  }
+
+  const fresh: DriftCase[] = [];
+  for (const c of cases) {
+    const key = `${(c.method ?? "?").toUpperCase()} ${c.path ?? "?"} ${c.expected}->${c.observed}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    fresh.push(c);
+  }
+
+  if (fresh.length === 0) return { written: 0, file };
+
+  let body = existing;
+  if (!/^drifts:\s*$/m.test(body)) {
+    body = body.trim();
+    if (body.length > 0) body += "\n";
+    body += "drifts:\n";
+  } else if (!body.endsWith("\n")) {
+    body += "\n";
+  }
+
+  for (const c of fresh) {
+    body += `  - method: ${c.method ?? "?"}\n`;
+    body += `    path: ${c.path ?? "?"}\n`;
+    body += `    expected: ${c.expected}\n`;
+    body += `    observed: ${c.observed}\n`;
+    body += `    note: ""\n`;
+  }
+
+  await mkdir(dirname(file), { recursive: true });
+  await writeFile(file, body, "utf-8");
+  return { written: fresh.length, file };
+}

package/src/core/runner/preflight-vars.ts

@@ -0,0 +1,153 @@
+import type { TestSuite, TestStep, AssertionRule } from "../parser/types.ts";
+import { GENERATORS } from "../parser/variables.ts";
+
+const VAR_PATTERN = /\{\{([^{}]+)\}\}/g;
+
+function scanRefs(value: unknown, out: Set<string>): void {
+  if (typeof value === "string") {
+    for (const match of value.matchAll(VAR_PATTERN)) {
+      const key = match[1]!.trim();
+      if (!key.startsWith("$")) out.add(key);
+    }
+  } else if (Array.isArray(value)) {
+    for (const item of value) scanRefs(item, out);
+  } else if (typeof value === "object" && value !== null) {
+    for (const v of Object.values(value)) scanRefs(v, out);
+  }
+}
+
+export function collectCapturesAndSets(step: TestStep, out: Set<string>): void {
+  if (step.set) {
+    for (const k of Object.keys(step.set)) out.add(k);
+  }
+  if (step.for_each?.var) out.add(step.for_each.var);
+  const scanRule = (rule: AssertionRule | undefined): void => {
+    if (!rule) return;
+    if (rule.capture) out.add(rule.capture);
+    if (rule.each) {
+      for (const r of Object.values(rule.each)) scanRule(r);
+    }
+    if (rule.contains_item) {
+      for (const r of Object.values(rule.contains_item)) scanRule(r);
+    }
+  };
+  if (step.expect?.body) {
+    for (const r of Object.values(step.expect.body)) scanRule(r);
+  }
+  if (step.expect?.headers) {
+    for (const v of Object.values(step.expect.headers)) {
+      if (typeof v === "object" && v !== null) scanRule(v as AssertionRule);
+    }
+  }
+}
+
+export interface MissingVarHit {
+  suite: string;
+  file?: string;
+  step?: string;
+  variable: string;
+}
+
+/**
+ * Pre-flight scan: find {{var}} references in suites that have no producer
+ * (env value, suite-level parameterize/set, prior-step capture). Excludes
+ * built-in $generators.
+ *
+ * Conservative: per-suite — accumulates all captures/sets across steps, so
+ * forward references inside the suite are tolerated (correctness requires
+ * runtime ordering checks anyway).
+ */
+/** All non-generator {{var}} references a step reads (path, headers, body,
+ *  query, skip_if, retry_until, set, for_each). Excludes `$generators`. */
+export function collectStepRefs(step: TestStep): Set<string> {
+  const refs = new Set<string>();
+  scanRefs(step.path, refs);
+  scanRefs(step.headers, refs);
+  scanRefs(step.json, refs);
+  scanRefs(step.form, refs);
+  scanRefs(step.multipart, refs);
+  scanRefs(step.query, refs);
+  if (step.skip_if) scanRefs(step.skip_if, refs);
+  if (step.retry_until) scanRefs(step.retry_until.condition, refs);
+  if (step.set) scanRefs(step.set, refs);
+  if (step.for_each) scanRefs(step.for_each.in, refs);
+  return refs;
+}
+
+export function preflightCheckVars(
+  suites: TestSuite[],
+  env: Record<string, string>,
+): MissingVarHit[] {
+  const hits: MissingVarHit[] = [];
+  const generatorKeys = new Set(Object.keys(GENERATORS));
+
+  for (const suite of suites) {
+    const known = new Set<string>(Object.keys(env));
+    if (suite.parameterize) {
+      for (const k of Object.keys(suite.parameterize)) known.add(k);
+    }
+    for (const step of suite.tests) collectCapturesAndSets(step, known);
+
+    const scanStepRefs = collectStepRefs;
+
+    const suiteRefs = new Set<string>();
+    if (suite.base_url) scanRefs(suite.base_url, suiteRefs);
+    if (suite.headers) scanRefs(suite.headers, suiteRefs);
+    for (const v of suiteRefs) {
+      if (!known.has(v) && !generatorKeys.has(v)) {
+        hits.push({ suite: suite.name, file: suite.filePath, variable: v });
+      }
+    }
+
+    for (const step of suite.tests) {
+      const refs = scanStepRefs(step);
+      for (const v of refs) {
+        if (!known.has(v) && !generatorKeys.has(v)) {
+          hits.push({
+            suite: suite.name,
+            file: suite.filePath,
+            step: step.name,
+            variable: v,
+          });
+        }
+      }
+    }
+  }
+
+  return hits;
+}
+
+export function formatMissingVarLine(hit: MissingVarHit): string {
+  const where = hit.step ? `${hit.suite} → ${hit.step}` : hit.suite;
+  const file = hit.file ? ` (${hit.file})` : "";
+  return `Undefined variable {{${hit.variable}}} in ${where}${file}`;
+}
+
+/**
+ * TASK-248: collapse per-(suite,step,variable) hits into one summary line per
+ * unique variable. When the same {{var}} is referenced in 8–12 places (typical
+ * when running outside a workspace and `.env.yaml` is missing), per-hit emit
+ * spams stderr — the aggregated form keeps signal (the *names* of the missing
+ * vars) while dropping noise.
+ */
+export function summarizeMissingVars(hits: MissingVarHit[]): string[] {
+  if (hits.length === 0) return [];
+  const byVar = new Map<string, { refs: number; suites: Set<string> }>();
+  for (const h of hits) {
+    let entry = byVar.get(h.variable);
+    if (!entry) {
+      entry = { refs: 0, suites: new Set() };
+      byVar.set(h.variable, entry);
+    }
+    entry.refs++;
+    entry.suites.add(h.suite);
+  }
+  const names = [...byVar.keys()].sort();
+  const totalRefs = hits.length;
+  const totalSuites = new Set(hits.map((h) => h.suite)).size;
+  const head = names.slice(0, 6).map((n) => `{{${n}}}`).join(", ");
+  const tail = names.length > 6 ? `, … and ${names.length - 6} more` : "";
+  return [
+    `Undefined variables: ${head}${tail} (${totalRefs} reference${totalRefs === 1 ? "" : "s"} across ${totalSuites} suite${totalSuites === 1 ? "" : "s"})`,
+  ];
+}

package/src/core/runner/progress-tracker.ts

@@ -0,0 +1,73 @@
+/**
+ * ARV-249: lightweight progress tracker for `zond run`. A separate module
+ * so the formatter can be unit-tested without spinning up an executor.
+ *
+ * Architecture: run.ts owns the tracker + `setInterval`. Every completed
+ * step calls `tracker.recordStep(...)` via `RunSuiteOptions.onStepDone`,
+ * the interval ticks every PROGRESS_INTERVAL_MS and writes one stderr
+ * line, and run.ts clears the interval before printing the final report.
+ */
+import { formatEta } from "../util/format-eta.ts";
+import type { StepResult } from "./types.ts";
+
+export const PROGRESS_INTERVAL_MS = 5000;
+/** Wait this long before emitting the first progress line. Short runs
+ *  that finish under the threshold stay silent. */
+export const PROGRESS_QUIET_MS = 5000;
+
+export interface ProgressSnapshot {
+  elapsedMs: number;
+  completedSteps: number;
+  totalSteps: number;
+  httpRequests: number;
+  effectiveRps: number;
+  etaSeconds: number;
+}
+
+export class ProgressTracker {
+  private completedSteps = 0;
+  private httpRequests = 0;
+  private readonly startedAt: number;
+
+  constructor(private readonly totalSteps: number, now: number = Date.now()) {
+    this.startedAt = now;
+  }
+
+  recordStep(step: StepResult): void {
+    this.completedSteps += 1;
+    if (step.status !== "skip" && step.response !== undefined) {
+      this.httpRequests += 1;
+    }
+  }
+
+  snapshot(now: number = Date.now()): ProgressSnapshot {
+    const elapsedMs = Math.max(0, now - this.startedAt);
+    const elapsedSec = elapsedMs / 1000;
+    const effectiveRps = elapsedSec > 0 ? this.httpRequests / elapsedSec : 0;
+    const remaining = Math.max(0, this.totalSteps - this.completedSteps);
+    const stepRate = elapsedSec > 0 ? this.completedSteps / elapsedSec : 0;
+    const etaSeconds = stepRate > 0 ? remaining / stepRate : Infinity;
+    return {
+      elapsedMs,
+      completedSteps: this.completedSteps,
+      totalSteps: this.totalSteps,
+      httpRequests: this.httpRequests,
+      effectiveRps,
+      etaSeconds,
+    };
+  }
+}
+
+/** Format a progress snapshot for stderr. Stable wording — agents may
+ *  grep the line. */
+export function formatProgressLine(snap: ProgressSnapshot): string {
+  const elapsed = formatEta(snap.elapsedMs / 1000);
+  const pct = snap.totalSteps > 0
+    ? Math.min(100, Math.floor((snap.completedSteps / snap.totalSteps) * 100))
+    : 0;
+  const rps = snap.effectiveRps >= 10
+    ? Math.round(snap.effectiveRps).toString()
+    : snap.effectiveRps.toFixed(1);
+  const eta = Number.isFinite(snap.etaSeconds) ? formatEta(snap.etaSeconds) : "?";
+  return `zond: [${elapsed}] ${snap.completedSteps}/${snap.totalSteps} steps (${pct}%), ${snap.httpRequests} req, ~${rps} req/s, ETA ${eta}`;
+}