@kirrosh/zond 0.16.0 → 0.17.0

package/src/cli/commands/sync.ts

@@ -0,0 +1,240 @@
+import { join } from "path";
+import { mkdir } from "fs/promises";
+import {
+  readOpenApiSpec,
+  extractEndpoints,
+  extractSecuritySchemes,
+  serializeSuite,
+} from "../../core/generator/index.ts";
+import { generateSuites } from "../../core/generator/suite-generator.ts";
+import { filterByTag } from "../../core/generator/chunker.ts";
+import { readMeta, writeMeta, hashSpec, buildFileMeta } from "../../core/meta/meta-store.ts";
+import { diffEndpoints } from "../../core/sync/spec-differ.ts";
+import { printError, printSuccess, printWarning } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+import { version as ZOND_VERSION } from "../../../package.json";
+import { getDb } from "../../db/schema.ts";
+import { findCollectionByTestPath, updateCollection } from "../../db/queries.ts";
+
+export interface SyncOptions {
+  specPath: string;
+  testsDir: string;
+  dryRun?: boolean;
+  tag?: string;
+  json?: boolean;
+}
+
+export async function syncCommand(options: SyncOptions): Promise<number> {
+  try {
+    // Load existing metadata
+    const meta = await readMeta(options.testsDir);
+    if (!meta) {
+      const msg =
+        "No .zond-meta.json found. Run `zond generate <spec> --output <dir>` first to initialize metadata.";
+      if (options.json) {
+        printJson(jsonError("sync", [msg]));
+      } else {
+        printError(msg);
+      }
+      return 2;
+    }
+
+    // Load current spec
+    const doc = await readOpenApiSpec(options.specPath);
+    const specContent = JSON.stringify(doc);
+    const currentHash = hashSpec(specContent);
+
+    if (currentHash === meta.specHash) {
+      const msg = "Spec unchanged — nothing to sync.";
+      if (options.json) {
+        printJson(jsonOk("sync", { newEndpoints: [], generatedFiles: [], removedKeys: [], specChanged: false }, [msg]));
+      } else {
+        console.log(msg);
+      }
+      return 0;
+    }
+
+    // Extract current endpoints
+    let currentEndpoints = extractEndpoints(doc);
+    const securitySchemes = extractSecuritySchemes(doc);
+
+    if (options.tag) {
+      currentEndpoints = filterByTag(currentEndpoints, options.tag);
+    }
+
+    // Collect all previously known endpoint keys from meta
+    const prevKeys = Object.values(meta.files).flatMap((f) => f.endpoints);
+
+    // Compute diff
+    const { newEndpoints, removedKeys } = diffEndpoints(prevKeys, currentEndpoints);
+
+    const warnings: string[] = [];
+
+    if (removedKeys.length > 0) {
+      for (const key of removedKeys) {
+        warnings.push(`Removed endpoint not deleted from tests (review manually): ${key}`);
+      }
+    }
+
+    if (newEndpoints.length === 0) {
+      const msg = "Spec changed (hash differs) but no new endpoints detected. Existing tests may need manual review.";
+      warnings.push(msg);
+      if (options.json) {
+        printJson(jsonOk("sync", {
+          newEndpoints: [],
+          removedKeys,
+          generatedFiles: [],
+          specChanged: true,
+        }, warnings));
+      } else {
+        console.log(msg);
+        for (const w of warnings) {
+          printWarning(w);
+        }
+      }
+      return 0;
+    }
+
+    // Generate suites for new endpoints only
+    const suites = generateSuites({ endpoints: newEndpoints, securitySchemes });
+
+    if (options.dryRun) {
+      const newEndpointKeys = newEndpoints.map((ep) => `${ep.method.toUpperCase()} ${ep.path}`);
+      const plannedFiles = suites.map((s) => ({
+        file: `${s.fileStem ?? s.name}.yaml`,
+        suite: s.name,
+        tests: s.tests.length,
+      }));
+
+      if (options.json) {
+        printJson(jsonOk("sync", {
+          dryRun: true,
+          newEndpoints: newEndpointKeys,
+          removedKeys,
+          plannedFiles,
+          specChanged: true,
+        }, warnings));
+      } else {
+        console.log(`[dry-run] Detected ${newEndpoints.length} new endpoint(s):`);
+        for (const ep of newEndpoints) {
+          console.log(`  + ${ep.method.toUpperCase()} ${ep.path}`);
+        }
+        console.log(`\nWould generate ${suites.length} new suite file(s):`);
+        for (const f of plannedFiles) {
+          console.log(`  ${f.file}  (${f.tests} tests)`);
+        }
+        if (removedKeys.length > 0) {
+          console.log("\nRemoved endpoints (not deleted — review tests):");
+          for (const key of removedKeys) {
+            console.log(`  - ${key}`);
+          }
+        }
+        console.log("\nNo files written (dry-run).");
+      }
+      return 0;
+    }
+
+    // Write new files (skip if file already exists)
+    await mkdir(options.testsDir, { recursive: true });
+
+    const generatedFiles: Array<{ file: string; suite: string; tests: number }> = [];
+    const skippedFiles: string[] = [];
+    const updatedMetaFiles: Record<string, import("../../core/meta/types.ts").FileMeta> = {};
+
+    for (const suite of suites) {
+      const fileName = `${suite.fileStem ?? suite.name}.yaml`;
+      const filePath = join(options.testsDir, fileName);
+      const existing = Bun.file(filePath);
+
+      if (await existing.exists()) {
+        skippedFiles.push(fileName);
+        warnings.push(`Skipped ${fileName} (already exists — add new endpoints manually)`);
+        continue;
+      }
+
+      const yaml = serializeSuite(suite);
+      await Bun.write(filePath, yaml);
+      generatedFiles.push({ file: filePath, suite: suite.name, tests: suite.tests.length });
+      updatedMetaFiles[fileName] = buildFileMeta(suite, ZOND_VERSION);
+    }
+
+    // Update metadata: merge new file entries, update hash and timestamp
+    await writeMeta(options.testsDir, {
+      zondVersion: ZOND_VERSION,
+      lastSyncedAt: new Date().toISOString(),
+      specUrl: options.specPath,
+      specHash: currentHash,
+      files: { ...meta.files, ...updatedMetaFiles },
+    });
+
+    // Sync DB collection if one is registered for this tests directory
+    try {
+      getDb();
+      const collection = findCollectionByTestPath(options.testsDir);
+      if (collection && collection.openapi_spec !== options.specPath) {
+        updateCollection(collection.id, { openapi_spec: options.specPath });
+        warnings.push(`Updated collection '${collection.name}' spec reference: ${collection.openapi_spec ?? "(none)"} → ${options.specPath}`);
+      }
+    } catch {
+      // DB unavailable (e.g. no zond.db yet) — not a fatal error for sync
+    }
+
+    const newEndpointKeys = newEndpoints.map((ep) => `${ep.method.toUpperCase()} ${ep.path}`);
+
+    if (options.json) {
+      printJson(jsonOk("sync", {
+        newEndpoints: newEndpointKeys,
+        removedKeys,
+        generatedFiles,
+        skippedFiles,
+        specChanged: true,
+      }, warnings));
+    } else {
+      console.log(`Spec changed. Detected ${newEndpoints.length} new endpoint(s):`);
+      for (const ep of newEndpoints) {
+        console.log(`  + ${ep.method.toUpperCase()} ${ep.path}`);
+      }
+
+      if (generatedFiles.length > 0) {
+        console.log(`\nGenerated ${generatedFiles.length} new suite file(s):`);
+        for (const f of generatedFiles) {
+          console.log(`  ${f.file}  (${f.tests} tests)`);
+        }
+      }
+
+      if (skippedFiles.length > 0) {
+        console.log("\nSkipped (file exists, review manually):");
+        for (const f of skippedFiles) {
+          console.log(`  ${f}`);
+        }
+      }
+
+      if (removedKeys.length > 0) {
+        console.log("\nRemoved endpoints (not deleted — review tests):");
+        for (const key of removedKeys) {
+          console.log(`  - ${key}`);
+        }
+      }
+
+      if (generatedFiles.length > 0) {
+        printSuccess(`\nSync complete. ${generatedFiles.length} file(s) written.`);
+      } else {
+        printWarning("No new files written — all target files already exist.");
+      }
+
+      for (const w of warnings) {
+        printWarning(w);
+      }
+    }
+
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("sync", [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/index.ts

@@ -3,7 +3,6 @@
 import { runCommand } from "./commands/run.ts";
 import { validateCommand } from "./commands/validate.ts";
 import { serveCommand } from "./commands/serve.ts";
-import { mcpCommand } from "./commands/mcp.ts";
 import { coverageCommand } from "./commands/coverage.ts";
 import { ciInitCommand } from "./commands/ci-init.ts";
 import { initCommand } from "./commands/init.ts";
@@ -12,6 +11,8 @@ import { dbCommand } from "./commands/db.ts";
 import { requestCommand } from "./commands/request.ts";
 import { guideCommand } from "./commands/guide.ts";
 import { generateCommand } from "./commands/generate.ts";
+import { exportCommand } from "./commands/export.ts";
+import { syncCommand } from "./commands/sync.ts";
 import { printError } from "./output.ts";
 import { getRuntimeInfo } from "./runtime.ts";
 import { getDb } from "../db/schema.ts";
@@ -103,9 +104,9 @@ Usage:
   zond guide <spec>     Generate test generation guide from OpenAPI spec
   zond serve            Start web dashboard
   zond ui               Alias for 'serve --open' (start dashboard & open browser)
-  zond mcp              Start MCP server (stdio transport for AI agents)
-                           --dir <path>  Set working directory (relative paths resolve here)
   zond ci init          Generate CI/CD workflow (GitHub Actions, GitLab CI)
+  zond export postman <path>  Export YAML tests as Postman Collection v2.1
+  zond sync <spec>      Detect new/removed endpoints and generate tests for new ones
 
 Options for 'run':
   --dry-run            Show requests without sending them (exit code always 0)
@@ -175,6 +176,16 @@ Options for 'ci init':
   --dir <path>         Project root directory (default: current directory)
   --force              Overwrite existing CI config
 
+Options for 'export postman':
+  --output <file>      Output file path (default: collection.postman.json)
+  --env <file>         Also export .env.yaml as Postman environment
+  --collection-name <name>  Collection name (default: derived from path)
+
+Options for 'sync':
+  --tests <dir>        Path to test files directory (required)
+  --dry-run            Show what would be generated without writing files
+  --tag <tag>          Limit sync to endpoints with this tag
+
 General:
   --json               Output in JSON envelope format (available for all commands)
   --help, -h           Show this help
@@ -308,13 +319,6 @@ async function main(): Promise<number> {
       });
     }
 
-    case "mcp": {
-      return mcpCommand({
-        dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
-        dir: typeof flags["dir"] === "string" ? flags["dir"] : undefined,
-      });
-    }
-
     case "ci": {
       const ciSub = positional[0];
       if (ciSub !== "init") {
@@ -504,6 +508,46 @@ async function main(): Promise<number> {
       });
     }
 
+    case "export": {
+      const subcommand = positional[0];
+      if (subcommand !== "postman") {
+        printError(`Unknown export subcommand: ${subcommand ?? "(none)"}. Usage: zond export postman <path>`);
+        return 2;
+      }
+      const testsPath = positional[1];
+      if (!testsPath) {
+        printError("Missing tests path. Usage: zond export postman <path> [--output <file>]");
+        return 2;
+      }
+      return exportCommand({
+        testsPath,
+        output: typeof flags["output"] === "string" ? flags["output"] : "collection.postman.json",
+        env: typeof flags["env"] === "string" ? flags["env"] : undefined,
+        collectionName: typeof flags["collection-name"] === "string" ? flags["collection-name"] : undefined,
+        json: jsonFlag,
+      });
+    }
+
+    case "sync": {
+      const specPath = positional[0];
+      if (!specPath) {
+        printError("Missing spec path. Usage: zond sync <spec> --tests <dir> [--dry-run] [--tag <tag>]");
+        return 2;
+      }
+      const testsDir = typeof flags["tests"] === "string" ? flags["tests"] : undefined;
+      if (!testsDir) {
+        printError("Missing --tests <dir>. Usage: zond sync <spec> --tests <dir>");
+        return 2;
+      }
+      return syncCommand({
+        specPath,
+        testsDir,
+        dryRun: flags["dry-run"] === true,
+        tag: typeof flags["tag"] === "string" ? flags["tag"] : undefined,
+        json: jsonFlag,
+      });
+    }
+
     default: {
       printError(`Unknown command: ${command}`);
       printUsage();

package/src/core/diagnostics/db-analysis.ts

@@ -1,7 +1,8 @@
 import { getDb } from "../../db/schema.ts";
 import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
 import { join } from "node:path";
-import { statusHint, classifyFailure, envHint, envCategory, schemaHint, computeSharedEnvIssue } from "./failure-hints.ts";
+import { statusHint, classifyFailure, envHint, envCategory, schemaHint, computeSharedEnvIssue, recommendedAction, softDeleteHint, type RecommendedAction } from "./failure-hints.ts";
+import { AUTH_PATH_RE } from "../runner/execute-run.ts";
 
 export function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
   if (!raw) return undefined;
@@ -116,11 +117,18 @@ export interface FailureGroup {
   pattern: string;
   count: number;
   failure_type: string;
+  recommended_action: RecommendedAction;
   hint?: string;
   examples: string[];
   response_status: number | null;
 }
 
+export interface CascadeSkipGroup {
+  capture_var: string;
+  count: number;
+  examples: string[];
+}
+
 export interface DiagnoseResult {
   run: {
     id: number;
@@ -136,13 +144,17 @@ export interface DiagnoseResult {
     assertion_failures: number;
     network_errors: number;
   };
+  agent_directive?: string;
   env_issue?: string;
+  auth_hint?: string;
+  cascade_skips?: CascadeSkipGroup[];
   failures: Array<{
     suite_name: string;
     test_name: string;
     suite_file?: string;
     status: string;
     failure_type: string;
+    recommended_action: RecommendedAction;
     error_message?: string;
     request_method: string | null;
     request_url: string | null;
@@ -174,8 +186,12 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
   const failures = allResults
     .filter(r => r.status === "fail" || r.status === "error")
     .map(r => {
-      const hint = envHint(r.request_url, r.error_message, envFilePath) ?? statusHint(r.response_status);
+      const parsedBody = parseBodySafe(r.response_body);
+      const hint = envHint(r.request_url, r.error_message, envFilePath) ??
+        softDeleteHint(r.response_status, r.request_method, parsedBody) ??
+        statusHint(r.response_status);
       const failure_type = classifyFailure(r.status, r.response_status);
+      const rec_action = recommendedAction(failure_type, r.response_status);
       const sHint = schemaHint(failure_type, r.response_status);
       return {
         suite_name: r.suite_name,
@@ -183,13 +199,14 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
         ...(r.suite_file ? { suite_file: r.suite_file } : {}),
         status: r.status,
         failure_type,
+        recommended_action: rec_action,
         error_message: truncateErrorMessage(r.error_message, verbose),
         request_method: r.request_method,
         request_url: r.request_url,
         response_status: r.response_status,
         ...(hint ? { hint } : {}),
         ...(sHint ? { schema_hint: sHint } : {}),
-        response_body: parseBodySafe(r.response_body),
+        response_body: parsedBody,
         response_headers: filterHeaders(r.response_headers),
         assertions: r.assertions,
         duration_ms: r.duration_ms,
@@ -198,9 +215,60 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
 
   const sharedEnvHint = computeSharedEnvIssue(failures, envFilePath);
 
-  const apiErrors = failures.filter(f => f.failure_type === "api_error").length;
-  const assertionFailures = failures.filter(f => f.failure_type === "assertion_failed").length;
-  const networkErrors = failures.filter(f => f.failure_type === "network_error").length;
+  let apiErrors = 0, assertionFailures = 0, networkErrors = 0;
+  let authFailureCount = 0;
+  for (const f of failures) {
+    if (f.failure_type === "api_error") apiErrors++;
+    else if (f.failure_type === "assertion_failed") assertionFailures++;
+    else if (f.failure_type === "network_error") networkErrors++;
+    if (f.response_status === 401 || f.response_status === 403) authFailureCount++;
+  }
+
+  let agent_directive: string | undefined;
+  if (apiErrors > 0) {
+    const fixable = assertionFailures + networkErrors;
+    agent_directive =
+      `${apiErrors} test${apiErrors === 1 ? "" : "s"} returned 5xx server errors. ` +
+      `Do NOT change test expectations to accept 5xx responses. ` +
+      `These are backend bugs, not test logic errors. ` +
+      `Stop iterating on these tests and report the failures to the API team.` +
+      (fixable > 0
+        ? ` The remaining ${fixable} failure${fixable === 1 ? "" : "s"} may be fixable in test logic.`
+        : "");
+  }
+
+  // Cascade skips: skipped tests due to missing captures from failed create steps
+  const CASCADE_RE = /^Depends on missing capture: (.+)$/;
+  const groupMap = new Map<string, string[]>();
+  for (const r of allResults) {
+    if (r.status !== "skip") continue;
+    const match = CASCADE_RE.exec(r.error_message ?? "");
+    if (!match) continue;
+    const captureVar = match[1]!;
+    const existing = groupMap.get(captureVar) ?? [];
+    existing.push(`${r.suite_name}/${r.test_name}`);
+    groupMap.set(captureVar, existing);
+  }
+  const cascade_skips: CascadeSkipGroup[] | undefined = groupMap.size > 0
+    ? [...groupMap.entries()].map(([capture_var, examples]) => ({
+        capture_var,
+        count: examples.length,
+        examples: examples.slice(0, 3),
+      }))
+    : undefined;
+
+  // Auth hint: when many tests fail with 401/403, suggest auth setup
+  let auth_hint: string | undefined;
+  if (authFailureCount >= 5 && authFailureCount / diagRun.total >= 0.3) {
+    const loginEndpoint = allResults.find(
+      r => r.request_method?.toUpperCase() === "POST" && AUTH_PATH_RE.test(r.request_url ?? "")
+    );
+    if (loginEndpoint) {
+      auth_hint = `${authFailureCount} tests failed with 401/403. Found auth endpoint: POST ${loginEndpoint.request_url} — add \`setup: true\` to your auth suite so its captured token is shared with all other suites, or set auth_token manually in .env.yaml`;
+    } else {
+      auth_hint = `${authFailureCount} tests failed with 401/403 — add \`setup: true\` to your auth suite so its captured token is shared with all other suites, or set auth_token in .env.yaml`;
+    }
+  }
 
   const { grouped_failures, compactFailures } = verbose
     ? { grouped_failures: undefined, compactFailures: failures }
@@ -221,13 +289,16 @@ export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string):
       assertion_failures: assertionFailures,
       network_errors: networkErrors,
     },
+    ...(agent_directive ? { agent_directive } : {}),
     ...(sharedEnvHint ? { env_issue: sharedEnvHint } : {}),
+    ...(auth_hint ? { auth_hint } : {}),
+    ...(cascade_skips ? { cascade_skips } : {}),
     failures: compactFailures,
     ...(grouped_failures ? { grouped_failures } : {}),
   };
 }
 
-type FailureItem = { suite_name: string; test_name: string; failure_type: string; hint?: string; response_status: number | null };
+type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; hint?: string; response_status: number | null };
 
 /** Group similar failures for compact output. Exported for testing. */
 export function groupFailures<T extends FailureItem>(failures: T[]): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
@@ -268,6 +339,7 @@ export function groupFailures<T extends FailureItem>(failures: T[]): { grouped_f
       pattern,
       count: group.items.length,
       failure_type: group.failure_type,
+      recommended_action: group.items[0]!.recommended_action,
       hint: group.hint,
       examples: group.items.slice(0, 2).map(f => `${f.suite_name}/${f.test_name}`),
       response_status: group.response_status,

package/src/core/diagnostics/failure-hints.ts

@@ -37,6 +37,26 @@ export function envHint(url: string | null, errorMessage: string | null, envFile
   return null;
 }
 
+export type RecommendedAction =
+  | "report_backend_bug"
+  | "fix_auth_config"
+  | "fix_test_logic"
+  | "fix_network_config";
+
+export function recommendedAction(
+  failureType: "api_error" | "assertion_failed" | "network_error",
+  responseStatus: number | null,
+): RecommendedAction {
+  if (failureType === "api_error") return "report_backend_bug";
+  if (failureType === "network_error") {
+    if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
+    return "fix_network_config";
+  }
+  // assertion_failed
+  if (responseStatus === 401 || responseStatus === 403) return "fix_auth_config";
+  return "fix_test_logic";
+}
+
 export function envCategory(hint: string | undefined): string | null {
   if (!hint) return null;
   if (hint.includes("base_url is not set") || hint.includes("base_url is missing") || hint.includes("base_url is not configured")) return "base_url_missing";
@@ -55,6 +75,25 @@ export function schemaHint(
   return null;
 }
 
+export function softDeleteHint(
+  actualStatus: number | null | undefined,
+  requestMethod: string | null | undefined,
+  responseBody: unknown,
+): string | null {
+  if (actualStatus !== 200 || requestMethod?.toUpperCase() !== "GET") return null;
+  if (responseBody && typeof responseBody === "object") {
+    const hasStatusField =
+      "status" in (responseBody as object) ||
+      "state" in (responseBody as object) ||
+      "deleted" in (responseBody as object) ||
+      "is_deleted" in (responseBody as object);
+    if (hasStatusField) {
+      return 'GET returned 200 with a status/state field after DELETE — likely soft delete. Update the test: remove the "Verify deleted → 404" step and instead assert the status field value (e.g. status: "cancelled")';
+    }
+  }
+  return null;
+}
+
 export function computeSharedEnvIssue(
   failures: Array<{ hint?: string }>,
   envFilePath?: string,