@kirrosh/zond 0.14.0 → 0.16.0

package/README.md

@@ -17,7 +17,7 @@ Zond reads your OpenAPI spec and gives your AI agent everything it needs to test
 
 Then say: _"Safely cover the API from openapi.json with tests"_
 
-You get skills, slash commands, and 12 MCP tools in one package.
+You get auto-validation hooks, CLI tools, and 8 MCP tools — all in one package.
 
 <details>
 <summary>Other installation methods (MCP, CLI, binary)</summary>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kirrosh/zond",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
   "license": "MIT",
   "module": "index.ts",
@@ -26,12 +26,13 @@
   "scripts": {
     "zond": "bun run src/cli/index.ts",
     "test": "bun run test:unit && bun run test:mocked",
-    "test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/safe-run.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/reporter/ tests/version-sync.test.ts",
+    "test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/safe-run.test.ts tests/cli/json-envelope.test.ts tests/cli/describe.test.ts tests/cli/db.test.ts tests/cli/request.test.ts tests/cli/init.test.ts tests/cli/guide.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/reporter/ tests/version-sync.test.ts",
     "test:mocked": "bun run scripts/run-mocked-tests.ts",
     "check": "tsc --noEmit --project tsconfig.json",
     "build": "bun build --compile src/cli/index.ts --outfile zond",
     "version:sync": "bun run scripts/sync-version.ts",
-    "postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json .claude-plugin/marketplace.json"
+    "postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json",
+    "bench:api": "bun benchmarks/api/server.ts"
   },
   "devDependencies": {
     "@types/bun": "latest"

package/src/cli/commands/ci-init.ts

@@ -1,11 +1,13 @@
 import { resolve, dirname } from "path";
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import { printSuccess, printError } from "../output.ts";
+import { jsonOk, printJson } from "../json-envelope.ts";
 
 export interface CiInitOptions {
   platform?: "github" | "gitlab";
   force: boolean;
   dir?: string;
+  json?: boolean;
 }
 
 const GH_ACTIONS_TEMPLATE = `name: API Tests
@@ -155,7 +157,16 @@ export async function ciInitCommand(options: CiInitOptions): Promise<number> {
     created = writeIfMissing(targetPath, GITLAB_CI_TEMPLATE, options.force);
   }
 
-  if (created) {
+  if (options.json) {
+    const targetPath = platform === "github"
+      ? resolve(cwd, ".github/workflows/api-tests.yml")
+      : resolve(cwd, ".gitlab-ci.yml");
+    printJson(jsonOk("ci init", {
+      platform,
+      filePath: targetPath,
+      created,
+    }));
+  } else if (created) {
     printSuccess("CI workflow created. Commit and push to activate.");
   }
 

package/src/cli/commands/coverage.ts

@@ -2,12 +2,14 @@ import { readOpenApiSpec, extractEndpoints, scanCoveredEndpoints, filterUncovere
 import { getDb } from "../../db/schema.ts";
 import { getResultsByRunId, getRunById } from "../../db/queries.ts";
 import { printError, printSuccess } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
 
 export interface CoverageOptions {
   spec: string;
   tests: string;
   failOnCoverage?: number;
   runId?: number;
+  json?: boolean;
 }
 
 const RESET = "\x1b[0m";
@@ -145,12 +147,30 @@ export async function coverageCommand(options: CoverageOptions): Promise<number>
       }
     }
 
+    if (options.json) {
+      const coveredEndpoints = allEndpoints.filter(ep => !uncovered.includes(ep)).map(ep => `${ep.method} ${ep.path}`);
+      const uncoveredEndpoints = uncovered.map(ep => `${ep.method} ${ep.path}`);
+      printJson(jsonOk("coverage", {
+        covered: coveredCount,
+        uncovered: uncovered.length,
+        total: allEndpoints.length,
+        percentage,
+        coveredEndpoints,
+        uncoveredEndpoints,
+      }));
+    }
+
     if (options.failOnCoverage !== undefined) {
       return percentage < options.failOnCoverage ? 1 : 0;
     }
     return uncovered.length > 0 ? 1 : 0;
   } catch (err) {
-    printError(err instanceof Error ? err.message : String(err));
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("coverage", [message]));
+    } else {
+      printError(message);
+    }
     return 2;
   }
 }

package/src/cli/commands/db.ts

@@ -0,0 +1,121 @@
+import { getCollections, getRuns, getRunDetail, diagnoseRun, compareRuns } from "../../core/diagnostics/db-analysis.ts";
+import { printError } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface DbOptions {
+  subcommand: string;
+  positional: string[];
+  limit?: number;
+  verbose?: boolean;
+  dbPath?: string;
+  json?: boolean;
+}
+
+export async function dbCommand(options: DbOptions): Promise<number> {
+  const { subcommand, positional, json } = options;
+
+  try {
+    switch (subcommand) {
+      case "collections": {
+        const collections = getCollections(options.dbPath);
+        if (json) {
+          printJson(jsonOk("db collections", { collections }));
+        } else {
+          if (collections.length === 0) {
+            console.log("No collections found.");
+          } else {
+            for (const c of collections) {
+              console.log(`[${(c as any).id}] ${(c as any).name} — ${(c as any).test_path ?? "no test path"}`);
+            }
+          }
+        }
+        return 0;
+      }
+
+      case "runs": {
+        const runs = getRuns(options.limit ?? 10, options.dbPath);
+        if (json) {
+          printJson(jsonOk("db runs", { runs }));
+        } else {
+          if (runs.length === 0) {
+            console.log("No runs found.");
+          } else {
+            for (const r of runs) {
+              const run = r as any;
+              const status = run.failed > 0 ? "FAIL" : "PASS";
+              console.log(`#${run.id} ${status} ${run.passed}/${run.total} passed (${run.started_at})`);
+            }
+          }
+        }
+        return 0;
+      }
+
+      case "run": {
+        const id = parseInt(positional[0] ?? "", 10);
+        if (isNaN(id)) {
+          const msg = "Missing run ID. Usage: zond db run <id>";
+          if (json) printJson(jsonError("db run", [msg]));
+          else printError(msg);
+          return 2;
+        }
+        const detail = getRunDetail(id, options.verbose, options.dbPath);
+        if (json) {
+          printJson(jsonOk("db run", detail));
+        } else {
+          console.log(JSON.stringify(detail, null, 2));
+        }
+        return 0;
+      }
+
+      case "diagnose": {
+        const id = parseInt(positional[0] ?? "", 10);
+        if (isNaN(id)) {
+          const msg = "Missing run ID. Usage: zond db diagnose <id>";
+          if (json) printJson(jsonError("db diagnose", [msg]));
+          else printError(msg);
+          return 2;
+        }
+        const result = diagnoseRun(id, options.verbose, options.dbPath);
+        if (json) {
+          printJson(jsonOk("db diagnose", result));
+        } else {
+          console.log(JSON.stringify(result, null, 2));
+        }
+        return 0;
+      }
+
+      case "compare": {
+        const idA = parseInt(positional[0] ?? "", 10);
+        const idB = parseInt(positional[1] ?? "", 10);
+        if (isNaN(idA) || isNaN(idB)) {
+          const msg = "Missing run IDs. Usage: zond db compare <idA> <idB>";
+          if (json) printJson(jsonError("db compare", [msg]));
+          else printError(msg);
+          return 2;
+        }
+        const result = compareRuns(idA, idB, options.dbPath);
+        if (json) {
+          printJson(jsonOk("db compare", result));
+        } else {
+          console.log(JSON.stringify(result, null, 2));
+        }
+        return 0;
+      }
+
+      default: {
+        const msg = `Unknown db subcommand: ${subcommand}. Available: collections, runs, run, diagnose, compare`;
+        if (json) printJson(jsonError("db", [msg]));
+        else printError(msg);
+        return 2;
+      }
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (json) {
+      printJson(jsonError(`db ${subcommand}`, [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/commands/describe.ts

@@ -0,0 +1,60 @@
+import { describeEndpoint, describeCompact } from "../../core/generator/describe.ts";
+import { printError } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface DescribeOptions {
+  specPath: string;
+  method?: string;
+  path?: string;
+  compact?: boolean;
+  json?: boolean;
+}
+
+export async function describeCommand(options: DescribeOptions): Promise<number> {
+  try {
+    if (options.compact) {
+      const endpoints = await describeCompact(options.specPath);
+
+      if (options.json) {
+        printJson(jsonOk("describe", { endpoints }));
+      } else {
+        for (const ep of endpoints) {
+          const parts = [ep.method.padEnd(7), ep.path];
+          if (ep.operationId) parts.push(`(${ep.operationId})`);
+          if (ep.summary) parts.push(`— ${ep.summary}`);
+          if (ep.deprecated) parts.push("[deprecated]");
+          console.log(parts.join(" "));
+        }
+        console.log(`\n${endpoints.length} endpoint(s)`);
+      }
+      return 0;
+    }
+
+    if (!options.method || !options.path) {
+      const msg = "Missing --method and --path. Use --compact for all endpoints, or specify --method and --path for one.";
+      if (options.json) {
+        printJson(jsonError("describe", [msg]));
+      } else {
+        printError(msg);
+      }
+      return 2;
+    }
+
+    const result = await describeEndpoint(options.specPath, options.method, options.path);
+
+    if (options.json) {
+      printJson(jsonOk("describe", result));
+    } else {
+      console.log(JSON.stringify(result, null, 2));
+    }
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("describe", [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/commands/generate.ts

@@ -0,0 +1,127 @@
+import { join, dirname } from "path";
+import { mkdir } from "fs/promises";
+import {
+  readOpenApiSpec,
+  extractEndpoints,
+  extractSecuritySchemes,
+  scanCoveredEndpoints,
+  filterUncoveredEndpoints,
+  serializeSuite,
+} from "../../core/generator/index.ts";
+import { generateSuites } from "../../core/generator/suite-generator.ts";
+import { filterByTag } from "../../core/generator/chunker.ts";
+import { parse } from "../../core/parser/yaml-parser.ts";
+import { printError, printSuccess } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface GenerateOptions {
+  specPath: string;
+  output: string;
+  tag?: string;
+  uncoveredOnly?: boolean;
+  json?: boolean;
+}
+
+export async function generateCommand(options: GenerateOptions): Promise<number> {
+  try {
+    const doc = await readOpenApiSpec(options.specPath);
+    let endpoints = extractEndpoints(doc);
+    const securitySchemes = extractSecuritySchemes(doc);
+    const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
+    const warnings: string[] = [];
+
+    // Filter to uncovered only
+    if (options.uncoveredOnly) {
+      const covered = await scanCoveredEndpoints(options.output);
+      const before = endpoints.length;
+      endpoints = filterUncoveredEndpoints(endpoints, covered);
+      const coveredCount = before - endpoints.length;
+      if (coveredCount > 0) {
+        warnings.push(`Skipped ${coveredCount} already-covered endpoints`);
+      }
+    }
+
+    // Filter by tag
+    if (options.tag) {
+      endpoints = filterByTag(endpoints, options.tag);
+    }
+
+    if (endpoints.length === 0) {
+      if (options.json) {
+        printJson(jsonOk("generate", { files: [], message: "No endpoints to generate tests for" }, warnings));
+      } else {
+        console.log("No endpoints to generate tests for.");
+      }
+      return 0;
+    }
+
+    // Generate suites
+    const suites = generateSuites({ endpoints, securitySchemes });
+
+    // Ensure output directory exists
+    await mkdir(options.output, { recursive: true });
+
+    // Write suite files
+    const createdFiles: Array<{ file: string; suite: string; tests: number }> = [];
+
+    for (const suite of suites) {
+      const yaml = serializeSuite(suite);
+      const fileName = `${suite.fileStem ?? suite.name}.yaml`;
+      const filePath = join(options.output, fileName);
+      await Bun.write(filePath, yaml);
+      createdFiles.push({ file: filePath, suite: suite.name, tests: suite.tests.length });
+    }
+
+    // Create .env.yaml with base_url if it doesn't exist
+    const envPath = join(options.output, ".env.yaml");
+    const envFile = Bun.file(envPath);
+    if (!(await envFile.exists()) && baseUrl) {
+      await Bun.write(envPath, `base_url: ${baseUrl}\n`);
+      warnings.push(`Created ${envPath} with base_url from spec`);
+    }
+
+    // Validate generated files
+    const validationErrors: string[] = [];
+    try {
+      await parse(options.output);
+    } catch (err) {
+      validationErrors.push(err instanceof Error ? err.message : String(err));
+    }
+
+    if (validationErrors.length > 0) {
+      warnings.push(`Validation warnings: ${validationErrors.join("; ")}`);
+    }
+
+    // Output
+    const totalTests = createdFiles.reduce((sum, f) => sum + f.tests, 0);
+
+    if (options.json) {
+      printJson(jsonOk("generate", {
+        files: createdFiles,
+        totalSuites: suites.length,
+        totalTests,
+        outputDir: options.output,
+      }, warnings));
+    } else {
+      printSuccess(`Generated ${suites.length} suite(s) with ${totalTests} test(s) in ${options.output}`);
+      for (const f of createdFiles) {
+        console.log(`  ${f.file} (${f.tests} tests)`);
+      }
+      if (warnings.length > 0) {
+        for (const w of warnings) {
+          console.log(`  ⚠ ${w}`);
+        }
+      }
+    }
+
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("generate", [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/commands/guide.ts

@@ -0,0 +1,127 @@
+import {
+  readOpenApiSpec,
+  extractEndpoints,
+  extractSecuritySchemes,
+  scanCoveredEndpoints,
+  filterUncoveredEndpoints,
+} from "../../core/generator/index.ts";
+import { compressEndpointsWithSchemas, buildGenerationGuide } from "../../core/generator/guide-builder.ts";
+import { planChunks, filterByTag } from "../../core/generator/chunker.ts";
+import { findCollectionBySpec } from "../../db/queries.ts";
+import { printError } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface GuideOptions {
+  specPath: string;
+  testsDir?: string;
+  tag?: string;
+  json?: boolean;
+}
+
+export async function guideCommand(options: GuideOptions): Promise<number> {
+  try {
+    const doc = await readOpenApiSpec(options.specPath);
+    let endpoints = extractEndpoints(doc);
+    const securitySchemes = extractSecuritySchemes(doc);
+    const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
+    const title = (doc as any).info?.title as string | undefined;
+
+    let outputDir = options.testsDir;
+    if (!outputDir) {
+      try {
+        const collection = findCollectionBySpec(options.specPath);
+        outputDir = collection?.test_path ?? "./tests/";
+      } catch {
+        outputDir = "./tests/";
+      }
+    }
+
+    let coverageInfo: { covered: number; total: number; percentage: number } | undefined;
+    if (options.testsDir) {
+      const totalBefore = endpoints.length;
+      const covered = await scanCoveredEndpoints(options.testsDir);
+      const uncovered = filterUncoveredEndpoints(endpoints, covered);
+      const coveredCount = totalBefore - uncovered.length;
+      const percentage = totalBefore > 0 ? Math.round((coveredCount / totalBefore) * 100) : 100;
+      coverageInfo = { covered: coveredCount, total: totalBefore, percentage };
+      endpoints = uncovered;
+    }
+
+    if (endpoints.length === 0) {
+      if (options.json) {
+        printJson(jsonOk("guide", { fullyCovered: true, ...coverageInfo }));
+      } else {
+        console.log("All endpoints are covered.");
+      }
+      return 0;
+    }
+
+    if (options.tag) {
+      endpoints = filterByTag(endpoints, options.tag);
+      if (endpoints.length === 0) {
+        const msg = `No endpoints found for tag "${options.tag}"`;
+        if (options.json) printJson(jsonError("guide", [msg]));
+        else printError(msg);
+        return 1;
+      }
+    }
+
+    const plan = planChunks(endpoints);
+
+    if (plan.needsChunking && !options.tag) {
+      if (options.json) {
+        printJson(jsonOk("guide", {
+          mode: "plan",
+          title: title ?? "API",
+          totalEndpoints: plan.totalEndpoints,
+          chunks: plan.chunks,
+          ...(coverageInfo ? { coverage: coverageInfo } : {}),
+        }));
+      } else {
+        console.log(`API has ${plan.totalEndpoints} endpoints across ${plan.chunks.length} tags.`);
+        console.log("Generate per-tag with --tag <name>:\n");
+        for (const chunk of plan.chunks) {
+          console.log(`  --tag ${chunk.tag} (${chunk.count} endpoints)`);
+        }
+      }
+      return 0;
+    }
+
+    const coverageHeader = coverageInfo
+      ? `## Coverage: ${coverageInfo.covered}/${coverageInfo.total} endpoints covered (${coverageInfo.percentage}%). Generating tests for ${endpoints.length} uncovered endpoints:`
+      : undefined;
+
+    const apiContext = compressEndpointsWithSchemas(endpoints, securitySchemes);
+    const guide = buildGenerationGuide({
+      title: options.tag ? `${title ?? "API"} — tag: ${options.tag}` : (title ?? "API"),
+      baseUrl,
+      apiContext,
+      outputDir,
+      securitySchemes,
+      endpointCount: endpoints.length,
+      coverageHeader,
+      includeFormat: true,
+    });
+
+    if (options.json) {
+      printJson(jsonOk("guide", {
+        title: title ?? "API",
+        endpointCount: endpoints.length,
+        outputDir,
+        guide,
+        ...(coverageInfo ? { coverage: coverageInfo } : {}),
+      }));
+    } else {
+      console.log(guide);
+    }
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("guide", [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/commands/init.ts

@@ -0,0 +1,57 @@
+import { setupApi } from "../../core/setup-api.ts";
+import { printError, printSuccess } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface InitOptions {
+  name?: string;
+  spec?: string;
+  baseUrl?: string;
+  dir?: string;
+  force?: boolean;
+  insecure?: boolean;
+  dbPath?: string;
+  json?: boolean;
+}
+
+export async function initCommand(options: InitOptions): Promise<number> {
+  try {
+    const envVars: Record<string, string> = {};
+    if (options.baseUrl) envVars.base_url = options.baseUrl;
+
+    const result = await setupApi({
+      name: options.name,
+      spec: options.spec,
+      dir: options.dir,
+      envVars: Object.keys(envVars).length > 0 ? envVars : undefined,
+      dbPath: options.dbPath,
+      force: options.force,
+      insecure: options.insecure,
+    });
+
+    if (options.json) {
+      printJson(jsonOk("init", {
+        collectionId: result.collectionId,
+        baseDir: result.baseDir,
+        testPath: result.testPath,
+        endpoints: result.specEndpoints,
+        warnings: result.warnings ?? [],
+      }, result.warnings));
+    } else {
+      printSuccess(`Created API '${options.name ?? "api"}' at ${result.baseDir} (${result.specEndpoints} endpoints)`);
+      if (result.warnings) {
+        for (const w of result.warnings) {
+          process.stderr.write(`Warning: ${w}\n`);
+        }
+      }
+    }
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("init", [message]));
+    } else {
+      printError(message);
+    }
+    return 2;
+  }
+}

package/src/cli/commands/request.ts

@@ -0,0 +1,57 @@
+import { sendAdHocRequest } from "../../core/runner/send-request.ts";
+import { printError } from "../output.ts";
+import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
+
+export interface RequestOptions {
+  method: string;
+  url: string;
+  headers?: string[];
+  body?: string;
+  timeout?: number;
+  env?: string;
+  api?: string;
+  jsonPath?: string;
+  dbPath?: string;
+  json?: boolean;
+}
+
+export async function requestCommand(options: RequestOptions): Promise<number> {
+  try {
+    const headers: Record<string, string> = {};
+    if (options.headers) {
+      for (const h of options.headers) {
+        const colonIdx = h.indexOf(":");
+        if (colonIdx > 0) {
+          headers[h.slice(0, colonIdx).trim()] = h.slice(colonIdx + 1).trim();
+        }
+      }
+    }
+
+    const result = await sendAdHocRequest({
+      method: options.method.toUpperCase(),
+      url: options.url,
+      headers: Object.keys(headers).length > 0 ? headers : undefined,
+      body: options.body,
+      timeout: options.timeout,
+      envName: options.env,
+      collectionName: options.api,
+      jsonPath: options.jsonPath,
+      dbPath: options.dbPath,
+    });
+
+    if (options.json) {
+      printJson(jsonOk("request", result));
+    } else {
+      console.log(JSON.stringify(result, null, 2));
+    }
+    return 0;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (options.json) {
+      printJson(jsonError("request", [message]));
+    } else {
+      printError(message);
+    }
+    return 1;
+  }
+}