@kirrosh/zond 0.14.0 → 0.16.0

package/src/mcp/tools/query-db.ts

@@ -1,63 +1,7 @@
 import { z } from "zod";
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { getDb } from "../../db/schema.ts";
-import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
-import { join } from "node:path";
 import { TOOL_DESCRIPTIONS } from "../descriptions.js";
-import { statusHint, classifyFailure, envHint, envCategory, schemaHint } from "../../core/diagnostics/failure-hints.ts";
-
-function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
-  if (!raw) return undefined;
-  if (verbose || raw.length < 500) return raw;
-  const lines = raw.split(/\r?\n/);
-  // First line is the error message itself
-  const msgLines = [lines[0]!];
-  // Grab up to 3 stack-trace lines (indented or starting with "at ")
-  let traceCount = 0;
-  for (let i = 1; i < lines.length && traceCount < 3; i++) {
-    const line = lines[i]!;
-    if (/^\s+/.test(line) || /^\s*at\s/.test(line)) {
-      msgLines.push(line);
-      traceCount++;
-    }
-  }
-  const remaining = lines.length - msgLines.length;
-  if (remaining > 0) {
-    msgLines.push(`...[truncated ${remaining} lines]`);
-  }
-  return msgLines.join("\n");
-}
-
-function parseBodySafe(raw: string | null | undefined): unknown {
-  if (!raw) return undefined;
-  const truncated = raw.length > 2000 ? raw.slice(0, 2000) + "…[truncated]" : raw;
-  try {
-    return JSON.parse(raw);
-  } catch {
-    return truncated;
-  }
-}
-
-const USEFUL_HEADERS = new Set([
-  "content-type", "content-length", "location", "retry-after",
-  "www-authenticate", "allow",
-]);
-const USEFUL_PREFIXES = ["x-", "ratelimit"];
-
-function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
-  if (!raw) return undefined;
-  try {
-    const h = JSON.parse(raw) as Record<string, string>;
-    const out: Record<string, string> = {};
-    for (const [k, v] of Object.entries(h)) {
-      const l = k.toLowerCase();
-      if (USEFUL_HEADERS.has(l) || USEFUL_PREFIXES.some(p => l.startsWith(p))) {
-        out[k] = v;
-      }
-    }
-    return Object.keys(out).length > 0 ? out : undefined;
-  } catch { return undefined; }
-}
+import { getCollections, getRuns, getRunDetail, diagnoseRun, compareRuns } from "../../core/diagnostics/db-analysis.ts";
 
 export function registerQueryDbTool(server: McpServer, dbPath?: string) {
   server.registerTool("query_db", {
@@ -76,18 +20,16 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
     },
   }, async ({ action, runId, runIdB, limit, verbose }) => {
     try {
-      getDb(dbPath);
-
       switch (action) {
         case "list_collections": {
-          const collections = listCollections();
+          const collections = getCollections(dbPath);
           return {
             content: [{ type: "text" as const, text: JSON.stringify(collections, null, 2) }],
           };
         }
 
         case "list_runs": {
-          const runs = listRuns(limit ?? 20);
+          const runs = getRuns(limit ?? 20, dbPath);
           return {
             content: [{ type: "text" as const, text: JSON.stringify(runs, null, 2) }],
           };
@@ -100,39 +42,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
               isError: true,
             };
           }
-          const run = getRunById(runId);
-          if (!run) {
-            return {
-              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run ${runId} not found` }, null, 2) }],
-              isError: true,
-            };
-          }
-          const results = getResultsByRunId(runId);
-          const detail = {
-            run: {
-              id: run.id,
-              started_at: run.started_at,
-              finished_at: run.finished_at,
-              total: run.total,
-              passed: run.passed,
-              failed: run.failed,
-              skipped: run.skipped,
-              trigger: run.trigger,
-              environment: run.environment,
-              duration_ms: run.duration_ms,
-            },
-            results: results.map(r => ({
-              suite_name: r.suite_name,
-              test_name: r.test_name,
-              status: r.status,
-              duration_ms: r.duration_ms,
-              request_method: r.request_method,
-              request_url: r.request_url,
-              response_status: r.response_status,
-              error_message: truncateErrorMessage(r.error_message, verbose),
-              assertions: r.assertions,
-            })),
-          };
+          const detail = getRunDetail(runId, verbose, dbPath);
           return {
             content: [{ type: "text" as const, text: JSON.stringify(detail, null, 2) }],
           };
@@ -145,81 +55,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
               isError: true,
             };
           }
-          const diagRun = getRunById(runId);
-          if (!diagRun) {
-            return {
-              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run ${runId} not found` }, null, 2) }],
-              isError: true,
-            };
-          }
-
-          // Resolve env file path from collection for actionable hints
-          let envFilePath: string | undefined;
-          if (diagRun.collection_id) {
-            const collection = getCollectionById(diagRun.collection_id);
-            if (collection?.base_dir) {
-              envFilePath = join(collection.base_dir, ".env.yaml").replace(/\\/g, "/");
-            }
-          }
-
-          const allResults = getResultsByRunId(runId);
-          const failures = allResults
-            .filter(r => r.status === "fail" || r.status === "error")
-            .map(r => {
-              // env issues take priority over generic status hints
-              const hint = envHint(r.request_url, r.error_message, envFilePath) ?? statusHint(r.response_status);
-              const failure_type = classifyFailure(r.status, r.response_status);
-              const sHint = schemaHint(failure_type, r.response_status);
-              return {
-                suite_name: r.suite_name,
-                test_name: r.test_name,
-                status: r.status,
-                failure_type,
-                error_message: truncateErrorMessage(r.error_message, verbose),
-                request_method: r.request_method,
-                request_url: r.request_url,
-                response_status: r.response_status,
-                ...(hint ? { hint } : {}),
-                ...(sHint ? { schema_hint: sHint } : {}),
-                response_body: parseBodySafe(r.response_body),
-                response_headers: filterHeaders(r.response_headers),
-                assertions: r.assertions,
-                duration_ms: r.duration_ms,
-              };
-            });
-
-          // Top-level env_issue when all failures have the same env problem category
-          const categories = new Set(failures.map(f => envCategory(f.hint)).filter(Boolean));
-          const sharedEnvHint = categories.size === 1
-            ? categories.has("base_url_missing")
-              ? `All failures: base_url is not set — add base_url to ${envFilePath ?? ".env.yaml"}`
-              : categories.has("unresolved_variable")
-              ? `All failures: some variables are not substituted — check variable names in ${envFilePath ?? ".env.yaml"}`
-              : [...failures.map(f => f.hint).filter(Boolean)][0]
-            : undefined;
-
-          const apiErrors = failures.filter(f => f.failure_type === "api_error").length;
-          const assertionFailures = failures.filter(f => f.failure_type === "assertion_failed").length;
-          const networkErrors = failures.filter(f => f.failure_type === "network_error").length;
-
-          const result = {
-            run: {
-              id: diagRun.id,
-              started_at: diagRun.started_at,
-              environment: diagRun.environment,
-              duration_ms: diagRun.duration_ms,
-            },
-            summary: {
-              total: diagRun.total,
-              passed: diagRun.passed,
-              failed: diagRun.failed,
-              api_errors: apiErrors,
-              assertion_failures: assertionFailures,
-              network_errors: networkErrors,
-            },
-            ...(sharedEnvHint ? { env_issue: sharedEnvHint } : {}),
-            failures,
-          };
+          const result = diagnoseRun(runId, verbose, dbPath);
           return {
             content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
           };
@@ -232,59 +68,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
               isError: true,
             };
           }
-          const runARecord = getRunById(runId);
-          const runBRecord = getRunById(runIdB);
-          if (!runARecord) {
-            return {
-              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run #${runId} not found` }, null, 2) }],
-              isError: true,
-            };
-          }
-          if (!runBRecord) {
-            return {
-              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run #${runIdB} not found` }, null, 2) }],
-              isError: true,
-            };
-          }
-
-          const resultsA = getResultsByRunId(runId);
-          const resultsB = getResultsByRunId(runIdB);
-
-          const mapA = new Map<string, string>();
-          const mapB = new Map<string, string>();
-          for (const r of resultsA) mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
-          for (const r of resultsB) mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
-
-          const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
-          const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
-          let unchanged = 0;
-          let newTests = 0;
-          let removedTests = 0;
-
-          for (const [key, statusB] of mapB) {
-            const statusA = mapA.get(key);
-            if (statusA === undefined) { newTests++; continue; }
-            const [suite, test] = key.split("::") as [string, string];
-            const wasPass = statusA === "pass";
-            const isPass = statusB === "pass";
-            const wasFail = statusA === "fail" || statusA === "error";
-            const isFail = statusB === "fail" || statusB === "error";
-            if (wasPass && isFail) regressions.push({ suite, test, before: statusA, after: statusB });
-            else if (wasFail && isPass) fixes.push({ suite, test, before: statusA, after: statusB });
-            else unchanged++;
-          }
-          for (const key of mapA.keys()) {
-            if (!mapB.has(key)) removedTests++;
-          }
-
-          const compareResult = {
-            runA: { id: runId, started_at: runARecord.started_at },
-            runB: { id: runIdB, started_at: runBRecord.started_at },
-            summary: { regressions: regressions.length, fixes: fixes.length, unchanged, newTests, removedTests },
-            regressions,
-            fixes,
-            hasRegressions: regressions.length > 0,
-          };
+          const compareResult = compareRuns(runId, runIdB, dbPath);
           return {
             content: [{ type: "text" as const, text: JSON.stringify(compareResult, null, 2) }],
           };

package/src/mcp/tools/run-tests.ts

@@ -56,6 +56,7 @@ export function registerRunTestsTool(server: McpServer, dbPath?: string) {
       r.steps.filter(s => s.status === "fail" || s.status === "error").map(s => ({
         suite: r.suite_name,
         test: s.name,
+        ...(r.suite_file ? { file: r.suite_file } : {}),
         status: s.status,
         error: s.error,
         assertions: s.assertions.filter(a => !a.passed).map(a => ({

package/src/mcp/tools/send-request.ts

@@ -1,29 +1,8 @@
 import { z } from "zod";
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { executeRequest } from "../../core/runner/http-client.ts";
-import { loadEnvironment, substituteString, substituteDeep } from "../../core/parser/variables.ts";
-import { getDb } from "../../db/schema.ts";
-import { findCollectionByNameOrId } from "../../db/queries.ts";
+import { sendAdHocRequest } from "../../core/runner/send-request.ts";
 import { TOOL_DESCRIPTIONS } from "../descriptions.js";
 
-function extractByPath(obj: unknown, path: string): unknown {
-  const segments = path.replace(/\[(\d+)\]/g, '.$1').split('.').filter(Boolean);
-  let current: unknown = obj;
-  for (const seg of segments) {
-    if (current === null || current === undefined) return undefined;
-    if (Array.isArray(current)) {
-      const idx = parseInt(seg, 10);
-      if (isNaN(idx)) return undefined;
-      current = current[idx];
-    } else if (typeof current === 'object') {
-      current = (current as Record<string, unknown>)[seg];
-    } else {
-      return undefined;
-    }
-  }
-  return current;
-}
-
 export function registerSendRequestTool(server: McpServer, dbPath?: string) {
   server.registerTool("send_request", {
     description: TOOL_DESCRIPTIONS.send_request,
@@ -40,48 +19,23 @@ export function registerSendRequestTool(server: McpServer, dbPath?: string) {
     },
   }, async ({ method, url, headers, body, timeout, envName, collectionName, jsonPath, maxResponseChars }) => {
     try {
-      let searchDir = process.cwd();
-      if (collectionName) {
-        getDb(dbPath);
-        const col = findCollectionByNameOrId(collectionName);
-        if (col?.base_dir) searchDir = col.base_dir;
-      }
-      const vars = await loadEnvironment(envName, searchDir);
-
-      const resolvedUrl = substituteString(url, vars) as string;
-      const parsedHeaders = headers ? JSON.parse(headers) as Record<string, string> : {};
-      const resolvedHeaders = Object.keys(parsedHeaders).length > 0 ? substituteDeep(parsedHeaders, vars) : {};
-      const resolvedBody = body ? substituteString(body, vars) as string : undefined;
-
-      const response = await executeRequest(
-        {
-          method,
-          url: resolvedUrl,
-          headers: resolvedHeaders,
-          body: resolvedBody,
-        },
-        timeout ? { timeout } : undefined,
-      );
-
-      let responseBody: unknown = response.body_parsed ?? response.body;
-
-      // Apply jsonPath filter
-      if (jsonPath && responseBody !== undefined) {
-        responseBody = extractByPath(responseBody, jsonPath);
-      }
-
-      const result = {
-        status: response.status,
-        headers: response.headers,
-        body: responseBody,
-        duration_ms: response.duration_ms,
-      };
+      const parsedHeaders = headers ? JSON.parse(headers) as Record<string, string> : undefined;
+
+      const result = await sendAdHocRequest({
+        method,
+        url,
+        headers: parsedHeaders,
+        body: body ?? undefined,
+        timeout,
+        envName,
+        collectionName,
+        jsonPath,
+        dbPath,
+      });
 
       let text = JSON.stringify(result, null, 2);
-
-      // Apply maxResponseChars truncation
       if (maxResponseChars && text.length > maxResponseChars) {
-        text = text.slice(0, maxResponseChars) + '\n…[truncated]';
+        text = text.slice(0, maxResponseChars) + '\n\u2026[truncated]';
       }
 
       return {

package/src/web/views/suites-tab.ts

@@ -8,7 +8,7 @@ import { basename } from "node:path";
 
 export function renderSuitesTab(state: CollectionState): string {
   if (state.suites.length === 0) {
-    return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>generate_and_save</code>.</div>`;
+    return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>zond guide</code> or use the test-generation skill.</div>`;
   }
 
   const rows = state.suites.map((s, i) => renderSuiteRow(s, i)).join("");

package/src/mcp/tools/generate-and-save.ts

@@ -1,202 +0,0 @@
-import { z } from "zod";
-import { join } from "node:path";
-import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import {
-  readOpenApiSpec,
-  extractEndpoints,
-  extractSecuritySchemes,
-  scanCoveredEndpoints,
-  filterUncoveredEndpoints,
-  serializeSuite,
-  generateSuites,
-  findUnresolvedVars,
-} from "../../core/generator/index.ts";
-import { loadEnvironment } from "../../core/parser/variables.ts";
-import { compressEndpointsWithSchemas, buildGenerationGuide } from "../../core/generator/guide-builder.ts";
-import { findCollectionBySpec } from "../../db/queries.ts";
-import { planChunks, filterByTag } from "../../core/generator/chunker.ts";
-import { TOOL_DESCRIPTIONS } from "../descriptions.js";
-import { validateAndSave } from "./save-test-suite.ts";
-
-export function registerGenerateAndSaveTool(server: McpServer) {
-  server.registerTool("generate_and_save", {
-    description: TOOL_DESCRIPTIONS.generate_and_save,
-    inputSchema: {
-      specPath: z.string().describe("Path or URL to OpenAPI spec file"),
-      outputDir: z.optional(z.string()).describe("Directory for saving test files (default: ./tests/)"),
-      tag: z.optional(z.string()).describe("Generate tests only for this tag's endpoints"),
-      methodFilter: z.optional(z.array(z.string())).describe("Only include endpoints with these HTTP methods (e.g. [\"GET\"] for smoke tests)"),
-      testsDir: z.optional(z.string()).describe("Path to existing tests directory — filters to uncovered endpoints only"),
-      overwrite: z.optional(z.boolean()).describe("Hint for save_test_suites overwrite behavior (default: false)"),
-      includeFormat: z.optional(z.boolean()).describe("Include YAML format reference (default: true, set false for subsequent tag chunks)"),
-      mode: z.optional(z.enum(["generate", "guide"])).describe(
-        "'generate' creates and saves YAML test files deterministically (default), 'guide' returns text for LLM-crafted tests"
-      ),
-    },
-  }, async ({ specPath, outputDir, tag, methodFilter, testsDir, overwrite, includeFormat, mode }) => {
-    try {
-      const doc = await readOpenApiSpec(specPath);
-      let endpoints = extractEndpoints(doc);
-      const securitySchemes = extractSecuritySchemes(doc);
-      const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
-      const title = (doc as any).info?.title as string | undefined;
-      let effectiveOutputDir = outputDir;
-      if (!effectiveOutputDir) {
-        const collection = findCollectionBySpec(specPath);
-        effectiveOutputDir = collection?.test_path ?? "./tests/";
-      }
-      const effectiveMode = mode ?? "generate";
-
-      // Apply method filter
-      if (methodFilter && methodFilter.length > 0) {
-        const methods = methodFilter.map(m => m.toUpperCase());
-        endpoints = endpoints.filter(ep => methods.includes(ep.method.toUpperCase()));
-      }
-
-      // Coverage filtering
-      let coverageInfo: { covered: number; total: number; percentage: number } | undefined;
-      if (testsDir) {
-        const totalBefore = endpoints.length;
-        const covered = await scanCoveredEndpoints(testsDir);
-        const uncovered = filterUncoveredEndpoints(endpoints, covered);
-        const coveredCount = totalBefore - uncovered.length;
-        const percentage = totalBefore > 0 ? Math.round((coveredCount / totalBefore) * 100) : 100;
-        coverageInfo = { covered: coveredCount, total: totalBefore, percentage };
-        endpoints = uncovered;
-      }
-
-      if (endpoints.length === 0) {
-        const msg = testsDir
-          ? { fullyCovered: true, ...coverageInfo }
-          : { error: "No endpoints found in the spec" };
-        return {
-          content: [{ type: "text" as const, text: JSON.stringify(msg, null, 2) }],
-          isError: !testsDir,
-        };
-      }
-
-      // Tag filtering
-      if (tag) {
-        endpoints = filterByTag(endpoints, tag);
-        if (endpoints.length === 0) {
-          return {
-            content: [{ type: "text" as const, text: JSON.stringify({ error: `No endpoints found for tag "${tag}"` }, null, 2) }],
-            isError: true,
-          };
-        }
-      }
-
-      const plan = planChunks(endpoints);
-
-      // Plan mode: large API without specific tag
-      if (plan.needsChunking && !tag) {
-        const result: Record<string, unknown> = {
-          mode: "plan",
-          title: title ?? "API",
-          totalEndpoints: plan.totalEndpoints,
-          chunks: plan.chunks,
-          instruction:
-            `This API has ${plan.totalEndpoints} endpoints across ${plan.chunks.length} tags. ` +
-            `Call generate_and_save with tag parameter for each chunk sequentially. ` +
-            (effectiveMode === "guide"
-              ? `Pass includeFormat: false for subsequent chunks to save tokens. `
-              : "") +
-            `Example: generate_and_save(specPath: '${specPath}', tag: '${plan.chunks[0]!.tag}'` +
-            (effectiveMode === "guide" ? `, mode: 'guide'` : "") + `)`,
-        };
-        if (coverageInfo) {
-          result.coverage = coverageInfo;
-        }
-        return {
-          content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
-        };
-      }
-
-      // ── Generate mode: deterministic YAML generation ──
-      if (effectiveMode === "generate") {
-        const suites = generateSuites({ endpoints, securitySchemes });
-
-        const files: Array<{
-          saved: boolean;
-          filePath: string;
-          tests: number;
-          error?: string;
-        }> = [];
-
-        for (const suite of suites) {
-          const yaml = serializeSuite(suite);
-          const fileName = (suite.fileStem ?? suite.name) + ".yaml";
-          const filePath = join(effectiveOutputDir, fileName);
-
-          const { result: saveResult } = await validateAndSave(filePath, yaml, overwrite ?? false);
-          files.push({
-            saved: saveResult.saved,
-            filePath: saveResult.filePath ?? filePath,
-            tests: suite.tests.length,
-            ...(saveResult.error ? { error: saveResult.error } : {}),
-          });
-        }
-
-        const warnings: string[] = [];
-        const env = await loadEnvironment(undefined, effectiveOutputDir);
-        const envKeys = new Set(Object.keys(env));
-        for (const suite of suites) {
-          const unresolved = findUnresolvedVars(suite, envKeys);
-          if (unresolved.length > 0)
-            warnings.push(`${suite.fileStem ?? suite.name}.yaml: unresolved [${unresolved.join(", ")}]`);
-        }
-
-        const response: Record<string, unknown> = {
-          mode: "generate",
-          suitesGenerated: suites.length,
-          files,
-          ...(warnings.length > 0 ? { warnings } : {}),
-          hint: files.some(f => !f.saved)
-            ? "Some files were not saved (already exist?). Use overwrite: true to replace."
-            : "Files saved. Run run_tests to verify. Use mode: 'guide' for LLM-crafted tests with more detail.",
-        };
-        if (coverageInfo) {
-          response.coverage = coverageInfo;
-        }
-
-        return {
-          content: [{ type: "text" as const, text: JSON.stringify(response, null, 2) }],
-        };
-      }
-
-      // ── Guide mode: text-based generation guide ──
-      const coverageHeader = coverageInfo
-        ? `## Coverage: ${coverageInfo.covered}/${coverageInfo.total} endpoints covered (${coverageInfo.percentage}%). Generating tests for ${endpoints.length} uncovered endpoints:`
-        : undefined;
-
-      const apiContext = compressEndpointsWithSchemas(endpoints, securitySchemes);
-      const guide = buildGenerationGuide({
-        title: tag ? `${title ?? "API"} — tag: ${tag}` : (title ?? "API"),
-        baseUrl,
-        apiContext,
-        outputDir: effectiveOutputDir,
-        securitySchemes,
-        endpointCount: endpoints.length,
-        coverageHeader,
-        includeFormat: includeFormat ?? true,
-      });
-
-      const saveInstructions = `
----
-## Save Instructions
-- Output directory: ${effectiveOutputDir}
-- Use \`save_test_suites\` to save all generated files in one call
-- Overwrite: ${overwrite ? "true" : "false (set overwrite: true in save_test_suites to replace existing files)"}
-- After saving, run \`run_tests\` to verify`;
-
-      return {
-        content: [{ type: "text" as const, text: guide + saveInstructions }],
-      };
-    } catch (err) {
-      return {
-        content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
-        isError: true,
-      };
-    }
-  });
-}