@kirrosh/zond 0.13.0 → 0.16.0

package/src/core/agent/tools/diagnose-failure.ts

@@ -1,51 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import { getDb } from "../../../db/schema.ts";
-import { getRunById, getResultsByRunId } from "../../../db/queries.ts";
-
-export const diagnoseFailureTool = tool({
-  description: "Diagnose failures in a test run by analyzing failed steps and their errors",
-  inputSchema: z.object({
-    runId: z.number().describe("Run ID to diagnose"),
-  }),
-  execute: async (args) => {
-    try {
-      getDb();
-
-      const run = getRunById(args.runId);
-      if (!run) return { error: `Run ${args.runId} not found` };
-
-      const results = getResultsByRunId(args.runId);
-      const failures = results
-        .filter((r) => r.status === "fail" || r.status === "error")
-        .map((r) => ({
-          suite_name: r.suite_name,
-          test_name: r.test_name,
-          status: r.status,
-          error_message: r.error_message,
-          request_method: r.request_method,
-          request_url: r.request_url,
-          response_status: r.response_status,
-          assertions: r.assertions,
-          duration_ms: r.duration_ms,
-        }));
-
-      return {
-        run: {
-          id: run.id,
-          started_at: run.started_at,
-          environment: run.environment,
-          duration_ms: run.duration_ms,
-        },
-        summary: {
-          total: run.total,
-          passed: run.passed,
-          failed: run.failed,
-        },
-        failures,
-      };
-    } catch (err) {
-      return { error: (err as Error).message };
-    }
-  },
-});

package/src/core/agent/tools/index.ts

@@ -1,42 +0,0 @@
-import { tool } from "ai";
-import { runTestsTool } from "./run-tests.ts";
-import { queryResultsTool } from "./query-results.ts";
-import { diagnoseFailureTool } from "./diagnose-failure.ts";
-import { sendRequestTool } from "./send-request.ts";
-import type { AgentConfig } from "../types.ts";
-
-export function buildAgentTools(config: AgentConfig) {
-  // In safe mode, wrap run_tests to force safe=true
-  const run_tests = config.safeMode
-    ? tool({
-        description: runTestsTool.description,
-        inputSchema: runTestsTool.inputSchema,
-        execute: async (args, options) => {
-          return runTestsTool.execute!({ ...args, safe: true }, options);
-        },
-      })
-    : runTestsTool;
-
-  // In safe mode, wrap send_request to only allow GET
-  const send_request = config.safeMode
-    ? tool({
-        description: sendRequestTool.description,
-        inputSchema: sendRequestTool.inputSchema,
-        execute: async (args, options) => {
-          if (args.method !== "GET") {
-            return { error: "Safe mode: only GET requests are allowed" };
-          }
-          return sendRequestTool.execute!(args, options);
-        },
-      })
-    : sendRequestTool;
-
-  return {
-    run_tests,
-    query_results: queryResultsTool,
-    diagnose_failure: diagnoseFailureTool,
-    send_request,
-  };
-}
-
-export { runTestsTool, queryResultsTool, diagnoseFailureTool, sendRequestTool };

package/src/core/agent/tools/query-results.ts

@@ -1,40 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import { getDb } from "../../../db/schema.ts";
-import { listRuns, getRunById, getResultsByRunId, listCollections } from "../../../db/queries.ts";
-
-export const queryResultsTool = tool({
-  description: "Query test run results and collections from the database",
-  inputSchema: z.object({
-    action: z.enum(["list_runs", "get_run", "list_collections"]).describe("Action to perform"),
-    runId: z.number().optional().describe("Run ID (for get_run action)"),
-    limit: z.number().optional().describe("Max results to return (default: 20)"),
-  }),
-  execute: async (args) => {
-    try {
-      getDb();
-
-      switch (args.action) {
-        case "list_runs": {
-          const runs = listRuns(args.limit ?? 20);
-          return { runs };
-        }
-        case "get_run": {
-          if (args.runId == null) return { error: "runId is required for get_run action" };
-          const run = getRunById(args.runId);
-          if (!run) return { error: `Run ${args.runId} not found` };
-          const results = getResultsByRunId(args.runId);
-          return { run, results };
-        }
-        case "list_collections": {
-          const collections = listCollections();
-          return { collections };
-        }
-        default:
-          return { error: `Unknown action: ${args.action}` };
-      }
-    } catch (err) {
-      return { error: (err as Error).message };
-    }
-  },
-});

package/src/core/agent/tools/run-tests.ts

@@ -1,38 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import { executeRun } from "../../runner/execute-run.ts";
-
-export const runTestsTool = tool({
-  description: "Run API test suites from a YAML file or directory and return results summary",
-  inputSchema: z.object({
-    testPath: z.string().describe("Path to test YAML file or directory"),
-    envName: z.string().optional().describe("Environment name (loads .env.<name>.yaml)"),
-    safe: z.boolean().optional().describe("Run only GET tests (read-only, safe mode)"),
-  }),
-  execute: async (args) => {
-    try {
-      const { runId, results } = await executeRun({
-        testPath: args.testPath,
-        envName: args.envName,
-        safe: args.safe,
-        trigger: "agent",
-      });
-
-      const total = results.reduce((s, r) => s + r.total, 0);
-      const passed = results.reduce((s, r) => s + r.passed, 0);
-      const failed = results.reduce((s, r) => s + r.failed, 0);
-      const skipped = results.reduce((s, r) => s + r.skipped, 0);
-
-      return {
-        runId,
-        total,
-        passed,
-        failed,
-        skipped,
-        status: failed > 0 ? "has_failures" : "all_passed",
-      };
-    } catch (err) {
-      return { error: (err as Error).message };
-    }
-  },
-});

package/src/core/agent/tools/send-request.ts

@@ -1,44 +0,0 @@
-import { tool } from "ai";
-import { z } from "zod";
-import { executeRequest } from "../../runner/http-client.ts";
-import { loadEnvironment, substituteString, substituteDeep } from "../../parser/variables.ts";
-
-export const sendRequestTool = tool({
-  description: "Send an ad-hoc HTTP request. Supports variable interpolation from environments (e.g. {{base_url}}).",
-  inputSchema: z.object({
-    method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]).describe("HTTP method"),
-    url: z.string().describe("Request URL (supports {{variable}} interpolation)"),
-    headers: z.record(z.string(), z.string()).optional().describe("Request headers"),
-    body: z.string().optional().describe("Request body (JSON string)"),
-    timeout: z.number().int().positive().optional().describe("Request timeout in ms"),
-    envName: z.string().optional().describe("Environment name for variable interpolation"),
-  }),
-  execute: async (args) => {
-    try {
-      const vars = await loadEnvironment(args.envName);
-
-      const resolvedUrl = substituteString(args.url, vars) as string;
-      const resolvedHeaders = args.headers ? substituteDeep(args.headers, vars) : {};
-      const resolvedBody = args.body ? substituteString(args.body, vars) as string : undefined;
-
-      const response = await executeRequest(
-        {
-          method: args.method,
-          url: resolvedUrl,
-          headers: resolvedHeaders,
-          body: resolvedBody,
-        },
-        args.timeout ? { timeout: args.timeout } : undefined,
-      );
-
-      // Compact output for agent — skip response headers
-      return {
-        status: response.status,
-        body: response.body_parsed ?? response.body,
-        duration_ms: response.duration_ms,
-      };
-    } catch (err) {
-      return { error: (err as Error).message };
-    }
-  },
-});

package/src/core/agent/types.ts

@@ -1,22 +0,0 @@
-import type { AIProviderConfig } from "../generator/ai/types.ts";
-
-export interface AgentConfig {
-  provider: AIProviderConfig;
-  safeMode?: boolean;
-  dbPath?: string;
-  maxSteps?: number;
-}
-
-export interface ToolEvent {
-  toolName: string;
-  args: Record<string, unknown>;
-  result: unknown;
-  timestamp: string;
-}
-
-export interface AgentTurnResult {
-  text: string;
-  toolEvents: ToolEvent[];
-  inputTokens: number;
-  outputTokens: number;
-}

package/src/core/generator/ai/ai-generator.ts

@@ -1,61 +0,0 @@
-import type { AIGenerateOptions, AIGenerateResult } from "./types.ts";
-import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../openapi-reader.ts";
-import { buildMessages } from "./prompt-builder.ts";
-import { chatCompletion } from "./llm-client.ts";
-import { parseAIResponse } from "./output-parser.ts";
-
-export async function generateWithAI(options: AIGenerateOptions): Promise<AIGenerateResult> {
-  // 1. Read OpenAPI spec
-  const doc = await readOpenApiSpec(options.specPath);
-
-  // 2. Extract endpoints + security schemes
-  let endpoints = extractEndpoints(doc);
-  if (endpoints.length === 0) {
-    throw new Error("No endpoints found in the OpenAPI spec");
-  }
-  const securitySchemes = extractSecuritySchemes(doc);
-
-  // Filter to single endpoint if requested
-  if (options.filterEndpoint) {
-    const { method, path } = options.filterEndpoint;
-    const filtered = endpoints.filter(
-      (ep) => ep.method === method.toUpperCase() && ep.path === path,
-    );
-    if (filtered.length === 0) {
-      throw new Error(`Endpoint ${method} ${path} not found in spec`);
-    }
-    endpoints = filtered;
-  }
-
-  // Determine base URL: explicit option, or from spec servers[0]
-  const baseUrl = options.baseUrl ?? (doc as any).servers?.[0]?.url as string | undefined;
-
-  // 3. Build prompt
-  const messages = buildMessages(endpoints, securitySchemes, options.prompt, baseUrl);
-
-  // 4. Call LLM
-  const startTime = Date.now();
-  const llmResult = await chatCompletion(options.provider, messages);
-  const durationMs = Date.now() - startTime;
-
-  // 5. Parse + validate output
-  const parsed = parseAIResponse(llmResult.content);
-
-  if (parsed.suites.length === 0) {
-    const errorDetail = parsed.errors.length > 0
-      ? parsed.errors.join("; ")
-      : "No valid suites in response";
-    throw new Error(`AI generation failed: ${errorDetail}`);
-  }
-
-  // If there are validation errors but we still got suites, include them as warnings
-  const yaml = parsed.yaml;
-
-  return {
-    yaml,
-    rawResponse: llmResult.content,
-    promptTokens: llmResult.usage.promptTokens,
-    completionTokens: llmResult.usage.completionTokens,
-    model: options.provider.model,
-  };
-}

package/src/core/generator/ai/llm-client.ts

@@ -1,159 +0,0 @@
-import type { AIProviderConfig } from "./types.ts";
-
-export interface ChatMessage {
-  role: "system" | "user" | "assistant";
-  content: string;
-}
-
-export interface ChatCompletionResult {
-  content: string;
-  usage: {
-    promptTokens?: number;
-    completionTokens?: number;
-  };
-}
-
-export async function chatCompletion(
-  config: AIProviderConfig,
-  messages: ChatMessage[],
-): Promise<ChatCompletionResult> {
-  if (config.provider === "anthropic") {
-    return callAnthropic(config, messages);
-  }
-  return callOpenAICompatible(config, messages);
-}
-
-async function callOpenAICompatible(
-  config: AIProviderConfig,
-  messages: ChatMessage[],
-): Promise<ChatCompletionResult> {
-  const url = `${config.baseUrl.replace(/\/+$/, "")}/chat/completions`;
-
-  // For ollama/custom providers, inject system prompt into first user message
-  // to avoid issues with thinking models (e.g. qwen3) that break with separate system messages
-  let apiMessages: Array<{ role: string; content: string }>;
-  if (config.provider === "ollama" || config.provider === "custom") {
-    const systemMsgs = messages.filter((m) => m.role === "system");
-    const nonSystem = messages.filter((m) => m.role !== "system");
-    if (systemMsgs.length > 0 && nonSystem.length > 0) {
-      const systemText = systemMsgs.map((m) => m.content).join("\n\n");
-      apiMessages = nonSystem.map((m, i) =>
-        i === 0 ? { role: m.role, content: `${systemText}\n\n${m.content}` } : { role: m.role, content: m.content }
-      );
-    } else {
-      apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
-    }
-  } else {
-    apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
-  }
-
-  const body: Record<string, unknown> = {
-    model: config.model,
-    messages: apiMessages,
-    temperature: config.temperature ?? 0.2,
-    max_tokens: config.maxTokens ?? 4096,
-  };
-
-  // Request JSON output where supported (OpenAI, newer Ollama models)
-  if (config.provider === "openai") {
-    body.response_format = { type: "json_object" };
-  }
-
-  const headers: Record<string, string> = {
-    "Content-Type": "application/json",
-  };
-  if (config.apiKey) {
-    headers["Authorization"] = `Bearer ${config.apiKey}`;
-  }
-
-  const resp = await fetch(url, {
-    method: "POST",
-    headers,
-    body: JSON.stringify(body),
-  });
-
-  if (!resp.ok) {
-    const text = await resp.text();
-    throw new Error(`LLM request failed (${resp.status}): ${text}`);
-  }
-
-  const data = (await resp.json()) as {
-    choices: Array<{ message: { content: string; reasoning?: string } }>;
-    usage?: { prompt_tokens?: number; completion_tokens?: number };
-  };
-
-  const msg = data.choices?.[0]?.message;
-  // Thinking models (e.g. qwen3) may put output in `reasoning` with empty `content`
-  const content = msg?.content || msg?.reasoning || "";
-  return {
-    content,
-    usage: {
-      promptTokens: data.usage?.prompt_tokens,
-      completionTokens: data.usage?.completion_tokens,
-    },
-  };
-}
-
-async function callAnthropic(
-  config: AIProviderConfig,
-  messages: ChatMessage[],
-): Promise<ChatCompletionResult> {
-  const url = `${config.baseUrl.replace(/\/+$/, "")}/v1/messages`;
-
-  // Separate system prompt from user/assistant messages
-  const systemMessages = messages.filter((m) => m.role === "system");
-  const nonSystemMessages = messages.filter((m) => m.role !== "system");
-
-  const systemText = systemMessages.map((m) => m.content).join("\n\n");
-
-  const body: Record<string, unknown> = {
-    model: config.model,
-    max_tokens: config.maxTokens ?? 4096,
-    temperature: config.temperature ?? 0.2,
-    messages: nonSystemMessages.map((m) => ({
-      role: m.role,
-      content: m.content,
-    })),
-  };
-
-  if (systemText) {
-    body.system = systemText;
-  }
-
-  const headers: Record<string, string> = {
-    "Content-Type": "application/json",
-    "anthropic-version": "2023-06-01",
-  };
-  if (config.apiKey) {
-    headers["x-api-key"] = config.apiKey;
-  }
-
-  const resp = await fetch(url, {
-    method: "POST",
-    headers,
-    body: JSON.stringify(body),
-  });
-
-  if (!resp.ok) {
-    const text = await resp.text();
-    throw new Error(`Anthropic request failed (${resp.status}): ${text}`);
-  }
-
-  const data = (await resp.json()) as {
-    content: Array<{ type: string; text: string }>;
-    usage?: { input_tokens?: number; output_tokens?: number };
-  };
-
-  const content = data.content
-    ?.filter((b) => b.type === "text")
-    .map((b) => b.text)
-    .join("") ?? "";
-
-  return {
-    content,
-    usage: {
-      promptTokens: data.usage?.input_tokens,
-      completionTokens: data.usage?.output_tokens,
-    },
-  };
-}