@kirrosh/zond 0.7.0

package/src/core/agent/context-manager.ts

@@ -0,0 +1,41 @@
+import type { CoreMessageFormat } from "../../db/queries.ts";
+
+const MAX_MESSAGES = 20;
+const KEEP_RECENT_TURNS = 6; // 6 turns = 12 messages (user + assistant pairs)
+const KEEP_RECENT_MESSAGES = KEEP_RECENT_TURNS * 2;
+
+export function trimContext(messages: CoreMessageFormat[]): CoreMessageFormat[] {
+  if (messages.length <= MAX_MESSAGES) {
+    return messages;
+  }
+
+  const oldMessages = messages.slice(0, messages.length - KEEP_RECENT_MESSAGES);
+  const recentMessages = messages.slice(messages.length - KEEP_RECENT_MESSAGES);
+
+  const summary = buildSummary(oldMessages);
+
+  // Use role "user" for the summary so that the conversation always starts with a user message.
+  // Some providers require conversations to begin with a user turn.
+  return [
+    { role: "user" as const, content: summary },
+    ...recentMessages,
+  ];
+}
+
+function buildSummary(messages: CoreMessageFormat[]): string {
+  const userMessages = messages.filter((m) => m.role === "user");
+  const topics = userMessages
+    .map((m) => m.content.slice(0, 80))
+    .slice(0, 5);
+
+  const topicList = topics.length > 0
+    ? topics.map((t) => `- ${t}`).join("\n")
+    : "- General conversation";
+
+  return `[Conversation summary — ${messages.length} earlier messages condensed]
+
+Topics discussed:
+${topicList}
+
+The conversation continues below with the most recent messages.`;
+}

package/src/core/agent/system-prompt.ts

@@ -0,0 +1,28 @@
+export const AGENT_SYSTEM_PROMPT = `You are an API testing assistant powered by zond. You help users run, create, validate, and diagnose API tests.
+
+You have access to the following tools:
+
+- **run_tests**: Execute API test suites from YAML files or directories. Returns pass/fail summary with run ID.
+- **validate_tests**: Validate YAML test files without executing them. Check syntax and structure.
+- **query_results**: Query historical test run results and collections from the database.
+- **diagnose_failure**: Analyze a failed test run to identify root causes and suggest fixes.
+
+Tool usage examples:
+- run_tests: { testPath: "tests/api.yaml" } or { testPath: "tests/", envName: "staging", safe: true }
+- validate_tests: { testPath: "tests/api.yaml" }
+- query_results: action must be "list_runs", "get_run" (requires runId), or "list_collections"
+  - List runs: { action: "list_runs", limit: 10 }
+  - Get run details: { action: "get_run", runId: 1 }
+  - List collections: { action: "list_collections" }
+- diagnose_failure: { runId: 1 }
+
+Guidelines:
+- When asked to run tests, use the run_tests tool and report results clearly.
+- When a test run has failures, proactively use diagnose_failure to analyze the issues.
+- When asked about past results, use query_results to look up run history.
+- Always provide actionable suggestions when tests fail.
+- Be concise but thorough in your explanations.
+- If a tool call fails with a validation error, re-read the tool schema and retry with corrected arguments.
+- When in safe mode, only GET (read-only) tests will be executed.
+- When using thinking/reasoning, keep your internal reasoning focused and share conclusions with the user.
+`;

package/src/core/agent/tools/diagnose-failure.ts

@@ -0,0 +1,51 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { getDb } from "../../../db/schema.ts";
+import { getRunById, getResultsByRunId } from "../../../db/queries.ts";
+
+export const diagnoseFailureTool = tool({
+  description: "Diagnose failures in a test run by analyzing failed steps and their errors",
+  inputSchema: z.object({
+    runId: z.number().describe("Run ID to diagnose"),
+  }),
+  execute: async (args) => {
+    try {
+      getDb();
+
+      const run = getRunById(args.runId);
+      if (!run) return { error: `Run ${args.runId} not found` };
+
+      const results = getResultsByRunId(args.runId);
+      const failures = results
+        .filter((r) => r.status === "fail" || r.status === "error")
+        .map((r) => ({
+          suite_name: r.suite_name,
+          test_name: r.test_name,
+          status: r.status,
+          error_message: r.error_message,
+          request_method: r.request_method,
+          request_url: r.request_url,
+          response_status: r.response_status,
+          assertions: r.assertions,
+          duration_ms: r.duration_ms,
+        }));
+
+      return {
+        run: {
+          id: run.id,
+          started_at: run.started_at,
+          environment: run.environment,
+          duration_ms: run.duration_ms,
+        },
+        summary: {
+          total: run.total,
+          passed: run.passed,
+          failed: run.failed,
+        },
+        failures,
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/explore-api.ts

@@ -0,0 +1,40 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../../generator/index.ts";
+
+export const exploreApiTool = tool({
+  description: "Explore an OpenAPI spec — list endpoints with method, path, and summary. Optionally filter by tag.",
+  inputSchema: z.object({
+    specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML)"),
+    tag: z.string().optional().describe("Filter endpoints by tag"),
+  }),
+  execute: async (args) => {
+    try {
+      const doc = await readOpenApiSpec(args.specPath);
+      const allEndpoints = extractEndpoints(doc);
+      const securitySchemes = extractSecuritySchemes(doc);
+      const servers = ((doc as any).servers ?? []) as Array<{ url: string }>;
+
+      const endpoints = args.tag
+        ? allEndpoints.filter(ep => ep.tags.includes(args.tag!))
+        : allEndpoints;
+
+      // Compact output — method + path + summary only
+      return {
+        title: (doc as any).info?.title,
+        version: (doc as any).info?.version,
+        servers: servers.map(s => s.url),
+        securitySchemes: securitySchemes.map(s => s.name),
+        totalEndpoints: allEndpoints.length,
+        ...(args.tag ? { filteredByTag: args.tag, matchingEndpoints: endpoints.length } : {}),
+        endpoints: endpoints.map(ep => ({
+          method: ep.method,
+          path: ep.path,
+          summary: ep.summary,
+        })),
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/index.ts

@@ -0,0 +1,46 @@
+import { tool } from "ai";
+import { runTestsTool } from "./run-tests.ts";
+import { validateTestsTool } from "./validate-tests.ts";
+import { queryResultsTool } from "./query-results.ts";
+import { diagnoseFailureTool } from "./diagnose-failure.ts";
+import { sendRequestTool } from "./send-request.ts";
+import { exploreApiTool } from "./explore-api.ts";
+import type { AgentConfig } from "../types.ts";
+
+export function buildAgentTools(config: AgentConfig) {
+  // In safe mode, wrap run_tests to force safe=true
+  const run_tests = config.safeMode
+    ? tool({
+        description: runTestsTool.description,
+        inputSchema: runTestsTool.inputSchema,
+        execute: async (args, options) => {
+          return runTestsTool.execute!({ ...args, safe: true }, options);
+        },
+      })
+    : runTestsTool;
+
+  // In safe mode, wrap send_request to only allow GET
+  const send_request = config.safeMode
+    ? tool({
+        description: sendRequestTool.description,
+        inputSchema: sendRequestTool.inputSchema,
+        execute: async (args, options) => {
+          if (args.method !== "GET") {
+            return { error: "Safe mode: only GET requests are allowed" };
+          }
+          return sendRequestTool.execute!(args, options);
+        },
+      })
+    : sendRequestTool;
+
+  return {
+    run_tests,
+    validate_tests: validateTestsTool,
+    query_results: queryResultsTool,
+    diagnose_failure: diagnoseFailureTool,
+    send_request,
+    explore_api: exploreApiTool,
+  };
+}
+
+export { runTestsTool, validateTestsTool, queryResultsTool, diagnoseFailureTool, sendRequestTool, exploreApiTool };

package/src/core/agent/tools/query-results.ts

@@ -0,0 +1,40 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { getDb } from "../../../db/schema.ts";
+import { listRuns, getRunById, getResultsByRunId, listCollections } from "../../../db/queries.ts";
+
+export const queryResultsTool = tool({
+  description: "Query test run results and collections from the database",
+  inputSchema: z.object({
+    action: z.enum(["list_runs", "get_run", "list_collections"]).describe("Action to perform"),
+    runId: z.number().optional().describe("Run ID (for get_run action)"),
+    limit: z.number().optional().describe("Max results to return (default: 20)"),
+  }),
+  execute: async (args) => {
+    try {
+      getDb();
+
+      switch (args.action) {
+        case "list_runs": {
+          const runs = listRuns(args.limit ?? 20);
+          return { runs };
+        }
+        case "get_run": {
+          if (args.runId == null) return { error: "runId is required for get_run action" };
+          const run = getRunById(args.runId);
+          if (!run) return { error: `Run ${args.runId} not found` };
+          const results = getResultsByRunId(args.runId);
+          return { run, results };
+        }
+        case "list_collections": {
+          const collections = listCollections();
+          return { collections };
+        }
+        default:
+          return { error: `Unknown action: ${args.action}` };
+      }
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/run-tests.ts

@@ -0,0 +1,38 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { executeRun } from "../../runner/execute-run.ts";
+
+export const runTestsTool = tool({
+  description: "Run API test suites from a YAML file or directory and return results summary",
+  inputSchema: z.object({
+    testPath: z.string().describe("Path to test YAML file or directory"),
+    envName: z.string().optional().describe("Environment name (loads .env.<name>.yaml)"),
+    safe: z.boolean().optional().describe("Run only GET tests (read-only, safe mode)"),
+  }),
+  execute: async (args) => {
+    try {
+      const { runId, results } = await executeRun({
+        testPath: args.testPath,
+        envName: args.envName,
+        safe: args.safe,
+        trigger: "agent",
+      });
+
+      const total = results.reduce((s, r) => s + r.total, 0);
+      const passed = results.reduce((s, r) => s + r.passed, 0);
+      const failed = results.reduce((s, r) => s + r.failed, 0);
+      const skipped = results.reduce((s, r) => s + r.skipped, 0);
+
+      return {
+        runId,
+        total,
+        passed,
+        failed,
+        skipped,
+        status: failed > 0 ? "has_failures" : "all_passed",
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/send-request.ts

@@ -0,0 +1,44 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { executeRequest } from "../../runner/http-client.ts";
+import { loadEnvironment, substituteString, substituteDeep } from "../../parser/variables.ts";
+
+export const sendRequestTool = tool({
+  description: "Send an ad-hoc HTTP request. Supports variable interpolation from environments (e.g. {{base_url}}).",
+  inputSchema: z.object({
+    method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]).describe("HTTP method"),
+    url: z.string().describe("Request URL (supports {{variable}} interpolation)"),
+    headers: z.record(z.string(), z.string()).optional().describe("Request headers"),
+    body: z.string().optional().describe("Request body (JSON string)"),
+    timeout: z.number().int().positive().optional().describe("Request timeout in ms"),
+    envName: z.string().optional().describe("Environment name for variable interpolation"),
+  }),
+  execute: async (args) => {
+    try {
+      const vars = await loadEnvironment(args.envName);
+
+      const resolvedUrl = substituteString(args.url, vars) as string;
+      const resolvedHeaders = args.headers ? substituteDeep(args.headers, vars) : {};
+      const resolvedBody = args.body ? substituteString(args.body, vars) as string : undefined;
+
+      const response = await executeRequest(
+        {
+          method: args.method,
+          url: resolvedUrl,
+          headers: resolvedHeaders,
+          body: resolvedBody,
+        },
+        args.timeout ? { timeout: args.timeout } : undefined,
+      );
+
+      // Compact output for agent — skip response headers
+      return {
+        status: response.status,
+        body: response.body_parsed ?? response.body,
+        duration_ms: response.duration_ms,
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/validate-tests.ts

@@ -0,0 +1,23 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { parse } from "../../parser/yaml-parser.ts";
+
+export const validateTestsTool = tool({
+  description: "Validate YAML test files without running them. Returns parsed suite info or validation errors.",
+  inputSchema: z.object({
+    testPath: z.string().describe("Path to test YAML file or directory"),
+  }),
+  execute: async (args) => {
+    try {
+      const suites = await parse(args.testPath);
+      return {
+        valid: true,
+        suiteCount: suites.length,
+        totalTests: suites.reduce((s, suite) => s + suite.tests.length, 0),
+        suites: suites.map((s) => ({ name: s.name, testCount: s.tests.length })),
+      };
+    } catch (err) {
+      return { valid: false, error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/types.ts

@@ -0,0 +1,22 @@
+import type { AIProviderConfig } from "../generator/ai/types.ts";
+
+export interface AgentConfig {
+  provider: AIProviderConfig;
+  safeMode?: boolean;
+  dbPath?: string;
+  maxSteps?: number;
+}
+
+export interface ToolEvent {
+  toolName: string;
+  args: Record<string, unknown>;
+  result: unknown;
+  timestamp: string;
+}
+
+export interface AgentTurnResult {
+  text: string;
+  toolEvents: ToolEvent[];
+  inputTokens: number;
+  outputTokens: number;
+}

package/src/core/diagnostics/failure-hints.ts

@@ -0,0 +1,63 @@
+/**
+ * Failure classification and diagnostic hints.
+ * Extracted from query-db.ts for reuse in Web UI.
+ */
+
+export function statusHint(status: number | null | undefined): string | null {
+  if (!status) return null;
+  if (status >= 500) return "Server-side error — inspect response_body for errorMessage/errorDetail; likely a backend bug";
+  if (status === 401 || status === 403) return "Auth failure — check auth_token/api_key in .env.yaml";
+  if (status === 404) return "Resource not found — verify the path and ID";
+  if (status === 400 || status === 422) return "Validation error — check request body fields match the schema";
+  return null;
+}
+
+export function classifyFailure(status: string, responseStatus: number | null): "api_error" | "assertion_failed" | "network_error" {
+  if (status === "error" && (responseStatus === null || responseStatus < 500)) return "network_error";
+  if (responseStatus !== null && responseStatus >= 500) return "api_error";
+  return "assertion_failed";
+}
+
+export function envHint(url: string | null, errorMessage: string | null, envFilePath?: string): string | null {
+  const envFile = envFilePath ? envFilePath : ".env.yaml in your API directory";
+
+  if (url && /\{\{[^}]+\}\}/.test(url)) {
+    return `URL contains unresolved variable: "${url}" — variable name may not match the key in ${envFile}`;
+  }
+  if (url && !url.startsWith("http://") && !url.startsWith("https://")) {
+    return `base_url is not set or empty — URL resolved to "${url}". Add base_url to ${envFile}`;
+  }
+  if (errorMessage?.includes("base_url is not configured")) {
+    return `base_url is missing or empty. Add base_url: https://your-api.com to ${envFile}`;
+  }
+  if (errorMessage?.includes("URL is invalid") || errorMessage?.includes("Failed to parse URL")) {
+    return `URL is malformed — likely base_url is empty or invalid. Check base_url in ${envFile}`;
+  }
+  return null;
+}
+
+export function envCategory(hint: string | undefined): string | null {
+  if (!hint) return null;
+  if (hint.includes("base_url is not set") || hint.includes("base_url is missing") || hint.includes("base_url is not configured")) return "base_url_missing";
+  if (hint.includes("unresolved variable")) return "unresolved_variable";
+  if (hint.includes("URL is malformed")) return "url_malformed";
+  return null;
+}
+
+export function computeSharedEnvIssue(
+  failures: Array<{ hint?: string }>,
+  envFilePath?: string,
+): string | null {
+  const categories = new Set(failures.map(f => envCategory(f.hint)).filter(Boolean));
+  if (categories.size !== 1) return null;
+
+  const envFile = envFilePath ?? ".env.yaml";
+  if (categories.has("base_url_missing")) {
+    return `All failures: base_url is not set — add base_url to ${envFile}`;
+  }
+  if (categories.has("unresolved_variable")) {
+    return `All failures: some variables are not substituted — check variable names in ${envFile}`;
+  }
+  // url_malformed
+  return [...failures.map(f => f.hint).filter(Boolean)][0] ?? null;
+}

package/src/core/generator/ai/ai-generator.ts

@@ -0,0 +1,61 @@
+import type { AIGenerateOptions, AIGenerateResult } from "./types.ts";
+import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../openapi-reader.ts";
+import { buildMessages } from "./prompt-builder.ts";
+import { chatCompletion } from "./llm-client.ts";
+import { parseAIResponse } from "./output-parser.ts";
+
+export async function generateWithAI(options: AIGenerateOptions): Promise<AIGenerateResult> {
+  // 1. Read OpenAPI spec
+  const doc = await readOpenApiSpec(options.specPath);
+
+  // 2. Extract endpoints + security schemes
+  let endpoints = extractEndpoints(doc);
+  if (endpoints.length === 0) {
+    throw new Error("No endpoints found in the OpenAPI spec");
+  }
+  const securitySchemes = extractSecuritySchemes(doc);
+
+  // Filter to single endpoint if requested
+  if (options.filterEndpoint) {
+    const { method, path } = options.filterEndpoint;
+    const filtered = endpoints.filter(
+      (ep) => ep.method === method.toUpperCase() && ep.path === path,
+    );
+    if (filtered.length === 0) {
+      throw new Error(`Endpoint ${method} ${path} not found in spec`);
+    }
+    endpoints = filtered;
+  }
+
+  // Determine base URL: explicit option, or from spec servers[0]
+  const baseUrl = options.baseUrl ?? (doc as any).servers?.[0]?.url as string | undefined;
+
+  // 3. Build prompt
+  const messages = buildMessages(endpoints, securitySchemes, options.prompt, baseUrl);
+
+  // 4. Call LLM
+  const startTime = Date.now();
+  const llmResult = await chatCompletion(options.provider, messages);
+  const durationMs = Date.now() - startTime;
+
+  // 5. Parse + validate output
+  const parsed = parseAIResponse(llmResult.content);
+
+  if (parsed.suites.length === 0) {
+    const errorDetail = parsed.errors.length > 0
+      ? parsed.errors.join("; ")
+      : "No valid suites in response";
+    throw new Error(`AI generation failed: ${errorDetail}`);
+  }
+
+  // If there are validation errors but we still got suites, include them as warnings
+  const yaml = parsed.yaml;
+
+  return {
+    yaml,
+    rawResponse: llmResult.content,
+    promptTokens: llmResult.usage.promptTokens,
+    completionTokens: llmResult.usage.completionTokens,
+    model: options.provider.model,
+  };
+}

package/src/core/generator/ai/llm-client.ts

@@ -0,0 +1,159 @@
+import type { AIProviderConfig } from "./types.ts";
+
+export interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+
+export interface ChatCompletionResult {
+  content: string;
+  usage: {
+    promptTokens?: number;
+    completionTokens?: number;
+  };
+}
+
+export async function chatCompletion(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  if (config.provider === "anthropic") {
+    return callAnthropic(config, messages);
+  }
+  return callOpenAICompatible(config, messages);
+}
+
+async function callOpenAICompatible(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  const url = `${config.baseUrl.replace(/\/+$/, "")}/chat/completions`;
+
+  // For ollama/custom providers, inject system prompt into first user message
+  // to avoid issues with thinking models (e.g. qwen3) that break with separate system messages
+  let apiMessages: Array<{ role: string; content: string }>;
+  if (config.provider === "ollama" || config.provider === "custom") {
+    const systemMsgs = messages.filter((m) => m.role === "system");
+    const nonSystem = messages.filter((m) => m.role !== "system");
+    if (systemMsgs.length > 0 && nonSystem.length > 0) {
+      const systemText = systemMsgs.map((m) => m.content).join("\n\n");
+      apiMessages = nonSystem.map((m, i) =>
+        i === 0 ? { role: m.role, content: `${systemText}\n\n${m.content}` } : { role: m.role, content: m.content }
+      );
+    } else {
+      apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
+    }
+  } else {
+    apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
+  }
+
+  const body: Record<string, unknown> = {
+    model: config.model,
+    messages: apiMessages,
+    temperature: config.temperature ?? 0.2,
+    max_tokens: config.maxTokens ?? 4096,
+  };
+
+  // Request JSON output where supported (OpenAI, newer Ollama models)
+  if (config.provider === "openai") {
+    body.response_format = { type: "json_object" };
+  }
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+  };
+  if (config.apiKey) {
+    headers["Authorization"] = `Bearer ${config.apiKey}`;
+  }
+
+  const resp = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`LLM request failed (${resp.status}): ${text}`);
+  }
+
+  const data = (await resp.json()) as {
+    choices: Array<{ message: { content: string; reasoning?: string } }>;
+    usage?: { prompt_tokens?: number; completion_tokens?: number };
+  };
+
+  const msg = data.choices?.[0]?.message;
+  // Thinking models (e.g. qwen3) may put output in `reasoning` with empty `content`
+  const content = msg?.content || msg?.reasoning || "";
+  return {
+    content,
+    usage: {
+      promptTokens: data.usage?.prompt_tokens,
+      completionTokens: data.usage?.completion_tokens,
+    },
+  };
+}
+
+async function callAnthropic(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  const url = `${config.baseUrl.replace(/\/+$/, "")}/v1/messages`;
+
+  // Separate system prompt from user/assistant messages
+  const systemMessages = messages.filter((m) => m.role === "system");
+  const nonSystemMessages = messages.filter((m) => m.role !== "system");
+
+  const systemText = systemMessages.map((m) => m.content).join("\n\n");
+
+  const body: Record<string, unknown> = {
+    model: config.model,
+    max_tokens: config.maxTokens ?? 4096,
+    temperature: config.temperature ?? 0.2,
+    messages: nonSystemMessages.map((m) => ({
+      role: m.role,
+      content: m.content,
+    })),
+  };
+
+  if (systemText) {
+    body.system = systemText;
+  }
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    "anthropic-version": "2023-06-01",
+  };
+  if (config.apiKey) {
+    headers["x-api-key"] = config.apiKey;
+  }
+
+  const resp = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`Anthropic request failed (${resp.status}): ${text}`);
+  }
+
+  const data = (await resp.json()) as {
+    content: Array<{ type: string; text: string }>;
+    usage?: { input_tokens?: number; output_tokens?: number };
+  };
+
+  const content = data.content
+    ?.filter((b) => b.type === "text")
+    .map((b) => b.text)
+    .join("") ?? "";
+
+  return {
+    content,
+    usage: {
+      promptTokens: data.usage?.input_tokens,
+      completionTokens: data.usage?.output_tokens,
+    },
+  };
+}