@kirrosh/apitool 0.4.3

package/src/cli/output.ts

@@ -0,0 +1,24 @@
+const RESET = "\x1b[0m";
+const RED = "\x1b[31m";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+
+function useColor(): boolean {
+  return process.stderr.isTTY ?? false;
+}
+
+export function printError(message: string): void {
+  const msg = useColor() ? `${RED}Error: ${message}${RESET}` : `Error: ${message}`;
+  process.stderr.write(msg + "\n");
+}
+
+export function printSuccess(message: string): void {
+  const color = process.stdout.isTTY ?? false;
+  const msg = color ? `${GREEN}${message}${RESET}` : message;
+  process.stdout.write(msg + "\n");
+}
+
+export function printWarning(message: string): void {
+  const msg = useColor() ? `${YELLOW}Warning: ${message}${RESET}` : `Warning: ${message}`;
+  process.stderr.write(msg + "\n");
+}

package/src/cli/runtime.ts

@@ -0,0 +1,7 @@
+export function isCompiledBinary(): boolean {
+  return process.argv[0] === "bun" && import.meta.path.includes("~BUN");
+}
+
+export function getRuntimeInfo(): string {
+  return isCompiledBinary() ? "standalone" : "bun";
+}

package/src/core/agent/agent-loop.ts

@@ -0,0 +1,116 @@
+// Suppress AI SDK v2 spec compatibility warnings for Ollama (cosmetic, tool calling works fine)
+(globalThis as any).AI_SDK_LOG_WARNINGS = false;
+
+import { generateText, stepCountIs } from "ai";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createAnthropic } from "@ai-sdk/anthropic";
+import { AGENT_SYSTEM_PROMPT } from "./system-prompt.ts";
+import { buildAgentTools } from "./tools/index.ts";
+import type { AgentConfig, AgentTurnResult, ToolEvent } from "./types.ts";
+import type { ModelMessage } from "ai";
+
+export function buildProvider(config: AgentConfig) {
+  const { provider } = config.provider;
+
+  if (provider === "anthropic") {
+    return createAnthropic({
+      apiKey: config.provider.apiKey,
+      baseURL: config.provider.baseUrl || undefined,
+    });
+  }
+
+  // openai, ollama, custom all use OpenAI-compatible API
+  return createOpenAI({
+    apiKey: config.provider.apiKey ?? "ollama",
+    baseURL: config.provider.baseUrl,
+  });
+}
+
+function buildModel(config: AgentConfig) {
+  const provider = buildProvider(config);
+  const { provider: providerType } = config.provider;
+
+  // For ollama/custom, use .chat() to avoid the responses API which they don't support.
+  if (providerType === "ollama" || providerType === "custom") {
+    return (provider as ReturnType<typeof createOpenAI>).chat(config.provider.model);
+  }
+
+  return provider(config.provider.model);
+}
+
+/**
+ * Prepare messages with system prompt.
+ * Some small/local models (e.g. qwen3 thinking mode via Ollama) break tool calling
+ * when a separate `system` message is present. For ollama/custom providers, we inject
+ * the system prompt into the first user message instead.
+ */
+function prepareMessages(
+  messages: ModelMessage[],
+  config: AgentConfig,
+): { system?: string; messages: ModelMessage[] } {
+  const { provider } = config.provider;
+
+  if (provider === "ollama" || provider === "custom") {
+    // Inject system prompt into first user message to avoid breaking tool calling
+    const prepared = [...messages];
+    const firstUserIdx = prepared.findIndex(
+      (m) => m.role === "user" && typeof m.content === "string",
+    );
+
+    if (firstUserIdx >= 0) {
+      const msg = prepared[firstUserIdx] as { role: "user"; content: string };
+      prepared[firstUserIdx] = {
+        ...msg,
+        content: `[System instructions]\n${AGENT_SYSTEM_PROMPT}\n[End instructions]\n\n${msg.content}`,
+      };
+    }
+
+    return { messages: prepared };
+  }
+
+  // For OpenAI/Anthropic, use the standard system parameter
+  return { system: AGENT_SYSTEM_PROMPT, messages };
+}
+
+export async function runAgentTurn(
+  messages: ModelMessage[],
+  config: AgentConfig,
+  onToolEvent?: (event: ToolEvent) => void,
+): Promise<AgentTurnResult> {
+  const model = buildModel(config);
+  const tools = buildAgentTools(config);
+  const { system, messages: prepared } = prepareMessages(messages, config);
+  const toolEvents: ToolEvent[] = [];
+
+  const result = await generateText({
+    model,
+    system,
+    messages: prepared,
+    tools,
+    stopWhen: stepCountIs(config.maxSteps ?? 10),
+    maxOutputTokens: config.provider.maxTokens ?? 4096,
+    onStepFinish: ({ toolCalls, toolResults }) => {
+      if (toolCalls) {
+        for (let i = 0; i < toolCalls.length; i++) {
+          const call = toolCalls[i]!;
+          const toolResult = toolResults?.[i];
+          const event: ToolEvent = {
+            toolName: call.toolName,
+            args: ("input" in call ? call.input : {}) as Record<string, unknown>,
+            result: toolResult ?? null,
+            timestamp: new Date().toISOString(),
+          };
+          toolEvents.push(event);
+          onToolEvent?.(event);
+        }
+      }
+    },
+  });
+
+  return {
+    text: result.text,
+    toolEvents,
+    inputTokens: result.usage?.inputTokens ?? 0,
+    outputTokens: result.usage?.outputTokens ?? 0,
+  };
+}

package/src/core/agent/context-manager.ts

@@ -0,0 +1,41 @@
+import type { CoreMessageFormat } from "../../db/queries.ts";
+
+const MAX_MESSAGES = 20;
+const KEEP_RECENT_TURNS = 6; // 6 turns = 12 messages (user + assistant pairs)
+const KEEP_RECENT_MESSAGES = KEEP_RECENT_TURNS * 2;
+
+export function trimContext(messages: CoreMessageFormat[]): CoreMessageFormat[] {
+  if (messages.length <= MAX_MESSAGES) {
+    return messages;
+  }
+
+  const oldMessages = messages.slice(0, messages.length - KEEP_RECENT_MESSAGES);
+  const recentMessages = messages.slice(messages.length - KEEP_RECENT_MESSAGES);
+
+  const summary = buildSummary(oldMessages);
+
+  // Use role "user" for the summary so that the conversation always starts with a user message.
+  // Some providers require conversations to begin with a user turn.
+  return [
+    { role: "user" as const, content: summary },
+    ...recentMessages,
+  ];
+}
+
+function buildSummary(messages: CoreMessageFormat[]): string {
+  const userMessages = messages.filter((m) => m.role === "user");
+  const topics = userMessages
+    .map((m) => m.content.slice(0, 80))
+    .slice(0, 5);
+
+  const topicList = topics.length > 0
+    ? topics.map((t) => `- ${t}`).join("\n")
+    : "- General conversation";
+
+  return `[Conversation summary — ${messages.length} earlier messages condensed]
+
+Topics discussed:
+${topicList}
+
+The conversation continues below with the most recent messages.`;
+}

package/src/core/agent/system-prompt.ts

@@ -0,0 +1,33 @@
+export const AGENT_SYSTEM_PROMPT = `You are an API testing assistant powered by apitool. You help users run, create, validate, and diagnose API tests.
+
+You have access to the following tools:
+
+- **run_tests**: Execute API test suites from YAML files or directories. Returns pass/fail summary with run ID.
+- **validate_tests**: Validate YAML test files without executing them. Check syntax and structure.
+- **query_results**: Query historical test run results and collections from the database.
+- **manage_environment**: List, get, or set environment variables used during test execution.
+- **diagnose_failure**: Analyze a failed test run to identify root causes and suggest fixes.
+
+Tool usage examples:
+- run_tests: { testPath: "tests/api.yaml" } or { testPath: "tests/", envName: "staging", safe: true }
+- validate_tests: { testPath: "tests/api.yaml" }
+- query_results: action must be "list_runs", "get_run" (requires runId), or "list_collections"
+  - List runs: { action: "list_runs", limit: 10 }
+  - Get run details: { action: "get_run", runId: 1 }
+  - List collections: { action: "list_collections" }
+- manage_environment: action must be "list", "get" (requires name), or "set" (requires name + variables)
+  - List environments: { action: "list" }
+  - Get environment: { action: "get", name: "staging" }
+  - Set variables: { action: "set", name: "staging", variables: { "base_url": "https://api.example.com" } }
+- diagnose_failure: { runId: 1 }
+
+Guidelines:
+- When asked to run tests, use the run_tests tool and report results clearly.
+- When a test run has failures, proactively use diagnose_failure to analyze the issues.
+- When asked about past results, use query_results to look up run history.
+- Always provide actionable suggestions when tests fail.
+- Be concise but thorough in your explanations.
+- If a tool call fails with a validation error, re-read the tool schema and retry with corrected arguments.
+- When in safe mode, only GET (read-only) tests will be executed.
+- When using thinking/reasoning, keep your internal reasoning focused and share conclusions with the user.
+`;

package/src/core/agent/tools/diagnose-failure.ts

@@ -0,0 +1,51 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { getDb } from "../../../db/schema.ts";
+import { getRunById, getResultsByRunId } from "../../../db/queries.ts";
+
+export const diagnoseFailureTool = tool({
+  description: "Diagnose failures in a test run by analyzing failed steps and their errors",
+  inputSchema: z.object({
+    runId: z.number().describe("Run ID to diagnose"),
+  }),
+  execute: async (args) => {
+    try {
+      getDb();
+
+      const run = getRunById(args.runId);
+      if (!run) return { error: `Run ${args.runId} not found` };
+
+      const results = getResultsByRunId(args.runId);
+      const failures = results
+        .filter((r) => r.status === "fail" || r.status === "error")
+        .map((r) => ({
+          suite_name: r.suite_name,
+          test_name: r.test_name,
+          status: r.status,
+          error_message: r.error_message,
+          request_method: r.request_method,
+          request_url: r.request_url,
+          response_status: r.response_status,
+          assertions: r.assertions,
+          duration_ms: r.duration_ms,
+        }));
+
+      return {
+        run: {
+          id: run.id,
+          started_at: run.started_at,
+          environment: run.environment,
+          duration_ms: run.duration_ms,
+        },
+        summary: {
+          total: run.total,
+          passed: run.passed,
+          failed: run.failed,
+        },
+        failures,
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/explore-api.ts

@@ -0,0 +1,40 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../../generator/index.ts";
+
+export const exploreApiTool = tool({
+  description: "Explore an OpenAPI spec — list endpoints with method, path, and summary. Optionally filter by tag.",
+  inputSchema: z.object({
+    specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML)"),
+    tag: z.string().optional().describe("Filter endpoints by tag"),
+  }),
+  execute: async (args) => {
+    try {
+      const doc = await readOpenApiSpec(args.specPath);
+      const allEndpoints = extractEndpoints(doc);
+      const securitySchemes = extractSecuritySchemes(doc);
+      const servers = ((doc as any).servers ?? []) as Array<{ url: string }>;
+
+      const endpoints = args.tag
+        ? allEndpoints.filter(ep => ep.tags.includes(args.tag!))
+        : allEndpoints;
+
+      // Compact output — method + path + summary only
+      return {
+        title: (doc as any).info?.title,
+        version: (doc as any).info?.version,
+        servers: servers.map(s => s.url),
+        securitySchemes: securitySchemes.map(s => s.name),
+        totalEndpoints: allEndpoints.length,
+        ...(args.tag ? { filteredByTag: args.tag, matchingEndpoints: endpoints.length } : {}),
+        endpoints: endpoints.map(ep => ({
+          method: ep.method,
+          path: ep.path,
+          summary: ep.summary,
+        })),
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/index.ts

@@ -0,0 +1,48 @@
+import { tool } from "ai";
+import { runTestsTool } from "./run-tests.ts";
+import { validateTestsTool } from "./validate-tests.ts";
+import { queryResultsTool } from "./query-results.ts";
+import { manageEnvironmentTool } from "./manage-environment.ts";
+import { diagnoseFailureTool } from "./diagnose-failure.ts";
+import { sendRequestTool } from "./send-request.ts";
+import { exploreApiTool } from "./explore-api.ts";
+import type { AgentConfig } from "../types.ts";
+
+export function buildAgentTools(config: AgentConfig) {
+  // In safe mode, wrap run_tests to force safe=true
+  const run_tests = config.safeMode
+    ? tool({
+        description: runTestsTool.description,
+        inputSchema: runTestsTool.inputSchema,
+        execute: async (args, options) => {
+          return runTestsTool.execute!({ ...args, safe: true }, options);
+        },
+      })
+    : runTestsTool;
+
+  // In safe mode, wrap send_request to only allow GET
+  const send_request = config.safeMode
+    ? tool({
+        description: sendRequestTool.description,
+        inputSchema: sendRequestTool.inputSchema,
+        execute: async (args, options) => {
+          if (args.method !== "GET") {
+            return { error: "Safe mode: only GET requests are allowed" };
+          }
+          return sendRequestTool.execute!(args, options);
+        },
+      })
+    : sendRequestTool;
+
+  return {
+    run_tests,
+    validate_tests: validateTestsTool,
+    query_results: queryResultsTool,
+    manage_environment: manageEnvironmentTool,
+    diagnose_failure: diagnoseFailureTool,
+    send_request,
+    explore_api: exploreApiTool,
+  };
+}
+
+export { runTestsTool, validateTestsTool, queryResultsTool, manageEnvironmentTool, diagnoseFailureTool, sendRequestTool, exploreApiTool };

package/src/core/agent/tools/manage-environment.ts

@@ -0,0 +1,40 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { getDb } from "../../../db/schema.ts";
+import { listEnvironmentRecords, getEnvironment, upsertEnvironment } from "../../../db/queries.ts";
+
+export const manageEnvironmentTool = tool({
+  description: "List, get, or set environment variables used for API test execution",
+  inputSchema: z.object({
+    action: z.enum(["list", "get", "set"]).describe("Action to perform"),
+    name: z.string().optional().describe("Environment name"),
+    variables: z.record(z.string(), z.string()).optional().describe("Variables to set (for set action)"),
+  }),
+  execute: async (args) => {
+    try {
+      getDb();
+
+      switch (args.action) {
+        case "list": {
+          const environments = listEnvironmentRecords();
+          return { environments };
+        }
+        case "get": {
+          if (!args.name) return { error: "name is required for get action" };
+          const variables = getEnvironment(args.name);
+          if (!variables) return { error: `Environment '${args.name}' not found` };
+          return { name: args.name, variables };
+        }
+        case "set": {
+          if (!args.name || !args.variables) return { error: "name and variables are required for set action" };
+          upsertEnvironment(args.name, args.variables);
+          return { success: true, name: args.name };
+        }
+        default:
+          return { error: `Unknown action: ${args.action}` };
+      }
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/query-results.ts

@@ -0,0 +1,40 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { getDb } from "../../../db/schema.ts";
+import { listRuns, getRunById, getResultsByRunId, listCollections } from "../../../db/queries.ts";
+
+export const queryResultsTool = tool({
+  description: "Query test run results and collections from the database",
+  inputSchema: z.object({
+    action: z.enum(["list_runs", "get_run", "list_collections"]).describe("Action to perform"),
+    runId: z.number().optional().describe("Run ID (for get_run action)"),
+    limit: z.number().optional().describe("Max results to return (default: 20)"),
+  }),
+  execute: async (args) => {
+    try {
+      getDb();
+
+      switch (args.action) {
+        case "list_runs": {
+          const runs = listRuns(args.limit ?? 20);
+          return { runs };
+        }
+        case "get_run": {
+          if (args.runId == null) return { error: "runId is required for get_run action" };
+          const run = getRunById(args.runId);
+          if (!run) return { error: `Run ${args.runId} not found` };
+          const results = getResultsByRunId(args.runId);
+          return { run, results };
+        }
+        case "list_collections": {
+          const collections = listCollections();
+          return { collections };
+        }
+        default:
+          return { error: `Unknown action: ${args.action}` };
+      }
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/run-tests.ts

@@ -0,0 +1,38 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { executeRun } from "../../runner/execute-run.ts";
+
+export const runTestsTool = tool({
+  description: "Run API test suites from a YAML file or directory and return results summary",
+  inputSchema: z.object({
+    testPath: z.string().describe("Path to test YAML file or directory"),
+    envName: z.string().optional().describe("Environment name (loads .env.<name>.yaml)"),
+    safe: z.boolean().optional().describe("Run only GET tests (read-only, safe mode)"),
+  }),
+  execute: async (args) => {
+    try {
+      const { runId, results } = await executeRun({
+        testPath: args.testPath,
+        envName: args.envName,
+        safe: args.safe,
+        trigger: "agent",
+      });
+
+      const total = results.reduce((s, r) => s + r.total, 0);
+      const passed = results.reduce((s, r) => s + r.passed, 0);
+      const failed = results.reduce((s, r) => s + r.failed, 0);
+      const skipped = results.reduce((s, r) => s + r.skipped, 0);
+
+      return {
+        runId,
+        total,
+        passed,
+        failed,
+        skipped,
+        status: failed > 0 ? "has_failures" : "all_passed",
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/send-request.ts

@@ -0,0 +1,44 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { executeRequest } from "../../runner/http-client.ts";
+import { loadEnvironment, substituteString, substituteDeep } from "../../parser/variables.ts";
+
+export const sendRequestTool = tool({
+  description: "Send an ad-hoc HTTP request. Supports variable interpolation from environments (e.g. {{base_url}}).",
+  inputSchema: z.object({
+    method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]).describe("HTTP method"),
+    url: z.string().describe("Request URL (supports {{variable}} interpolation)"),
+    headers: z.record(z.string(), z.string()).optional().describe("Request headers"),
+    body: z.string().optional().describe("Request body (JSON string)"),
+    timeout: z.number().int().positive().optional().describe("Request timeout in ms"),
+    envName: z.string().optional().describe("Environment name for variable interpolation"),
+  }),
+  execute: async (args) => {
+    try {
+      const vars = await loadEnvironment(args.envName);
+
+      const resolvedUrl = substituteString(args.url, vars) as string;
+      const resolvedHeaders = args.headers ? substituteDeep(args.headers, vars) : {};
+      const resolvedBody = args.body ? substituteString(args.body, vars) as string : undefined;
+
+      const response = await executeRequest(
+        {
+          method: args.method,
+          url: resolvedUrl,
+          headers: resolvedHeaders,
+          body: resolvedBody,
+        },
+        args.timeout ? { timeout: args.timeout } : undefined,
+      );
+
+      // Compact output for agent — skip response headers
+      return {
+        status: response.status,
+        body: response.body_parsed ?? response.body,
+        duration_ms: response.duration_ms,
+      };
+    } catch (err) {
+      return { error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/tools/validate-tests.ts

@@ -0,0 +1,23 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { parse } from "../../parser/yaml-parser.ts";
+
+export const validateTestsTool = tool({
+  description: "Validate YAML test files without running them. Returns parsed suite info or validation errors.",
+  inputSchema: z.object({
+    testPath: z.string().describe("Path to test YAML file or directory"),
+  }),
+  execute: async (args) => {
+    try {
+      const suites = await parse(args.testPath);
+      return {
+        valid: true,
+        suiteCount: suites.length,
+        totalTests: suites.reduce((s, suite) => s + suite.tests.length, 0),
+        suites: suites.map((s) => ({ name: s.name, testCount: s.tests.length })),
+      };
+    } catch (err) {
+      return { valid: false, error: (err as Error).message };
+    }
+  },
+});

package/src/core/agent/types.ts

@@ -0,0 +1,22 @@
+import type { AIProviderConfig } from "../generator/ai/types.ts";
+
+export interface AgentConfig {
+  provider: AIProviderConfig;
+  safeMode?: boolean;
+  dbPath?: string;
+  maxSteps?: number;
+}
+
+export interface ToolEvent {
+  toolName: string;
+  args: Record<string, unknown>;
+  result: unknown;
+  timestamp: string;
+}
+
+export interface AgentTurnResult {
+  text: string;
+  toolEvents: ToolEvent[];
+  inputTokens: number;
+  outputTokens: number;
+}

package/src/core/generator/ai/ai-generator.ts

@@ -0,0 +1,61 @@
+import type { AIGenerateOptions, AIGenerateResult } from "./types.ts";
+import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../openapi-reader.ts";
+import { buildMessages } from "./prompt-builder.ts";
+import { chatCompletion } from "./llm-client.ts";
+import { parseAIResponse } from "./output-parser.ts";
+
+export async function generateWithAI(options: AIGenerateOptions): Promise<AIGenerateResult> {
+  // 1. Read OpenAPI spec
+  const doc = await readOpenApiSpec(options.specPath);
+
+  // 2. Extract endpoints + security schemes
+  let endpoints = extractEndpoints(doc);
+  if (endpoints.length === 0) {
+    throw new Error("No endpoints found in the OpenAPI spec");
+  }
+  const securitySchemes = extractSecuritySchemes(doc);
+
+  // Filter to single endpoint if requested
+  if (options.filterEndpoint) {
+    const { method, path } = options.filterEndpoint;
+    const filtered = endpoints.filter(
+      (ep) => ep.method === method.toUpperCase() && ep.path === path,
+    );
+    if (filtered.length === 0) {
+      throw new Error(`Endpoint ${method} ${path} not found in spec`);
+    }
+    endpoints = filtered;
+  }
+
+  // Determine base URL: explicit option, or from spec servers[0]
+  const baseUrl = options.baseUrl ?? (doc as any).servers?.[0]?.url as string | undefined;
+
+  // 3. Build prompt
+  const messages = buildMessages(endpoints, securitySchemes, options.prompt, baseUrl);
+
+  // 4. Call LLM
+  const startTime = Date.now();
+  const llmResult = await chatCompletion(options.provider, messages);
+  const durationMs = Date.now() - startTime;
+
+  // 5. Parse + validate output
+  const parsed = parseAIResponse(llmResult.content);
+
+  if (parsed.suites.length === 0) {
+    const errorDetail = parsed.errors.length > 0
+      ? parsed.errors.join("; ")
+      : "No valid suites in response";
+    throw new Error(`AI generation failed: ${errorDetail}`);
+  }
+
+  // If there are validation errors but we still got suites, include them as warnings
+  const yaml = parsed.yaml;
+
+  return {
+    yaml,
+    rawResponse: llmResult.content,
+    promptTokens: llmResult.usage.promptTokens,
+    completionTokens: llmResult.usage.completionTokens,
+    model: options.provider.model,
+  };
+}