@kirrosh/apitool 0.4.3

package/tests/agent/tools/query-results.test.ts

@@ -0,0 +1,77 @@
+import { describe, test, expect, mock, beforeEach, afterAll } from "bun:test";
+
+const mockListRuns = mock(() => [
+  { id: 1, started_at: "2024-01-01", total: 5, passed: 4, failed: 1, skipped: 0 },
+  { id: 2, started_at: "2024-01-02", total: 3, passed: 3, failed: 0, skipped: 0 },
+]);
+const mockGetRunById = mock((): unknown => ({
+  id: 1, started_at: "2024-01-01", finished_at: "2024-01-01", total: 5, passed: 4, failed: 1, skipped: 0,
+  trigger: "cli", environment: "staging", duration_ms: 1234,
+}));
+const mockGetResultsByRunId = mock(() => [
+  { suite_name: "API", test_name: "GET /pets", status: "pass", duration_ms: 100 },
+  { suite_name: "API", test_name: "POST /pets", status: "fail", duration_ms: 200, error_message: "404" },
+]);
+const mockListCollections = mock(() => [
+  { id: 1, name: "Petstore", test_path: "./tests/", total_runs: 5 },
+]);
+
+mock.module("../../../src/db/queries.ts", () => ({
+  listRuns: mockListRuns,
+  getRunById: mockGetRunById,
+  getResultsByRunId: mockGetResultsByRunId,
+  listCollections: mockListCollections,
+}));
+
+mock.module("../../../src/db/schema.ts", () => ({
+  getDb: mock(() => ({})),
+}));
+
+afterAll(() => { mock.restore(); });
+
+import { queryResultsTool } from "../../../src/core/agent/tools/query-results.ts";
+
+const toolOpts = { toolCallId: "test", messages: [] as any[] };
+
+describe("queryResultsTool", () => {
+  beforeEach(() => {
+    mockListRuns.mockClear();
+    mockGetRunById.mockClear();
+    mockGetResultsByRunId.mockClear();
+    mockListCollections.mockClear();
+  });
+
+  test("is an AI SDK v6 tool with inputSchema", () => {
+    expect(queryResultsTool).toHaveProperty("inputSchema");
+    expect(queryResultsTool).toHaveProperty("execute");
+  });
+
+  test("list_runs action returns runs", async () => {
+    const result = await queryResultsTool.execute!({ action: "list_runs" }, toolOpts) as any;
+    expect(result).toEqual({ runs: mockListRuns() });
+  });
+
+  test("get_run action returns run details with results", async () => {
+    const result = await queryResultsTool.execute!({ action: "get_run", runId: 1 }, toolOpts);
+    expect(result).toHaveProperty("run");
+    expect(result).toHaveProperty("results");
+    expect(mockGetRunById).toHaveBeenCalledWith(1);
+    expect(mockGetResultsByRunId).toHaveBeenCalledWith(1);
+  });
+
+  test("get_run with missing run returns error", async () => {
+    mockGetRunById.mockReturnValueOnce(null);
+    const result = await queryResultsTool.execute!({ action: "get_run", runId: 999 }, toolOpts);
+    expect(result).toEqual({ error: "Run 999 not found" });
+  });
+
+  test("list_collections action returns collections", async () => {
+    const result = await queryResultsTool.execute!({ action: "list_collections" }, toolOpts) as any;
+    expect(result).toEqual({ collections: mockListCollections() });
+  });
+
+  test("unknown action returns error", async () => {
+    const result = await queryResultsTool.execute!({ action: "unknown" as any }, toolOpts);
+    expect(result).toEqual({ error: "Unknown action: unknown" });
+  });
+});

package/tests/agent/tools/run-tests.test.ts

@@ -0,0 +1,89 @@
+import { describe, test, expect, mock, beforeEach, afterAll } from "bun:test";
+
+// Mock executeRun before importing the tool
+mock.module("../../../src/core/runner/execute-run.ts", () => ({
+  executeRun: mock(() => Promise.resolve({
+    runId: 1,
+    results: [{
+      suite_name: "Pet API",
+      started_at: "2024-01-01T00:00:00Z",
+      finished_at: "2024-01-01T00:00:01Z",
+      total: 3,
+      passed: 2,
+      failed: 1,
+      skipped: 0,
+      steps: [],
+    }],
+  })),
+}));
+
+afterAll(() => { mock.restore(); });
+
+import { runTestsTool } from "../../../src/core/agent/tools/run-tests.ts";
+import { executeRun } from "../../../src/core/runner/execute-run.ts";
+
+const toolOpts = { toolCallId: "test", messages: [] as any[] };
+
+describe("runTestsTool", () => {
+  beforeEach(() => {
+    (executeRun as ReturnType<typeof mock>).mockClear();
+  });
+
+  test("is an AI SDK v6 tool with inputSchema", () => {
+    expect(runTestsTool).toHaveProperty("inputSchema");
+    expect(runTestsTool).toHaveProperty("execute");
+    expect(runTestsTool).toHaveProperty("description");
+  });
+
+  test("returns structured result on success", async () => {
+    const result = await runTestsTool.execute!({ testPath: "tests/api.yaml" }, toolOpts);
+    expect(result).toEqual({
+      runId: 1,
+      total: 3,
+      passed: 2,
+      failed: 1,
+      skipped: 0,
+      status: "has_failures",
+    });
+    expect(executeRun).toHaveBeenCalledWith({
+      testPath: "tests/api.yaml",
+      envName: undefined,
+      safe: undefined,
+      trigger: "agent",
+    });
+  });
+
+  test("passes envName and safe mode", async () => {
+    await runTestsTool.execute!({ testPath: "tests/", envName: "staging", safe: true }, toolOpts);
+    expect(executeRun).toHaveBeenCalledWith({
+      testPath: "tests/",
+      envName: "staging",
+      safe: true,
+      trigger: "agent",
+    });
+  });
+
+  test("returns all_passed when no failures", async () => {
+    (executeRun as ReturnType<typeof mock>).mockResolvedValueOnce({
+      runId: 2,
+      results: [{
+        suite_name: "OK",
+        started_at: "2024-01-01T00:00:00Z",
+        finished_at: "2024-01-01T00:00:01Z",
+        total: 5,
+        passed: 5,
+        failed: 0,
+        skipped: 0,
+        steps: [],
+      }],
+    });
+    const result = await runTestsTool.execute!({ testPath: "tests/" }, toolOpts);
+    expect((result as any).status).toBe("all_passed");
+  });
+
+  test("returns structured error on failure", async () => {
+    (executeRun as ReturnType<typeof mock>).mockRejectedValueOnce(new Error("No test files found"));
+    const result = await runTestsTool.execute!({ testPath: "bad/path" }, toolOpts);
+    expect(result).toEqual({ error: "No test files found" });
+  });
+});

package/tests/agent/tools/send-request.test.ts

@@ -0,0 +1,78 @@
+import { describe, test, expect, mock, beforeEach, afterAll } from "bun:test";
+
+mock.module("../../../src/core/runner/http-client.ts", () => ({
+  executeRequest: mock(() => Promise.resolve({
+    status: 200,
+    headers: { "content-type": "application/json" },
+    body: '{"ok":true}',
+    body_parsed: { ok: true },
+    duration_ms: 30,
+  })),
+  DEFAULT_FETCH_OPTIONS: { timeout: 30000, retries: 0, retry_delay: 1000, follow_redirects: true },
+}));
+
+mock.module("../../../src/core/parser/variables.ts", () => ({
+  loadEnvironment: mock(() => Promise.resolve({ base_url: "https://api.test.com" })),
+  substituteString: mock((template: string, vars: Record<string, unknown>) => {
+    return template.replace(/\{\{(.+?)\}\}/g, (_, key: string) => String(vars[key] ?? `{{${key}}}`));
+  }),
+  substituteDeep: mock((value: any, vars: Record<string, unknown>) => {
+    if (typeof value === "object" && value !== null) {
+      const result: Record<string, unknown> = {};
+      for (const [k, v] of Object.entries(value)) {
+        result[k] = typeof v === "string"
+          ? v.replace(/\{\{(.+?)\}\}/g, (_, key: string) => String(vars[key] ?? `{{${key}}}`))
+          : v;
+      }
+      return result;
+    }
+    return value;
+  }),
+  GENERATORS: {},
+}));
+
+afterAll(() => { mock.restore(); });
+
+import { sendRequestTool } from "../../../src/core/agent/tools/send-request.ts";
+import { executeRequest } from "../../../src/core/runner/http-client.ts";
+
+const toolOpts = { toolCallId: "test", messages: [] as any[] };
+
+describe("sendRequestTool", () => {
+  beforeEach(() => {
+    (executeRequest as ReturnType<typeof mock>).mockClear();
+  });
+
+  test("is an AI SDK v6 tool with inputSchema", () => {
+    expect(sendRequestTool).toHaveProperty("inputSchema");
+    expect(sendRequestTool).toHaveProperty("execute");
+    expect(sendRequestTool).toHaveProperty("description");
+  });
+
+  test("sends request and returns compact result (no headers)", async () => {
+    const result = await sendRequestTool.execute!({ method: "GET", url: "https://api.test.com/data" }, toolOpts);
+    expect(result).toEqual({
+      status: 200,
+      body: { ok: true },
+      duration_ms: 30,
+    });
+  });
+
+  test("passes headers and body", async () => {
+    await sendRequestTool.execute!({
+      method: "POST",
+      url: "https://api.test.com/data",
+      headers: { "Authorization": "Bearer token" },
+      body: '{"name":"test"}',
+    }, toolOpts);
+    expect(executeRequest).toHaveBeenCalledTimes(1);
+    const call = (executeRequest as ReturnType<typeof mock>).mock.calls[0]![0];
+    expect(call.method).toBe("POST");
+  });
+
+  test("returns error on failure", async () => {
+    (executeRequest as ReturnType<typeof mock>).mockRejectedValueOnce(new Error("connection refused"));
+    const result = await sendRequestTool.execute!({ method: "GET", url: "https://bad.host/" }, toolOpts);
+    expect(result).toEqual({ error: "connection refused" });
+  });
+});

package/tests/agent/tools/validate-tests.test.ts

@@ -0,0 +1,59 @@
+import { describe, test, expect, mock, beforeEach, afterAll } from "bun:test";
+
+mock.module("../../../src/core/parser/yaml-parser.ts", () => ({
+  parse: mock(() => Promise.resolve([
+    { name: "Suite A", tests: [{ name: "t1" }, { name: "t2" }] },
+  ])),
+  parseDirectorySafe: mock(() => Promise.resolve({
+    suites: [{ name: "Suite A", tests: [{ name: "t1" }] }],
+    errors: [],
+  })),
+}));
+
+afterAll(() => { mock.restore(); });
+
+import { validateTestsTool } from "../../../src/core/agent/tools/validate-tests.ts";
+import { parse } from "../../../src/core/parser/yaml-parser.ts";
+
+const toolOpts = { toolCallId: "test", messages: [] as any[] };
+
+describe("validateTestsTool", () => {
+  beforeEach(() => {
+    (parse as ReturnType<typeof mock>).mockClear();
+  });
+
+  test("is an AI SDK v6 tool with inputSchema", () => {
+    expect(validateTestsTool).toHaveProperty("inputSchema");
+    expect(validateTestsTool).toHaveProperty("execute");
+  });
+
+  test("returns valid result for correct YAML", async () => {
+    const result = await validateTestsTool.execute!({ testPath: "tests/api.yaml" }, toolOpts);
+    expect(result).toEqual({
+      valid: true,
+      suiteCount: 1,
+      totalTests: 2,
+      suites: [{ name: "Suite A", testCount: 2 }],
+    });
+  });
+
+  test("returns structured error on parse failure", async () => {
+    (parse as ReturnType<typeof mock>).mockRejectedValueOnce(new Error("Invalid YAML in test.yaml: bad indent"));
+    const result = await validateTestsTool.execute!({ testPath: "bad.yaml" }, toolOpts);
+    expect(result).toEqual({
+      valid: false,
+      error: "Invalid YAML in test.yaml: bad indent",
+    });
+  });
+
+  test("returns valid=true for empty suite list", async () => {
+    (parse as ReturnType<typeof mock>).mockResolvedValueOnce([]);
+    const result = await validateTestsTool.execute!({ testPath: "empty/" }, toolOpts);
+    expect(result).toEqual({
+      valid: true,
+      suiteCount: 0,
+      totalTests: 0,
+      suites: [],
+    });
+  });
+});

package/tests/ai/ai-generator.integration.test.ts

@@ -0,0 +1,131 @@
+import { describe, test, expect, beforeAll } from "bun:test";
+import { generateWithAI } from "../../src/core/generator/ai/ai-generator.ts";
+import { resolveProviderConfig } from "../../src/core/generator/ai/types.ts";
+import { validateSuite } from "../../src/core/parser/schema.ts";
+
+// This test requires a running Ollama instance with llama3.2:3b model
+// Skip if Ollama is not available
+
+let ollamaAvailable = false;
+
+beforeAll(async () => {
+  try {
+    const resp = await fetch("http://localhost:11434/api/version");
+    if (resp.ok) {
+      // Check if model is available
+      const tagsResp = await fetch("http://localhost:11434/api/tags");
+      if (tagsResp.ok) {
+        const tags = await tagsResp.json() as { models: Array<{ name: string }> };
+        ollamaAvailable = tags.models?.some(m => m.name.startsWith("llama3.2")) ?? false;
+      }
+    }
+  } catch {
+    ollamaAvailable = false;
+  }
+
+  if (!ollamaAvailable) {
+    console.log("SKIP: Ollama not available or llama3.2 not installed");
+  }
+});
+
+describe("AI Generator E2E with Ollama", () => {
+  test("generates valid YAML from petstore spec", async () => {
+    if (!ollamaAvailable) return;
+
+    const provider = resolveProviderConfig({
+      provider: "ollama",
+      model: "llama3.2:3b",
+    });
+
+    const result = await generateWithAI({
+      specPath: "tests/fixtures/petstore-simple.json",
+      prompt: "Create a pet and then get it by ID to verify it exists",
+      provider,
+    });
+
+    // Should produce non-empty YAML
+    expect(result.yaml.length).toBeGreaterThan(50);
+    expect(result.model).toBe("llama3.2:3b");
+    expect(result.rawResponse.length).toBeGreaterThan(0);
+
+    // YAML should contain expected keywords
+    expect(result.yaml).toContain("name:");
+    expect(result.yaml).toContain("tests:");
+    expect(result.yaml).toContain("expect:");
+
+    // Should contain at least one HTTP method
+    const hasMethod = /\b(GET|POST|PUT|PATCH|DELETE):/m.test(result.yaml);
+    expect(hasMethod).toBe(true);
+
+    console.log("Generated YAML:\n" + result.yaml);
+  }, 120_000); // 2 min timeout for LLM
+
+  test("generated YAML parses and validates with Zod schema", async () => {
+    if (!ollamaAvailable) return;
+
+    const provider = resolveProviderConfig({
+      provider: "ollama",
+      model: "llama3.2:3b",
+    });
+
+    const result = await generateWithAI({
+      specPath: "tests/fixtures/petstore-simple.json",
+      prompt: "Create a new pet with name and species, verify creation with GET",
+      provider,
+    });
+
+    // Split multi-document YAML
+    const docs = result.yaml.split(/\n---\n/).filter(Boolean);
+    expect(docs.length).toBeGreaterThanOrEqual(1);
+
+    for (const doc of docs) {
+      const parsed = Bun.YAML.parse(doc);
+      // Should not throw — validates with Zod
+      const suite = validateSuite(parsed);
+      expect(suite.name).toBeTruthy();
+      expect(suite.tests.length).toBeGreaterThanOrEqual(1);
+
+      for (const step of suite.tests) {
+        expect(step.name).toBeTruthy();
+        expect(step.method).toBeTruthy();
+        expect(step.path).toBeTruthy();
+      }
+    }
+  }, 120_000);
+
+  test("generates test with 409 conflict scenario (retry up to 3 times)", async () => {
+    if (!ollamaAvailable) return;
+
+    const provider = resolveProviderConfig({
+      provider: "ollama",
+      model: "llama3.2:3b",
+    });
+
+    // Small models sometimes produce invalid JSON, retry a few times
+    let lastError: Error | null = null;
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      try {
+        const result = await generateWithAI({
+          specPath: "tests/fixtures/petstore-simple.json",
+          prompt: "Test uniqueness: create a pet, then create a duplicate pet with the same name and expect 409 conflict error response",
+          provider,
+        });
+
+        expect(result.yaml.length).toBeGreaterThan(50);
+        expect(result.yaml).toContain("POST:");
+        // Should reference 409 or conflict scenario
+        const has409 = result.yaml.includes("409") || result.yaml.toLowerCase().includes("conflict") || result.yaml.toLowerCase().includes("duplicate");
+        expect(has409).toBe(true);
+
+        console.log(`409 Conflict YAML (attempt ${attempt}):\n` + result.yaml);
+        return; // success
+      } catch (err) {
+        lastError = err as Error;
+        console.log(`Attempt ${attempt} failed: ${lastError.message}`);
+      }
+    }
+
+    // If all retries failed, skip gracefully (small model limitation)
+    console.log(`SKIP: All 3 attempts failed for 409 scenario. Last error: ${lastError?.message}`);
+  }, 300_000);
+});

package/tests/ai/llm-client.test.ts

@@ -0,0 +1,145 @@
+import { describe, test, expect } from "bun:test";
+import { chatCompletion } from "../../src/core/generator/ai/llm-client.ts";
+import type { AIProviderConfig } from "../../src/core/generator/ai/types.ts";
+
+describe("llm-client", () => {
+  test("sends correct headers for openai provider", async () => {
+    let capturedRequest: { url: string; headers: Record<string, string>; body: any } | null = null;
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = (async (input: any, init: any) => {
+      capturedRequest = {
+        url: typeof input === "string" ? input : input.url,
+        headers: Object.fromEntries(Object.entries(init?.headers ?? {})),
+        body: JSON.parse(init?.body ?? "{}"),
+      };
+      return new Response(JSON.stringify({
+        choices: [{ message: { content: '{"suites":[]}' } }],
+        usage: { prompt_tokens: 10, completion_tokens: 5 },
+      }), { status: 200 });
+    }) as unknown as typeof fetch;
+
+    try {
+      const config: AIProviderConfig = {
+        provider: "openai",
+        baseUrl: "https://api.openai.com/v1",
+        apiKey: "sk-test-key",
+        model: "gpt-4o",
+        temperature: 0.2,
+        maxTokens: 4096,
+      };
+
+      const result = await chatCompletion(config, [
+        { role: "system", content: "You are helpful" },
+        { role: "user", content: "Hello" },
+      ]);
+
+      expect(capturedRequest).not.toBeNull();
+      expect(capturedRequest!.url).toBe("https://api.openai.com/v1/chat/completions");
+      expect(capturedRequest!.headers["Authorization"]).toBe("Bearer sk-test-key");
+      expect(capturedRequest!.headers["Content-Type"]).toBe("application/json");
+      expect(capturedRequest!.body.model).toBe("gpt-4o");
+      expect(capturedRequest!.body.messages.length).toBe(2);
+      expect(capturedRequest!.body.response_format).toEqual({ type: "json_object" });
+      expect(result.content).toBe('{"suites":[]}');
+      expect(result.usage.promptTokens).toBe(10);
+      expect(result.usage.completionTokens).toBe(5);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("sends correct format for anthropic provider", async () => {
+    let capturedRequest: { url: string; headers: Record<string, string>; body: any } | null = null;
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = (async (input: any, init: any) => {
+      capturedRequest = {
+        url: typeof input === "string" ? input : input.url,
+        headers: Object.fromEntries(Object.entries(init?.headers ?? {})),
+        body: JSON.parse(init?.body ?? "{}"),
+      };
+      return new Response(JSON.stringify({
+        content: [{ type: "text", text: '{"suites":[]}' }],
+        usage: { input_tokens: 20, output_tokens: 10 },
+      }), { status: 200 });
+    }) as unknown as typeof fetch;
+
+    try {
+      const config: AIProviderConfig = {
+        provider: "anthropic",
+        baseUrl: "https://api.anthropic.com",
+        apiKey: "sk-ant-test",
+        model: "claude-sonnet-4-20250514",
+        temperature: 0.2,
+        maxTokens: 4096,
+      };
+
+      const result = await chatCompletion(config, [
+        { role: "system", content: "System prompt" },
+        { role: "user", content: "User message" },
+      ]);
+
+      expect(capturedRequest).not.toBeNull();
+      expect(capturedRequest!.url).toBe("https://api.anthropic.com/v1/messages");
+      expect(capturedRequest!.headers["x-api-key"]).toBe("sk-ant-test");
+      expect(capturedRequest!.headers["anthropic-version"]).toBe("2023-06-01");
+      // System should be top-level, not in messages
+      expect(capturedRequest!.body.system).toBe("System prompt");
+      expect(capturedRequest!.body.messages.length).toBe(1); // only user message
+      expect(capturedRequest!.body.messages[0].role).toBe("user");
+      expect(result.content).toBe('{"suites":[]}');
+      expect(result.usage.promptTokens).toBe(20);
+      expect(result.usage.completionTokens).toBe(10);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("uses ollama path without Authorization when no apiKey", async () => {
+    let capturedHeaders: Record<string, string> = {};
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = (async (_input: any, init: any) => {
+      capturedHeaders = Object.fromEntries(Object.entries(init?.headers ?? {}));
+      return new Response(JSON.stringify({
+        choices: [{ message: { content: "test" } }],
+      }), { status: 200 });
+    }) as unknown as typeof fetch;
+
+    try {
+      const config: AIProviderConfig = {
+        provider: "ollama",
+        baseUrl: "http://localhost:11434/v1",
+        model: "llama3.2",
+      };
+
+      await chatCompletion(config, [{ role: "user", content: "hi" }]);
+      expect(capturedHeaders["Authorization"]).toBeUndefined();
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test("throws on HTTP error", async () => {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = (async () => {
+      return new Response("Rate limit exceeded", { status: 429 });
+    }) as unknown as typeof fetch;
+
+    try {
+      const config: AIProviderConfig = {
+        provider: "openai",
+        baseUrl: "https://api.openai.com/v1",
+        apiKey: "sk-test",
+        model: "gpt-4o",
+      };
+
+      await expect(
+        chatCompletion(config, [{ role: "user", content: "hi" }])
+      ).rejects.toThrow("429");
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});