@kirrosh/apitool 0.4.3

package/src/core/generator/ai/llm-client.ts

@@ -0,0 +1,159 @@
+import type { AIProviderConfig } from "./types.ts";
+
+export interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+
+export interface ChatCompletionResult {
+  content: string;
+  usage: {
+    promptTokens?: number;
+    completionTokens?: number;
+  };
+}
+
+export async function chatCompletion(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  if (config.provider === "anthropic") {
+    return callAnthropic(config, messages);
+  }
+  return callOpenAICompatible(config, messages);
+}
+
+async function callOpenAICompatible(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  const url = `${config.baseUrl.replace(/\/+$/, "")}/chat/completions`;
+
+  // For ollama/custom providers, inject system prompt into first user message
+  // to avoid issues with thinking models (e.g. qwen3) that break with separate system messages
+  let apiMessages: Array<{ role: string; content: string }>;
+  if (config.provider === "ollama" || config.provider === "custom") {
+    const systemMsgs = messages.filter((m) => m.role === "system");
+    const nonSystem = messages.filter((m) => m.role !== "system");
+    if (systemMsgs.length > 0 && nonSystem.length > 0) {
+      const systemText = systemMsgs.map((m) => m.content).join("\n\n");
+      apiMessages = nonSystem.map((m, i) =>
+        i === 0 ? { role: m.role, content: `${systemText}\n\n${m.content}` } : { role: m.role, content: m.content }
+      );
+    } else {
+      apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
+    }
+  } else {
+    apiMessages = messages.map((m) => ({ role: m.role, content: m.content }));
+  }
+
+  const body: Record<string, unknown> = {
+    model: config.model,
+    messages: apiMessages,
+    temperature: config.temperature ?? 0.2,
+    max_tokens: config.maxTokens ?? 4096,
+  };
+
+  // Request JSON output where supported (OpenAI, newer Ollama models)
+  if (config.provider === "openai") {
+    body.response_format = { type: "json_object" };
+  }
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+  };
+  if (config.apiKey) {
+    headers["Authorization"] = `Bearer ${config.apiKey}`;
+  }
+
+  const resp = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`LLM request failed (${resp.status}): ${text}`);
+  }
+
+  const data = (await resp.json()) as {
+    choices: Array<{ message: { content: string; reasoning?: string } }>;
+    usage?: { prompt_tokens?: number; completion_tokens?: number };
+  };
+
+  const msg = data.choices?.[0]?.message;
+  // Thinking models (e.g. qwen3) may put output in `reasoning` with empty `content`
+  const content = msg?.content || msg?.reasoning || "";
+  return {
+    content,
+    usage: {
+      promptTokens: data.usage?.prompt_tokens,
+      completionTokens: data.usage?.completion_tokens,
+    },
+  };
+}
+
+async function callAnthropic(
+  config: AIProviderConfig,
+  messages: ChatMessage[],
+): Promise<ChatCompletionResult> {
+  const url = `${config.baseUrl.replace(/\/+$/, "")}/v1/messages`;
+
+  // Separate system prompt from user/assistant messages
+  const systemMessages = messages.filter((m) => m.role === "system");
+  const nonSystemMessages = messages.filter((m) => m.role !== "system");
+
+  const systemText = systemMessages.map((m) => m.content).join("\n\n");
+
+  const body: Record<string, unknown> = {
+    model: config.model,
+    max_tokens: config.maxTokens ?? 4096,
+    temperature: config.temperature ?? 0.2,
+    messages: nonSystemMessages.map((m) => ({
+      role: m.role,
+      content: m.content,
+    })),
+  };
+
+  if (systemText) {
+    body.system = systemText;
+  }
+
+  const headers: Record<string, string> = {
+    "Content-Type": "application/json",
+    "anthropic-version": "2023-06-01",
+  };
+  if (config.apiKey) {
+    headers["x-api-key"] = config.apiKey;
+  }
+
+  const resp = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`Anthropic request failed (${resp.status}): ${text}`);
+  }
+
+  const data = (await resp.json()) as {
+    content: Array<{ type: string; text: string }>;
+    usage?: { input_tokens?: number; output_tokens?: number };
+  };
+
+  const content = data.content
+    ?.filter((b) => b.type === "text")
+    .map((b) => b.text)
+    .join("") ?? "";
+
+  return {
+    content,
+    usage: {
+      promptTokens: data.usage?.input_tokens,
+      completionTokens: data.usage?.output_tokens,
+    },
+  };
+}

package/src/core/generator/ai/output-parser.ts

@@ -0,0 +1,307 @@
+import type { RawSuite } from "../serializer.ts";
+import { serializeSuite } from "../serializer.ts";
+import { TestSuiteSchema } from "../../parser/schema.ts";
+
+export interface ParseResult {
+  suites: RawSuite[];
+  yaml: string;
+  errors: string[];
+}
+
+export function parseAIResponse(raw: string): ParseResult {
+  const errors: string[] = [];
+
+  // Sanitize first (fix template vars, NaN, etc.) so extractJson's
+  // bracket-matching isn't confused by bare {{...}} tokens
+  const sanitized = sanitizeJson(raw);
+
+  // Extract JSON from response (handle fences, leading text)
+  const json = extractJson(sanitized);
+  if (!json) {
+    return { suites: [], yaml: "", errors: ["Could not find valid JSON in LLM response"] };
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(json);
+  } catch (e) {
+    return { suites: [], yaml: "", errors: [`Invalid JSON: ${(e as Error).message}`] };
+  }
+
+  // Normalize to array of suite objects
+  let suiteObjects: unknown[];
+  if (Array.isArray(parsed)) {
+    suiteObjects = parsed;
+  } else if (typeof parsed === "object" && parsed !== null) {
+    const obj = parsed as Record<string, unknown>;
+    if (Array.isArray(obj.suites)) {
+      suiteObjects = obj.suites;
+    } else if (obj.name && Array.isArray(obj.tests)) {
+      // Single suite object
+      suiteObjects = [obj];
+    } else {
+      return { suites: [], yaml: "", errors: ["JSON does not contain a valid suite structure"] };
+    }
+  } else {
+    return { suites: [], yaml: "", errors: ["Expected JSON object or array"] };
+  }
+
+  const validSuites: RawSuite[] = [];
+  const yamlParts: string[] = [];
+
+  for (let i = 0; i < suiteObjects.length; i++) {
+    const suiteObj = suiteObjects[i];
+    if (typeof suiteObj !== "object" || suiteObj === null) {
+      errors.push(`Suite ${i + 1}: not a valid object`);
+      continue;
+    }
+
+    // Transform method keys to the format our schema expects
+    const rawSuite = transformSuite(suiteObj as Record<string, unknown>);
+
+    // Skip suites without tests — can't serialize them
+    if (!Array.isArray((rawSuite as any).tests) || (rawSuite as any).tests.length === 0) {
+      errors.push(`Suite "${(rawSuite as any).name ?? i + 1}": no tests defined, skipped`);
+      continue;
+    }
+
+    // Validate against Zod schema
+    const result = TestSuiteSchema.safeParse(rawSuite);
+    if (!result.success) {
+      // Try to auto-fix and re-validate
+      const fixed = autoFixSuite(rawSuite);
+      const retry = TestSuiteSchema.safeParse(fixed);
+      if (retry.success) {
+        validSuites.push(fixed as unknown as RawSuite);
+        yamlParts.push(serializeSuite(fixed as unknown as RawSuite));
+        continue;
+      }
+      const issues = result.error.issues.map((issue) =>
+        `${issue.path.join(".")}: ${issue.message}`
+      ).join("; ");
+      errors.push(`Suite "${(rawSuite as any).name ?? i + 1}" validation: ${issues}`);
+      continue;
+    }
+
+    validSuites.push(rawSuite as unknown as RawSuite);
+    yamlParts.push(serializeSuite(rawSuite as unknown as RawSuite));
+  }
+
+  if (validSuites.length === 0 && errors.length === 0) {
+    errors.push("No test suites found in LLM response");
+  }
+
+  return {
+    suites: validSuites,
+    yaml: yamlParts.join("\n---\n"),
+    errors,
+  };
+}
+
+/** Fix common LLM JSON mistakes before parsing */
+function sanitizeJson(json: string): string {
+  let s = json;
+
+  // 1. Fix broken template vars: {{$randomString} → {{$randomString}}
+  s = s.replace(/\{\{([$]?\w+)\}(?!\})/g, '{{$1}}');
+
+  // 2. Quote bare (unquoted) {{...}} template vars outside of JSON strings.
+  //    Walk character-by-character to track string context.
+  s = quoteBareTemplateVars(s);
+
+  // 3. Replace bare NaN / Infinity / undefined with null
+  s = s.replace(/\bNaN\b/g, 'null');
+  s = s.replace(/\bundefined\b/g, 'null');
+  s = s.replace(/-?\bInfinity\b/g, 'null');
+
+  // 4. Remove trailing commas before } or ]
+  s = s.replace(/,(\s*[}\]])/g, '$1');
+
+  // 5. Try parsing; if it fails, attempt to fix unbalanced brackets
+  try { JSON.parse(s); } catch {
+    s = fixUnbalancedBrackets(s);
+  }
+
+  return s;
+}
+
+/** Wrap bare {{...}} tokens (outside JSON strings) in quotes */
+function quoteBareTemplateVars(json: string): string {
+  const result: string[] = [];
+  let inString = false;
+  let escape = false;
+
+  for (let i = 0; i < json.length; i++) {
+    const ch = json[i]!;
+
+    if (escape) { escape = false; result.push(ch); continue; }
+    if (ch === '\\' && inString) { escape = true; result.push(ch); continue; }
+    if (ch === '"') { inString = !inString; result.push(ch); continue; }
+
+    // Inside a JSON string, {{ }} are fine — leave them as-is
+    if (inString) { result.push(ch); continue; }
+
+    // Outside a string: check for {{ start
+    if (ch === '{' && json[i + 1] === '{') {
+      // Find the closing }}
+      const end = json.indexOf('}}', i + 2);
+      if (end !== -1) {
+        const tpl = json.slice(i, end + 2); // e.g. "{{$randomInt}}"
+        result.push('"', tpl, '"');
+        i = end + 1; // skip past }}
+        continue;
+      }
+    }
+
+    result.push(ch);
+  }
+
+  return result.join('');
+}
+
+/** Remove excess closing brackets/braces by tracking depth */
+function fixUnbalancedBrackets(json: string): string {
+  const result: string[] = [];
+  let inString = false;
+  let escape = false;
+  const stack: string[] = [];
+
+  for (let i = 0; i < json.length; i++) {
+    const ch = json[i]!;
+    if (escape) { escape = false; result.push(ch); continue; }
+    if (ch === '\\' && inString) { escape = true; result.push(ch); continue; }
+    if (ch === '"') { inString = !inString; result.push(ch); continue; }
+    if (inString) { result.push(ch); continue; }
+
+    if (ch === '{' || ch === '[') {
+      stack.push(ch);
+      result.push(ch);
+    } else if (ch === '}' || ch === ']') {
+      const expected = ch === '}' ? '{' : '[';
+      if (stack.length > 0 && stack[stack.length - 1] === expected) {
+        stack.pop();
+        result.push(ch);
+      }
+      // else: skip the excess closing bracket
+    } else {
+      result.push(ch);
+    }
+  }
+
+  return result.join('');
+}
+
+function extractJson(raw: string): string | null {
+  // Try 1: Extract from ```json ... ``` fences (use last match for thinking models)
+  const fenceMatches = [...raw.matchAll(/```(?:json)?\s*\n?([\s\S]*?)```/g)];
+  if (fenceMatches.length > 0) {
+    return fenceMatches[fenceMatches.length - 1]![1]!.trim();
+  }
+
+  // Try 2: Find all balanced JSON blocks, return the largest one
+  // (thinking models put explanatory JSON snippets before the final answer)
+  const candidates: string[] = [];
+  for (let pos = 0; pos < raw.length; pos++) {
+    const ch = raw[pos];
+    if (ch !== "{" && ch !== "[") continue;
+
+    const open = ch;
+    const close = ch === "{" ? "}" : "]";
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+
+    for (let i = pos; i < raw.length; i++) {
+      const c = raw[i];
+      if (escape) { escape = false; continue; }
+      if (c === "\\") { escape = true; continue; }
+      if (c === '"') { inString = !inString; continue; }
+      if (inString) continue;
+
+      if (c === open) depth++;
+      if (c === close) {
+        depth--;
+        if (depth === 0) {
+          candidates.push(raw.slice(pos, i + 1));
+          pos = i; // skip past this block
+          break;
+        }
+      }
+    }
+  }
+
+  if (candidates.length === 0) return null;
+
+  // Return the largest candidate (most likely the full answer, not a snippet)
+  return candidates.reduce((a, b) => a.length >= b.length ? a : b);
+}
+
+function fixBodyAssertions(body: Record<string, unknown>): Record<string, unknown> {
+  const fixed: Record<string, unknown> = {};
+  for (const [key, val] of Object.entries(body)) {
+    if (val === null || val === undefined) {
+      // null/undefined → { exists: true }
+      fixed[key] = { exists: true };
+    } else if (typeof val === "string") {
+      // bare string → { type: string }
+      fixed[key] = { type: val };
+    } else if (typeof val === "object" && val !== null) {
+      const rule = val as Record<string, unknown>;
+      // Coerce "true"/"false" strings to boolean for `exists`
+      if (typeof rule.exists === "string") {
+        rule.exists = rule.exists === "true";
+      }
+      fixed[key] = rule;
+    } else {
+      fixed[key] = val;
+    }
+  }
+  return fixed;
+}
+
+function transformSuite(obj: Record<string, unknown>): Record<string, unknown> {
+  const tests = obj.tests as Array<Record<string, unknown>> | undefined;
+  if (!Array.isArray(tests)) return obj;
+
+  const transformedTests = tests.map((step) => {
+    // Ensure expect exists
+    if (!step.expect) {
+      step.expect = {};
+    }
+
+    const expect = step.expect as Record<string, unknown>;
+    if (expect.body && typeof expect.body === "object" && expect.body !== null) {
+      expect.body = fixBodyAssertions(expect.body as Record<string, unknown>);
+    }
+
+    return step;
+  });
+
+  return { ...obj, tests: transformedTests };
+}
+
+/** Deep-fix a suite that failed validation — coerce types LLMs commonly get wrong */
+function autoFixSuite(obj: Record<string, unknown>): Record<string, unknown> {
+  const tests = obj.tests as Array<Record<string, unknown>> | undefined;
+  if (!Array.isArray(tests)) return obj;
+
+  const fixedTests = tests.map((step) => {
+    const expect = step.expect as Record<string, unknown> | undefined;
+    if (!expect) return step;
+
+    // Fix status as string → number
+    if (typeof expect.status === "string") {
+      const n = parseInt(expect.status as string, 10);
+      if (!isNaN(n)) expect.status = n;
+    }
+
+    // Fix body assertions again (in case transformSuite missed edge cases)
+    if (expect.body && typeof expect.body === "object" && expect.body !== null) {
+      expect.body = fixBodyAssertions(expect.body as Record<string, unknown>);
+    }
+
+    return step;
+  });
+
+  return { ...obj, tests: fixedTests };
+}

package/src/core/generator/ai/prompt-builder.ts

@@ -0,0 +1,153 @@
+import type { EndpointInfo, SecuritySchemeInfo } from "../types.ts";
+import type { ChatMessage } from "./llm-client.ts";
+import { compressSchema, formatParam } from "../schema-utils.ts";
+
+const SYSTEM_PROMPT = `You are an API test generator. You produce JSON output that represents test suites for an API testing tool.
+
+OUTPUT FORMAT — return a JSON object with a single key "suites" containing an array of suite objects:
+{
+  "suites": [
+    {
+      "name": "Suite Name",
+      "base_url": "{{base_url}}",
+      "headers": {
+        "Authorization": "Bearer {{auth_token}}"
+      },
+      "tests": [
+        {
+          "name": "Authenticate",
+          "POST": "/auth/login",
+          "json": { "username": "{{auth_username}}", "password": "{{auth_password}}" },
+          "expect": {
+            "status": 200,
+            "body": {
+              "token": { "type": "string", "capture": "auth_token" }
+            }
+          }
+        },
+        {
+          "name": "Create item",
+          "POST": "/items",
+          "json": { "name": "{{$randomString}}" },
+          "expect": {
+            "status": 201,
+            "body": {
+              "id": { "type": "number", "capture": "created_id" }
+            }
+          }
+        },
+        {
+          "name": "Verify created",
+          "GET": "/items/{{created_id}}",
+          "expect": {
+            "status": 200,
+            "body": {
+              "id": { "equals": "{{created_id}}" }
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
+
+RULES:
+1. Each test step has exactly ONE HTTP method key: GET, POST, PUT, PATCH, or DELETE. The value is the path.
+2. Use "json" for request bodies (objects). Use "form" for form data. Use "query" for query parameters.
+3. "expect" contains "status" (number) and optional "body" with field assertions.
+4. Body assertions are objects with these optional keys:
+   - "type": "string" | "number" | "integer" | "boolean" | "array" | "object"
+   - "equals": exact value match
+   - "contains": substring match (strings)
+   - "matches": regex pattern
+   - "gt": greater than (numbers)
+   - "lt": less than (numbers)
+   - "exists": true (must be boolean true, NEVER a string)
+   - "capture": variable name — SAVES the response value into a variable for later steps
+5. Use {{variable}} syntax to reference captured values in paths, bodies, and assertions.
+6. ONLY these built-in generators exist: {{$randomInt}}, {{$uuid}}, {{$timestamp}}, {{$randomEmail}}, {{$randomString}}, {{$randomName}}. Do NOT invent others like $randomString(N) or $randomWord — they do not exist.
+7. Use {{base_url}} for the base URL — never hardcode it.
+8. Steps execute sequentially — a capture in step 1 is available in step 2+.
+9. Generate realistic test data. Use generators for uniqueness where needed.
+10. Output ONLY the JSON object, no markdown fences or extra text.
+
+CRITICAL — common mistakes to avoid:
+- NEVER use "equals" to save a value. "equals" COMPARES, "capture" SAVES. To extract a token: {"capture": "auth_token"} NOT {"equals": "{{auth_token}}"}.
+- NEVER prefix double braces with a dollar sign. Correct: {{my_var}}. Wrong: $` + `{{my_var}}. Generators also use plain braces: {{$randomString}} not $` + `{{$randomString}}.
+- If the API has authentication (JWT, Bearer token), ALWAYS add a login step FIRST that captures the token, then set suite-level "headers": {"Authorization": "Bearer {{auth_token}}"}.
+- For login credentials, use environment variables {{auth_username}} and {{auth_password}}, NOT generators like {{$randomEmail}}.
+- "exists" value MUST be boolean true or false, NEVER the string "true".`;
+
+export function buildMessages(
+  endpoints: EndpointInfo[],
+  securitySchemes: SecuritySchemeInfo[],
+  userPrompt: string,
+  baseUrl?: string,
+): ChatMessage[] {
+  const apiContext = compressEndpoints(endpoints, securitySchemes);
+
+  const userMessage = `API SPECIFICATION:
+${apiContext}
+${baseUrl ? `\nBase URL: ${baseUrl}` : ""}
+
+USER REQUEST:
+${userPrompt}
+
+Generate test suites as JSON following the rules in your instructions.`;
+
+  return [
+    { role: "system", content: SYSTEM_PROMPT },
+    { role: "user", content: userMessage },
+  ];
+}
+
+function compressEndpoints(
+  endpoints: EndpointInfo[],
+  securitySchemes: SecuritySchemeInfo[],
+): string {
+  const lines: string[] = [];
+
+  // Security schemes
+  if (securitySchemes.length > 0) {
+    lines.push("SECURITY:");
+    for (const s of securitySchemes) {
+      let desc = `  ${s.name}: ${s.type}`;
+      if (s.scheme) desc += ` (${s.scheme})`;
+      if (s.in && s.apiKeyName) desc += ` (${s.apiKeyName} in ${s.in})`;
+      lines.push(desc);
+    }
+    lines.push("");
+  }
+
+  // Endpoints
+  lines.push("ENDPOINTS:");
+  for (const ep of endpoints) {
+    const summary = ep.summary ? ` — ${ep.summary}` : "";
+    const security = ep.security.length > 0 ? `  [auth: ${ep.security.join(", ")}]` : "";
+    lines.push(`${ep.method} ${ep.path}${summary}${security}`);
+
+    // Parameters
+    const pathParams = ep.parameters.filter((p) => p.in === "path");
+    const queryParams = ep.parameters.filter((p) => p.in === "query");
+    if (pathParams.length > 0) {
+      lines.push(`  Path params: ${pathParams.map((p) => formatParam(p)).join(", ")}`);
+    }
+    if (queryParams.length > 0) {
+      lines.push(`  Query params: ${queryParams.map((p) => formatParam(p)).join(", ")}`);
+    }
+
+    // Request body
+    if (ep.requestBodySchema) {
+      lines.push(`  Body: ${compressSchema(ep.requestBodySchema)}`);
+    }
+
+    // Responses
+    for (const resp of ep.responses) {
+      const schemaStr = resp.schema ? ` ${compressSchema(resp.schema)}` : "";
+      lines.push(`  ${resp.statusCode}: ${resp.description}${schemaStr}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+

package/src/core/generator/ai/types.ts

@@ -0,0 +1,56 @@
+export interface AIProviderConfig {
+  provider: "ollama" | "openai" | "anthropic" | "custom";
+  baseUrl: string;
+  apiKey?: string;
+  model: string;
+  temperature?: number;
+  maxTokens?: number;
+}
+
+export interface AIGenerateOptions {
+  specPath: string;
+  prompt: string;
+  provider: AIProviderConfig;
+  baseUrl?: string;
+  collectionId?: number;
+  /** Filter to a single endpoint by method+path */
+  filterEndpoint?: { method: string; path: string };
+}
+
+export interface AIGenerateResult {
+  yaml: string;
+  rawResponse: string;
+  promptTokens?: number;
+  completionTokens?: number;
+  model: string;
+}
+
+export const PROVIDER_DEFAULTS: Record<string, Partial<AIProviderConfig>> = {
+  ollama: {
+    baseUrl: "http://localhost:11434/v1",
+    model: "llama3.2:3b",
+  },
+  openai: {
+    baseUrl: "https://api.openai.com/v1",
+    model: "gpt-4o",
+  },
+  anthropic: {
+    baseUrl: "https://api.anthropic.com",
+    model: "claude-sonnet-4-20250514",
+  },
+};
+
+export function resolveProviderConfig(partial: Partial<AIProviderConfig> & { provider: AIProviderConfig["provider"] }): AIProviderConfig {
+  const defaults = PROVIDER_DEFAULTS[partial.provider] ?? {};
+  // Thinking models (e.g. qwen3) use reasoning tokens that count toward max_tokens,
+  // so local models need a higher budget than cloud APIs
+  const defaultMaxTokens = partial.provider === "ollama" || partial.provider === "custom" ? 8192 : 4096;
+  return {
+    provider: partial.provider,
+    baseUrl: partial.baseUrl ?? defaults.baseUrl ?? "",
+    apiKey: partial.apiKey,
+    model: partial.model ?? defaults.model ?? "",
+    temperature: partial.temperature ?? 0.2,
+    maxTokens: partial.maxTokens ?? defaultMaxTokens,
+  };
+}