@kirrosh/apitool 0.4.3

package/src/core/reporter/junit.ts

@@ -0,0 +1,78 @@
+import type { TestRunResult, StepResult } from "../runner/types.ts";
+import type { Reporter, ReporterOptions } from "./types.ts";
+
+function escapeXml(str: string): string {
+  return str
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+function formatTime(ms: number): string {
+  return (ms / 1000).toFixed(3);
+}
+
+function renderTestcase(step: StepResult): string {
+  const time = formatTime(step.duration_ms);
+  const name = escapeXml(step.name);
+
+  if (step.status === "pass") {
+    return `    <testcase name="${name}" time="${time}"/>`;
+  }
+
+  if (step.status === "skip") {
+    return `    <testcase name="${name}" time="${time}">\n      <skipped/>\n    </testcase>`;
+  }
+
+  if (step.status === "fail") {
+    const failedAssertions = step.assertions.filter((a) => !a.passed);
+    const message = failedAssertions.length > 0
+      ? escapeXml(`${failedAssertions[0]!.rule}: expected ${JSON.stringify(failedAssertions[0]!.expected)}, got ${JSON.stringify(failedAssertions[0]!.actual)}`)
+      : escapeXml(step.error ?? "Assertion failed");
+    const body = failedAssertions
+      .map((a) => escapeXml(`${a.rule}: expected ${JSON.stringify(a.expected)}, got ${JSON.stringify(a.actual)}`))
+      .join("\n");
+    return `    <testcase name="${name}" time="${time}">\n      <failure message="${message}">${body}</failure>\n    </testcase>`;
+  }
+
+  // error
+  const message = escapeXml(step.error ?? "Unknown error");
+  return `    <testcase name="${name}" time="${time}">\n      <error message="${message}">${message}</error>\n    </testcase>`;
+}
+
+function renderTestsuite(result: TestRunResult): string {
+  const name = escapeXml(result.suite_name);
+  const failures = result.failed;
+  const tests = result.total;
+  const errors = result.steps.filter((s) => s.status === "error").length;
+  const skipped = result.skipped;
+  const time = formatTime(result.steps.reduce((sum, s) => sum + s.duration_ms, 0));
+
+  const testcases = result.steps.map(renderTestcase).join("\n");
+
+  return `  <testsuite name="${name}" tests="${tests}" failures="${failures}" errors="${errors}" skipped="${skipped}" time="${time}">\n${testcases}\n  </testsuite>`;
+}
+
+export function generateJunitXml(results: TestRunResult[]): string {
+  const totalTests = results.reduce((s, r) => s + r.total, 0);
+  const totalFailures = results.reduce((s, r) => s + r.failed, 0);
+  const totalErrors = results.reduce((s, r) => s + r.steps.filter((s) => s.status === "error").length, 0);
+  const totalTime = formatTime(results.reduce((s, r) => s + r.steps.reduce((ss, step) => ss + step.duration_ms, 0), 0));
+
+  const suites = results.map(renderTestsuite).join("\n");
+
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<testsuites tests="${totalTests}" failures="${totalFailures}" errors="${totalErrors}" time="${totalTime}">`,
+    suites,
+    `</testsuites>`,
+  ].join("\n");
+}
+
+export const junitReporter: Reporter = {
+  report(results: TestRunResult[], _options?: ReporterOptions): void {
+    console.log(generateJunitXml(results));
+  },
+};

package/src/core/reporter/types.ts

@@ -0,0 +1,12 @@
+import type { TestRunResult } from "../runner/types.ts";
+
+export interface ReporterOptions {
+  /** Whether to use ANSI colors. Default: auto-detect via isTTY. */
+  color?: boolean;
+}
+
+export type ReporterName = "console" | "json" | "junit";
+
+export interface Reporter {
+  report(results: TestRunResult[], options?: ReporterOptions): void;
+}

package/src/core/runner/assertions.ts

@@ -0,0 +1,172 @@
+import type { TestStepExpect, AssertionRule } from "../parser/types.ts";
+import type { HttpResponse, AssertionResult } from "./types.ts";
+import { getByPath } from "../utils.ts";
+
+function checkType(value: unknown, expectedType: string): boolean {
+  switch (expectedType) {
+    case "string": return typeof value === "string";
+    case "integer": return typeof value === "number" && Number.isInteger(value);
+    case "number": return typeof value === "number";
+    case "boolean": return typeof value === "boolean";
+    case "array": return Array.isArray(value);
+    case "object": return typeof value === "object" && value !== null && !Array.isArray(value);
+    default: return false;
+  }
+}
+
+function describeType(value: unknown): string {
+  if (value === null) return "null";
+  if (value === undefined) return "undefined";
+  if (Array.isArray(value)) return "array";
+  if (typeof value === "number" && Number.isInteger(value)) return "integer";
+  return typeof value;
+}
+
+function deepEquals(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  // Loose numeric comparison: "123" == 123
+  if (typeof a === "number" && typeof b === "string") return a === Number(b);
+  if (typeof a === "string" && typeof b === "number") return Number(a) === b;
+  if (typeof a !== typeof b) return false;
+  if (typeof a !== "object" || a === null || b === null) return false;
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+function checkRule(path: string, rule: AssertionRule, actual: unknown): AssertionResult[] {
+  const results: AssertionResult[] = [];
+  const field = `body.${path}`;
+
+  if (rule.exists !== undefined) {
+    const doesExist = actual !== undefined && actual !== null;
+    results.push({
+      field, rule: `exists ${rule.exists}`,
+      passed: doesExist === rule.exists, actual: doesExist, expected: rule.exists,
+    });
+  }
+
+  if (rule.type !== undefined) {
+    results.push({
+      field, rule: `type ${rule.type}`,
+      passed: checkType(actual, rule.type), actual: describeType(actual), expected: rule.type,
+    });
+  }
+
+  if (rule.equals !== undefined) {
+    results.push({
+      field, rule: `equals ${JSON.stringify(rule.equals)}`,
+      passed: deepEquals(actual, rule.equals), actual, expected: rule.equals,
+    });
+  }
+
+  if (rule.contains !== undefined) {
+    const passed = typeof actual === "string" && actual.includes(rule.contains);
+    results.push({
+      field, rule: `contains "${rule.contains}"`,
+      passed, actual, expected: rule.contains,
+    });
+  }
+
+  if (rule.matches !== undefined) {
+    const passed = typeof actual === "string" && new RegExp(rule.matches).test(actual);
+    results.push({
+      field, rule: `matches ${rule.matches}`,
+      passed, actual, expected: rule.matches,
+    });
+  }
+
+  if (rule.gt !== undefined) {
+    const passed = typeof actual === "number" && actual > rule.gt;
+    results.push({
+      field, rule: `gt ${rule.gt}`,
+      passed, actual, expected: rule.gt,
+    });
+  }
+
+  if (rule.lt !== undefined) {
+    const passed = typeof actual === "number" && actual < rule.lt;
+    results.push({
+      field, rule: `lt ${rule.lt}`,
+      passed, actual, expected: rule.lt,
+    });
+  }
+
+  return results;
+}
+
+export function checkAssertions(expect: TestStepExpect, response: HttpResponse): AssertionResult[] {
+  const results: AssertionResult[] = [];
+
+  if (expect.status !== undefined) {
+    results.push({
+      field: "status",
+      rule: `equals ${expect.status}`,
+      passed: response.status === expect.status,
+      actual: response.status,
+      expected: expect.status,
+    });
+  }
+
+  if (expect.duration !== undefined) {
+    results.push({
+      field: "duration",
+      rule: `lte ${expect.duration}ms`,
+      passed: response.duration_ms <= expect.duration,
+      actual: response.duration_ms,
+      expected: expect.duration,
+    });
+  }
+
+  if (expect.headers) {
+    for (const [key, expectedValue] of Object.entries(expect.headers)) {
+      const actual = response.headers[key.toLowerCase()];
+      results.push({
+        field: `headers.${key}`,
+        rule: `equals "${expectedValue}"`,
+        passed: actual === expectedValue,
+        actual,
+        expected: expectedValue,
+      });
+    }
+  }
+
+  if (expect.body && response.body_parsed !== undefined) {
+    for (const [path, rule] of Object.entries(expect.body)) {
+      let actual: unknown;
+      if (path === "_body") {
+        actual = response.body_parsed;
+      } else if (path.startsWith("_body.")) {
+        actual = getByPath(response.body_parsed, path.slice(6));
+      } else {
+        actual = getByPath(response.body_parsed, path);
+      }
+      results.push(...checkRule(path, rule, actual));
+    }
+  }
+
+  return results;
+}
+
+export function extractCaptures(
+  bodyRules: Record<string, AssertionRule> | undefined,
+  responseBody: unknown,
+): Record<string, unknown> {
+  const captures: Record<string, unknown> = {};
+  if (!bodyRules || responseBody === undefined) return captures;
+
+  for (const [path, rule] of Object.entries(bodyRules)) {
+    if (rule.capture) {
+      let value: unknown;
+      if (path === "_body") {
+        value = responseBody;
+      } else if (path.startsWith("_body.")) {
+        value = getByPath(responseBody, path.slice(6));
+      } else {
+        value = getByPath(responseBody, path);
+      }
+      if (value !== undefined) {
+        captures[rule.capture] = value;
+      }
+    }
+  }
+  return captures;
+}

package/src/core/runner/execute-run.ts

@@ -0,0 +1,75 @@
+import { parse } from "../parser/yaml-parser.ts";
+import { loadEnvironment } from "../parser/variables.ts";
+import { filterSuitesByTags } from "../parser/filter.ts";
+import { runSuite } from "./executor.ts";
+import { getDb } from "../../db/schema.ts";
+import { createRun, finalizeRun, saveResults, findCollectionByTestPath } from "../../db/queries.ts";
+import { dirname, resolve } from "path";
+import { stat } from "node:fs/promises";
+import type { TestRunResult } from "./types.ts";
+
+export interface ExecuteRunOptions {
+  testPath: string;
+  envName?: string;
+  trigger?: string;  // "cli" | "webui" | "mcp"
+  dbPath?: string;
+  safe?: boolean;
+  tag?: string[];
+}
+
+export interface ExecuteRunResult {
+  runId: number;
+  results: TestRunResult[];
+}
+
+export async function executeRun(options: ExecuteRunOptions): Promise<ExecuteRunResult> {
+  const { testPath, envName, trigger = "cli", dbPath, safe, tag } = options;
+
+  let suites = await parse(testPath);
+  if (suites.length === 0) {
+    throw new Error("No test files found");
+  }
+
+  // Tag filter
+  if (tag && tag.length > 0) {
+    suites = filterSuitesByTags(suites, tag);
+    if (suites.length === 0) {
+      throw new Error("No suites match the specified tags");
+    }
+  }
+
+  // Safe mode: filter to GET-only tests
+  if (safe) {
+    for (const suite of suites) {
+      suite.tests = suite.tests.filter(t => t.method === "GET");
+    }
+    suites = suites.filter(s => s.tests.length > 0);
+    if (suites.length === 0) {
+      throw new Error("No GET tests found. Nothing to run in safe mode.");
+    }
+  }
+
+  const fileStat = await stat(testPath).catch(() => null);
+  const envDir = fileStat?.isDirectory() ? testPath : dirname(testPath);
+
+  getDb(dbPath);
+  const resolvedPath = resolve(testPath);
+  const collection = findCollectionByTestPath(resolvedPath)
+    ?? (fileStat?.isFile() ? findCollectionByTestPath(resolve(dirname(testPath))) : null);
+
+  // If no envName given but a collection exists, fall back to "default" for DB lookup
+  const effectiveEnvName = envName ?? (collection ? "default" : undefined);
+  const env = await loadEnvironment(effectiveEnvName, envDir, collection?.id);
+  const results = await Promise.all(suites.map((s) => runSuite(s, env)));
+
+  const runId = createRun({
+    started_at: results[0]?.started_at ?? new Date().toISOString(),
+    environment: effectiveEnvName,
+    trigger,
+    collection_id: collection?.id,
+  });
+  finalizeRun(runId, results);
+  saveResults(runId, results);
+
+  return { runId, results };
+}

package/src/core/runner/executor.ts

@@ -0,0 +1,150 @@
+import type { TestSuite, Environment } from "../parser/types.ts";
+import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
+import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
+import { executeRequest, type FetchOptions } from "./http-client.ts";
+import { checkAssertions, extractCaptures } from "./assertions.ts";
+
+function buildUrl(baseUrl: string | undefined, path: string, query?: Record<string, string>): string {
+  let url = baseUrl ? `${baseUrl.replace(/\/+$/, "")}${path}` : path;
+  if (query && Object.keys(query).length > 0) {
+    const params = new URLSearchParams(query);
+    url += `?${params.toString()}`;
+  }
+  return url;
+}
+
+function makeSkippedResult(stepName: string, reason: string): StepResult {
+  return {
+    name: stepName,
+    status: "skip",
+    duration_ms: 0,
+    request: { method: "", url: "", headers: {} },
+    assertions: [],
+    captures: {},
+    error: reason,
+  };
+}
+
+export async function runSuite(suite: TestSuite, env: Environment = {}): Promise<TestRunResult> {
+  const startedAt = new Date().toISOString();
+  const steps: StepResult[] = [];
+  const variables: Record<string, unknown> = { ...env };
+  const failedCaptures = new Set<string>();
+
+  const fetchOptions: Partial<FetchOptions> = {
+    timeout: suite.config.timeout,
+    retries: suite.config.retries,
+    retry_delay: suite.config.retry_delay,
+    follow_redirects: suite.config.follow_redirects,
+  };
+
+  for (const step of suite.tests) {
+    // Skip check: if step references a failed capture variable, skip it
+    const referencedVars = extractVariableReferences(step);
+    const missingCapture = referencedVars.find((v) => failedCaptures.has(v));
+    if (missingCapture) {
+      steps.push(makeSkippedResult(step.name, `Depends on missing capture: ${missingCapture}`));
+      continue;
+    }
+
+    // Substitute variables
+    const resolved = substituteStep(step, variables);
+
+    // Build request — substitute base_url and suite headers with current variables
+    const resolvedBaseUrl = suite.base_url ? substituteString(suite.base_url, variables) as string : undefined;
+    const resolvedSuiteHeaders = suite.headers ? substituteDeep(suite.headers, variables) : undefined;
+    const url = buildUrl(resolvedBaseUrl, resolved.path, resolved.query);
+    const headers: Record<string, string> = { ...resolvedSuiteHeaders, ...resolved.headers };
+    let body: string | undefined;
+
+    if (resolved.json !== undefined) {
+      body = JSON.stringify(resolved.json);
+      if (!headers["Content-Type"] && !headers["content-type"]) {
+        headers["Content-Type"] = "application/json";
+      }
+    } else if (resolved.form) {
+      body = new URLSearchParams(resolved.form).toString();
+      if (!headers["Content-Type"] && !headers["content-type"]) {
+        headers["Content-Type"] = "application/x-www-form-urlencoded";
+      }
+    }
+
+    const request: HttpRequest = { method: resolved.method, url, headers, body };
+
+    try {
+      const response = await executeRequest(request, fetchOptions);
+
+      // Extract captures
+      const captures = extractCaptures(resolved.expect.body, response.body_parsed);
+      Object.assign(variables, captures);
+
+      // Track expected captures that weren't obtained
+      if (resolved.expect.body) {
+        for (const rule of Object.values(resolved.expect.body)) {
+          if (rule.capture && !(rule.capture in captures)) {
+            failedCaptures.add(rule.capture);
+          }
+        }
+      }
+
+      // Run assertions
+      const assertions = checkAssertions(resolved.expect, response);
+      const allPassed = assertions.every((a) => a.passed);
+
+      steps.push({
+        name: step.name,
+        status: allPassed ? "pass" : "fail",
+        duration_ms: response.duration_ms,
+        request,
+        response,
+        assertions,
+        captures,
+      });
+
+      // If step failed, mark its captures as unreliable
+      if (!allPassed && resolved.expect.body) {
+        for (const rule of Object.values(resolved.expect.body)) {
+          if (rule.capture) {
+            failedCaptures.add(rule.capture);
+          }
+        }
+      }
+    } catch (err) {
+      const errorMsg = err instanceof Error ? err.message : String(err);
+      steps.push({
+        name: step.name,
+        status: "error",
+        duration_ms: 0,
+        request,
+        assertions: [],
+        captures: {},
+        error: errorMsg,
+      });
+
+      // Mark any captures from this step as failed
+      if (step.expect.body) {
+        for (const rule of Object.values(step.expect.body)) {
+          if (rule.capture) failedCaptures.add(rule.capture);
+        }
+      }
+    }
+  }
+
+  const finishedAt = new Date().toISOString();
+  return {
+    suite_name: suite.name,
+    suite_tags: suite.tags,
+    suite_description: suite.description,
+    started_at: startedAt,
+    finished_at: finishedAt,
+    total: steps.length,
+    passed: steps.filter((s) => s.status === "pass").length,
+    failed: steps.filter((s) => s.status === "fail").length,
+    skipped: steps.filter((s) => s.status === "skip").length,
+    steps,
+  };
+}
+
+export async function runSuites(suites: TestSuite[], env: Environment = {}): Promise<TestRunResult[]> {
+  return Promise.all(suites.map((suite) => runSuite(suite, env)));
+}

package/src/core/runner/http-client.ts

@@ -0,0 +1,69 @@
+import type { HttpRequest, HttpResponse } from "./types.ts";
+
+export interface FetchOptions {
+  timeout: number;
+  retries: number;
+  retry_delay: number;
+  follow_redirects: boolean;
+}
+
+export const DEFAULT_FETCH_OPTIONS: FetchOptions = {
+  timeout: 30000,
+  retries: 0,
+  retry_delay: 1000,
+  follow_redirects: true,
+};
+
+export async function executeRequest(
+  request: HttpRequest,
+  options?: Partial<FetchOptions>,
+): Promise<HttpResponse> {
+  const opts = { ...DEFAULT_FETCH_OPTIONS, ...options };
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt <= opts.retries; attempt++) {
+    if (attempt > 0) {
+      await Bun.sleep(opts.retry_delay);
+    }
+
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), opts.timeout);
+      const start = performance.now();
+
+      const response = await fetch(request.url, {
+        method: request.method,
+        headers: request.headers,
+        body: request.body ?? undefined,
+        signal: controller.signal,
+        redirect: opts.follow_redirects ? "follow" : "manual",
+        tls: { rejectUnauthorized: false },
+      });
+
+      clearTimeout(timeoutId);
+      const duration_ms = Math.round(performance.now() - start);
+
+      const bodyText = await response.text();
+      let body_parsed: unknown = undefined;
+      const contentType = response.headers.get("content-type") ?? "";
+      if (contentType.includes("application/json")) {
+        try {
+          body_parsed = JSON.parse(bodyText);
+        } catch {
+          // Body is not valid JSON despite content-type
+        }
+      }
+
+      const headers: Record<string, string> = {};
+      response.headers.forEach((v, k) => {
+        headers[k] = v;
+      });
+
+      return { status: response.status, headers, body: bodyText, body_parsed, duration_ms };
+    } catch (err) {
+      lastError = err instanceof Error ? err : new Error(String(err));
+    }
+  }
+
+  throw lastError!;
+}

package/src/core/runner/index.ts

@@ -0,0 +1,12 @@
+export type {
+  StepStatus,
+  HttpRequest,
+  HttpResponse,
+  AssertionResult,
+  StepResult,
+  TestRunResult,
+} from "./types.ts";
+
+export { executeRequest, type FetchOptions, DEFAULT_FETCH_OPTIONS } from "./http-client.ts";
+export { checkAssertions, extractCaptures } from "./assertions.ts";
+export { runSuite, runSuites } from "./executor.ts";

package/src/core/runner/types.ts

@@ -0,0 +1,48 @@
+export type StepStatus = "pass" | "fail" | "skip" | "error";
+
+export interface HttpRequest {
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+  body?: string;
+}
+
+export interface HttpResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+  body_parsed?: unknown;
+  duration_ms: number;
+}
+
+export interface AssertionResult {
+  field: string;
+  rule: string;
+  passed: boolean;
+  actual: unknown;
+  expected: unknown;
+}
+
+export interface StepResult {
+  name: string;
+  status: StepStatus;
+  duration_ms: number;
+  request: HttpRequest;
+  response?: HttpResponse;
+  assertions: AssertionResult[];
+  captures: Record<string, unknown>;
+  error?: string;
+}
+
+export interface TestRunResult {
+  suite_name: string;
+  suite_tags?: string[];
+  suite_description?: string;
+  started_at: string;
+  finished_at: string;
+  total: number;
+  passed: number;
+  failed: number;
+  skipped: number;
+  steps: StepResult[];
+}