@kirrosh/apitool 0.4.3 → 0.5.1

package/src/core/runner/executor.ts

@@ -25,7 +25,7 @@ function makeSkippedResult(stepName: string, reason: string): StepResult {
   };
 }
 
-export async function runSuite(suite: TestSuite, env: Environment = {}): Promise<TestRunResult> {
+export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun = false): Promise<TestRunResult> {
   const startedAt = new Date().toISOString();
   const steps: StepResult[] = [];
   const variables: Record<string, unknown> = { ...env };
@@ -71,6 +71,20 @@ export async function runSuite(suite: TestSuite, env: Environment = {}): Promise
 
     const request: HttpRequest = { method: resolved.method, url, headers, body };
 
+    if (dryRun) {
+      const bodyPreview = body ? ` ${body.slice(0, 200)}` : "";
+      steps.push({
+        name: step.name,
+        status: "pass",
+        duration_ms: 0,
+        request,
+        assertions: [],
+        captures: {},
+        error: `[DRY RUN] ${resolved.method} ${url}${bodyPreview}`,
+      });
+      continue;
+    }
+
     try {
       const response = await executeRequest(request, fetchOptions);
 
@@ -145,6 +159,6 @@ export async function runSuite(suite: TestSuite, env: Environment = {}): Promise
   };
 }
 
-export async function runSuites(suites: TestSuite[], env: Environment = {}): Promise<TestRunResult[]> {
-  return Promise.all(suites.map((suite) => runSuite(suite, env)));
+export async function runSuites(suites: TestSuite[], env: Environment = {}, dryRun = false): Promise<TestRunResult[]> {
+  return Promise.all(suites.map((suite) => runSuite(suite, env, dryRun)));
 }

package/src/db/schema.ts

@@ -38,6 +38,12 @@ export function closeDb(): void {
   }
 }
 
+export function resetDb(): void {
+  if (_db) { try { _db.close(); } catch {} }
+  _db = null;
+  _dbPath = null;
+}
+
 // ──────────────────────────────────────────────
 // Schema
 // ──────────────────────────────────────────────

package/src/mcp/server.ts

@@ -7,12 +7,14 @@ import { registerSendRequestTool } from "./tools/send-request.ts";
 import { registerExploreApiTool } from "./tools/explore-api.ts";
 import { registerManageEnvironmentTool } from "./tools/manage-environment.ts";
 import { registerCoverageAnalysisTool } from "./tools/coverage-analysis.ts";
-import { registerSaveTestSuiteTool } from "./tools/save-test-suite.ts";
+import { registerSaveTestSuiteTool, registerSaveTestSuitesTool } from "./tools/save-test-suite.ts";
 import { registerGenerateTestsGuideTool } from "./tools/generate-tests-guide.ts";
 import { registerSetupApiTool } from "./tools/setup-api.ts";
 import { registerGenerateMissingTestsTool } from "./tools/generate-missing-tests.ts";
 import { registerManageServerTool } from "./tools/manage-server.ts";
 import { registerCiInitTool } from "./tools/ci-init.ts";
+import { registerSetWorkDirTool } from "./tools/set-work-dir.ts";
+import { registerDescribeEndpointTool } from "./tools/describe-endpoint.ts";
 
 export interface McpServerOptions {
   dbPath?: string;
@@ -35,11 +37,14 @@ export async function startMcpServer(options: McpServerOptions = {}): Promise<vo
   registerManageEnvironmentTool(server, dbPath);
   registerCoverageAnalysisTool(server);
   registerSaveTestSuiteTool(server, dbPath);
+  registerSaveTestSuitesTool(server, dbPath);
   registerGenerateTestsGuideTool(server);
   registerSetupApiTool(server, dbPath);
   registerGenerateMissingTestsTool(server);
   registerManageServerTool(server, dbPath);
   registerCiInitTool(server);
+  registerSetWorkDirTool(server);
+  registerDescribeEndpointTool(server);
 
   // Connect via stdio transport
   const transport = new StdioServerTransport();

package/src/mcp/tools/coverage-analysis.ts

@@ -9,8 +9,9 @@ export function registerCoverageAnalysisTool(server: McpServer) {
     inputSchema: {
       specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML)"),
       testsDir: z.string().describe("Path to directory with test YAML files"),
+      failThreshold: z.optional(z.number().min(0).max(100)).describe("Return isError when coverage % is below this threshold (0–100)"),
     },
-  }, async ({ specPath, testsDir }) => {
+  }, async ({ specPath, testsDir, failThreshold }) => {
     try {
       const doc = await readOpenApiSpec(specPath);
       const allEndpoints = extractEndpoints(doc);
@@ -45,8 +46,10 @@ export function registerCoverageAnalysisTool(server: McpServer) {
         })),
       };
 
+      const belowThreshold = failThreshold !== undefined && percentage < failThreshold;
       return {
         content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
+        ...(belowThreshold ? { isError: true } : {}),
       };
     } catch (err) {
       return {

package/src/mcp/tools/describe-endpoint.ts

@@ -0,0 +1,159 @@
+import { z } from "zod";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { OpenAPIV3 } from "openapi-types";
+import { readOpenApiSpec } from "../../core/generator/index.ts";
+
+export function registerDescribeEndpointTool(server: McpServer) {
+  server.registerTool("describe_endpoint", {
+    description:
+      "Full details for one endpoint: params grouped by type, request body schema, " +
+      "all response schemas + response headers, security, deprecated flag. " +
+      "Use when a test fails and you need complete endpoint spec without reading the whole file.",
+    inputSchema: {
+      specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML) or HTTP URL"),
+      method: z.string().describe('HTTP method, e.g. "GET", "POST", "PUT"'),
+      path: z.string().describe('Endpoint path, e.g. "/pets/{petId}"'),
+    },
+  }, async ({ specPath, method, path: endpointPath }) => {
+    try {
+      const doc = await readOpenApiSpec(specPath) as OpenAPIV3.Document;
+
+      // Normalize inputs
+      const methodLower = method.toLowerCase() as OpenAPIV3.HttpMethods;
+      const normalizedPath = endpointPath.replace(/\/+$/, "") || "/";
+
+      // Find operation — try exact match first, then case-insensitive path match
+      let operation: OpenAPIV3.OperationObject | undefined;
+      let resolvedPath = normalizedPath;
+
+      const paths = doc.paths ?? {};
+
+      if (paths[normalizedPath]?.[methodLower]) {
+        operation = paths[normalizedPath][methodLower] as OpenAPIV3.OperationObject;
+      } else {
+        // Case-insensitive fallback
+        const lowerTarget = normalizedPath.toLowerCase();
+        for (const [p, pathItem] of Object.entries(paths)) {
+          if (p.toLowerCase() === lowerTarget && pathItem?.[methodLower]) {
+            operation = pathItem[methodLower] as OpenAPIV3.OperationObject;
+            resolvedPath = p;
+            break;
+          }
+        }
+      }
+
+      if (!operation) {
+        const available = Object.entries(paths).flatMap(([p, pathItem]) =>
+          Object.keys(pathItem ?? {})
+            .filter(k => ["get","post","put","patch","delete","head","options","trace"].includes(k))
+            .map(k => `${k.toUpperCase()} ${p}`)
+        ).sort();
+        return {
+          content: [{
+            type: "text" as const,
+            text: JSON.stringify({
+              error: `Endpoint ${method.toUpperCase()} ${endpointPath} not found in spec`,
+              availableEndpoints: available,
+            }, null, 2),
+          }],
+          isError: true,
+        };
+      }
+
+      const pathItem = paths[resolvedPath] ?? {};
+
+      // Merge path-level and operation-level parameters (operation overrides by name+in)
+      const pathLevelParams = (pathItem.parameters ?? []) as OpenAPIV3.ParameterObject[];
+      const opLevelParams = (operation.parameters ?? []) as OpenAPIV3.ParameterObject[];
+
+      const paramMap = new Map<string, OpenAPIV3.ParameterObject>();
+      for (const p of pathLevelParams) paramMap.set(`${p.in}:${p.name}`, p);
+      for (const p of opLevelParams) paramMap.set(`${p.in}:${p.name}`, p); // operation overrides
+
+      // Group by "in"
+      const grouped: Record<string, object[]> = { path: [], query: [], header: [], cookie: [] };
+      for (const p of paramMap.values()) {
+        const loc = p.in in grouped ? p.in : "query";
+        const schema = p.schema as OpenAPIV3.SchemaObject | undefined;
+        grouped[loc]!.push({
+          name: p.name,
+          required: p.required ?? false,
+          ...(schema?.type ? { type: schema.type } : {}),
+          ...(schema?.format ? { format: schema.format } : {}),
+          ...(schema?.enum ? { enum: schema.enum } : {}),
+          ...(schema?.default !== undefined ? { default: schema.default } : {}),
+          ...(p.description ? { description: p.description } : {}),
+        });
+      }
+
+      // Request body
+      let requestBody: object | undefined;
+      if (operation.requestBody) {
+        const rb = operation.requestBody as OpenAPIV3.RequestBodyObject;
+        const contentTypes = Object.keys(rb.content ?? {});
+        const preferredCt = contentTypes.find(ct => ct.includes("application/json")) ?? contentTypes[0];
+        const mediaObj = preferredCt ? rb.content[preferredCt] : undefined;
+        requestBody = {
+          required: rb.required ?? false,
+          ...(preferredCt ? { contentType: preferredCt } : {}),
+          ...(mediaObj?.schema ? { schema: mediaObj.schema } : {}),
+          ...(rb.description ? { description: rb.description } : {}),
+        };
+      }
+
+      // Responses
+      const responses: Record<string, object> = {};
+      for (const [statusCode, respObj] of Object.entries(operation.responses ?? {})) {
+        const resp = respObj as OpenAPIV3.ResponseObject;
+        const contentTypes = Object.keys(resp.content ?? {});
+        const preferredCt = contentTypes.find(ct => ct.includes("application/json")) ?? contentTypes[0];
+        const mediaObj = preferredCt ? resp.content?.[preferredCt] : undefined;
+
+        // Response headers
+        const headers: Record<string, object> = {};
+        for (const [hName, hObj] of Object.entries(resp.headers ?? {})) {
+          const h = hObj as OpenAPIV3.HeaderObject;
+          headers[hName] = {
+            ...(h.description ? { description: h.description } : {}),
+            ...(h.schema ? { schema: h.schema } : {}),
+          };
+        }
+
+        responses[statusCode] = {
+          description: resp.description,
+          headers,
+          ...(preferredCt ? { contentType: preferredCt } : {}),
+          ...(mediaObj?.schema ? { schema: mediaObj.schema } : {}),
+        };
+      }
+
+      // Security — merge doc-level and operation-level
+      const docSecurity = (doc.security ?? []) as OpenAPIV3.SecurityRequirementObject[];
+      const opSecurity = (operation.security ?? docSecurity) as OpenAPIV3.SecurityRequirementObject[];
+      const securityNames = [...new Set(opSecurity.flatMap(req => Object.keys(req)))];
+
+      const result = {
+        method: method.toUpperCase(),
+        path: resolvedPath,
+        ...(operation.operationId ? { operationId: operation.operationId } : {}),
+        ...(operation.summary ? { summary: operation.summary } : {}),
+        ...(operation.description ? { description: operation.description } : {}),
+        ...(operation.tags?.length ? { tags: operation.tags } : {}),
+        deprecated: operation.deprecated ?? false,
+        security: securityNames,
+        parameters: grouped,
+        ...(requestBody ? { requestBody } : {}),
+        responses,
+      };
+
+      return {
+        content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
+      };
+    } catch (err) {
+      return {
+        content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
+        isError: true,
+      };
+    }
+  });
+}

package/src/mcp/tools/generate-missing-tests.ts

@@ -18,15 +18,22 @@ export function registerGenerateMissingTestsTool(server: McpServer) {
       specPath: z.string().describe("Path or URL to OpenAPI spec file"),
       testsDir: z.string().describe("Path to directory with existing test YAML files"),
       outputDir: z.optional(z.string()).describe("Directory for saving new test files (default: same as testsDir)"),
+      methodFilter: z.optional(z.array(z.string())).describe("Only include endpoints with these HTTP methods (e.g. [\"GET\"] for smoke tests)"),
     },
-  }, async ({ specPath, testsDir, outputDir }) => {
+  }, async ({ specPath, testsDir, outputDir, methodFilter }) => {
     try {
       const doc = await readOpenApiSpec(specPath);
-      const allEndpoints = extractEndpoints(doc);
+      let allEndpoints = extractEndpoints(doc);
       const securitySchemes = extractSecuritySchemes(doc);
       const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
       const title = (doc as any).info?.title as string | undefined;
 
+      // Apply method filter before coverage check
+      if (methodFilter && methodFilter.length > 0) {
+        const methods = methodFilter.map(m => m.toUpperCase());
+        allEndpoints = allEndpoints.filter(ep => methods.includes(ep.method.toUpperCase()));
+      }
+
       if (allEndpoints.length === 0) {
         return {
           content: [{ type: "text" as const, text: JSON.stringify({ error: "No endpoints found in the spec" }, null, 2) }],

package/src/mcp/tools/generate-tests-guide.ts

@@ -14,11 +14,16 @@ export function registerGenerateTestsGuideTool(server: McpServer) {
     inputSchema: {
       specPath: z.string().describe("Path or URL to OpenAPI spec file"),
       outputDir: z.optional(z.string()).describe("Directory for saving test files (default: ./tests/)"),
+      methodFilter: z.optional(z.array(z.string())).describe("Only include endpoints with these HTTP methods (e.g. [\"GET\"] for smoke tests)"),
     },
-  }, async ({ specPath, outputDir }) => {
+  }, async ({ specPath, outputDir, methodFilter }) => {
     try {
       const doc = await readOpenApiSpec(specPath);
-      const endpoints = extractEndpoints(doc);
+      let endpoints = extractEndpoints(doc);
+      if (methodFilter && methodFilter.length > 0) {
+        const methods = methodFilter.map(m => m.toUpperCase());
+        endpoints = endpoints.filter(ep => methods.includes(ep.method.toUpperCase()));
+      }
       const securitySchemes = extractSecuritySchemes(doc);
       const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
       const title = (doc as any).info?.title as string | undefined;
@@ -302,6 +307,21 @@ ${hasAuth ? `**Auth suite** (\`auth.yaml\`):
 
 ---
 
+## Tag Conventions
+
+Use standard tags to enable safe filtering:
+
+| Tag | HTTP Methods | Safe for |
+|-----|-------------|---------|
+| \`smoke\` | GET only | Production (read-only, zero risk) |
+| \`crud\` | POST/PUT/PATCH | Staging only (state-changing) |
+| \`destructive\` | DELETE | Explicit opt-in, run last |
+| \`auth\` | Any (auth flows) | Run first to capture tokens |
+
+Example: \`apitool run --tag smoke --safe\` → reads-only, safe against production.
+
+---
+
 ## Practical Tips
 
 - **int64 IDs**: For APIs returning large auto-generated IDs (int64), prefer setting fixed IDs in request bodies rather than capturing auto-generated ones, as JSON number precision may cause mismatches.
@@ -312,6 +332,26 @@ ${hasAuth ? `**Auth suite** (\`auth.yaml\`):
 - **Error responses**: Assert that error bodies contain useful info (\`message: { exists: true }\`), not just status codes.
 - **Bulk operations**: After bulk create (createWithArray, createWithList), add GET steps to verify resources were actually created.
 - **204 No Content**: When an endpoint returns 204, omit \`body:\` assertions entirely — an empty response IS the correct behavior. Adding body assertions on 204 will always fail.
+- **Cleanup pattern**: Always delete test data in the same suite. Use a create → read → delete lifecycle so tests are idempotent:
+  \`\`\`yaml
+  tests:
+    - name: Create test resource
+      POST: /users
+      json: { name: "apitool-test-{{$randomString}}" }
+      expect:
+        status: 201
+        body:
+          id: { capture: user_id }
+    - name: Read created resource
+      GET: /users/{{user_id}}
+      expect:
+        status: 200
+    - name: Cleanup - delete test resource
+      DELETE: /users/{{user_id}}
+      expect:
+        status: 204
+  \`\`\`
+- **Identifiable test data**: Prefix test data with \`apitool-test-\` or use \`{{$uuid}}\` / \`apitool-test-{{$randomString}}\` so you can identify and clean up leftover test data if needed.
 
 ---
 

package/src/mcp/tools/query-db.ts

@@ -8,16 +8,19 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
     description:
       "Query the apitool database. Actions: list_collections (all APIs with run stats), " +
       "list_runs (recent test runs), get_run_results (full detail for a run), " +
-      "diagnose_failure (only failed/errored steps for a run).",
+      "diagnose_failure (only failed/errored steps for a run), " +
+      "compare_runs (regressions and fixes between two runs).",
     inputSchema: {
-      action: z.enum(["list_collections", "list_runs", "get_run_results", "diagnose_failure"])
+      action: z.enum(["list_collections", "list_runs", "get_run_results", "diagnose_failure", "compare_runs"])
         .describe("Query action to perform"),
       runId: z.optional(z.number().int())
         .describe("Run ID (required for get_run_results and diagnose_failure)"),
+      runIdB: z.optional(z.number().int())
+        .describe("Second run ID (required for compare_runs — this is the newer run)"),
       limit: z.optional(z.number().int().min(1).max(100))
         .describe("Max number of runs to return (default: 20, only for list_runs)"),
     },
-  }, async ({ action, runId, limit }) => {
+  }, async ({ action, runId, runIdB, limit }) => {
     try {
       getDb(dbPath);
 
@@ -130,6 +133,71 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
             content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
           };
         }
+
+        case "compare_runs": {
+          if (runId == null || runIdB == null) {
+            return {
+              content: [{ type: "text" as const, text: JSON.stringify({ error: "Both runId (run A) and runIdB (run B) are required for compare_runs" }, null, 2) }],
+              isError: true,
+            };
+          }
+          const runARecord = getRunById(runId);
+          const runBRecord = getRunById(runIdB);
+          if (!runARecord) {
+            return {
+              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run #${runId} not found` }, null, 2) }],
+              isError: true,
+            };
+          }
+          if (!runBRecord) {
+            return {
+              content: [{ type: "text" as const, text: JSON.stringify({ error: `Run #${runIdB} not found` }, null, 2) }],
+              isError: true,
+            };
+          }
+
+          const resultsA = getResultsByRunId(runId);
+          const resultsB = getResultsByRunId(runIdB);
+
+          const mapA = new Map<string, string>();
+          const mapB = new Map<string, string>();
+          for (const r of resultsA) mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
+          for (const r of resultsB) mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
+
+          const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
+          const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
+          let unchanged = 0;
+          let newTests = 0;
+          let removedTests = 0;
+
+          for (const [key, statusB] of mapB) {
+            const statusA = mapA.get(key);
+            if (statusA === undefined) { newTests++; continue; }
+            const [suite, test] = key.split("::") as [string, string];
+            const wasPass = statusA === "pass";
+            const isPass = statusB === "pass";
+            const wasFail = statusA === "fail" || statusA === "error";
+            const isFail = statusB === "fail" || statusB === "error";
+            if (wasPass && isFail) regressions.push({ suite, test, before: statusA, after: statusB });
+            else if (wasFail && isPass) fixes.push({ suite, test, before: statusA, after: statusB });
+            else unchanged++;
+          }
+          for (const key of mapA.keys()) {
+            if (!mapB.has(key)) removedTests++;
+          }
+
+          const compareResult = {
+            runA: { id: runId, started_at: runARecord.started_at },
+            runB: { id: runIdB, started_at: runBRecord.started_at },
+            summary: { regressions: regressions.length, fixes: fixes.length, unchanged, newTests, removedTests },
+            regressions,
+            fixes,
+            hasRegressions: regressions.length > 0,
+          };
+          return {
+            content: [{ type: "text" as const, text: JSON.stringify(compareResult, null, 2) }],
+          };
+        }
       }
     } catch (err) {
       return {

package/src/mcp/tools/run-tests.ts

@@ -11,8 +11,10 @@ export function registerRunTestsTool(server: McpServer, dbPath?: string) {
       envName: z.optional(z.string()).describe("Environment name (loads .env.<name>.yaml)"),
       safe: z.optional(z.boolean()).describe("Run only GET tests (read-only, safe mode)"),
       tag: z.optional(z.array(z.string())).describe("Filter suites by tag (OR logic)"),
+      envVars: z.optional(z.record(z.string(), z.string())).describe("Environment variables to inject (override env file, e.g. {\"TOKEN\": \"xxx\"})"),
+      dryRun: z.optional(z.boolean()).describe("Show requests without sending them (always exits 0)"),
     },
-  }, async ({ testPath, envName, safe, tag }) => {
+  }, async ({ testPath, envName, safe, tag, envVars, dryRun }) => {
     const { runId, results } = await executeRun({
       testPath,
       envName,
@@ -20,6 +22,8 @@ export function registerRunTestsTool(server: McpServer, dbPath?: string) {
       dbPath,
       safe,
       tag,
+      envVars,
+      dryRun,
     });
 
     const total = results.reduce((s, r) => s + r.total, 0);