@kirrosh/zond 0.9.4 → 0.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kirrosh/zond",
-  "version": "0.9.4",
+  "version": "0.11.0",
   "description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
   "license": "MIT",
   "module": "index.ts",

package/src/core/generator/index.ts

@@ -12,3 +12,4 @@ export { compressEndpointsWithSchemas, buildGenerationGuide } from "./guide-buil
 export type { GuideOptions } from "./guide-builder.ts";
 export type { EndpointWarning, WarningCode } from "./endpoint-warnings.ts";
 export type { EndpointInfo, ResponseInfo, GenerateOptions, SecuritySchemeInfo, CrudGroup } from "./types.ts";
+export { generateSuites, generateStep, detectCrudGroups, generateCrudSuite } from "./suite-generator.ts";

package/src/core/generator/serializer.ts

@@ -34,6 +34,7 @@ export interface RawStep {
 
 export interface RawSuite {
   name: string;
+  tags?: string[];
   folder?: string;
   fileStem?: string;
   base_url?: string;
@@ -48,6 +49,9 @@ export interface RawSuite {
 export function serializeSuite(suite: RawSuite): string {
   const lines: string[] = [];
   lines.push(`name: ${yamlScalar(suite.name)}`);
+  if (suite.tags && suite.tags.length > 0) {
+    lines.push(`tags: [${suite.tags.join(", ")}]`);
+  }
   if (suite.base_url) {
     lines.push(`base_url: ${yamlScalar(suite.base_url)}`);
   }

package/src/core/generator/suite-generator.ts

@@ -0,0 +1,388 @@
+import type { OpenAPIV3 } from "openapi-types";
+import type { EndpointInfo, SecuritySchemeInfo, CrudGroup } from "./types.ts";
+import type { RawSuite, RawStep } from "./serializer.ts";
+import { generateFromSchema } from "./data-factory.ts";
+import { groupEndpointsByTag } from "./chunker.ts";
+
+// ──────────────────────────────────────────────
+// Helpers
+// ──────────────────────────────────────────────
+
+/** Convert OpenAPI path params {param} to test interpolation {{param}} */
+function convertPath(path: string): string {
+  return path.replace(/\{([^}]+)\}/g, "{{$1}}");
+}
+
+function slugify(s: string): string {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}
+
+function escapeRegex(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function getExpectedStatus(ep: EndpointInfo): number {
+  const success = ep.responses.find(r => r.statusCode >= 200 && r.statusCode < 300);
+  if (success) return success.statusCode;
+  if (ep.responses.length > 0) return ep.responses[0].statusCode;
+  return 200;
+}
+
+function getSuccessSchema(ep: EndpointInfo): OpenAPIV3.SchemaObject | undefined {
+  return ep.responses.find(r => r.statusCode >= 200 && r.statusCode < 300)?.schema;
+}
+
+function getBodyAssertions(ep: EndpointInfo): Record<string, Record<string, string>> | undefined {
+  const schema = getSuccessSchema(ep);
+  if (!schema) return undefined;
+
+  if (schema.type === "array") {
+    return { _body: { type: "array" } };
+  }
+
+  if (schema.properties) {
+    const assertions: Record<string, Record<string, string>> = {};
+    const props = Object.keys(schema.properties).slice(0, 5);
+    for (const prop of props) {
+      assertions[prop] = { exists: "true" };
+    }
+    return Object.keys(assertions).length > 0 ? assertions : undefined;
+  }
+
+  return undefined;
+}
+
+function getAuthHeaders(
+  ep: EndpointInfo,
+  schemes: SecuritySchemeInfo[],
+): Record<string, string> | undefined {
+  if (ep.security.length === 0) return undefined;
+
+  for (const secName of ep.security) {
+    const scheme = schemes.find(s => s.name === secName);
+    if (!scheme) continue;
+
+    if (scheme.type === "http" && scheme.scheme === "bearer") {
+      return { Authorization: "Bearer {{auth_token}}" };
+    }
+    if (scheme.type === "apiKey" && scheme.in === "header" && scheme.apiKeyName) {
+      return { [scheme.apiKeyName]: "{{api_key}}" };
+    }
+  }
+
+  return undefined;
+}
+
+function getRequiredQueryParams(ep: EndpointInfo): Record<string, unknown> | undefined {
+  const queryParams = ep.parameters.filter(p => p.in === "query" && p.required);
+  if (queryParams.length === 0) return undefined;
+
+  const query: Record<string, unknown> = {};
+  for (const p of queryParams) {
+    if (p.schema) {
+      query[p.name] = generateFromSchema(p.schema as OpenAPIV3.SchemaObject, p.name);
+    } else {
+      query[p.name] = "{{$randomString}}";
+    }
+  }
+  return query;
+}
+
+/** Check if all endpoints share the same auth headers → suite-level */
+function getSuiteHeaders(
+  endpoints: EndpointInfo[],
+  schemes: SecuritySchemeInfo[],
+): Record<string, string> | undefined {
+  if (endpoints.length === 0) return undefined;
+
+  const headerSets = endpoints.map(ep => getAuthHeaders(ep, schemes));
+  const first = headerSets[0];
+  if (!first) return undefined;
+
+  const firstJson = JSON.stringify(first);
+  const allSame = headerSets.every(h => JSON.stringify(h) === firstJson);
+  return allSame ? first : undefined;
+}
+
+/** Find the best field to capture from POST response (for CRUD chains) */
+function getCaptureField(ep: EndpointInfo): string {
+  const schema = getSuccessSchema(ep);
+  if (schema?.properties) {
+    if ("id" in schema.properties) return "id";
+    for (const [name, propSchema] of Object.entries(schema.properties)) {
+      const s = propSchema as OpenAPIV3.SchemaObject;
+      if (s.type === "integer" || s.format === "uuid") return name;
+    }
+  }
+  return "id";
+}
+
+// ──────────────────────────────────────────────
+// Public API
+// ──────────────────────────────────────────────
+
+/** Generate a single test step from an EndpointInfo */
+export function generateStep(
+  ep: EndpointInfo,
+  securitySchemes: SecuritySchemeInfo[],
+): RawStep {
+  const method = ep.method.toUpperCase();
+  const name = ep.operationId ?? ep.summary ?? `${method} ${ep.path}`;
+  const path = convertPath(ep.path);
+
+  const step: RawStep = {
+    name,
+    [method]: path,
+    expect: {
+      status: getExpectedStatus(ep),
+    },
+  };
+
+  const authHeaders = getAuthHeaders(ep, securitySchemes);
+  if (authHeaders) {
+    step.headers = authHeaders;
+  }
+
+  if (["POST", "PUT", "PATCH"].includes(method) && ep.requestBodySchema) {
+    step.json = generateFromSchema(ep.requestBodySchema);
+  }
+
+  const query = getRequiredQueryParams(ep);
+  if (query) {
+    step.query = query;
+  }
+
+  const body = getBodyAssertions(ep);
+  if (body) {
+    step.expect.body = body;
+  }
+
+  return step;
+}
+
+/** Detect CRUD groups from a list of endpoints */
+export function detectCrudGroups(endpoints: EndpointInfo[]): CrudGroup[] {
+  const groups: CrudGroup[] = [];
+  const postEndpoints = endpoints.filter(ep => ep.method.toUpperCase() === "POST" && !ep.deprecated);
+
+  for (const createEp of postEndpoints) {
+    const basePath = createEp.path;
+
+    // Find item endpoints: basePath/{param}
+    const itemPattern = new RegExp(`^${escapeRegex(basePath)}/\\{([^}]+)\\}$`);
+    const itemEndpoints = endpoints.filter(ep => !ep.deprecated && itemPattern.test(ep.path));
+
+    if (itemEndpoints.length === 0) continue;
+
+    const itemPath = itemEndpoints[0].path;
+    const idMatch = itemPath.match(/\{([^}]+)\}$/);
+    if (!idMatch) continue;
+    const idParam = idMatch[1];
+
+    const read = itemEndpoints.find(ep => ep.method.toUpperCase() === "GET");
+    if (!read) continue; // Minimum: POST + GET/{id}
+
+    const update = itemEndpoints.find(ep => ["PUT", "PATCH"].includes(ep.method.toUpperCase()));
+    const del = itemEndpoints.find(ep => ep.method.toUpperCase() === "DELETE");
+    const list = endpoints.find(ep => ep.method.toUpperCase() === "GET" && ep.path === basePath && !ep.deprecated);
+
+    const resource = basePath.split("/").filter(Boolean).pop() ?? "resource";
+
+    groups.push({
+      resource,
+      basePath,
+      itemPath,
+      idParam,
+      create: createEp,
+      list,
+      read,
+      update,
+      delete: del,
+    });
+  }
+
+  return groups;
+}
+
+/** Generate a CRUD chain suite from a CrudGroup */
+export function generateCrudSuite(
+  group: CrudGroup,
+  securitySchemes: SecuritySchemeInfo[],
+): RawSuite {
+  const captureField = group.create ? getCaptureField(group.create) : "id";
+  const captureVar = `${group.resource.replace(/s$/, "")}_id`;
+  const tests: RawStep[] = [];
+
+  const allEps = [group.create, group.list, group.read, group.update, group.delete].filter(Boolean) as EndpointInfo[];
+  const suiteHeaders = getSuiteHeaders(allEps, securitySchemes);
+
+  // 1. Create
+  if (group.create) {
+    const step = generateStep(group.create, securitySchemes);
+    if (!step.expect.body) step.expect.body = {};
+    step.expect.body[captureField] = { capture: captureVar };
+    if (suiteHeaders) delete (step as any).headers;
+    tests.push(step);
+  }
+
+  // 2. Read created
+  if (group.read) {
+    const step: RawStep = {
+      name: group.read.operationId ?? `Read created ${group.resource.replace(/s$/, "")}`,
+      GET: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
+      expect: {
+        status: getExpectedStatus(group.read),
+        body: getBodyAssertions(group.read),
+      },
+    };
+    tests.push(step);
+  }
+
+  // 3. Update
+  if (group.update) {
+    const method = group.update.method.toUpperCase();
+    const step: RawStep = {
+      name: group.update.operationId ?? `Update ${group.resource.replace(/s$/, "")}`,
+      [method]: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
+      expect: {
+        status: getExpectedStatus(group.update),
+      },
+    };
+    if (group.update.requestBodySchema) {
+      step.json = generateFromSchema(group.update.requestBodySchema);
+    }
+    tests.push(step);
+  }
+
+  // 4. Delete
+  if (group.delete) {
+    const step: RawStep = {
+      name: group.delete.operationId ?? `Delete ${group.resource.replace(/s$/, "")}`,
+      DELETE: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
+      expect: {
+        status: getExpectedStatus(group.delete),
+      },
+    };
+    tests.push(step);
+
+    // 5. Verify deleted
+    if (group.read) {
+      tests.push({
+        name: `Verify ${group.resource.replace(/s$/, "")} deleted`,
+        GET: convertPath(group.itemPath).replace(`{{${group.idParam}}}`, `{{${captureVar}}}`),
+        expect: {
+          status: 404,
+        },
+      });
+    }
+  }
+
+  const suite: RawSuite = {
+    name: `${group.resource}-crud`,
+    tags: ["crud"],
+    fileStem: `crud-${slugify(group.resource)}`,
+    base_url: "{{base_url}}",
+    tests,
+  };
+
+  if (suiteHeaders) {
+    suite.headers = suiteHeaders;
+  }
+
+  return suite;
+}
+
+/** Main entry point: generate all suites from endpoints */
+export function generateSuites(opts: {
+  endpoints: EndpointInfo[];
+  securitySchemes: SecuritySchemeInfo[];
+}): RawSuite[] {
+  const { endpoints, securitySchemes } = opts;
+
+  // Filter deprecated
+  const active = endpoints.filter(ep => !ep.deprecated);
+
+  // 1. Detect CRUD groups
+  const crudGroups = detectCrudGroups(active);
+
+  // Collect endpoints consumed by CRUD groups
+  const crudEndpointKeys = new Set<string>();
+  for (const g of crudGroups) {
+    if (g.create) crudEndpointKeys.add(`${g.create.method.toUpperCase()} ${g.create.path}`);
+    if (g.list) crudEndpointKeys.add(`${g.list.method.toUpperCase()} ${g.list.path}`);
+    if (g.read) crudEndpointKeys.add(`${g.read.method.toUpperCase()} ${g.read.path}`);
+    if (g.update) crudEndpointKeys.add(`${g.update.method.toUpperCase()} ${g.update.path}`);
+    if (g.delete) crudEndpointKeys.add(`${g.delete.method.toUpperCase()} ${g.delete.path}`);
+  }
+
+  // Remaining endpoints (not in any CRUD group)
+  const remaining = active.filter(ep => !crudEndpointKeys.has(`${ep.method.toUpperCase()} ${ep.path}`));
+
+  const suites: RawSuite[] = [];
+
+  // 2. Group remaining by tag → smoke + smoke-unsafe
+  const byTag = groupEndpointsByTag(remaining);
+
+  for (const [tag, tagEndpoints] of byTag) {
+    const tagSlug = slugify(tag) || "api";
+
+    // GET endpoints → smoke suite
+    const getEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() === "GET");
+    if (getEndpoints.length > 0) {
+      const tests = getEndpoints.map(ep => generateStep(ep, securitySchemes));
+      const headers = getSuiteHeaders(getEndpoints, securitySchemes);
+
+      const suite: RawSuite = {
+        name: `${tagSlug}-smoke`,
+        tags: ["smoke"],
+        fileStem: `smoke-${tagSlug}`,
+        base_url: "{{base_url}}",
+        tests,
+      };
+
+      if (headers) {
+        suite.headers = headers;
+        for (const t of tests) {
+          if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
+            delete (t as any).headers;
+          }
+        }
+      }
+
+      suites.push(suite);
+    }
+
+    // Non-GET endpoints → smoke-unsafe suite
+    const unsafeEndpoints = tagEndpoints.filter(ep => ep.method.toUpperCase() !== "GET");
+    if (unsafeEndpoints.length > 0) {
+      const tests = unsafeEndpoints.map(ep => generateStep(ep, securitySchemes));
+      const headers = getSuiteHeaders(unsafeEndpoints, securitySchemes);
+
+      const suite: RawSuite = {
+        name: `${tagSlug}-smoke-unsafe`,
+        tags: ["smoke", "unsafe"],
+        fileStem: `smoke-${tagSlug}-unsafe`,
+        base_url: "{{base_url}}",
+        tests,
+      };
+
+      if (headers) {
+        suite.headers = headers;
+        for (const t of tests) {
+          if (t.headers && JSON.stringify(t.headers) === JSON.stringify(headers)) {
+            delete (t as any).headers;
+          }
+        }
+      }
+
+      suites.push(suite);
+    }
+  }
+
+  // 3. CRUD suites
+  for (const group of crudGroups) {
+    suites.push(generateCrudSuite(group, securitySchemes));
+  }
+
+  return suites;
+}

package/src/mcp/descriptions.ts

@@ -57,7 +57,9 @@ export const TOOL_DESCRIPTIONS = {
     "and return a focused test generation guide. For large APIs returns a chunking plan — " +
     "call again with tag parameter for each chunk. Use testsDir param to only generate for uncovered endpoints. " +
     "After generating YAML, use save_test_suites to save files, then run_tests to verify. " +
-    "Includes YAML format cheatsheet by default; pass includeFormat: false for subsequent tag chunks to save tokens.",
+    "Includes YAML format cheatsheet by default; pass includeFormat: false for subsequent tag chunks to save tokens. " +
+    "Use mode: 'generate' to auto-generate and save deterministic YAML test files (smoke + CRUD) without LLM. " +
+    "Default mode is 'generate'; use mode: 'guide' for the text-based generation guide.",
 
   ci_init:
     "Generate a CI/CD workflow file for running API tests automatically on push, PR, and schedule. " +

package/src/mcp/tools/generate-and-save.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { join } from "node:path";
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import {
   readOpenApiSpec,
@@ -6,10 +7,13 @@ import {
   extractSecuritySchemes,
   scanCoveredEndpoints,
   filterUncoveredEndpoints,
+  serializeSuite,
+  generateSuites,
 } from "../../core/generator/index.ts";
 import { compressEndpointsWithSchemas, buildGenerationGuide } from "../../core/generator/guide-builder.ts";
 import { planChunks, filterByTag } from "../../core/generator/chunker.ts";
 import { TOOL_DESCRIPTIONS } from "../descriptions.js";
+import { validateAndSave } from "./save-test-suite.ts";
 
 export function registerGenerateAndSaveTool(server: McpServer) {
   server.registerTool("generate_and_save", {
@@ -22,8 +26,11 @@ export function registerGenerateAndSaveTool(server: McpServer) {
       testsDir: z.optional(z.string()).describe("Path to existing tests directory — filters to uncovered endpoints only"),
       overwrite: z.optional(z.boolean()).describe("Hint for save_test_suites overwrite behavior (default: false)"),
       includeFormat: z.optional(z.boolean()).describe("Include YAML format reference (default: true, set false for subsequent tag chunks)"),
+      mode: z.optional(z.enum(["generate", "guide"])).describe(
+        "'generate' creates and saves YAML test files deterministically (default), 'guide' returns text for LLM-crafted tests"
+      ),
     },
-  }, async ({ specPath, outputDir, tag, methodFilter, testsDir, overwrite, includeFormat }) => {
+  }, async ({ specPath, outputDir, tag, methodFilter, testsDir, overwrite, includeFormat, mode }) => {
     try {
       const doc = await readOpenApiSpec(specPath);
       let endpoints = extractEndpoints(doc);
@@ -31,6 +38,7 @@ export function registerGenerateAndSaveTool(server: McpServer) {
       const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
       const title = (doc as any).info?.title as string | undefined;
       const effectiveOutputDir = outputDir ?? "./tests/";
+      const effectiveMode = mode ?? "generate";
 
       // Apply method filter
       if (methodFilter && methodFilter.length > 0) {
@@ -83,8 +91,11 @@ export function registerGenerateAndSaveTool(server: McpServer) {
           instruction:
             `This API has ${plan.totalEndpoints} endpoints across ${plan.chunks.length} tags. ` +
             `Call generate_and_save with tag parameter for each chunk sequentially. ` +
-            `Pass includeFormat: false for subsequent chunks to save tokens. ` +
-            `Example: generate_and_save(specPath: '${specPath}', tag: '${plan.chunks[0].tag}')`,
+            (effectiveMode === "guide"
+              ? `Pass includeFormat: false for subsequent chunks to save tokens. `
+              : "") +
+            `Example: generate_and_save(specPath: '${specPath}', tag: '${plan.chunks[0].tag}'` +
+            (effectiveMode === "guide" ? `, mode: 'guide'` : "") + `)`,
         };
         if (coverageInfo) {
           result.coverage = coverageInfo;
@@ -94,7 +105,49 @@ export function registerGenerateAndSaveTool(server: McpServer) {
         };
       }
 
-      // Guide mode: small API or specific tag
+      // ── Generate mode: deterministic YAML generation ──
+      if (effectiveMode === "generate") {
+        const suites = generateSuites({ endpoints, securitySchemes });
+
+        const files: Array<{
+          saved: boolean;
+          filePath: string;
+          tests: number;
+          error?: string;
+        }> = [];
+
+        for (const suite of suites) {
+          const yaml = serializeSuite(suite);
+          const fileName = (suite.fileStem ?? suite.name) + ".yaml";
+          const filePath = join(effectiveOutputDir, fileName);
+
+          const result = await validateAndSave(filePath, yaml, overwrite ?? false);
+          files.push({
+            saved: result.saved,
+            filePath: result.filePath ?? filePath,
+            tests: suite.tests.length,
+            ...(result.error ? { error: result.error } : {}),
+          });
+        }
+
+        const response: Record<string, unknown> = {
+          mode: "generate",
+          suitesGenerated: suites.length,
+          files,
+          hint: files.some(f => !f.saved)
+            ? "Some files were not saved (already exist?). Use overwrite: true to replace."
+            : "Files saved. Run run_tests to verify. Use mode: 'guide' for LLM-crafted tests with more detail.",
+        };
+        if (coverageInfo) {
+          response.coverage = coverageInfo;
+        }
+
+        return {
+          content: [{ type: "text" as const, text: JSON.stringify(response, null, 2) }],
+        };
+      }
+
+      // ── Guide mode: text-based generation guide ──
       const coverageHeader = coverageInfo
         ? `## Coverage: ${coverageInfo.covered}/${coverageInfo.total} endpoints covered (${coverageInfo.percentage}%). Generating tests for ${endpoints.length} uncovered endpoints:`
         : undefined;