@kirrosh/zond 0.7.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -3
- package/src/core/agent/system-prompt.ts +2 -3
- package/src/core/agent/tools/index.ts +1 -5
- package/src/core/diagnostics/failure-hints.ts +10 -0
- package/src/{mcp/tools/generate-tests-guide.ts → core/generator/guide-builder.ts} +25 -57
- package/src/core/generator/index.ts +2 -0
- package/src/core/generator/schema-utils.ts +30 -0
- package/src/mcp/descriptions.ts +1 -23
- package/src/mcp/server.ts +1 -9
- package/src/mcp/tools/describe-endpoint.ts +2 -1
- package/src/mcp/tools/generate-and-save.ts +2 -1
- package/src/mcp/tools/query-db.ts +3 -1
- package/src/mcp/tools/run-tests.ts +6 -0
- package/src/mcp/tools/save-test-suite.ts +1 -1
- package/src/web/views/suites-tab.ts +1 -1
- package/src/core/agent/tools/explore-api.ts +0 -40
- package/src/core/agent/tools/validate-tests.ts +0 -23
- package/src/mcp/tools/explore-api.ts +0 -84
- package/src/mcp/tools/generate-missing-tests.ts +0 -91
- package/src/mcp/tools/validate-tests.ts +0 -43
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kirrosh/zond",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "index.ts",
|
|
@@ -26,11 +26,13 @@
|
|
|
26
26
|
"scripts": {
|
|
27
27
|
"zond": "bun run src/cli/index.ts",
|
|
28
28
|
"test": "bun run test:unit && bun run test:mocked",
|
|
29
|
-
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/chat.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/doctor.test.ts tests/cli/init.test.ts tests/cli/runs.test.ts tests/cli/safe-run.test.ts tests/cli/update.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/agent/agent-loop.test.ts tests/agent/context-manager.test.ts tests/agent/system-prompt.test.ts tests/reporter/",
|
|
29
|
+
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/chat.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/doctor.test.ts tests/cli/init.test.ts tests/cli/runs.test.ts tests/cli/safe-run.test.ts tests/cli/update.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/agent/agent-loop.test.ts tests/agent/context-manager.test.ts tests/agent/system-prompt.test.ts tests/reporter/ tests/version-sync.test.ts",
|
|
30
30
|
"test:mocked": "bun run scripts/run-mocked-tests.ts",
|
|
31
31
|
"test:ai": "bun test tests/ai/",
|
|
32
32
|
"check": "tsc --noEmit --project tsconfig.json",
|
|
33
|
-
"build": "bun build --compile src/cli/index.ts --outfile zond"
|
|
33
|
+
"build": "bun build --compile src/cli/index.ts --outfile zond",
|
|
34
|
+
"version:sync": "bun run scripts/sync-version.ts",
|
|
35
|
+
"postversion": "bun run scripts/sync-version.ts && git add .claude-plugin/plugin.json .claude-plugin/marketplace.json"
|
|
34
36
|
},
|
|
35
37
|
"devDependencies": {
|
|
36
38
|
"@types/bun": "latest"
|
|
@@ -1,15 +1,14 @@
|
|
|
1
|
-
export const AGENT_SYSTEM_PROMPT = `You are an API testing assistant powered by zond. You help users run, create,
|
|
1
|
+
export const AGENT_SYSTEM_PROMPT = `You are an API testing assistant powered by zond. You help users run, create, and diagnose API tests.
|
|
2
2
|
|
|
3
3
|
You have access to the following tools:
|
|
4
4
|
|
|
5
5
|
- **run_tests**: Execute API test suites from YAML files or directories. Returns pass/fail summary with run ID.
|
|
6
|
-
- **validate_tests**: Validate YAML test files without executing them. Check syntax and structure.
|
|
7
6
|
- **query_results**: Query historical test run results and collections from the database.
|
|
8
7
|
- **diagnose_failure**: Analyze a failed test run to identify root causes and suggest fixes.
|
|
8
|
+
- **send_request**: Send an ad-hoc HTTP request for quick testing.
|
|
9
9
|
|
|
10
10
|
Tool usage examples:
|
|
11
11
|
- run_tests: { testPath: "tests/api.yaml" } or { testPath: "tests/", envName: "staging", safe: true }
|
|
12
|
-
- validate_tests: { testPath: "tests/api.yaml" }
|
|
13
12
|
- query_results: action must be "list_runs", "get_run" (requires runId), or "list_collections"
|
|
14
13
|
- List runs: { action: "list_runs", limit: 10 }
|
|
15
14
|
- Get run details: { action: "get_run", runId: 1 }
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { tool } from "ai";
|
|
2
2
|
import { runTestsTool } from "./run-tests.ts";
|
|
3
|
-
import { validateTestsTool } from "./validate-tests.ts";
|
|
4
3
|
import { queryResultsTool } from "./query-results.ts";
|
|
5
4
|
import { diagnoseFailureTool } from "./diagnose-failure.ts";
|
|
6
5
|
import { sendRequestTool } from "./send-request.ts";
|
|
7
|
-
import { exploreApiTool } from "./explore-api.ts";
|
|
8
6
|
import type { AgentConfig } from "../types.ts";
|
|
9
7
|
|
|
10
8
|
export function buildAgentTools(config: AgentConfig) {
|
|
@@ -35,12 +33,10 @@ export function buildAgentTools(config: AgentConfig) {
|
|
|
35
33
|
|
|
36
34
|
return {
|
|
37
35
|
run_tests,
|
|
38
|
-
validate_tests: validateTestsTool,
|
|
39
36
|
query_results: queryResultsTool,
|
|
40
37
|
diagnose_failure: diagnoseFailureTool,
|
|
41
38
|
send_request,
|
|
42
|
-
explore_api: exploreApiTool,
|
|
43
39
|
};
|
|
44
40
|
}
|
|
45
41
|
|
|
46
|
-
export { runTestsTool,
|
|
42
|
+
export { runTestsTool, queryResultsTool, diagnoseFailureTool, sendRequestTool };
|
|
@@ -44,6 +44,16 @@ export function envCategory(hint: string | undefined): string | null {
|
|
|
44
44
|
return null;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
export function schemaHint(
|
|
48
|
+
failureType: string,
|
|
49
|
+
responseStatus: number | null | undefined,
|
|
50
|
+
): string | null {
|
|
51
|
+
if (failureType === "assertion_failed" || responseStatus === 400 || responseStatus === 422) {
|
|
52
|
+
return "Use describe_endpoint(specPath, method, path) to verify expected request/response schema";
|
|
53
|
+
}
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
47
57
|
export function computeSharedEnvIssue(
|
|
48
58
|
failures: Array<{ hint?: string }>,
|
|
49
59
|
envFilePath?: string,
|
|
@@ -1,56 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../../core/generator/index.ts";
|
|
4
|
-
import type { EndpointInfo, SecuritySchemeInfo } from "../../core/generator/types.ts";
|
|
5
|
-
import { compressSchema, formatParam, isAnySchema } from "../../core/generator/schema-utils.ts";
|
|
6
|
-
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
7
|
-
|
|
8
|
-
export function registerGenerateTestsGuideTool(server: McpServer) {
|
|
9
|
-
server.registerTool("generate_tests_guide", {
|
|
10
|
-
description: TOOL_DESCRIPTIONS.generate_tests_guide,
|
|
11
|
-
inputSchema: {
|
|
12
|
-
specPath: z.string().describe("Path or URL to OpenAPI spec file"),
|
|
13
|
-
outputDir: z.optional(z.string()).describe("Directory for saving test files (default: ./tests/)"),
|
|
14
|
-
methodFilter: z.optional(z.array(z.string())).describe("Only include endpoints with these HTTP methods (e.g. [\"GET\"] for smoke tests)"),
|
|
15
|
-
tag: z.optional(z.string()).describe("Filter endpoints by tag"),
|
|
16
|
-
},
|
|
17
|
-
}, async ({ specPath, outputDir, methodFilter, tag }) => {
|
|
18
|
-
try {
|
|
19
|
-
const doc = await readOpenApiSpec(specPath);
|
|
20
|
-
let endpoints = extractEndpoints(doc);
|
|
21
|
-
if (methodFilter && methodFilter.length > 0) {
|
|
22
|
-
const methods = methodFilter.map(m => m.toUpperCase());
|
|
23
|
-
endpoints = endpoints.filter(ep => methods.includes(ep.method.toUpperCase()));
|
|
24
|
-
}
|
|
25
|
-
if (tag) {
|
|
26
|
-
const lower = tag.toLowerCase();
|
|
27
|
-
endpoints = endpoints.filter(ep => ep.tags.some(t => t.toLowerCase() === lower));
|
|
28
|
-
}
|
|
29
|
-
const securitySchemes = extractSecuritySchemes(doc);
|
|
30
|
-
const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
|
|
31
|
-
const title = (doc as any).info?.title as string | undefined;
|
|
32
|
-
|
|
33
|
-
const apiContext = compressEndpointsWithSchemas(endpoints, securitySchemes);
|
|
34
|
-
const guide = buildGenerationGuide({
|
|
35
|
-
title: title ?? "API",
|
|
36
|
-
baseUrl,
|
|
37
|
-
apiContext,
|
|
38
|
-
outputDir: outputDir ?? "./tests/",
|
|
39
|
-
securitySchemes,
|
|
40
|
-
endpointCount: endpoints.length,
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
return {
|
|
44
|
-
content: [{ type: "text" as const, text: guide }],
|
|
45
|
-
};
|
|
46
|
-
} catch (err) {
|
|
47
|
-
return {
|
|
48
|
-
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
49
|
-
isError: true,
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
});
|
|
53
|
-
}
|
|
1
|
+
import type { EndpointInfo, SecuritySchemeInfo } from "./types.ts";
|
|
2
|
+
import { compressSchema, formatParam, isAnySchema } from "./schema-utils.ts";
|
|
54
3
|
|
|
55
4
|
export function compressEndpointsWithSchemas(
|
|
56
5
|
endpoints: EndpointInfo[],
|
|
@@ -118,11 +67,30 @@ export interface GuideOptions {
|
|
|
118
67
|
securitySchemes: SecuritySchemeInfo[];
|
|
119
68
|
endpointCount: number;
|
|
120
69
|
coverageHeader?: string;
|
|
70
|
+
compact?: boolean;
|
|
121
71
|
}
|
|
122
72
|
|
|
123
73
|
export function buildGenerationGuide(opts: GuideOptions): string {
|
|
124
74
|
const hasAuth = opts.securitySchemes.length > 0;
|
|
125
75
|
|
|
76
|
+
if (opts.compact) {
|
|
77
|
+
const securitySummary = hasAuth
|
|
78
|
+
? `Security: ${opts.securitySchemes.map(s => `${s.name} (${s.type}${s.scheme ? `/${s.scheme}` : ""})`).join(", ")}`
|
|
79
|
+
: "Security: none";
|
|
80
|
+
|
|
81
|
+
return `# Test Generation Guide for ${opts.title}
|
|
82
|
+
${opts.coverageHeader ? `\n${opts.coverageHeader}\n` : ""}
|
|
83
|
+
## API Specification (${opts.endpointCount} endpoints)
|
|
84
|
+
${opts.baseUrl ? `Base URL: ${opts.baseUrl}` : "Base URL: use {{base_url}} environment variable"}
|
|
85
|
+
${securitySummary}
|
|
86
|
+
|
|
87
|
+
${opts.apiContext}
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
> **Note:** Refer to the full YAML format reference, generation algorithm, tag conventions, and practical tips from the initial \`generate_and_save\` call (without \`tag\`). Only the API endpoints above are unique to this chunk.`;
|
|
92
|
+
}
|
|
93
|
+
|
|
126
94
|
return `# Test Generation Guide for ${opts.title}
|
|
127
95
|
${opts.coverageHeader ? `\n${opts.coverageHeader}\n` : ""}
|
|
128
96
|
## API Specification (${opts.endpointCount} endpoints)
|
|
@@ -262,7 +230,7 @@ Never put actual key values directly in YAML test files.
|
|
|
262
230
|
- Note required fields in request bodies
|
|
263
231
|
|
|
264
232
|
### Step 2: Plan Test Suites
|
|
265
|
-
Before generating, check coverage with \`coverage_analysis\` to avoid duplicating existing tests. Use \`
|
|
233
|
+
Before generating, check coverage with \`coverage_analysis\` to avoid duplicating existing tests. Use \`generate_and_save(testsDir=...)\` for incremental generation of uncovered endpoints only.
|
|
266
234
|
|
|
267
235
|
> **Coverage note**: coverage is a static scan of YAML files — an endpoint is "covered" if a test file contains a matching METHOD + path line, regardless of whether tests pass or actually run.
|
|
268
236
|
|
|
@@ -372,13 +340,13 @@ Example: \`zond run --tag smoke --safe\` → reads-only, safe against production
|
|
|
372
340
|
| Tool | When |
|
|
373
341
|
|------|------|
|
|
374
342
|
| \`setup_api\` | Register a new API (creates dirs, reads spec, sets up env) |
|
|
375
|
-
| \`
|
|
376
|
-
| \`generate_missing_tests\` | Get guide for only uncovered endpoints |
|
|
343
|
+
| \`generate_and_save\` | Get test generation guide (with auto-chunking for large APIs) |
|
|
377
344
|
| \`save_test_suite\` | Save generated YAML (validates before writing) |
|
|
345
|
+
| \`save_test_suites\` | Save multiple YAML files in one call |
|
|
378
346
|
| \`run_tests\` | Execute saved test suites |
|
|
379
347
|
| \`query_db\` | Query runs, collections, results, diagnose failures |
|
|
380
348
|
| \`coverage_analysis\` | Find untested endpoints for incremental generation |
|
|
381
|
-
| \`
|
|
349
|
+
| \`describe_endpoint\` | Get full details for one endpoint when debugging |
|
|
382
350
|
| \`ci_init\` | Generate CI/CD workflow (GitHub Actions / GitLab CI) to run tests on push |
|
|
383
351
|
|
|
384
352
|
## Workflow After Tests Pass
|
|
@@ -8,5 +8,7 @@ export type { AIProviderConfig, AIGenerateOptions, AIGenerateResult } from "./ai
|
|
|
8
8
|
export { scanCoveredEndpoints, filterUncoveredEndpoints, normalizePath, specPathToRegex } from "./coverage-scanner.ts";
|
|
9
9
|
export type { CoveredEndpoint } from "./coverage-scanner.ts";
|
|
10
10
|
export { analyzeEndpoints } from "./endpoint-warnings.ts";
|
|
11
|
+
export { compressEndpointsWithSchemas, buildGenerationGuide } from "./guide-builder.ts";
|
|
12
|
+
export type { GuideOptions } from "./guide-builder.ts";
|
|
11
13
|
export type { EndpointWarning, WarningCode } from "./endpoint-warnings.ts";
|
|
12
14
|
export type { EndpointInfo, ResponseInfo, GenerateOptions, SecuritySchemeInfo, CrudGroup } from "./types.ts";
|
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
import type { OpenAPIV3 } from "openapi-types";
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Deep-clone an object, replacing circular references with `{ "$ref": "[Circular]" }`.
|
|
5
|
+
* Uses WeakSet to track visited objects.
|
|
6
|
+
*/
|
|
7
|
+
export function decycleSchema(obj: unknown): unknown {
|
|
8
|
+
const seen = new WeakSet<object>();
|
|
9
|
+
|
|
10
|
+
function walk(value: unknown): unknown {
|
|
11
|
+
if (value === null || typeof value !== "object") return value;
|
|
12
|
+
|
|
13
|
+
if (Array.isArray(value)) {
|
|
14
|
+
return value.map(item => walk(item));
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const obj = value as Record<string, unknown>;
|
|
18
|
+
if (seen.has(obj)) {
|
|
19
|
+
return { $ref: "[Circular]" };
|
|
20
|
+
}
|
|
21
|
+
seen.add(obj);
|
|
22
|
+
|
|
23
|
+
const result: Record<string, unknown> = {};
|
|
24
|
+
for (const key of Object.keys(obj)) {
|
|
25
|
+
result[key] = walk(obj[key]);
|
|
26
|
+
}
|
|
27
|
+
return result;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return walk(obj);
|
|
31
|
+
}
|
|
32
|
+
|
|
3
33
|
/**
|
|
4
34
|
* Returns true if the schema is effectively `any` — no type, no properties, no constraints.
|
|
5
35
|
*/
|
package/src/mcp/descriptions.ts
CHANGED
|
@@ -14,42 +14,20 @@ export const TOOL_DESCRIPTIONS = {
|
|
|
14
14
|
"sets up environment variables, and creates a collection in the database. " +
|
|
15
15
|
"Use this before generating tests for a new API.",
|
|
16
16
|
|
|
17
|
-
explore_api:
|
|
18
|
-
"Explore an OpenAPI spec — list endpoints, servers, and security schemes. " +
|
|
19
|
-
"Use with includeSchemas=true when generating tests to get full request/response body schemas.",
|
|
20
|
-
|
|
21
17
|
describe_endpoint:
|
|
22
18
|
"Full details for one endpoint: params grouped by type, request body schema, " +
|
|
23
19
|
"all response schemas + response headers, security, deprecated flag. " +
|
|
24
20
|
"Use when a test fails and you need complete endpoint spec without reading the whole file.",
|
|
25
21
|
|
|
26
|
-
generate_tests_guide:
|
|
27
|
-
"Get a comprehensive guide for generating API test suites. " +
|
|
28
|
-
"Returns the full API specification (with request/response schemas) and a step-by-step algorithm " +
|
|
29
|
-
"for creating YAML test files. Use this BEFORE generating tests — it gives you " +
|
|
30
|
-
"everything you need to write high-quality test suites. " +
|
|
31
|
-
"After generating, use save_test_suite to save, run_tests to execute, " +
|
|
32
|
-
"manage_server(action: 'start') to view results in the Web UI, " +
|
|
33
|
-
"and query_db(action: 'diagnose_failure') to debug failures.",
|
|
34
|
-
|
|
35
|
-
generate_missing_tests:
|
|
36
|
-
"Analyze test coverage and generate a test guide for only the uncovered endpoints. " +
|
|
37
|
-
"Combines coverage_analysis + generate_tests_guide — returns a focused guide for missing tests. " +
|
|
38
|
-
"Use this for incremental test generation to avoid duplicating existing tests. " +
|
|
39
|
-
"After saving and running new tests, use manage_server(action: 'start') to view results in the Web UI.",
|
|
40
|
-
|
|
41
22
|
save_test_suite:
|
|
42
23
|
"Save a YAML test suite file with validation. Parses and validates the YAML content " +
|
|
43
24
|
"before writing. Returns structured errors if validation fails so you can fix and retry. " +
|
|
44
|
-
"Use after generating test content with
|
|
25
|
+
"Use after generating test content with generate_and_save.",
|
|
45
26
|
|
|
46
27
|
save_test_suites:
|
|
47
28
|
"Save multiple YAML test suite files in a single call. Each file is validated before writing. " +
|
|
48
29
|
"Returns per-file results. Use when you have generated multiple suites at once.",
|
|
49
30
|
|
|
50
|
-
validate_tests:
|
|
51
|
-
"Validate YAML test files without running them. Returns parsed suite info or validation errors.",
|
|
52
|
-
|
|
53
31
|
run_tests:
|
|
54
32
|
"Execute API tests from a YAML file or directory and return results summary with failures. " +
|
|
55
33
|
"Use after saving test suites with save_test_suite. Check query_db(action: 'diagnose_failure') for detailed failure analysis.",
|
package/src/mcp/server.ts
CHANGED
|
@@ -1,15 +1,11 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
2
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
3
|
import { registerRunTestsTool } from "./tools/run-tests.ts";
|
|
4
|
-
import { registerValidateTestsTool } from "./tools/validate-tests.ts";
|
|
5
4
|
import { registerQueryDbTool } from "./tools/query-db.ts";
|
|
6
5
|
import { registerSendRequestTool } from "./tools/send-request.ts";
|
|
7
|
-
import { registerExploreApiTool } from "./tools/explore-api.ts";
|
|
8
6
|
import { registerCoverageAnalysisTool } from "./tools/coverage-analysis.ts";
|
|
9
7
|
import { registerSaveTestSuiteTool, registerSaveTestSuitesTool } from "./tools/save-test-suite.ts";
|
|
10
|
-
import { registerGenerateTestsGuideTool } from "./tools/generate-tests-guide.ts";
|
|
11
8
|
import { registerSetupApiTool } from "./tools/setup-api.ts";
|
|
12
|
-
import { registerGenerateMissingTestsTool } from "./tools/generate-missing-tests.ts";
|
|
13
9
|
import { registerManageServerTool } from "./tools/manage-server.ts";
|
|
14
10
|
import { registerCiInitTool } from "./tools/ci-init.ts";
|
|
15
11
|
import { registerSetWorkDirTool } from "./tools/set-work-dir.ts";
|
|
@@ -25,21 +21,17 @@ export async function startMcpServer(options: McpServerOptions = {}): Promise<vo
|
|
|
25
21
|
|
|
26
22
|
const server = new McpServer({
|
|
27
23
|
name: "zond",
|
|
28
|
-
version: "0.
|
|
24
|
+
version: "0.8.0",
|
|
29
25
|
});
|
|
30
26
|
|
|
31
27
|
// Register all tools
|
|
32
28
|
registerRunTestsTool(server, dbPath);
|
|
33
|
-
registerValidateTestsTool(server);
|
|
34
29
|
registerQueryDbTool(server, dbPath);
|
|
35
30
|
registerSendRequestTool(server, dbPath);
|
|
36
|
-
registerExploreApiTool(server);
|
|
37
31
|
registerCoverageAnalysisTool(server, dbPath);
|
|
38
32
|
registerSaveTestSuiteTool(server, dbPath);
|
|
39
33
|
registerSaveTestSuitesTool(server, dbPath);
|
|
40
|
-
registerGenerateTestsGuideTool(server);
|
|
41
34
|
registerSetupApiTool(server, dbPath);
|
|
42
|
-
registerGenerateMissingTestsTool(server);
|
|
43
35
|
registerManageServerTool(server, dbPath);
|
|
44
36
|
registerCiInitTool(server);
|
|
45
37
|
registerSetWorkDirTool(server);
|
|
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
3
|
import type { OpenAPIV3 } from "openapi-types";
|
|
4
4
|
import { readOpenApiSpec } from "../../core/generator/index.ts";
|
|
5
|
+
import { decycleSchema } from "../../core/generator/schema-utils.ts";
|
|
5
6
|
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
6
7
|
|
|
7
8
|
function generateTestSnippet(params: {
|
|
@@ -229,7 +230,7 @@ export function registerDescribeEndpointTool(server: McpServer) {
|
|
|
229
230
|
};
|
|
230
231
|
|
|
231
232
|
return {
|
|
232
|
-
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
233
|
+
content: [{ type: "text" as const, text: JSON.stringify(decycleSchema(result), null, 2) }],
|
|
233
234
|
};
|
|
234
235
|
} catch (err) {
|
|
235
236
|
return {
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
scanCoveredEndpoints,
|
|
8
8
|
filterUncoveredEndpoints,
|
|
9
9
|
} from "../../core/generator/index.ts";
|
|
10
|
-
import { compressEndpointsWithSchemas, buildGenerationGuide } from "
|
|
10
|
+
import { compressEndpointsWithSchemas, buildGenerationGuide } from "../../core/generator/guide-builder.ts";
|
|
11
11
|
import { planChunks, filterByTag } from "../../core/generator/chunker.ts";
|
|
12
12
|
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
13
13
|
|
|
@@ -106,6 +106,7 @@ export function registerGenerateAndSaveTool(server: McpServer) {
|
|
|
106
106
|
securitySchemes,
|
|
107
107
|
endpointCount: endpoints.length,
|
|
108
108
|
coverageHeader,
|
|
109
|
+
compact: !!tag,
|
|
109
110
|
});
|
|
110
111
|
|
|
111
112
|
const saveInstructions = `
|
|
@@ -4,7 +4,7 @@ import { getDb } from "../../db/schema.ts";
|
|
|
4
4
|
import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
7
|
-
import { statusHint, classifyFailure, envHint, envCategory } from "../../core/diagnostics/failure-hints.ts";
|
|
7
|
+
import { statusHint, classifyFailure, envHint, envCategory, schemaHint } from "../../core/diagnostics/failure-hints.ts";
|
|
8
8
|
|
|
9
9
|
function parseBodySafe(raw: string | null | undefined): unknown {
|
|
10
10
|
if (!raw) return undefined;
|
|
@@ -124,6 +124,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
|
|
|
124
124
|
// env issues take priority over generic status hints
|
|
125
125
|
const hint = envHint(r.request_url, r.error_message, envFilePath) ?? statusHint(r.response_status);
|
|
126
126
|
const failure_type = classifyFailure(r.status, r.response_status);
|
|
127
|
+
const sHint = schemaHint(failure_type, r.response_status);
|
|
127
128
|
return {
|
|
128
129
|
suite_name: r.suite_name,
|
|
129
130
|
test_name: r.test_name,
|
|
@@ -134,6 +135,7 @@ export function registerQueryDbTool(server: McpServer, dbPath?: string) {
|
|
|
134
135
|
request_url: r.request_url,
|
|
135
136
|
response_status: r.response_status,
|
|
136
137
|
...(hint ? { hint } : {}),
|
|
138
|
+
...(sHint ? { schema_hint: sHint } : {}),
|
|
137
139
|
response_body: parseBodySafe(r.response_body),
|
|
138
140
|
response_headers: r.response_headers
|
|
139
141
|
? JSON.parse(r.response_headers)
|
|
@@ -48,6 +48,12 @@ export function registerRunTestsTool(server: McpServer, dbPath?: string) {
|
|
|
48
48
|
const hints: string[] = [];
|
|
49
49
|
if (failedSteps.length > 0) {
|
|
50
50
|
hints.push("Use query_db(action: 'diagnose_failure', runId: " + runId + ") for detailed failure analysis");
|
|
51
|
+
const hasAssertionFailures = failedSteps.some(s => s.assertions.length > 0);
|
|
52
|
+
if (hasAssertionFailures) {
|
|
53
|
+
hints.push(
|
|
54
|
+
"Some tests have assertion failures — use describe_endpoint(specPath, method, path) to verify expected schemas"
|
|
55
|
+
);
|
|
56
|
+
}
|
|
51
57
|
}
|
|
52
58
|
hints.push("Use manage_server(action: 'start') to launch the Web UI and view results visually in a browser at http://localhost:8080");
|
|
53
59
|
hints.push("Ask the user if they want to set up CI/CD to run these tests automatically on push. If yes, use ci_init to generate a workflow and help them push to GitHub/GitLab.");
|
|
@@ -138,7 +138,7 @@ export async function validateAndSave(
|
|
|
138
138
|
|
|
139
139
|
const coverage: Record<string, unknown> = { percentage, covered: coveredCount, total, uncoveredCount: uncovered.length };
|
|
140
140
|
if (percentage < 80 && uncovered.length > 0) {
|
|
141
|
-
coverage.suggestion = `Use
|
|
141
|
+
coverage.suggestion = `Use generate_and_save with testsDir to cover ${uncovered.length} remaining endpoint${uncovered.length > 1 ? "s" : ""}`;
|
|
142
142
|
}
|
|
143
143
|
result.coverage = coverage;
|
|
144
144
|
}
|
|
@@ -8,7 +8,7 @@ import { basename } from "node:path";
|
|
|
8
8
|
|
|
9
9
|
export function renderSuitesTab(state: CollectionState): string {
|
|
10
10
|
if (state.suites.length === 0) {
|
|
11
|
-
return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>
|
|
11
|
+
return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>generate_and_save</code>.</div>`;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
const rows = state.suites.map((s, i) => renderSuiteRow(s, i)).join("");
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
import { tool } from "ai";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../../generator/index.ts";
|
|
4
|
-
|
|
5
|
-
export const exploreApiTool = tool({
|
|
6
|
-
description: "Explore an OpenAPI spec — list endpoints with method, path, and summary. Optionally filter by tag.",
|
|
7
|
-
inputSchema: z.object({
|
|
8
|
-
specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML)"),
|
|
9
|
-
tag: z.string().optional().describe("Filter endpoints by tag"),
|
|
10
|
-
}),
|
|
11
|
-
execute: async (args) => {
|
|
12
|
-
try {
|
|
13
|
-
const doc = await readOpenApiSpec(args.specPath);
|
|
14
|
-
const allEndpoints = extractEndpoints(doc);
|
|
15
|
-
const securitySchemes = extractSecuritySchemes(doc);
|
|
16
|
-
const servers = ((doc as any).servers ?? []) as Array<{ url: string }>;
|
|
17
|
-
|
|
18
|
-
const endpoints = args.tag
|
|
19
|
-
? allEndpoints.filter(ep => ep.tags.includes(args.tag!))
|
|
20
|
-
: allEndpoints;
|
|
21
|
-
|
|
22
|
-
// Compact output — method + path + summary only
|
|
23
|
-
return {
|
|
24
|
-
title: (doc as any).info?.title,
|
|
25
|
-
version: (doc as any).info?.version,
|
|
26
|
-
servers: servers.map(s => s.url),
|
|
27
|
-
securitySchemes: securitySchemes.map(s => s.name),
|
|
28
|
-
totalEndpoints: allEndpoints.length,
|
|
29
|
-
...(args.tag ? { filteredByTag: args.tag, matchingEndpoints: endpoints.length } : {}),
|
|
30
|
-
endpoints: endpoints.map(ep => ({
|
|
31
|
-
method: ep.method,
|
|
32
|
-
path: ep.path,
|
|
33
|
-
summary: ep.summary,
|
|
34
|
-
})),
|
|
35
|
-
};
|
|
36
|
-
} catch (err) {
|
|
37
|
-
return { error: (err as Error).message };
|
|
38
|
-
}
|
|
39
|
-
},
|
|
40
|
-
});
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { tool } from "ai";
|
|
2
|
-
import { z } from "zod";
|
|
3
|
-
import { parse } from "../../parser/yaml-parser.ts";
|
|
4
|
-
|
|
5
|
-
export const validateTestsTool = tool({
|
|
6
|
-
description: "Validate YAML test files without running them. Returns parsed suite info or validation errors.",
|
|
7
|
-
inputSchema: z.object({
|
|
8
|
-
testPath: z.string().describe("Path to test YAML file or directory"),
|
|
9
|
-
}),
|
|
10
|
-
execute: async (args) => {
|
|
11
|
-
try {
|
|
12
|
-
const suites = await parse(args.testPath);
|
|
13
|
-
return {
|
|
14
|
-
valid: true,
|
|
15
|
-
suiteCount: suites.length,
|
|
16
|
-
totalTests: suites.reduce((s, suite) => s + suite.tests.length, 0),
|
|
17
|
-
suites: suites.map((s) => ({ name: s.name, testCount: s.tests.length })),
|
|
18
|
-
};
|
|
19
|
-
} catch (err) {
|
|
20
|
-
return { valid: false, error: (err as Error).message };
|
|
21
|
-
}
|
|
22
|
-
},
|
|
23
|
-
});
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
-
import { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "../../core/generator/index.ts";
|
|
4
|
-
import { compressSchema, formatParam } from "../../core/generator/schema-utils.ts";
|
|
5
|
-
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
6
|
-
|
|
7
|
-
export function registerExploreApiTool(server: McpServer) {
|
|
8
|
-
server.registerTool("explore_api", {
|
|
9
|
-
description: TOOL_DESCRIPTIONS.explore_api,
|
|
10
|
-
inputSchema: {
|
|
11
|
-
specPath: z.string().describe("Path to OpenAPI spec file (JSON or YAML)"),
|
|
12
|
-
tag: z.optional(z.string()).describe("Filter endpoints by tag"),
|
|
13
|
-
includeSchemas: z.optional(z.boolean()).describe("Include request/response body schemas and parameter types (default: false)"),
|
|
14
|
-
},
|
|
15
|
-
}, async ({ specPath, tag, includeSchemas }) => {
|
|
16
|
-
try {
|
|
17
|
-
const doc = await readOpenApiSpec(specPath);
|
|
18
|
-
const allEndpoints = extractEndpoints(doc);
|
|
19
|
-
const securitySchemes = extractSecuritySchemes(doc);
|
|
20
|
-
const servers = ((doc as any).servers ?? []) as Array<{ url: string; description?: string }>;
|
|
21
|
-
|
|
22
|
-
const endpoints = tag
|
|
23
|
-
? allEndpoints.filter(ep => ep.tags.includes(tag))
|
|
24
|
-
: allEndpoints;
|
|
25
|
-
|
|
26
|
-
const result = {
|
|
27
|
-
title: (doc as any).info?.title,
|
|
28
|
-
version: (doc as any).info?.version,
|
|
29
|
-
servers: servers.map(s => ({ url: s.url, description: s.description })),
|
|
30
|
-
securitySchemes: securitySchemes.map(s => ({
|
|
31
|
-
name: s.name,
|
|
32
|
-
type: s.type,
|
|
33
|
-
...(s.scheme ? { scheme: s.scheme } : {}),
|
|
34
|
-
...(s.in ? { in: s.in, keyName: s.apiKeyName } : {}),
|
|
35
|
-
})),
|
|
36
|
-
totalEndpoints: allEndpoints.length,
|
|
37
|
-
...(tag ? { filteredByTag: tag, matchingEndpoints: endpoints.length } : {}),
|
|
38
|
-
endpoints: endpoints.map(ep => {
|
|
39
|
-
const base: Record<string, unknown> = {
|
|
40
|
-
method: ep.method,
|
|
41
|
-
path: ep.path,
|
|
42
|
-
summary: ep.summary,
|
|
43
|
-
tags: ep.tags,
|
|
44
|
-
parameters: ep.parameters.map(p => ({
|
|
45
|
-
name: p.name,
|
|
46
|
-
in: p.in,
|
|
47
|
-
required: p.required ?? false,
|
|
48
|
-
...(includeSchemas ? { type: formatParam(p).split(": ")[1] } : {}),
|
|
49
|
-
})),
|
|
50
|
-
hasRequestBody: !!ep.requestBodySchema,
|
|
51
|
-
responses: ep.responses.map(r => ({
|
|
52
|
-
statusCode: r.statusCode,
|
|
53
|
-
description: r.description,
|
|
54
|
-
...(includeSchemas && r.schema ? { schema: compressSchema(r.schema) } : {}),
|
|
55
|
-
})),
|
|
56
|
-
};
|
|
57
|
-
|
|
58
|
-
if (includeSchemas) {
|
|
59
|
-
if (ep.requestBodySchema) {
|
|
60
|
-
base.requestBodySchema = compressSchema(ep.requestBodySchema);
|
|
61
|
-
}
|
|
62
|
-
if (ep.requestBodyContentType) {
|
|
63
|
-
base.requestBodyContentType = ep.requestBodyContentType;
|
|
64
|
-
}
|
|
65
|
-
if (ep.security.length > 0) {
|
|
66
|
-
base.security = ep.security;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return base;
|
|
71
|
-
}),
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
return {
|
|
75
|
-
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
76
|
-
};
|
|
77
|
-
} catch (err) {
|
|
78
|
-
return {
|
|
79
|
-
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
80
|
-
isError: true,
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
});
|
|
84
|
-
}
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
-
import {
|
|
4
|
-
readOpenApiSpec,
|
|
5
|
-
extractEndpoints,
|
|
6
|
-
extractSecuritySchemes,
|
|
7
|
-
scanCoveredEndpoints,
|
|
8
|
-
filterUncoveredEndpoints,
|
|
9
|
-
} from "../../core/generator/index.ts";
|
|
10
|
-
import { compressEndpointsWithSchemas, buildGenerationGuide } from "./generate-tests-guide.ts";
|
|
11
|
-
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
12
|
-
|
|
13
|
-
export function registerGenerateMissingTestsTool(server: McpServer) {
|
|
14
|
-
server.registerTool("generate_missing_tests", {
|
|
15
|
-
description: TOOL_DESCRIPTIONS.generate_missing_tests,
|
|
16
|
-
inputSchema: {
|
|
17
|
-
specPath: z.string().describe("Path or URL to OpenAPI spec file"),
|
|
18
|
-
testsDir: z.string().describe("Path to directory with existing test YAML files"),
|
|
19
|
-
outputDir: z.optional(z.string()).describe("Directory for saving new test files (default: same as testsDir)"),
|
|
20
|
-
methodFilter: z.optional(z.array(z.string())).describe("Only include endpoints with these HTTP methods (e.g. [\"GET\"] for smoke tests)"),
|
|
21
|
-
tag: z.optional(z.string()).describe("Filter endpoints by tag"),
|
|
22
|
-
},
|
|
23
|
-
}, async ({ specPath, testsDir, outputDir, methodFilter, tag }) => {
|
|
24
|
-
try {
|
|
25
|
-
const doc = await readOpenApiSpec(specPath);
|
|
26
|
-
let allEndpoints = extractEndpoints(doc);
|
|
27
|
-
const securitySchemes = extractSecuritySchemes(doc);
|
|
28
|
-
const baseUrl = ((doc as any).servers?.[0]?.url) as string | undefined;
|
|
29
|
-
const title = (doc as any).info?.title as string | undefined;
|
|
30
|
-
|
|
31
|
-
// Apply method filter before coverage check
|
|
32
|
-
if (methodFilter && methodFilter.length > 0) {
|
|
33
|
-
const methods = methodFilter.map(m => m.toUpperCase());
|
|
34
|
-
allEndpoints = allEndpoints.filter(ep => methods.includes(ep.method.toUpperCase()));
|
|
35
|
-
}
|
|
36
|
-
if (tag) {
|
|
37
|
-
const lower = tag.toLowerCase();
|
|
38
|
-
allEndpoints = allEndpoints.filter(ep => ep.tags.some(t => t.toLowerCase() === lower));
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
if (allEndpoints.length === 0) {
|
|
42
|
-
return {
|
|
43
|
-
content: [{ type: "text" as const, text: JSON.stringify({ error: "No endpoints found in the spec" }, null, 2) }],
|
|
44
|
-
isError: true,
|
|
45
|
-
};
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const covered = await scanCoveredEndpoints(testsDir);
|
|
49
|
-
const uncovered = filterUncoveredEndpoints(allEndpoints, covered);
|
|
50
|
-
const coveredCount = allEndpoints.length - uncovered.length;
|
|
51
|
-
const percentage = Math.round((coveredCount / allEndpoints.length) * 100);
|
|
52
|
-
|
|
53
|
-
if (uncovered.length === 0) {
|
|
54
|
-
return {
|
|
55
|
-
content: [{
|
|
56
|
-
type: "text" as const,
|
|
57
|
-
text: JSON.stringify({
|
|
58
|
-
fullyCovered: true,
|
|
59
|
-
percentage: 100,
|
|
60
|
-
totalEndpoints: allEndpoints.length,
|
|
61
|
-
covered: coveredCount,
|
|
62
|
-
}, null, 2),
|
|
63
|
-
}],
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// Build guide for uncovered endpoints only
|
|
68
|
-
const apiContext = compressEndpointsWithSchemas(uncovered, securitySchemes);
|
|
69
|
-
const coverageHeader = `## Coverage: ${coveredCount}/${allEndpoints.length} endpoints covered (${percentage}%). Generating tests for ${uncovered.length} uncovered endpoints:`;
|
|
70
|
-
|
|
71
|
-
const guide = buildGenerationGuide({
|
|
72
|
-
title: title ?? "API",
|
|
73
|
-
baseUrl,
|
|
74
|
-
apiContext,
|
|
75
|
-
outputDir: outputDir ?? testsDir,
|
|
76
|
-
securitySchemes,
|
|
77
|
-
endpointCount: uncovered.length,
|
|
78
|
-
coverageHeader,
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
return {
|
|
82
|
-
content: [{ type: "text" as const, text: guide }],
|
|
83
|
-
};
|
|
84
|
-
} catch (err) {
|
|
85
|
-
return {
|
|
86
|
-
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
87
|
-
isError: true,
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
});
|
|
91
|
-
}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
-
import { parse } from "../../core/parser/yaml-parser.ts";
|
|
4
|
-
import { TOOL_DESCRIPTIONS } from "../descriptions.js";
|
|
5
|
-
|
|
6
|
-
export function registerValidateTestsTool(server: McpServer) {
|
|
7
|
-
server.registerTool("validate_tests", {
|
|
8
|
-
description: TOOL_DESCRIPTIONS.validate_tests,
|
|
9
|
-
inputSchema: {
|
|
10
|
-
testPath: z.string().describe("Path to test YAML file or directory"),
|
|
11
|
-
},
|
|
12
|
-
}, async ({ testPath }) => {
|
|
13
|
-
try {
|
|
14
|
-
const suites = await parse(testPath);
|
|
15
|
-
const totalSteps = suites.reduce((sum, s) => sum + s.tests.length, 0);
|
|
16
|
-
|
|
17
|
-
const result = {
|
|
18
|
-
valid: true,
|
|
19
|
-
suites: suites.length,
|
|
20
|
-
tests: totalSteps,
|
|
21
|
-
details: suites.map(s => ({
|
|
22
|
-
name: s.name,
|
|
23
|
-
tests: s.tests.length,
|
|
24
|
-
tags: s.tags,
|
|
25
|
-
description: s.description,
|
|
26
|
-
source: (s as any)._source,
|
|
27
|
-
})),
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
return {
|
|
31
|
-
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
32
|
-
};
|
|
33
|
-
} catch (err) {
|
|
34
|
-
return {
|
|
35
|
-
content: [{ type: "text" as const, text: JSON.stringify({
|
|
36
|
-
valid: false,
|
|
37
|
-
error: (err as Error).message,
|
|
38
|
-
}, null, 2) }],
|
|
39
|
-
isError: true,
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
});
|
|
43
|
-
}
|