@kirrosh/apitool 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +27 -0
- package/.github/workflows/release.yml +97 -0
- package/.mcp.json +9 -0
- package/APITOOL.md +195 -0
- package/BACKLOG.md +62 -0
- package/CHANGELOG.md +88 -0
- package/LICENSE +21 -0
- package/README.md +105 -0
- package/bun.lock +291 -0
- package/docs/GLOSSARY.md +182 -0
- package/docs/INDEX.md +21 -0
- package/docs/agent.md +135 -0
- package/docs/archive/APITOOL-pre-M22.md +831 -0
- package/docs/archive/BACKLOG-AI-NATIVE.md +56 -0
- package/docs/archive/M1-M2-parser-runner.md +216 -0
- package/docs/archive/M4-M7-reporter-cli.md +179 -0
- package/docs/archive/M5-M7-storage-junit.md +300 -0
- package/docs/archive/M6-webui.md +339 -0
- package/docs/ci.md +274 -0
- package/docs/generation-issues.md +67 -0
- package/generated/.env.yaml +3 -0
- package/install.ps1 +80 -0
- package/install.sh +113 -0
- package/package.json +46 -0
- package/scripts/run-mocked-tests.ts +45 -0
- package/seed-demo.ts +53 -0
- package/self-tests/auth.yaml +18 -0
- package/self-tests/collections-crud.yaml +46 -0
- package/self-tests/environments-crud.yaml +48 -0
- package/self-tests/export.yaml +32 -0
- package/self-tests/runs.yaml +16 -0
- package/src/bun-types.d.ts +5 -0
- package/src/cli/commands/add-api.ts +51 -0
- package/src/cli/commands/ai-generate.ts +106 -0
- package/src/cli/commands/chat.ts +43 -0
- package/src/cli/commands/ci-init.ts +126 -0
- package/src/cli/commands/collections.ts +41 -0
- package/src/cli/commands/coverage.ts +65 -0
- package/src/cli/commands/doctor.ts +127 -0
- package/src/cli/commands/envs.ts +218 -0
- package/src/cli/commands/init.ts +84 -0
- package/src/cli/commands/mcp.ts +16 -0
- package/src/cli/commands/run.ts +137 -0
- package/src/cli/commands/runs.ts +108 -0
- package/src/cli/commands/serve.ts +22 -0
- package/src/cli/commands/update.ts +142 -0
- package/src/cli/commands/validate.ts +18 -0
- package/src/cli/index.ts +500 -0
- package/src/cli/output.ts +24 -0
- package/src/cli/runtime.ts +7 -0
- package/src/core/agent/agent-loop.ts +116 -0
- package/src/core/agent/context-manager.ts +41 -0
- package/src/core/agent/system-prompt.ts +33 -0
- package/src/core/agent/tools/diagnose-failure.ts +51 -0
- package/src/core/agent/tools/explore-api.ts +40 -0
- package/src/core/agent/tools/index.ts +48 -0
- package/src/core/agent/tools/manage-environment.ts +40 -0
- package/src/core/agent/tools/query-results.ts +40 -0
- package/src/core/agent/tools/run-tests.ts +38 -0
- package/src/core/agent/tools/send-request.ts +44 -0
- package/src/core/agent/tools/validate-tests.ts +23 -0
- package/src/core/agent/types.ts +22 -0
- package/src/core/generator/ai/ai-generator.ts +61 -0
- package/src/core/generator/ai/llm-client.ts +159 -0
- package/src/core/generator/ai/output-parser.ts +307 -0
- package/src/core/generator/ai/prompt-builder.ts +153 -0
- package/src/core/generator/ai/types.ts +56 -0
- package/src/core/generator/coverage-scanner.ts +87 -0
- package/src/core/generator/data-factory.ts +115 -0
- package/src/core/generator/index.ts +10 -0
- package/src/core/generator/openapi-reader.ts +142 -0
- package/src/core/generator/schema-utils.ts +52 -0
- package/src/core/generator/serializer.ts +189 -0
- package/src/core/generator/types.ts +47 -0
- package/src/core/parser/filter.ts +14 -0
- package/src/core/parser/index.ts +21 -0
- package/src/core/parser/schema.ts +175 -0
- package/src/core/parser/types.ts +50 -0
- package/src/core/parser/variables.ts +146 -0
- package/src/core/parser/yaml-parser.ts +85 -0
- package/src/core/reporter/console.ts +175 -0
- package/src/core/reporter/index.ts +23 -0
- package/src/core/reporter/json.ts +9 -0
- package/src/core/reporter/junit.ts +78 -0
- package/src/core/reporter/types.ts +12 -0
- package/src/core/runner/assertions.ts +172 -0
- package/src/core/runner/execute-run.ts +75 -0
- package/src/core/runner/executor.ts +150 -0
- package/src/core/runner/http-client.ts +69 -0
- package/src/core/runner/index.ts +12 -0
- package/src/core/runner/types.ts +48 -0
- package/src/core/setup-api.ts +97 -0
- package/src/core/utils.ts +9 -0
- package/src/db/queries.ts +868 -0
- package/src/db/schema.ts +215 -0
- package/src/mcp/server.ts +47 -0
- package/src/mcp/tools/ci-init.ts +57 -0
- package/src/mcp/tools/coverage-analysis.ts +58 -0
- package/src/mcp/tools/explore-api.ts +84 -0
- package/src/mcp/tools/generate-missing-tests.ts +80 -0
- package/src/mcp/tools/generate-tests-guide.ts +353 -0
- package/src/mcp/tools/manage-environment.ts +123 -0
- package/src/mcp/tools/manage-server.ts +87 -0
- package/src/mcp/tools/query-db.ts +141 -0
- package/src/mcp/tools/run-tests.ts +66 -0
- package/src/mcp/tools/save-test-suite.ts +164 -0
- package/src/mcp/tools/send-request.ts +53 -0
- package/src/mcp/tools/setup-api.ts +49 -0
- package/src/mcp/tools/validate-tests.ts +42 -0
- package/src/tui/chat-ui.ts +150 -0
- package/src/web/routes/api.ts +234 -0
- package/src/web/routes/dashboard.ts +348 -0
- package/src/web/routes/runs.ts +64 -0
- package/src/web/schemas.ts +121 -0
- package/src/web/server.ts +134 -0
- package/src/web/static/htmx.min.js +1 -0
- package/src/web/static/style.css +265 -0
- package/src/web/views/layout.ts +46 -0
- package/src/web/views/results.ts +209 -0
- package/tests/agent/agent-loop.test.ts +61 -0
- package/tests/agent/context-manager.test.ts +59 -0
- package/tests/agent/system-prompt.test.ts +42 -0
- package/tests/agent/tools/diagnose-failure.test.ts +85 -0
- package/tests/agent/tools/explore-api.test.ts +59 -0
- package/tests/agent/tools/manage-environment.test.ts +78 -0
- package/tests/agent/tools/query-results.test.ts +77 -0
- package/tests/agent/tools/run-tests.test.ts +89 -0
- package/tests/agent/tools/send-request.test.ts +78 -0
- package/tests/agent/tools/validate-tests.test.ts +59 -0
- package/tests/ai/ai-generator.integration.test.ts +131 -0
- package/tests/ai/llm-client.test.ts +145 -0
- package/tests/ai/output-parser.test.ts +132 -0
- package/tests/ai/prompt-builder.test.ts +67 -0
- package/tests/ai/types.test.ts +55 -0
- package/tests/cli/args.test.ts +63 -0
- package/tests/cli/chat.test.ts +38 -0
- package/tests/cli/ci-init.test.ts +112 -0
- package/tests/cli/commands.test.ts +316 -0
- package/tests/cli/coverage.test.ts +58 -0
- package/tests/cli/doctor.test.ts +39 -0
- package/tests/cli/envs.test.ts +181 -0
- package/tests/cli/init.test.ts +80 -0
- package/tests/cli/runs.test.ts +94 -0
- package/tests/cli/safe-run.test.ts +103 -0
- package/tests/cli/update.test.ts +32 -0
- package/tests/core/generator/schema-utils.test.ts +108 -0
- package/tests/core/parser/nested-assertions.test.ts +80 -0
- package/tests/core/runner/root-body-assertions.test.ts +70 -0
- package/tests/db/chat-queries.test.ts +88 -0
- package/tests/db/chat-schema.test.ts +37 -0
- package/tests/db/environments.test.ts +131 -0
- package/tests/db/queries.test.ts +409 -0
- package/tests/db/schema.test.ts +141 -0
- package/tests/fixtures/.env.yaml +3 -0
- package/tests/fixtures/auth-token-test.yaml +8 -0
- package/tests/fixtures/bail/suite-a.yaml +6 -0
- package/tests/fixtures/bail/suite-b.yaml +6 -0
- package/tests/fixtures/crud.yaml +35 -0
- package/tests/fixtures/invalid-missing-name.yaml +5 -0
- package/tests/fixtures/invalid-no-method.yaml +6 -0
- package/tests/fixtures/petstore-auth.json +295 -0
- package/tests/fixtures/petstore-simple.json +151 -0
- package/tests/fixtures/post-only.yaml +12 -0
- package/tests/fixtures/simple.yaml +6 -0
- package/tests/fixtures/valid/.env.yaml +1 -0
- package/tests/fixtures/valid/a.yaml +5 -0
- package/tests/fixtures/valid/b.yml +5 -0
- package/tests/generator/coverage-scanner.test.ts +129 -0
- package/tests/generator/data-factory.test.ts +133 -0
- package/tests/generator/openapi-reader.test.ts +131 -0
- package/tests/integration/auth-flow.test.ts +217 -0
- package/tests/mcp/coverage-analysis.test.ts +64 -0
- package/tests/mcp/explore-api-schemas.test.ts +105 -0
- package/tests/mcp/explore-api.test.ts +49 -0
- package/tests/mcp/generate-missing-tests.test.ts +69 -0
- package/tests/mcp/manage-environment.test.ts +89 -0
- package/tests/mcp/save-test-suite.test.ts +116 -0
- package/tests/mcp/send-request.test.ts +79 -0
- package/tests/mcp/setup-api.test.ts +106 -0
- package/tests/mcp/tools.test.ts +248 -0
- package/tests/parser/schema.test.ts +134 -0
- package/tests/parser/variables.test.ts +227 -0
- package/tests/parser/yaml-parser.test.ts +69 -0
- package/tests/reporter/console.test.ts +256 -0
- package/tests/reporter/json.test.ts +98 -0
- package/tests/reporter/junit.test.ts +284 -0
- package/tests/runner/assertions.test.ts +262 -0
- package/tests/runner/executor.test.ts +310 -0
- package/tests/runner/http-client.test.ts +138 -0
- package/tests/web/routes.test.ts +160 -0
- package/tsconfig.json +31 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { getDb } from "../../db/schema.ts";
|
|
4
|
+
import { listCollections, listRuns, getRunById, getResultsByRunId } from "../../db/queries.ts";
|
|
5
|
+
|
|
6
|
+
export function registerQueryDbTool(server: McpServer, dbPath?: string) {
|
|
7
|
+
server.registerTool("query_db", {
|
|
8
|
+
description:
|
|
9
|
+
"Query the apitool database. Actions: list_collections (all APIs with run stats), " +
|
|
10
|
+
"list_runs (recent test runs), get_run_results (full detail for a run), " +
|
|
11
|
+
"diagnose_failure (only failed/errored steps for a run).",
|
|
12
|
+
inputSchema: {
|
|
13
|
+
action: z.enum(["list_collections", "list_runs", "get_run_results", "diagnose_failure"])
|
|
14
|
+
.describe("Query action to perform"),
|
|
15
|
+
runId: z.optional(z.number().int())
|
|
16
|
+
.describe("Run ID (required for get_run_results and diagnose_failure)"),
|
|
17
|
+
limit: z.optional(z.number().int().min(1).max(100))
|
|
18
|
+
.describe("Max number of runs to return (default: 20, only for list_runs)"),
|
|
19
|
+
},
|
|
20
|
+
}, async ({ action, runId, limit }) => {
|
|
21
|
+
try {
|
|
22
|
+
getDb(dbPath);
|
|
23
|
+
|
|
24
|
+
switch (action) {
|
|
25
|
+
case "list_collections": {
|
|
26
|
+
const collections = listCollections();
|
|
27
|
+
return {
|
|
28
|
+
content: [{ type: "text" as const, text: JSON.stringify(collections, null, 2) }],
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
case "list_runs": {
|
|
33
|
+
const runs = listRuns(limit ?? 20);
|
|
34
|
+
return {
|
|
35
|
+
content: [{ type: "text" as const, text: JSON.stringify(runs, null, 2) }],
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
case "get_run_results": {
|
|
40
|
+
if (runId == null) {
|
|
41
|
+
return {
|
|
42
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: "runId is required for get_run_results" }, null, 2) }],
|
|
43
|
+
isError: true,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
const run = getRunById(runId);
|
|
47
|
+
if (!run) {
|
|
48
|
+
return {
|
|
49
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: `Run ${runId} not found` }, null, 2) }],
|
|
50
|
+
isError: true,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
const results = getResultsByRunId(runId);
|
|
54
|
+
const detail = {
|
|
55
|
+
run: {
|
|
56
|
+
id: run.id,
|
|
57
|
+
started_at: run.started_at,
|
|
58
|
+
finished_at: run.finished_at,
|
|
59
|
+
total: run.total,
|
|
60
|
+
passed: run.passed,
|
|
61
|
+
failed: run.failed,
|
|
62
|
+
skipped: run.skipped,
|
|
63
|
+
trigger: run.trigger,
|
|
64
|
+
environment: run.environment,
|
|
65
|
+
duration_ms: run.duration_ms,
|
|
66
|
+
},
|
|
67
|
+
results: results.map(r => ({
|
|
68
|
+
suite_name: r.suite_name,
|
|
69
|
+
test_name: r.test_name,
|
|
70
|
+
status: r.status,
|
|
71
|
+
duration_ms: r.duration_ms,
|
|
72
|
+
request_method: r.request_method,
|
|
73
|
+
request_url: r.request_url,
|
|
74
|
+
response_status: r.response_status,
|
|
75
|
+
error_message: r.error_message,
|
|
76
|
+
assertions: r.assertions,
|
|
77
|
+
})),
|
|
78
|
+
};
|
|
79
|
+
return {
|
|
80
|
+
content: [{ type: "text" as const, text: JSON.stringify(detail, null, 2) }],
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
case "diagnose_failure": {
|
|
85
|
+
if (runId == null) {
|
|
86
|
+
return {
|
|
87
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: "runId is required for diagnose_failure" }, null, 2) }],
|
|
88
|
+
isError: true,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
const diagRun = getRunById(runId);
|
|
92
|
+
if (!diagRun) {
|
|
93
|
+
return {
|
|
94
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: `Run ${runId} not found` }, null, 2) }],
|
|
95
|
+
isError: true,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
const allResults = getResultsByRunId(runId);
|
|
99
|
+
const failures = allResults
|
|
100
|
+
.filter(r => r.status === "fail" || r.status === "error")
|
|
101
|
+
.map(r => ({
|
|
102
|
+
suite_name: r.suite_name,
|
|
103
|
+
test_name: r.test_name,
|
|
104
|
+
status: r.status,
|
|
105
|
+
error_message: r.error_message,
|
|
106
|
+
request_method: r.request_method,
|
|
107
|
+
request_url: r.request_url,
|
|
108
|
+
response_status: r.response_status,
|
|
109
|
+
response_headers: r.response_headers
|
|
110
|
+
? JSON.parse(r.response_headers)
|
|
111
|
+
: undefined,
|
|
112
|
+
assertions: r.assertions,
|
|
113
|
+
duration_ms: r.duration_ms,
|
|
114
|
+
}));
|
|
115
|
+
const result = {
|
|
116
|
+
run: {
|
|
117
|
+
id: diagRun.id,
|
|
118
|
+
started_at: diagRun.started_at,
|
|
119
|
+
environment: diagRun.environment,
|
|
120
|
+
duration_ms: diagRun.duration_ms,
|
|
121
|
+
},
|
|
122
|
+
summary: {
|
|
123
|
+
total: diagRun.total,
|
|
124
|
+
passed: diagRun.passed,
|
|
125
|
+
failed: diagRun.failed,
|
|
126
|
+
},
|
|
127
|
+
failures,
|
|
128
|
+
};
|
|
129
|
+
return {
|
|
130
|
+
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
} catch (err) {
|
|
135
|
+
return {
|
|
136
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
137
|
+
isError: true,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { executeRun } from "../../core/runner/execute-run.ts";
|
|
4
|
+
|
|
5
|
+
export function registerRunTestsTool(server: McpServer, dbPath?: string) {
|
|
6
|
+
server.registerTool("run_tests", {
|
|
7
|
+
description: "Execute API tests from a YAML file or directory and return results summary with failures. " +
|
|
8
|
+
"Use after saving test suites with save_test_suite. Check query_db(action: 'diagnose_failure') for detailed failure analysis.",
|
|
9
|
+
inputSchema: {
|
|
10
|
+
testPath: z.string().describe("Path to test YAML file or directory"),
|
|
11
|
+
envName: z.optional(z.string()).describe("Environment name (loads .env.<name>.yaml)"),
|
|
12
|
+
safe: z.optional(z.boolean()).describe("Run only GET tests (read-only, safe mode)"),
|
|
13
|
+
tag: z.optional(z.array(z.string())).describe("Filter suites by tag (OR logic)"),
|
|
14
|
+
},
|
|
15
|
+
}, async ({ testPath, envName, safe, tag }) => {
|
|
16
|
+
const { runId, results } = await executeRun({
|
|
17
|
+
testPath,
|
|
18
|
+
envName,
|
|
19
|
+
trigger: "mcp",
|
|
20
|
+
dbPath,
|
|
21
|
+
safe,
|
|
22
|
+
tag,
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const total = results.reduce((s, r) => s + r.total, 0);
|
|
26
|
+
const passed = results.reduce((s, r) => s + r.passed, 0);
|
|
27
|
+
const failed = results.reduce((s, r) => s + r.failed, 0);
|
|
28
|
+
const skipped = results.reduce((s, r) => s + r.skipped, 0);
|
|
29
|
+
|
|
30
|
+
const failedSteps = results.flatMap(r =>
|
|
31
|
+
r.steps.filter(s => s.status === "fail" || s.status === "error").map(s => ({
|
|
32
|
+
suite: r.suite_name,
|
|
33
|
+
test: s.name,
|
|
34
|
+
status: s.status,
|
|
35
|
+
error: s.error,
|
|
36
|
+
assertions: s.assertions.filter(a => !a.passed).map(a => ({
|
|
37
|
+
field: a.field,
|
|
38
|
+
expected: a.expected,
|
|
39
|
+
actual: a.actual,
|
|
40
|
+
})),
|
|
41
|
+
}))
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
const hints: string[] = [];
|
|
45
|
+
if (failedSteps.length > 0) {
|
|
46
|
+
hints.push("Use query_db(action: 'diagnose_failure', runId: " + runId + ") for detailed failure analysis");
|
|
47
|
+
}
|
|
48
|
+
hints.push("Ask the user if they want to set up CI/CD to run these tests automatically on push. If yes, use ci_init to generate a workflow and help them push to GitHub/GitLab.");
|
|
49
|
+
|
|
50
|
+
const summary = {
|
|
51
|
+
runId,
|
|
52
|
+
total,
|
|
53
|
+
passed,
|
|
54
|
+
failed,
|
|
55
|
+
skipped,
|
|
56
|
+
suites: results.length,
|
|
57
|
+
status: failed === 0 ? "all_passed" : "has_failures",
|
|
58
|
+
...(failedSteps.length > 0 ? { failures: failedSteps } : {}),
|
|
59
|
+
hints,
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
content: [{ type: "text" as const, text: JSON.stringify(summary, null, 2) }],
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { validateSuite } from "../../core/parser/schema.ts";
|
|
4
|
+
import { join, dirname } from "node:path";
|
|
5
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
6
|
+
import YAML from "yaml";
|
|
7
|
+
|
|
8
|
+
export function registerSaveTestSuiteTool(server: McpServer, dbPath?: string) {
|
|
9
|
+
server.registerTool("save_test_suite", {
|
|
10
|
+
description: "Save a YAML test suite file with validation. Parses and validates the YAML content " +
|
|
11
|
+
"before writing. Returns structured errors if validation fails so you can fix and retry. " +
|
|
12
|
+
"Use after generating test content with generate_tests_guide.",
|
|
13
|
+
inputSchema: {
|
|
14
|
+
filePath: z.string().describe("Path for saving the YAML test file (e.g. apis/petstore/tests/pets-crud.yaml)"),
|
|
15
|
+
content: z.string().describe("YAML content of the test suite"),
|
|
16
|
+
overwrite: z.optional(z.boolean()).describe("Overwrite existing file (default: false)"),
|
|
17
|
+
},
|
|
18
|
+
}, async ({ filePath, content, overwrite }) => {
|
|
19
|
+
try {
|
|
20
|
+
// Parse YAML
|
|
21
|
+
let parsed: unknown;
|
|
22
|
+
try {
|
|
23
|
+
parsed = YAML.parse(content);
|
|
24
|
+
} catch (err) {
|
|
25
|
+
return {
|
|
26
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
27
|
+
saved: false,
|
|
28
|
+
error: `YAML parse error: ${(err as Error).message}`,
|
|
29
|
+
hint: "Check YAML syntax — indentation, colons, and quoting",
|
|
30
|
+
}, null, 2) }],
|
|
31
|
+
isError: true,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Validate against test suite schema
|
|
36
|
+
try {
|
|
37
|
+
validateSuite(parsed);
|
|
38
|
+
} catch (err) {
|
|
39
|
+
const message = (err as Error).message;
|
|
40
|
+
// Extract Zod error details
|
|
41
|
+
let hint = "Check the test suite structure matches the expected format";
|
|
42
|
+
if (message.includes("status")) {
|
|
43
|
+
hint = "Status codes must be numbers, not strings (e.g. status: 200, not status: \"200\")";
|
|
44
|
+
} else if (message.includes("exists")) {
|
|
45
|
+
hint = "exists must be boolean true/false, not string \"true\"/\"false\"";
|
|
46
|
+
} else if (message.includes("tests")) {
|
|
47
|
+
hint = "Suite must have a 'tests' array with at least one test step";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
52
|
+
saved: false,
|
|
53
|
+
error: `Validation: ${message}`,
|
|
54
|
+
hint,
|
|
55
|
+
}, null, 2) }],
|
|
56
|
+
isError: true,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Detect hardcoded credentials — long opaque strings in auth headers
|
|
61
|
+
const credentialPattern = /Authorization\s*:\s*["']?(Basic|Bearer)\s+([A-Za-z0-9+/=_\-]{20,})["']?/g;
|
|
62
|
+
const credMatches = [...content.matchAll(credentialPattern)];
|
|
63
|
+
const suspiciousCredentials = credMatches.filter(m => {
|
|
64
|
+
const value = m[2]!;
|
|
65
|
+
// Allow {{variable}} references — flag only literal tokens
|
|
66
|
+
return !value.startsWith("{{") && !value.endsWith("}}");
|
|
67
|
+
});
|
|
68
|
+
if (suspiciousCredentials.length > 0) {
|
|
69
|
+
return {
|
|
70
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
71
|
+
saved: false,
|
|
72
|
+
error: "Hardcoded credentials detected in Authorization header(s)",
|
|
73
|
+
hint: "Never put literal API keys or tokens in YAML files. Store them in the environment instead: use manage_environment(action: \"set\", name: \"default\", collectionName: \"...\", variables: {\"api_key\": \"...\"}) and reference as {{api_key}} in headers.",
|
|
74
|
+
detected: suspiciousCredentials.map(m => `${m[1]} <redacted>`),
|
|
75
|
+
}, null, 2) }],
|
|
76
|
+
isError: true,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Resolve path
|
|
81
|
+
const resolvedPath = filePath.startsWith("/") || /^[a-zA-Z]:/.test(filePath)
|
|
82
|
+
? filePath
|
|
83
|
+
: join(process.cwd(), filePath);
|
|
84
|
+
|
|
85
|
+
// Check existing file
|
|
86
|
+
if (!overwrite && existsSync(resolvedPath)) {
|
|
87
|
+
return {
|
|
88
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
89
|
+
saved: false,
|
|
90
|
+
error: `File already exists: ${resolvedPath}`,
|
|
91
|
+
hint: "Use overwrite: true to replace the existing file",
|
|
92
|
+
}, null, 2) }],
|
|
93
|
+
isError: true,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Create directories
|
|
98
|
+
const dir = dirname(resolvedPath);
|
|
99
|
+
if (!existsSync(dir)) {
|
|
100
|
+
mkdirSync(dir, { recursive: true });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Write original YAML content (preserve formatting/comments)
|
|
104
|
+
await Bun.write(resolvedPath, content);
|
|
105
|
+
|
|
106
|
+
// Extract summary info
|
|
107
|
+
const suite = parsed as Record<string, unknown>;
|
|
108
|
+
const tests = (suite.tests as unknown[]) ?? [];
|
|
109
|
+
|
|
110
|
+
const result: Record<string, unknown> = {
|
|
111
|
+
saved: true,
|
|
112
|
+
filePath: resolvedPath,
|
|
113
|
+
suite: {
|
|
114
|
+
name: suite.name,
|
|
115
|
+
tests: tests.length,
|
|
116
|
+
base_url: suite.base_url ?? null,
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
// CI hint
|
|
121
|
+
result.hint = "After tests are ready, ask the user if they want to set up CI/CD with ci_init to run tests automatically on push.";
|
|
122
|
+
|
|
123
|
+
// Attempt to compute coverage hint
|
|
124
|
+
try {
|
|
125
|
+
const testDir = dirname(resolvedPath);
|
|
126
|
+
const { findCollectionByTestPath } = await import("../../db/queries.ts");
|
|
127
|
+
const { getDb } = await import("../../db/schema.ts");
|
|
128
|
+
getDb(dbPath);
|
|
129
|
+
const collection = findCollectionByTestPath(testDir);
|
|
130
|
+
if (collection?.openapi_spec) {
|
|
131
|
+
const { readOpenApiSpec, extractEndpoints } = await import("../../core/generator/openapi-reader.ts");
|
|
132
|
+
const { scanCoveredEndpoints, filterUncoveredEndpoints } = await import("../../core/generator/coverage-scanner.ts");
|
|
133
|
+
|
|
134
|
+
const doc = await readOpenApiSpec(collection.openapi_spec);
|
|
135
|
+
const allEndpoints = extractEndpoints(doc);
|
|
136
|
+
const covered = await scanCoveredEndpoints(testDir);
|
|
137
|
+
const uncovered = filterUncoveredEndpoints(allEndpoints, covered);
|
|
138
|
+
|
|
139
|
+
const total = allEndpoints.length;
|
|
140
|
+
const coveredCount = total - uncovered.length;
|
|
141
|
+
const percentage = total > 0 ? Math.round((coveredCount / total) * 100) : 0;
|
|
142
|
+
|
|
143
|
+
const coverage: Record<string, unknown> = { percentage, covered: coveredCount, total, uncoveredCount: uncovered.length };
|
|
144
|
+
if (percentage < 80 && uncovered.length > 0) {
|
|
145
|
+
coverage.suggestion = `Use generate_missing_tests to cover ${uncovered.length} remaining endpoint${uncovered.length > 1 ? "s" : ""}`;
|
|
146
|
+
}
|
|
147
|
+
result.coverage = coverage;
|
|
148
|
+
}
|
|
149
|
+
} catch { /* silently skip coverage if unavailable */ }
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
153
|
+
};
|
|
154
|
+
} catch (err) {
|
|
155
|
+
return {
|
|
156
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
157
|
+
saved: false,
|
|
158
|
+
error: (err as Error).message,
|
|
159
|
+
}, null, 2) }],
|
|
160
|
+
isError: true,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { executeRequest } from "../../core/runner/http-client.ts";
|
|
4
|
+
import { loadEnvironment, substituteString, substituteDeep } from "../../core/parser/variables.ts";
|
|
5
|
+
|
|
6
|
+
export function registerSendRequestTool(server: McpServer) {
|
|
7
|
+
server.registerTool("send_request", {
|
|
8
|
+
description: "Send an ad-hoc HTTP request. Supports variable interpolation from environments (e.g. {{base_url}}).",
|
|
9
|
+
inputSchema: {
|
|
10
|
+
method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]).describe("HTTP method"),
|
|
11
|
+
url: z.string().describe("Request URL (supports {{variable}} interpolation)"),
|
|
12
|
+
headers: z.optional(z.string()).describe("Request headers as JSON string (e.g. '{\"Content-Type\": \"application/json\"}')"),
|
|
13
|
+
body: z.optional(z.string()).describe("Request body (JSON string)"),
|
|
14
|
+
timeout: z.optional(z.number().int().positive()).describe("Request timeout in ms"),
|
|
15
|
+
envName: z.optional(z.string()).describe("Environment name for variable interpolation"),
|
|
16
|
+
},
|
|
17
|
+
}, async ({ method, url, headers, body, timeout, envName }) => {
|
|
18
|
+
try {
|
|
19
|
+
const vars = await loadEnvironment(envName);
|
|
20
|
+
|
|
21
|
+
const resolvedUrl = substituteString(url, vars) as string;
|
|
22
|
+
const parsedHeaders = headers ? JSON.parse(headers) as Record<string, string> : {};
|
|
23
|
+
const resolvedHeaders = Object.keys(parsedHeaders).length > 0 ? substituteDeep(parsedHeaders, vars) : {};
|
|
24
|
+
const resolvedBody = body ? substituteString(body, vars) as string : undefined;
|
|
25
|
+
|
|
26
|
+
const response = await executeRequest(
|
|
27
|
+
{
|
|
28
|
+
method,
|
|
29
|
+
url: resolvedUrl,
|
|
30
|
+
headers: resolvedHeaders,
|
|
31
|
+
body: resolvedBody,
|
|
32
|
+
},
|
|
33
|
+
timeout ? { timeout } : undefined,
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
const result = {
|
|
37
|
+
status: response.status,
|
|
38
|
+
headers: response.headers,
|
|
39
|
+
body: response.body_parsed ?? response.body,
|
|
40
|
+
duration_ms: response.duration_ms,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
45
|
+
};
|
|
46
|
+
} catch (err) {
|
|
47
|
+
return {
|
|
48
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
49
|
+
isError: true,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { setupApi } from "../../core/setup-api.ts";
|
|
4
|
+
|
|
5
|
+
export function registerSetupApiTool(server: McpServer, dbPath?: string) {
|
|
6
|
+
server.registerTool("setup_api", {
|
|
7
|
+
description: "Register a new API for testing. Creates directory structure, reads OpenAPI spec, " +
|
|
8
|
+
"sets up environment variables, and creates a collection in the database. " +
|
|
9
|
+
"Use this before generating tests for a new API.",
|
|
10
|
+
inputSchema: {
|
|
11
|
+
name: z.string().describe("API name (e.g. 'petstore')"),
|
|
12
|
+
specPath: z.optional(z.string()).describe("Path or URL to OpenAPI spec"),
|
|
13
|
+
dir: z.optional(z.string()).describe("Base directory (default: ./apis/<name>/)"),
|
|
14
|
+
envVars: z.optional(z.string()).describe("Environment variables as JSON string (e.g. '{\"base_url\": \"...\", \"token\": \"...\"}')"),
|
|
15
|
+
force: z.optional(z.boolean()).describe("If true, delete existing API with same name and recreate from scratch"),
|
|
16
|
+
},
|
|
17
|
+
}, async ({ name, specPath, dir, envVars, force }) => {
|
|
18
|
+
try {
|
|
19
|
+
let parsedEnvVars: Record<string, string> | undefined;
|
|
20
|
+
if (envVars) {
|
|
21
|
+
try {
|
|
22
|
+
parsedEnvVars = JSON.parse(envVars);
|
|
23
|
+
} catch {
|
|
24
|
+
return {
|
|
25
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: "envVars must be a valid JSON string" }, null, 2) }],
|
|
26
|
+
isError: true,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const result = await setupApi({
|
|
31
|
+
name,
|
|
32
|
+
spec: specPath,
|
|
33
|
+
dir,
|
|
34
|
+
envVars: parsedEnvVars,
|
|
35
|
+
dbPath,
|
|
36
|
+
force,
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
41
|
+
};
|
|
42
|
+
} catch (err) {
|
|
43
|
+
return {
|
|
44
|
+
content: [{ type: "text" as const, text: JSON.stringify({ error: (err as Error).message }, null, 2) }],
|
|
45
|
+
isError: true,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { parse } from "../../core/parser/yaml-parser.ts";
|
|
4
|
+
|
|
5
|
+
export function registerValidateTestsTool(server: McpServer) {
|
|
6
|
+
server.registerTool("validate_tests", {
|
|
7
|
+
description: "Validate YAML test files without running them. Returns parsed suite info or validation errors.",
|
|
8
|
+
inputSchema: {
|
|
9
|
+
testPath: z.string().describe("Path to test YAML file or directory"),
|
|
10
|
+
},
|
|
11
|
+
}, async ({ testPath }) => {
|
|
12
|
+
try {
|
|
13
|
+
const suites = await parse(testPath);
|
|
14
|
+
const totalSteps = suites.reduce((sum, s) => sum + s.tests.length, 0);
|
|
15
|
+
|
|
16
|
+
const result = {
|
|
17
|
+
valid: true,
|
|
18
|
+
suites: suites.length,
|
|
19
|
+
tests: totalSteps,
|
|
20
|
+
details: suites.map(s => ({
|
|
21
|
+
name: s.name,
|
|
22
|
+
tests: s.tests.length,
|
|
23
|
+
tags: s.tags,
|
|
24
|
+
description: s.description,
|
|
25
|
+
source: (s as any)._source,
|
|
26
|
+
})),
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
|
|
31
|
+
};
|
|
32
|
+
} catch (err) {
|
|
33
|
+
return {
|
|
34
|
+
content: [{ type: "text" as const, text: JSON.stringify({
|
|
35
|
+
valid: false,
|
|
36
|
+
error: (err as Error).message,
|
|
37
|
+
}, null, 2) }],
|
|
38
|
+
isError: true,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { createInterface } from "readline";
|
|
2
|
+
import { runAgentTurn } from "../core/agent/agent-loop.ts";
|
|
3
|
+
import { trimContext } from "../core/agent/context-manager.ts";
|
|
4
|
+
import type { AgentConfig, ToolEvent } from "../core/agent/types.ts";
|
|
5
|
+
import type { ModelMessage } from "ai";
|
|
6
|
+
|
|
7
|
+
// ── ANSI helpers ──
|
|
8
|
+
const RESET = "\x1b[0m";
|
|
9
|
+
const BOLD = "\x1b[1m";
|
|
10
|
+
const DIM = "\x1b[2m";
|
|
11
|
+
const CYAN = "\x1b[36m";
|
|
12
|
+
const GREEN = "\x1b[32m";
|
|
13
|
+
const YELLOW = "\x1b[33m";
|
|
14
|
+
const MAGENTA = "\x1b[35m";
|
|
15
|
+
|
|
16
|
+
// Mouse reporting escape sequences
|
|
17
|
+
const DISABLE_MOUSE = "\x1b[?1000l\x1b[?1002l\x1b[?1003l\x1b[?1006l";
|
|
18
|
+
// Regex to strip mouse escape sequences from input
|
|
19
|
+
// Covers SGR extended, X10, urxvt mouse protocols
|
|
20
|
+
const MOUSE_SEQ_RE = /\x1b\[\<[\d;]*[mM]|\x1b\[M[\s\S]{3}|\x1b\[\d+;\d+;\d+M/g;
|
|
21
|
+
|
|
22
|
+
function printToolEvent(event: ToolEvent) {
|
|
23
|
+
const args = Object.entries(event.args)
|
|
24
|
+
.filter(([, v]) => v !== undefined)
|
|
25
|
+
.map(([k, v]) => `${k}=${JSON.stringify(v)}`)
|
|
26
|
+
.slice(0, 3)
|
|
27
|
+
.join(", ");
|
|
28
|
+
|
|
29
|
+
const result = event.result as Record<string, unknown> | null;
|
|
30
|
+
let summary = "done";
|
|
31
|
+
if (result) {
|
|
32
|
+
if ("error" in result) summary = `error: ${result.error}`;
|
|
33
|
+
else if ("status" in result) summary = String(result.status);
|
|
34
|
+
else if ("valid" in result) summary = result.valid ? "valid" : "invalid";
|
|
35
|
+
else if ("runId" in result) summary = `run #${result.runId}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
process.stdout.write(` ${MAGENTA}↳ ${event.toolName}${RESET}${DIM}(${args})${RESET} → ${summary}\n`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function startChatUI(config: AgentConfig): Promise<void> {
|
|
42
|
+
const { provider } = config.provider;
|
|
43
|
+
const model = config.provider.model;
|
|
44
|
+
const safeLabel = config.safeMode ? ` ${YELLOW}[SAFE]${RESET}` : "";
|
|
45
|
+
|
|
46
|
+
// Disable mouse reporting that may be left over from other TUI apps
|
|
47
|
+
process.stdout.write(DISABLE_MOUSE);
|
|
48
|
+
|
|
49
|
+
console.log(`\n${BOLD}${CYAN}apitool chat${RESET} — ${provider}/${model}${safeLabel}`);
|
|
50
|
+
console.log(`${DIM}Commands: /clear, /tokens, /quit | Ctrl+C to exit${RESET}\n`);
|
|
51
|
+
|
|
52
|
+
const messages: ModelMessage[] = [];
|
|
53
|
+
let totalIn = 0;
|
|
54
|
+
let totalOut = 0;
|
|
55
|
+
let busy = false;
|
|
56
|
+
|
|
57
|
+
const rl = createInterface({
|
|
58
|
+
input: process.stdin,
|
|
59
|
+
output: process.stdout,
|
|
60
|
+
prompt: `${BOLD}${CYAN}> ${RESET}`,
|
|
61
|
+
terminal: true,
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
rl.prompt();
|
|
65
|
+
|
|
66
|
+
rl.on("line", async (raw: string) => {
|
|
67
|
+
// Strip mouse escape sequences that leak into input
|
|
68
|
+
const text = raw.replace(MOUSE_SEQ_RE, "").trim();
|
|
69
|
+
|
|
70
|
+
if (!text || busy) {
|
|
71
|
+
if (!busy) rl.prompt();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (text === "/quit" || text === "/exit") {
|
|
76
|
+
rl.close();
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (text === "/clear") {
|
|
81
|
+
messages.length = 0;
|
|
82
|
+
totalIn = 0;
|
|
83
|
+
totalOut = 0;
|
|
84
|
+
console.log(`${DIM}Conversation cleared.${RESET}\n`);
|
|
85
|
+
rl.prompt();
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (text === "/tokens") {
|
|
90
|
+
console.log(`${DIM}Tokens: ${totalIn} in / ${totalOut} out${RESET}\n`);
|
|
91
|
+
rl.prompt();
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
busy = true;
|
|
96
|
+
messages.push({ role: "user", content: text });
|
|
97
|
+
|
|
98
|
+
// Trim context if conversation is long
|
|
99
|
+
const trimmed = trimContext(
|
|
100
|
+
messages.map((m) => ({
|
|
101
|
+
role: m.role as "user" | "assistant",
|
|
102
|
+
content: typeof m.content === "string" ? m.content : "",
|
|
103
|
+
})),
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
process.stdout.write(`\n${DIM}thinking...${RESET}`);
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
const result = await runAgentTurn(
|
|
110
|
+
trimmed.map((m) => ({ role: m.role, content: m.content })),
|
|
111
|
+
config,
|
|
112
|
+
(event) => {
|
|
113
|
+
process.stdout.write(`\r\x1b[K`);
|
|
114
|
+
printToolEvent(event);
|
|
115
|
+
process.stdout.write(`${DIM}thinking...${RESET}`);
|
|
116
|
+
},
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
process.stdout.write(`\r\x1b[K`);
|
|
120
|
+
|
|
121
|
+
const responseText = result.text || "(no response)";
|
|
122
|
+
console.log(`\n${GREEN}${BOLD}AI${RESET} ${responseText}\n`);
|
|
123
|
+
|
|
124
|
+
totalIn += result.inputTokens;
|
|
125
|
+
totalOut += result.outputTokens;
|
|
126
|
+
messages.push({ role: "assistant", content: responseText });
|
|
127
|
+
} catch (err) {
|
|
128
|
+
process.stdout.write(`\r\x1b[K`);
|
|
129
|
+
console.log(`\n${YELLOW}Error: ${(err as Error).message}${RESET}\n`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
busy = false;
|
|
133
|
+
rl.prompt();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
rl.on("close", () => {
|
|
137
|
+
// Disable mouse reporting on exit too
|
|
138
|
+
process.stdout.write(DISABLE_MOUSE);
|
|
139
|
+
console.log(`\n${DIM}Bye! (${totalIn} in / ${totalOut} out tokens)${RESET}`);
|
|
140
|
+
process.exit(0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// Also disable mouse on SIGINT
|
|
144
|
+
process.on("SIGINT", () => {
|
|
145
|
+
process.stdout.write(DISABLE_MOUSE);
|
|
146
|
+
rl.close();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
await new Promise<void>(() => {});
|
|
150
|
+
}
|