@skyramp/mcp 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/startTraceCollectionPrompts.js +0 -3
- package/build/prompts/testGenerationPrompt.js +0 -3
- package/build/services/TestGenerationService.js +55 -113
- package/build/tools/executeSkyrampTestTool.js +25 -5
- package/build/tools/generateContractRestTool.js +6 -1
- package/build/tools/generateE2ERestTool.js +8 -1
- package/build/tools/generateFuzzRestTool.js +6 -1
- package/build/tools/generateIntegrationRestTool.js +6 -1
- package/build/tools/generateLoadRestTool.js +23 -14
- package/build/tools/generateSmokeRestTool.js +18 -2
- package/build/tools/generateUIRestTool.js +5 -1
- package/build/tools/loginTool.js +5 -1
- package/build/tools/logoutTool.js +4 -1
- package/build/tools/startTraceCollectionTool.js +33 -3
- package/build/tools/stopTraceCollectionTool.js +27 -1
- package/build/types/TestTypes.js +25 -27
- package/build/utils/language-helper.js +69 -0
- package/build/utils/modularization-prompts.js +15 -0
- package/package.json +1 -1
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
// src/prompts/skyrampPrompt.ts
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { logger } from "../utils/logger.js";
|
|
4
|
-
// read env variable and use it to set custom prompt
|
|
5
|
-
const isDockerEnv = process.env.MCP_SERVER_RUNTIME === "docker";
|
|
6
|
-
logger.info("MCP server runtime environment", { isDockerEnv });
|
|
7
4
|
export function registerStartTraceCollectionPrompt(mcpServer) {
|
|
8
5
|
logger.info("registering start trace collection prompt");
|
|
9
6
|
mcpServer.prompt("skyramp_trace_prompt", "Skyramp trace collection prompt", {
|
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
// src/prompts/skyrampPrompt.ts
|
|
2
2
|
import { logger } from "../utils/logger.js";
|
|
3
|
-
// read env variable and use it to set custom prompt
|
|
4
|
-
const isDockerEnv = process.env.MCP_SERVER_RUNTIME === "docker";
|
|
5
|
-
logger.info("MCP server runtime environment", { isDockerEnv });
|
|
6
3
|
export function registerTestGenerationPrompt(mcpServer) {
|
|
7
4
|
logger.info("test generation prompt");
|
|
8
5
|
mcpServer.prompt("skyramp_test_generation_prompt", "Skyramp test generation prompt", {}, () => ({
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { SkyrampClient } from "@skyramp/skyramp";
|
|
2
2
|
import { analyzeOpenAPIWithGivenEndpoint } from "../utils/analyze-openapi.js";
|
|
3
3
|
import { getPathParameterValidationError, OUTPUT_DIR_FIELD_NAME, PATH_PARAMS_FIELD_NAME, QUERY_PARAMS_FIELD_NAME, FORM_PARAMS_FIELD_NAME, validateParams, validatePath, validateRequestData, TELEMETRY_entrypoint_FIELD_NAME, } from "../utils/utils.js";
|
|
4
|
+
import { getLanguageSteps } from "../utils/language-helper.js";
|
|
5
|
+
import { getModularizationPrompt } from "../utils/modularization-prompts.js";
|
|
4
6
|
export class TestGenerationService {
|
|
5
7
|
client;
|
|
6
8
|
constructor() {
|
|
@@ -13,100 +15,21 @@ export class TestGenerationService {
|
|
|
13
15
|
return validationResult;
|
|
14
16
|
}
|
|
15
17
|
const generateOptions = this.buildGenerationOptions(params);
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
// TODO- understand and make the logic better
|
|
19
|
+
// const apiAnalysisResult = await this.handleApiAnalysis(
|
|
20
|
+
// params,
|
|
21
|
+
// generateOptions
|
|
22
|
+
// );
|
|
23
|
+
// if (apiAnalysisResult) {
|
|
24
|
+
// return apiAnalysisResult;
|
|
25
|
+
// }
|
|
20
26
|
const result = await this.executeGeneration(generateOptions);
|
|
21
|
-
let modularizedResultPrompt = `
|
|
22
|
-
**IMPORTANT: Now please modularize the code by**
|
|
23
|
-
|
|
24
|
-
1. Group related lines logically (e.g., login steps together) to improve readability.
|
|
25
|
-
2. Parameterize the test function(s) with meaningful parameters to make it more flexible. Preserve the original order of code lines within each function
|
|
26
|
-
3. Do not change or remove any existing lines of code, variable names, function names, or function structures.
|
|
27
|
-
4. Only extract reusable logical sections into helper functions or parameterized structures where applicable.
|
|
28
|
-
5. Do not create any new test cases but only helper functions.
|
|
29
|
-
6. Make sure the logic is same and the test works as expected.
|
|
30
|
-
7. Do not add any new logic or classes or any other files.
|
|
31
|
-
|
|
32
|
-
Please refactor the code with these modularization principles.
|
|
33
|
-
`;
|
|
34
|
-
let languageSteps = "";
|
|
35
|
-
switch (params.language) {
|
|
36
|
-
case "python":
|
|
37
|
-
// python steps
|
|
38
|
-
languageSteps += `
|
|
39
|
-
# Install skyramp dependency
|
|
40
|
-
pip3 install skyramp
|
|
41
|
-
`;
|
|
42
|
-
// e2e and ui tests
|
|
43
|
-
if (params.testType == "e2e" || params.testType == "ui") {
|
|
44
|
-
languageSteps += `
|
|
45
|
-
# Install playwright dependencies
|
|
46
|
-
pip3 install pytest-playwright
|
|
47
|
-
# Execute the test file
|
|
48
|
-
python3 -m pytest --browser chromium <test-file>.py
|
|
49
|
-
`;
|
|
50
|
-
}
|
|
51
|
-
//framework pytest
|
|
52
|
-
languageSteps += `
|
|
53
|
-
# Install pytest dependency
|
|
54
|
-
pip3 install pytest
|
|
55
|
-
|
|
56
|
-
# Execution test file
|
|
57
|
-
python3 -m pytest <test-file>.py
|
|
58
|
-
`;
|
|
59
|
-
// robot framework
|
|
60
|
-
languageSteps += `
|
|
61
|
-
# Install robotframework dependency
|
|
62
|
-
pip3 install robotframework
|
|
63
|
-
|
|
64
|
-
# Execution of test file
|
|
65
|
-
python3 -m robot <robot-file>.robot
|
|
66
|
-
`;
|
|
67
|
-
break;
|
|
68
|
-
case "javascript":
|
|
69
|
-
languageSteps += `
|
|
70
|
-
# install dependencies for package.json
|
|
71
|
-
npm install @skyramp/skyramp @playwright/test
|
|
72
|
-
|
|
73
|
-
# Execution of test file
|
|
74
|
-
npx playwright test <test-file>.spec.js --reporter=list
|
|
75
|
-
`;
|
|
76
|
-
break;
|
|
77
|
-
case "typescript":
|
|
78
|
-
languageSteps += `
|
|
79
|
-
# install dependencies for package.json
|
|
80
|
-
npm install @skyramp/skyramp @playwright/test
|
|
81
|
-
|
|
82
|
-
# Execution of test file
|
|
83
|
-
npx playwright test <test-file>.spec.ts --reporter=list
|
|
84
|
-
`;
|
|
85
|
-
break;
|
|
86
|
-
case "java":
|
|
87
|
-
languageSteps += `
|
|
88
|
-
# Prerequisites
|
|
89
|
-
# Compile the test file
|
|
90
|
-
javac -cp "./lib/*" -d target <test-file>.java
|
|
91
|
-
|
|
92
|
-
# Execution of test file
|
|
93
|
-
JUNIT_PLATFORM_VER="1.9.3"
|
|
94
|
-
classpath="target:$(echo ./lib/*.jar | tr ' ' ':')"
|
|
95
|
-
java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \\
|
|
96
|
-
--classpath "\$classpath" \\
|
|
97
|
-
--include-engine=junit-jupiter \\
|
|
98
|
-
--select-class=<test-file> \\
|
|
99
|
-
--reports-dir=target/test-results
|
|
100
|
-
`;
|
|
101
|
-
break;
|
|
102
|
-
}
|
|
103
|
-
let isError = false;
|
|
104
|
-
// check the result include 'Successfully generated/updated tests files:'
|
|
105
|
-
if (!result.includes("Successfully generated/updated tests files:")) {
|
|
106
|
-
isError = true;
|
|
107
|
-
}
|
|
108
27
|
const testType = this.getTestType();
|
|
109
|
-
const shouldModularize = testType == "ui" || testType == "e2e";
|
|
28
|
+
const shouldModularize = testType == "ui" || testType == "e2e" || testType == "integration";
|
|
29
|
+
const languageSteps = getLanguageSteps({
|
|
30
|
+
language: params.language || "",
|
|
31
|
+
testType: params.testType,
|
|
32
|
+
});
|
|
110
33
|
return {
|
|
111
34
|
content: [
|
|
112
35
|
{
|
|
@@ -114,7 +37,7 @@ java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \
|
|
|
114
37
|
text: `${shouldModularize
|
|
115
38
|
? "Generated the test and WILL NOW MODULARIZE THE CODE USING THE FOLLOWING MODULARIZATION PRINCIPLES"
|
|
116
39
|
: "Generated the test and DO NOT MODULARIZE THE CODE"}
|
|
117
|
-
${result} ${shouldModularize ?
|
|
40
|
+
${result} ${shouldModularize ? getModularizationPrompt() : ""}
|
|
118
41
|
${languageSteps}
|
|
119
42
|
|
|
120
43
|
- Notify the user that they can also use **Execute with Skyramp** tool to run the test. If this is your first test you are trying to execute with Skyramp, it may take a little bit longer than usual.
|
|
@@ -122,11 +45,20 @@ java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \
|
|
|
122
45
|
Authentication: Please provide me with your authentication token (by default we assume a Bearer Token). If you don't require any token for your service please just say: 'No token required'`,
|
|
123
46
|
},
|
|
124
47
|
],
|
|
125
|
-
isError:
|
|
48
|
+
isError: false,
|
|
126
49
|
};
|
|
127
50
|
}
|
|
128
51
|
catch (error) {
|
|
129
|
-
|
|
52
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
53
|
+
return {
|
|
54
|
+
content: [
|
|
55
|
+
{
|
|
56
|
+
type: "text",
|
|
57
|
+
text: errorMessage,
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
isError: true,
|
|
61
|
+
};
|
|
130
62
|
}
|
|
131
63
|
}
|
|
132
64
|
validateInputs(params) {
|
|
@@ -154,7 +86,7 @@ java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \
|
|
|
154
86
|
: errList;
|
|
155
87
|
}
|
|
156
88
|
async handleApiAnalysis(params, generateOptions) {
|
|
157
|
-
if (params.apiSchema && params.endpointURL) {
|
|
89
|
+
if (params.apiSchema && params.endpointURL && params.method != "") {
|
|
158
90
|
try {
|
|
159
91
|
const requiredPathParams = await analyzeOpenAPIWithGivenEndpoint(params.apiSchema, params.endpointURL, params.pathParams ?? "");
|
|
160
92
|
if (requiredPathParams && requiredPathParams.trim().length > 0) {
|
|
@@ -170,33 +102,43 @@ java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \
|
|
|
170
102
|
}
|
|
171
103
|
}
|
|
172
104
|
catch (error) {
|
|
173
|
-
|
|
105
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
106
|
+
return {
|
|
107
|
+
content: [
|
|
108
|
+
{
|
|
109
|
+
type: "text",
|
|
110
|
+
text: `Error analyzing OpenAPI schema: ${errorMessage}`,
|
|
111
|
+
},
|
|
112
|
+
],
|
|
113
|
+
isError: true,
|
|
114
|
+
};
|
|
174
115
|
}
|
|
175
116
|
}
|
|
176
117
|
return null;
|
|
177
118
|
}
|
|
178
119
|
async executeGeneration(generateOptions) {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
120
|
+
try {
|
|
121
|
+
const result = await this.client.generateRestTest(generateOptions);
|
|
122
|
+
// Check if the result indicates failure
|
|
123
|
+
if (result && result.length > 0) {
|
|
124
|
+
const lowerResult = result.toLowerCase();
|
|
125
|
+
if (!lowerResult.includes("success")) {
|
|
126
|
+
throw new Error(`Test generation failed: ${result}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return `**Generated Test Details:**
|
|
183
130
|
- Test Type: ${this.getTestType()}
|
|
184
131
|
- Output Directory: ${generateOptions.outputDir}
|
|
185
132
|
- Language: ${generateOptions.language}
|
|
186
133
|
- Framework: ${generateOptions.framework}
|
|
187
134
|
|
|
188
|
-
${result}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
text: `Error generating ${this.getTestType()} test: ${error.message}`,
|
|
196
|
-
},
|
|
197
|
-
],
|
|
198
|
-
isError: true,
|
|
199
|
-
};
|
|
135
|
+
${result}
|
|
136
|
+
`;
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
140
|
+
throw new Error(`Failed to execute test generation: ${errorMessage}`);
|
|
141
|
+
}
|
|
200
142
|
}
|
|
201
143
|
buildBaseGenerationOptions(params) {
|
|
202
144
|
return {
|
|
@@ -12,11 +12,31 @@ const LANGUAGE_PATHS = {
|
|
|
12
12
|
};
|
|
13
13
|
export function registerExecuteSkyrampTestTool(server) {
|
|
14
14
|
server.registerTool("skyramp_execute_test", {
|
|
15
|
-
description: `Execute a Skyramp test
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing.
|
|
16
|
+
|
|
17
|
+
Skyramp is a comprehensive testing platform that generates reliable, out-of-the-box functional and performance tests with AI-powered scaffolding. The test execution engine runs your generated tests in controlled, isolated environments to ensure consistent results.
|
|
18
|
+
|
|
19
|
+
KEY FEATURES:
|
|
20
|
+
• Isolated Execution: Tests run in containerized environments for consistency
|
|
21
|
+
• Multi-Language Support: Execute tests written in Python, Java, JavaScript, or TypeScript
|
|
22
|
+
• Out-of-the-Box Execution: Generated tests work immediately without modification
|
|
23
|
+
|
|
24
|
+
REQUIRED PARAMETERS:
|
|
25
|
+
- language: Programming language of your test file (python, javascript, typescript, java)
|
|
26
|
+
- testType: Type of test to execute (smoke, contract, fuzz, integration, load, ui, e2e)
|
|
27
|
+
- testFile: Absolute path to the generated test file to execute
|
|
28
|
+
- token: Authentication token for your service (use empty string if no authentication required)
|
|
29
|
+
|
|
30
|
+
AUTHENTICATION:
|
|
31
|
+
Provide your authentication token (typically a Bearer token) for services that require authentication. Use an empty string for services that don't require authentication.
|
|
32
|
+
|
|
33
|
+
IMPORTANT NOTES:
|
|
34
|
+
- First-time usage may take longer as Docker images are downloaded
|
|
35
|
+
- Tests run in isolated containers for maximum reliability
|
|
36
|
+
- Generated tests are designed to work out-of-the-box without modification
|
|
37
|
+
- Results include detailed execution logs and test outcomes
|
|
38
|
+
|
|
39
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
|
|
20
40
|
inputSchema: {
|
|
21
41
|
language: z
|
|
22
42
|
.string()
|
|
@@ -25,7 +25,12 @@ export class ContractTestService extends TestGenerationService {
|
|
|
25
25
|
}
|
|
26
26
|
export function registerContractTestTool(server) {
|
|
27
27
|
server.registerTool("skyramp_contract_test_generation", {
|
|
28
|
-
description:
|
|
28
|
+
description: `Generate a contract test using Skyramp's deterministic test generation platform.
|
|
29
|
+
|
|
30
|
+
Contract tests ensure your API implementation matches its OpenAPI/Swagger specification exactly. They validate request/response schemas, status codes, headers, and data types to prevent contract violations and API breaking changes.
|
|
31
|
+
|
|
32
|
+
If the user says create tests for all methods in an endpoint then use "" as method value and skyramp will create all test in one file
|
|
33
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/contract-test`,
|
|
29
34
|
inputSchema: contractTestSchema,
|
|
30
35
|
}, async (params) => {
|
|
31
36
|
const service = new ContractTestService();
|
|
@@ -22,7 +22,14 @@ export class E2ETestService extends TestGenerationService {
|
|
|
22
22
|
}
|
|
23
23
|
export function registerEndtoEndTestTool(server) {
|
|
24
24
|
server.registerTool("skyramp_e2e_test_generation", {
|
|
25
|
-
description:
|
|
25
|
+
description: `Generate an End-to-End (E2E) test using Skyramp's deterministic test generation platform.
|
|
26
|
+
|
|
27
|
+
End-to-End tests validate complete user journeys by testing the entire application flow from frontend UI interactions to backend API responses. They ensure that all components work together correctly in realistic user scenarios, providing the highest confidence in application functionality.
|
|
28
|
+
|
|
29
|
+
TRACE & UI INTEGRATION:
|
|
30
|
+
E2E tests require both trace files (capturing backend API interactions) and Playwright recordings (capturing UI interactions) to generate comprehensive tests that validate the complete user experience.
|
|
31
|
+
|
|
32
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/e2e-test`,
|
|
26
33
|
inputSchema: e2eTestSchema,
|
|
27
34
|
annotations: {
|
|
28
35
|
keywords: ["e2e test", "end-to-end test"],
|
|
@@ -20,7 +20,12 @@ export class FuzzTestService extends TestGenerationService {
|
|
|
20
20
|
}
|
|
21
21
|
export function registerFuzzTestTool(server) {
|
|
22
22
|
server.registerTool("skyramp_fuzz_test_generation", {
|
|
23
|
-
description:
|
|
23
|
+
description: `Generate a fuzz test using Skyramp's deterministic test generation platform.
|
|
24
|
+
|
|
25
|
+
Fuzz tests improve application security and reliability by sending invalid, malformed, or unexpected data to your API endpoints. They help discover edge cases, input validation issues, error handling problems, and potential security vulnerabilities.
|
|
26
|
+
|
|
27
|
+
If the user says create tests for all methods in an endpoint then use "" as method value and skyramp will create all test in one file
|
|
28
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/fuzz-test`,
|
|
24
29
|
inputSchema: fuzzTestSchema,
|
|
25
30
|
annotations: {
|
|
26
31
|
keywords: ["negative test", "random test"],
|
|
@@ -20,7 +20,12 @@ export class IntegrationTestService extends TestGenerationService {
|
|
|
20
20
|
}
|
|
21
21
|
export function registerIntegrationTestTool(server) {
|
|
22
22
|
server.registerTool("skyramp_integration_test_generation", {
|
|
23
|
-
description:
|
|
23
|
+
description: `Generate an integration test using Skyramp's deterministic test generation platform.
|
|
24
|
+
|
|
25
|
+
Integration tests validate that multiple services, components, or modules work together correctly. They test complex user workflows, service interactions, data flow between systems, and ensure that integrated components function as expected in realistic scenarios.
|
|
26
|
+
|
|
27
|
+
If the user says create tests for all methods in an endpoint then use "" as method value and skyramp will create all test in one file
|
|
28
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/integration-test`,
|
|
24
29
|
inputSchema: integrationTestSchema,
|
|
25
30
|
}, async (params) => {
|
|
26
31
|
const service = new IntegrationTestService();
|
|
@@ -5,31 +5,32 @@ const loadTestSchema = {
|
|
|
5
5
|
...baseTestSchema,
|
|
6
6
|
loadCount: z
|
|
7
7
|
.string()
|
|
8
|
-
.
|
|
9
|
-
.describe("Total
|
|
8
|
+
.default("")
|
|
9
|
+
.describe("Total string of iteration to be excuted for the test. DO NOT ASSUME VALUE"),
|
|
10
10
|
loadDuration: z
|
|
11
11
|
.string()
|
|
12
|
-
.
|
|
12
|
+
.default("5")
|
|
13
13
|
.describe("Duration of the load test in seconds (default: 5)"),
|
|
14
14
|
loadNumThreads: z
|
|
15
15
|
.string()
|
|
16
|
-
.
|
|
16
|
+
.default("1")
|
|
17
17
|
.describe("Number of concurrent threads/users for the load test (default: 1)"),
|
|
18
18
|
loadRampupDuration: z
|
|
19
19
|
.string()
|
|
20
|
-
.
|
|
21
|
-
.describe("Time in seconds to gradually increase load to target level"),
|
|
20
|
+
.default("")
|
|
21
|
+
.describe("Time in seconds to gradually increase load to target level. DO NOT ASSUME VALUE"),
|
|
22
22
|
loadRampupInterval: z
|
|
23
23
|
.string()
|
|
24
|
-
.
|
|
25
|
-
.describe("Interval in seconds between ramping up threads"),
|
|
24
|
+
.default("")
|
|
25
|
+
.describe("Interval in seconds between ramping up threads. DO NOT ASSUME VALUE"),
|
|
26
26
|
loadTargetRPS: z
|
|
27
27
|
.string()
|
|
28
|
-
.
|
|
29
|
-
.describe("Target requests per second for the load test"),
|
|
30
|
-
trace: baseTraceSchema.shape.trace.
|
|
31
|
-
include: baseTraceSchema.shape.include
|
|
32
|
-
exclude: baseTraceSchema.shape.exclude
|
|
28
|
+
.default("")
|
|
29
|
+
.describe("Target requests per second for the load test. DO NOT ASSUME VALUE"),
|
|
30
|
+
trace: baseTraceSchema.shape.trace.default(""),
|
|
31
|
+
include: baseTraceSchema.shape.include,
|
|
32
|
+
exclude: baseTraceSchema.shape.exclude,
|
|
33
|
+
endpointURL: baseTestSchema.endpointURL.default(""),
|
|
33
34
|
};
|
|
34
35
|
export class LoadTestService extends TestGenerationService {
|
|
35
36
|
getTestType() {
|
|
@@ -49,7 +50,15 @@ export class LoadTestService extends TestGenerationService {
|
|
|
49
50
|
}
|
|
50
51
|
export function registerLoadTestTool(server) {
|
|
51
52
|
server.registerTool("skyramp_load_test_generation", {
|
|
52
|
-
description:
|
|
53
|
+
description: `Generate a load test using Skyramp's deterministic test generation platform.
|
|
54
|
+
|
|
55
|
+
Load tests evaluate your application's performance, scalability, and stability under various traffic conditions. They simulate multiple concurrent users, measure response times, identify performance bottlenecks, and ensure your system can handle expected traffic volumes.
|
|
56
|
+
|
|
57
|
+
***IMPORTANT**
|
|
58
|
+
-At any given time you can provide duration of the test or test execution count
|
|
59
|
+
-If the user says create tests for all methods in an endpoint then use "" as method value and skyramp will create all test in one file
|
|
60
|
+
|
|
61
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/load-test`,
|
|
53
62
|
inputSchema: loadTestSchema,
|
|
54
63
|
annotations: {
|
|
55
64
|
keywords: ["load test", "performance test"],
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { baseTestSchema } from "../types/TestTypes.js";
|
|
2
2
|
import { TestGenerationService, } from "../services/TestGenerationService.js";
|
|
3
|
+
import z from "zod";
|
|
3
4
|
// Concrete implementations for each test type
|
|
4
5
|
export class SmokeTestService extends TestGenerationService {
|
|
5
6
|
getTestType() {
|
|
@@ -9,10 +10,25 @@ export class SmokeTestService extends TestGenerationService {
|
|
|
9
10
|
return super.buildBaseGenerationOptions(params);
|
|
10
11
|
}
|
|
11
12
|
}
|
|
13
|
+
const smokeTestSchema = {
|
|
14
|
+
...baseTestSchema,
|
|
15
|
+
endpointURL: z
|
|
16
|
+
.string()
|
|
17
|
+
.describe(baseTestSchema.endpointURL.description || "The endpoint URL to test"),
|
|
18
|
+
apiSchema: z
|
|
19
|
+
.string()
|
|
20
|
+
.describe(baseTestSchema.apiSchema.description || "The OpenAPI schema to test"),
|
|
21
|
+
};
|
|
12
22
|
export function registerSmokeTestTool(server) {
|
|
13
23
|
server.registerTool("skyramp_smoke_test_generation", {
|
|
14
|
-
description:
|
|
15
|
-
|
|
24
|
+
description: `Generate a smoke test using Skyramp's deterministic test generation platform.
|
|
25
|
+
|
|
26
|
+
Smoke testing is a preliminary check used to verify that an endpoint is accessible and returns a valid response. Smoke tests are useful for quickly identifying critical defects after significant changes. They provide rapid validation for basic functionality verification and endpoint accessibility testing.
|
|
27
|
+
|
|
28
|
+
If the user says create tests for all methods in an endpoint then use "" as method value and skyramp will create all test in one file
|
|
29
|
+
|
|
30
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/smoke-test`,
|
|
31
|
+
inputSchema: smokeTestSchema,
|
|
16
32
|
annotations: {
|
|
17
33
|
keywords: ["smoke test", "quick test"],
|
|
18
34
|
},
|
|
@@ -25,7 +25,11 @@ const uiTestSchema = {
|
|
|
25
25
|
};
|
|
26
26
|
export function registerUITestTool(server) {
|
|
27
27
|
server.registerTool("skyramp_ui_test_generation", {
|
|
28
|
-
description:
|
|
28
|
+
description: `Generate a UI test using Skyramp's deterministic test generation platform.
|
|
29
|
+
|
|
30
|
+
UI tests validate user interface functionality by simulating real user interactions with your web application. They test user workflows, form submissions, navigation, responsive design, and ensure that your frontend works correctly across different browsers and devices. UI tests use Playwright recordings as input to generate comprehensive test suites that replay user interactions, validate UI elements, and verify expected behaviors in browser environments.
|
|
31
|
+
|
|
32
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/ui-test`,
|
|
29
33
|
inputSchema: uiTestSchema,
|
|
30
34
|
annotations: {
|
|
31
35
|
keywords: ["ui test", "playwright"],
|
package/build/tools/loginTool.js
CHANGED
|
@@ -2,7 +2,11 @@ import { SkyrampClient } from "@skyramp/skyramp";
|
|
|
2
2
|
import { logger } from "../utils/logger.js";
|
|
3
3
|
export function registerLoginTool(server) {
|
|
4
4
|
server.registerTool("skyramp_login", {
|
|
5
|
-
description:
|
|
5
|
+
description: `Login to Skyramp platform
|
|
6
|
+
|
|
7
|
+
Logging into Skyramp provides access to additional platform features and services.
|
|
8
|
+
|
|
9
|
+
`,
|
|
6
10
|
inputSchema: {},
|
|
7
11
|
annotations: {
|
|
8
12
|
keywords: ["login", "authenticate", "skyramp login"],
|
|
@@ -2,7 +2,10 @@ import { SkyrampClient } from "@skyramp/skyramp";
|
|
|
2
2
|
import { logger } from "../utils/logger.js";
|
|
3
3
|
export function registerLogoutTool(server) {
|
|
4
4
|
server.registerTool("skyramp_logout", {
|
|
5
|
-
description:
|
|
5
|
+
description: `Logout from Skyramp platform
|
|
6
|
+
|
|
7
|
+
Logout from Skyramp platform to end your authenticated session.
|
|
8
|
+
`,
|
|
6
9
|
inputSchema: {},
|
|
7
10
|
annotations: {
|
|
8
11
|
keywords: ["logout", "sign out", "skyramp logout"],
|
|
@@ -4,7 +4,37 @@ import openProxyTerminalTracked from "../utils/proxy-terminal.js";
|
|
|
4
4
|
import { TELEMETRY_entrypoint_FIELD_NAME } from "../utils/utils.js";
|
|
5
5
|
export function registerTraceTool(server) {
|
|
6
6
|
server.registerTool("skyramp_start_trace_generation", {
|
|
7
|
-
description:
|
|
7
|
+
description: `Start trace collection using Skyramp's comprehensive tracing capabilities.
|
|
8
|
+
|
|
9
|
+
Trace collection monitors your application in real-time to capture user interactions, API calls, service communications, and data flows. This captured data is then used to generate comprehensive integration, load, and E2E tests that reflect real-world usage patterns.
|
|
10
|
+
|
|
11
|
+
COLLECTION TYPES:
|
|
12
|
+
• UI-Only Tracing: Captures frontend user interactions for UI test generation
|
|
13
|
+
• Backend-Only Tracing: Records API calls and service communications for integration tests
|
|
14
|
+
• Full-Stack Tracing: Combines UI and backend tracing for comprehensive E2E test generation
|
|
15
|
+
|
|
16
|
+
REQUIRED PARAMETERS:
|
|
17
|
+
None - All parameters are optional with sensible defaults
|
|
18
|
+
|
|
19
|
+
OPTIONAL PARAMETERS:
|
|
20
|
+
- playwright: Enable Playwright for UI interaction tracing (default: true)
|
|
21
|
+
- runtime: Execution environment (docker, kubernetes, local) - Defaults to docker
|
|
22
|
+
- include: List of service names or patterns to include in trace collection
|
|
23
|
+
- exclude: List of service names or patterns to exclude from trace collection
|
|
24
|
+
- noProxy: List of hosts or patterns that should bypass proxy during tracing
|
|
25
|
+
- dockerNetwork: Docker network name for containerized trace collection
|
|
26
|
+
- dockerWorkerPort: Port number for the Docker worker service (default: 35142)
|
|
27
|
+
|
|
28
|
+
RUNTIME ENVIRONMENTS:
|
|
29
|
+
• Docker: Containerized tracing with automatic service discovery
|
|
30
|
+
|
|
31
|
+
WORKFLOW:
|
|
32
|
+
1. Start trace collection with desired configuration
|
|
33
|
+
2. Interact with your application (UI clicks, API calls, workflows)
|
|
34
|
+
3. Stop trace collection to save captured data
|
|
35
|
+
4. Use traces to generate realistic test scenarios
|
|
36
|
+
|
|
37
|
+
For detailed documentation visit:https://www.skyramp.dev/docs/load-test/advanced-generation#start-trace-collection`,
|
|
8
38
|
inputSchema: {
|
|
9
39
|
playwright: z
|
|
10
40
|
.boolean()
|
|
@@ -20,11 +50,11 @@ export function registerTraceTool(server) {
|
|
|
20
50
|
.describe("List of service names or patterns to include in trace collection"),
|
|
21
51
|
exclude: z
|
|
22
52
|
.array(z.string())
|
|
23
|
-
.
|
|
53
|
+
.default([])
|
|
24
54
|
.describe("List of service names or patterns to exclude from trace collection"),
|
|
25
55
|
noProxy: z
|
|
26
56
|
.array(z.string())
|
|
27
|
-
.
|
|
57
|
+
.default([])
|
|
28
58
|
.describe("List of hosts or patterns that should bypass proxy during tracing"),
|
|
29
59
|
dockerNetwork: z
|
|
30
60
|
.string()
|
|
@@ -5,7 +5,33 @@ import { PLAYWRIGHT_OUTPUT_FILE_FIELD_NAME, TELEMETRY_entrypoint_FIELD_NAME, TRA
|
|
|
5
5
|
import { logger } from "../utils/logger.js";
|
|
6
6
|
export function registerTraceStopTool(server) {
|
|
7
7
|
server.registerTool("skyramp_stop_trace_generation", {
|
|
8
|
-
description:
|
|
8
|
+
description: `Stop trace collection and save captured data using Skyramp's comprehensive tracing capabilities.
|
|
9
|
+
|
|
10
|
+
Stopping trace collection finalizes the monitoring process and exports all captured data to specified files. This includes API calls, service communications, user interactions, and network traffic that occurred during the collection period.
|
|
11
|
+
|
|
12
|
+
REQUIRED PARAMETERS:
|
|
13
|
+
- traceOutputFile: Absolute path to save trace data (JSON format) - This file contains collected application interactions for test generation
|
|
14
|
+
- playwright: Boolean flag indicating whether Playwright was used during collection (must match start configuration)
|
|
15
|
+
- playwrightOutput: Absolute path to save Playwright data (ZIP format) - Required when playwright is true, contains UI interaction recordings
|
|
16
|
+
|
|
17
|
+
OUTPUT FILES:
|
|
18
|
+
• Trace File (JSON): Contains backend API calls, service communications, and network traffic data
|
|
19
|
+
• Playwright File (ZIP): Contains UI interaction recordings and browser automation data (when enabled)
|
|
20
|
+
|
|
21
|
+
USAGE WORKFLOW:
|
|
22
|
+
1. Ensure this matches the configuration used when starting trace collection
|
|
23
|
+
2. Provide absolute paths for output files
|
|
24
|
+
3. Generated files can be used immediately for test generation
|
|
25
|
+
4. Trace data reflects real application usage patterns
|
|
26
|
+
|
|
27
|
+
INTEGRATION WITH TEST GENERATION:
|
|
28
|
+
The generated trace and Playwright files can be used as inputs for:
|
|
29
|
+
- Integration test generation (using trace files)
|
|
30
|
+
- Load test generation (using trace files)
|
|
31
|
+
- E2E test generation (using both trace and Playwright files)
|
|
32
|
+
- UI test generation (using Playwright files)
|
|
33
|
+
|
|
34
|
+
For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advanced-generation#end-trace-collection`,
|
|
9
35
|
inputSchema: {
|
|
10
36
|
traceOutputFile: z
|
|
11
37
|
.string()
|
package/build/types/TestTypes.js
CHANGED
|
@@ -2,11 +2,10 @@ import { z } from "zod";
|
|
|
2
2
|
export const baseSchema = z.object({
|
|
3
3
|
language: z
|
|
4
4
|
.string()
|
|
5
|
-
.optional()
|
|
6
5
|
.describe("Programming language for the generated test (default: python)"),
|
|
7
6
|
framework: z
|
|
8
7
|
.string()
|
|
9
|
-
.
|
|
8
|
+
.default("")
|
|
10
9
|
.describe("Testing framework to use (e.g., pytest for python, playwright for javascript, junit for java, etc.)"),
|
|
11
10
|
output: z.string().optional().describe("Name of the output test file"),
|
|
12
11
|
outputDir: z
|
|
@@ -17,36 +16,36 @@ export const baseSchema = z.object({
|
|
|
17
16
|
.describe("Whether to overwrite existing files in the output directory"),
|
|
18
17
|
deployDashboard: z
|
|
19
18
|
.boolean()
|
|
20
|
-
.
|
|
19
|
+
.default(false)
|
|
21
20
|
.describe("Whether to deploy the Skyramp dashboard for test monitoring"),
|
|
22
21
|
runtime: z
|
|
23
22
|
.string()
|
|
24
|
-
.
|
|
25
|
-
.describe("Runtime environment (docker, kubernetes,
|
|
23
|
+
.default("local")
|
|
24
|
+
.describe("Runtime environment (docker, kubernetes, local)"),
|
|
26
25
|
dockerNetwork: z
|
|
27
26
|
.string()
|
|
28
|
-
.
|
|
27
|
+
.default("")
|
|
29
28
|
.describe("Docker network name to use for containerized testing"),
|
|
30
29
|
dockerWorkerPort: z
|
|
31
30
|
.number()
|
|
32
|
-
.
|
|
31
|
+
.default(0)
|
|
33
32
|
.describe("Port number for the Docker worker service"),
|
|
34
33
|
k8sNamespace: z
|
|
35
34
|
.string()
|
|
36
|
-
.
|
|
35
|
+
.default("")
|
|
37
36
|
.describe("Kubernetes namespace for deployment"),
|
|
38
37
|
k8sConfig: z
|
|
39
38
|
.string()
|
|
40
|
-
.
|
|
39
|
+
.default("")
|
|
41
40
|
.describe("Path to Kubernetes configuration file"),
|
|
42
|
-
k8sContext: z.string().
|
|
41
|
+
k8sContext: z.string().default("").describe("Kubernetes context to use"),
|
|
43
42
|
authHeader: z
|
|
44
43
|
.string()
|
|
45
|
-
.
|
|
44
|
+
.default("")
|
|
46
45
|
.describe("Authorization header value for authenticated requests"),
|
|
47
46
|
insecure: z
|
|
48
47
|
.boolean()
|
|
49
|
-
.
|
|
48
|
+
.default(false)
|
|
50
49
|
.describe("Whether to allow insecure SSL connections for HTTPS endpoints"),
|
|
51
50
|
});
|
|
52
51
|
export const baseTraceSchema = z.object({
|
|
@@ -56,46 +55,45 @@ export const baseTraceSchema = z.object({
|
|
|
56
55
|
.describe("MUST be absolute path to trace file for test generation like /path/to/trace.json and MUST be a json file"),
|
|
57
56
|
include: z
|
|
58
57
|
.array(z.string())
|
|
59
|
-
.
|
|
60
|
-
.describe("List of endpoints or patterns to include in test generation"),
|
|
58
|
+
.default([])
|
|
59
|
+
.describe("List of endpoints or patterns to include in test generation. DO NOT ASSUME ANYTHING IF NOT PROVIDED"),
|
|
61
60
|
exclude: z
|
|
62
61
|
.array(z.string())
|
|
63
|
-
.
|
|
64
|
-
.describe("List of endpoints or patterns to exclude from test generation"),
|
|
62
|
+
.default([])
|
|
63
|
+
.describe("List of endpoints or patterns to exclude from test generation. DO NOT ASSUME ANYTHING IF NOT PROVIDED"),
|
|
65
64
|
});
|
|
66
65
|
// Base schema that all test tools share
|
|
67
66
|
export const baseTestSchema = {
|
|
68
67
|
...baseSchema.shape,
|
|
69
68
|
endpointURL: z
|
|
70
69
|
.string()
|
|
71
|
-
.optional()
|
|
72
70
|
.describe("The endpoint URL to test (e.g., https://api.example.com/v1/products)"),
|
|
73
71
|
method: z
|
|
74
72
|
.string()
|
|
75
|
-
.
|
|
76
|
-
.describe("HTTP method to use (GET, POST, PUT, DELETE, etc.). DO NOT ASSUME
|
|
73
|
+
.default("")
|
|
74
|
+
.describe("HTTP method to use (GET, POST, PUT, DELETE, etc.). DO NOT ASSUME VALUE IF NOT PROVIDED"),
|
|
77
75
|
apiSchema: z
|
|
78
76
|
.string()
|
|
79
|
-
.
|
|
80
|
-
.describe("MUST be absolute path to the OpenAPI/Swagger schema file"),
|
|
77
|
+
.default("")
|
|
78
|
+
.describe("MUST be absolute path(/path/to/openapi.json) to the OpenAPI/Swagger schema file or a URL to the OpenAPI/Swagger schema file(e.g. https://demoshop.skyramp.dev/openapi.json). DO NOT TRY TO ASSUME THE OPENAPI SCHEMA IF NOT PROVIDED"),
|
|
81
79
|
pathParams: z
|
|
82
80
|
.string()
|
|
83
|
-
.
|
|
81
|
+
.default("")
|
|
84
82
|
.describe("MUST be string of comma separated values like 'id=1,name=John' for URL path parameters"),
|
|
85
83
|
queryParams: z
|
|
86
84
|
.string()
|
|
87
|
-
.
|
|
85
|
+
.default("")
|
|
88
86
|
.describe("MUST be string of comma separated values like 'id=1,name=John' for URL query parameters"),
|
|
89
87
|
formParams: z
|
|
90
88
|
.string()
|
|
91
|
-
.
|
|
89
|
+
.default("")
|
|
92
90
|
.describe("MUST be string of comma separated values like 'id=1,name=John' for form data parameters"),
|
|
93
91
|
requestData: z
|
|
94
92
|
.string()
|
|
95
|
-
.
|
|
93
|
+
.default("")
|
|
96
94
|
.describe("Sample request body data, provided either as an inline JSON/YAML string or as an absolute file path prefixed with '@' (e.g., @/absolute/path/to/file)."),
|
|
97
95
|
responseStatusCode: z
|
|
98
96
|
.string()
|
|
99
|
-
.
|
|
100
|
-
.describe("Expected HTTP response status code (e.g., '200', '201', '404'). DO NOT ASSUME STATUS CODE IF NOT PROVIDED
|
|
97
|
+
.default("")
|
|
98
|
+
.describe("Expected HTTP response status code (e.g., '200', '201', '404'). DO NOT ASSUME STATUS CODE IF NOT PROVIDED"),
|
|
101
99
|
};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
export function getLanguageSteps(params) {
|
|
2
|
+
const { language, testType } = params;
|
|
3
|
+
switch (language) {
|
|
4
|
+
case "python":
|
|
5
|
+
let pythonSteps = `
|
|
6
|
+
# Install skyramp dependency
|
|
7
|
+
pip3 install skyramp
|
|
8
|
+
`;
|
|
9
|
+
// e2e and ui tests
|
|
10
|
+
if (testType === "e2e" || testType === "ui") {
|
|
11
|
+
pythonSteps += `
|
|
12
|
+
# Install playwright dependencies
|
|
13
|
+
pip3 install pytest-playwright
|
|
14
|
+
# Execute the test file
|
|
15
|
+
python3 -m pytest --browser chromium <test-file>.py
|
|
16
|
+
`;
|
|
17
|
+
}
|
|
18
|
+
// framework pytest
|
|
19
|
+
pythonSteps += `
|
|
20
|
+
# Install pytest dependency
|
|
21
|
+
pip3 install pytest
|
|
22
|
+
|
|
23
|
+
# Execution test file
|
|
24
|
+
python3 -m pytest <test-file>.py
|
|
25
|
+
`;
|
|
26
|
+
// robot framework
|
|
27
|
+
pythonSteps += `
|
|
28
|
+
# Install robotframework dependency
|
|
29
|
+
pip3 install robotframework
|
|
30
|
+
|
|
31
|
+
# Execution of test file
|
|
32
|
+
python3 -m robot <robot-file>.robot
|
|
33
|
+
`;
|
|
34
|
+
return pythonSteps;
|
|
35
|
+
case "javascript":
|
|
36
|
+
return `
|
|
37
|
+
# install dependencies for package.json
|
|
38
|
+
npm install @skyramp/skyramp @playwright/test
|
|
39
|
+
|
|
40
|
+
# Execution of test file
|
|
41
|
+
npx playwright test <test-file>.spec.js --reporter=list
|
|
42
|
+
`;
|
|
43
|
+
case "typescript":
|
|
44
|
+
return `
|
|
45
|
+
# install dependencies for package.json
|
|
46
|
+
npm install @skyramp/skyramp @playwright/test
|
|
47
|
+
|
|
48
|
+
# Execution of test file
|
|
49
|
+
npx playwright test <test-file>.spec.ts --reporter=list
|
|
50
|
+
`;
|
|
51
|
+
case "java":
|
|
52
|
+
return `
|
|
53
|
+
# Prerequisites
|
|
54
|
+
# Compile the test file
|
|
55
|
+
javac -cp "./lib/*" -d target <test-file>.java
|
|
56
|
+
|
|
57
|
+
# Execution of test file
|
|
58
|
+
JUNIT_PLATFORM_VER="1.9.3"
|
|
59
|
+
classpath="target:$(echo ./lib/*.jar | tr ' ' ':')"
|
|
60
|
+
java -jar "./lib/junit-platform-console-standalone-\${JUNIT_PLATFORM_VER}.jar" \\
|
|
61
|
+
--classpath "\$classpath" \\
|
|
62
|
+
--include-engine=junit-jupiter \\
|
|
63
|
+
--select-class=<test-file> \\
|
|
64
|
+
--reports-dir=target/test-results
|
|
65
|
+
`;
|
|
66
|
+
default:
|
|
67
|
+
return "";
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export function getModularizationPrompt() {
|
|
2
|
+
return `
|
|
3
|
+
**IMPORTANT: Now please modularize the code by**
|
|
4
|
+
|
|
5
|
+
1. Group related lines logically (e.g., login steps together) to improve readability and parameterize the test function(s) with meaningful parameters to make it more flexible.
|
|
6
|
+
2. Parameterize the test function(s) with meaningful parameters to make it more flexible. Preserve the original order of code lines within each function
|
|
7
|
+
3. Do not change or remove any existing lines of code, variable names, function names, or function structures.
|
|
8
|
+
4. Only extract reusable logical sections into helper functions or parameterized structures where applicable.
|
|
9
|
+
5. Do not create any new test cases but only helper functions.
|
|
10
|
+
6. Make sure the logic is same and the test works as expected.
|
|
11
|
+
7. Do not add any new logic or classes or any other files.
|
|
12
|
+
|
|
13
|
+
Please refactor the code with these modularization principles. Respond with the complete, modularized code only.
|
|
14
|
+
`;
|
|
15
|
+
}
|