@skyramp/mcp 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +56 -9
- package/build/prompts/startTraceCollectionPrompts.js +20 -2
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +42 -9
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +91 -85
- package/build/prompts/testbot/testbot-prompts.js +85 -18
- package/build/services/AnalyticsService.js +31 -12
- package/build/services/DriftAnalysisService.js +139 -13
- package/build/services/DriftAnalysisService.test.js +168 -0
- package/build/services/TestExecutionService.js +1 -1
- package/build/services/TestGenerationService.js +7 -2
- package/build/services/TestHealthService.js +38 -3
- package/build/services/TestHealthService.test.js +211 -0
- package/build/tools/submitReportTool.js +12 -3
- package/build/tools/submitReportTool.test.js +59 -1
- package/build/tools/test-maintenance/actionsTool.js +115 -9
- package/build/tools/test-maintenance/actionsTool.test.js +93 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +221 -14
- package/build/tools/test-recommendation/mapTestsTool.js +44 -22
- package/build/tools/test-recommendation/recommendTestsTool.js +87 -33
- package/build/tools/trace/startTraceCollectionTool.js +5 -2
- package/build/tools/trace/stopTraceCollectionTool.js +2 -2
- package/build/tools/workspace/initializeWorkspaceTool.js +222 -0
- package/build/types/RepositoryAnalysis.js +20 -0
- package/build/types/TestTypes.js +6 -2
- package/build/utils/initAgent.js +2 -1
- package/build/utils/scoring-engine.js +136 -18
- package/build/utils/telemetry.js +23 -0
- package/build/utils/utils.js +0 -1
- package/package.json +2 -2
package/build/index.js
CHANGED
|
@@ -30,9 +30,10 @@ import { registerExecuteBatchTestsTool } from "./tools/test-maintenance/executeB
|
|
|
30
30
|
import { registerCalculateHealthScoresTool } from "./tools/test-maintenance/calculateHealthScoresTool.js";
|
|
31
31
|
import { registerActionsTool } from "./tools/test-maintenance/actionsTool.js";
|
|
32
32
|
import { registerStateCleanupTool } from "./tools/test-maintenance/stateCleanupTool.js";
|
|
33
|
-
import { registerTestbotPrompt } from "./prompts/testbot/testbot-prompts.js";
|
|
33
|
+
import { registerTestbotPrompt, registerTestbotResource, } from "./prompts/testbot/testbot-prompts.js";
|
|
34
34
|
import { registerInitTestbotTool } from "./tools/initTestbotTool.js";
|
|
35
35
|
import { registerSubmitReportTool } from "./tools/submitReportTool.js";
|
|
36
|
+
import { registerInitializeWorkspaceTool } from "./tools/workspace/initializeWorkspaceTool.js";
|
|
36
37
|
import { AnalyticsService } from "./services/AnalyticsService.js";
|
|
37
38
|
import { initCheck } from "./utils/initAgent.js";
|
|
38
39
|
const server = new McpServer({
|
|
@@ -47,19 +48,62 @@ const server = new McpServer({
|
|
|
47
48
|
listChanged: true,
|
|
48
49
|
},
|
|
49
50
|
},
|
|
51
|
+
instructions: `Skyramp MCP Server — generates and executes API tests (smoke, fuzz, contract, load, integration, E2E, UI).
|
|
52
|
+
|
|
53
|
+
## Rules
|
|
54
|
+
- NEVER show CLI commands. ALWAYS use the MCP tools provided.
|
|
55
|
+
- For UI and E2E tests, use the trace collection start/stop tools.
|
|
56
|
+
|
|
57
|
+
## Workspace Initialization (before ANY other Skyramp tool)
|
|
58
|
+
Follow this flow EVERY time before calling any Skyramp tool:
|
|
59
|
+
|
|
60
|
+
1. **Check**: Does .skyramp/workspace.yml exist at the workspace root?
|
|
61
|
+
2. **If YES** → proceed with the requested tool.
|
|
62
|
+
3. **If NO** → you MUST call \`skyramp_initialize_workspace\` BEFORE doing anything else.
|
|
63
|
+
- Do NOT skip this step. Do NOT proceed to the requested tool first.
|
|
64
|
+
- Scan the repo for ALL services (see the tool description for detailed steps).
|
|
65
|
+
- A fullstack or monorepo MUST produce multiple services — never just one.
|
|
66
|
+
- After workspace init completes, THEN proceed with the originally requested tool.
|
|
67
|
+
4. **ONLY skip init if the user EXPLICITLY says** "no", "skip", "don't create workspace", or similar.
|
|
68
|
+
- A request like "execute tests" or "generate tests" is NOT a signal to skip init.
|
|
69
|
+
- If the user does decline, respect it — do NOT ask again, and proceed with the requested tool.
|
|
70
|
+
|
|
71
|
+
## Workspace Defaults for Test Generation (MANDATORY)
|
|
72
|
+
Before calling ANY test generation tool, you MUST follow this flow:
|
|
73
|
+
|
|
74
|
+
1. **Read** the .skyramp/workspace.yml file to get the configured defaults.
|
|
75
|
+
2. **Extract** the \`language\`, \`framework\`, \`outputDir\`, and \`api.baseUrl\` from the services section.
|
|
76
|
+
3. **Use those values** as defaults for the test generation tool call. Do NOT ask the user for these values if they are already configured in the workspace file.
|
|
77
|
+
4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
|
|
78
|
+
5. **If the workspace file does not exist**, or the needed values (language, framework, outputDir) are missing from the workspace config, ASK the user which language and framework they want before calling the tool.
|
|
79
|
+
6. The user can always override workspace defaults by explicitly specifying values in their request.
|
|
80
|
+
`,
|
|
50
81
|
});
|
|
51
82
|
// Check for first-time invocation after version update (runs in background, doesn't block)
|
|
52
|
-
let
|
|
83
|
+
let initCheckInFlight = false;
|
|
84
|
+
let initCheckDone = false;
|
|
53
85
|
const originalRegisterTool = server.registerTool.bind(server);
|
|
54
86
|
server.registerTool = function (name, definition, handler) {
|
|
55
87
|
const wrappedHandler = async (...args) => {
|
|
56
|
-
if (!
|
|
57
|
-
initCheck()
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
88
|
+
if (!initCheckDone && !initCheckInFlight) {
|
|
89
|
+
// Guard with inFlight so concurrent tool calls don't each spawn a new initCheck(),
|
|
90
|
+
// but allow retry on failure (initCheckInFlight is reset to false on error).
|
|
91
|
+
// SkyrampClient constructor calls checkForUpdate("npm") via synchronous koffi FFI,
|
|
92
|
+
// which can block the event loop for up to 60 s if the update-check server is
|
|
93
|
+
// unreachable. Deferring via setImmediate ensures the tool response is written to
|
|
94
|
+
// stdout (and acknowledged by the MCP client) before any blocking FFI call runs.
|
|
95
|
+
initCheckInFlight = true;
|
|
96
|
+
setImmediate(() => {
|
|
97
|
+
initCheck()
|
|
98
|
+
.then(() => {
|
|
99
|
+
initCheckDone = true;
|
|
100
|
+
})
|
|
101
|
+
.catch((err) => {
|
|
102
|
+
logger.error("Background initialization check failed", { error: err });
|
|
103
|
+
})
|
|
104
|
+
.finally(() => {
|
|
105
|
+
initCheckInFlight = false;
|
|
106
|
+
});
|
|
63
107
|
});
|
|
64
108
|
}
|
|
65
109
|
return handler(...args);
|
|
@@ -75,6 +119,7 @@ const prompts = [
|
|
|
75
119
|
];
|
|
76
120
|
if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
|
|
77
121
|
prompts.push(registerTestbotPrompt);
|
|
122
|
+
registerTestbotResource(server);
|
|
78
123
|
logger.info("TestBot prompt enabled via SKYRAMP_FEATURE_TESTBOT");
|
|
79
124
|
}
|
|
80
125
|
prompts.forEach((registerPrompt) => registerPrompt(server));
|
|
@@ -109,6 +154,8 @@ registerExecuteBatchTestsTool(server);
|
|
|
109
154
|
registerCalculateHealthScoresTool(server);
|
|
110
155
|
registerActionsTool(server);
|
|
111
156
|
registerStateCleanupTool(server);
|
|
157
|
+
// Register workspace management tools
|
|
158
|
+
registerInitializeWorkspaceTool(server);
|
|
112
159
|
// Register other Skyramp tools
|
|
113
160
|
const infrastructureTools = [
|
|
114
161
|
registerLoginTool,
|
|
@@ -30,13 +30,25 @@ When playwright is enabled for trace collection, you can optionally configure:
|
|
|
30
30
|
* 'webkit' - Safari/WebKit browser
|
|
31
31
|
- Use firefox or webkit when you need to test cross-browser compatibility or specific browser behaviors
|
|
32
32
|
|
|
33
|
-
2. **
|
|
33
|
+
2. **Device** (device):
|
|
34
|
+
- Device to emulate for mobile/tablet testing
|
|
35
|
+
- Supported devices include:
|
|
36
|
+
* 'iPhone 13' - Apple iPhone 13
|
|
37
|
+
* 'iPhone 13 Pro' - Apple iPhone 13 Pro
|
|
38
|
+
* 'Pixel 5' - Google Pixel 5
|
|
39
|
+
* 'Galaxy S9+' - Samsung Galaxy S9 Plus
|
|
40
|
+
* 'iPad Pro 11' - Apple iPad Pro
|
|
41
|
+
* And many more Playwright device profiles
|
|
42
|
+
- Leave empty (default) for desktop/no device emulation
|
|
43
|
+
- Use specific device names when testing mobile-responsive applications or generating mobile UI tests
|
|
44
|
+
|
|
45
|
+
3. **Playwright Storage Path** (playwrightStoragePath):
|
|
34
46
|
- Path to a playwright session storage file containing authentication data (cookies, localStorage, sessionStorage, etc.)
|
|
35
47
|
- MUST be an absolute path like /path/to/storage.json
|
|
36
48
|
- Use this when you have manually created a session from the login flow and want to reuse it for future trace collections to avoid manual login every time
|
|
37
49
|
- The session file should be created beforehand using Playwright's storageState feature during the login flow
|
|
38
50
|
|
|
39
|
-
|
|
51
|
+
4. **Playwright Viewport Size** (playwrightViewportSize):
|
|
40
52
|
- Defines the browser window size for trace collection
|
|
41
53
|
- Supported formats:
|
|
42
54
|
* 'hd' - 1280x720
|
|
@@ -52,6 +64,12 @@ When playwright is enabled for trace collection, you can optionally configure:
|
|
|
52
64
|
* To start a playwright trace collection session using agent, run the following command:
|
|
53
65
|
Start playwright trace collection with default settings.
|
|
54
66
|
|
|
67
|
+
**Example usage prompt for trace collection with device emulation:**
|
|
68
|
+
* To start a trace collection session with mobile device emulation:
|
|
69
|
+
Start trace collection with iPhone 13 device
|
|
70
|
+
Collect trace using Pixel 5 device emulation
|
|
71
|
+
Start playwright trace with iPad Pro device
|
|
72
|
+
|
|
55
73
|
**Example usage prompt for trace collection with playwright storage and viewport:**
|
|
56
74
|
* Start playwright trace collection with storage path /Users/dev/session-storage.json and viewport size full-hd
|
|
57
75
|
|
|
@@ -2,20 +2,35 @@
|
|
|
2
2
|
* Repository Analysis Prompt
|
|
3
3
|
* Comprehensive prompt for analyzing code repositories
|
|
4
4
|
*/
|
|
5
|
-
export function getRepositoryAnalysisPrompt(repositoryPath) {
|
|
5
|
+
export function getRepositoryAnalysisPrompt(repositoryPath, analysisScope = "full_repo", parsedDiff) {
|
|
6
|
+
const isDiffScope = analysisScope === "current_branch_diff";
|
|
6
7
|
return `
|
|
7
8
|
Analyze this repository systematically using the guide below.
|
|
8
9
|
|
|
9
10
|
REPOSITORY PATH: ${repositoryPath}
|
|
11
|
+
ANALYSIS SCOPE: ${analysisScope}${isDiffScope ? " (focus on branch changes only)" : " (full repository)"}
|
|
10
12
|
|
|
11
13
|
IMPORTANT OUTPUT FORMAT:
|
|
12
14
|
- Return analysis as valid JSON matching the RepositoryAnalysis interface
|
|
13
15
|
- Use empty arrays [] for unavailable data rather than omitting fields
|
|
14
16
|
- Use "unknown" for unknown string values
|
|
15
17
|
- Provide evidence (file paths, line numbers) where possible
|
|
18
|
+
${isDiffScope ? '- MUST include the "branchDiffContext" field in the output JSON' : ""}
|
|
16
19
|
|
|
17
20
|
## CRITICAL RULES
|
|
18
21
|
- If user flows are defined in documentation explicitly, follow them strictly and do not generate new ones.
|
|
22
|
+
${isDiffScope ? `- Do NOT run git commands — the diff has already been parsed by the tool
|
|
23
|
+
- Still gather full repo context (tech stack, infra, auth, existing tests) for accurate scoring
|
|
24
|
+
- For branchDiffContext, use the following pre-parsed metadata:
|
|
25
|
+
- currentBranch: "${parsedDiff?.currentBranch || "unknown"}"
|
|
26
|
+
- baseBranch: "${parsedDiff?.baseBranch || "main"}"
|
|
27
|
+
- changedFiles: [${parsedDiff?.changedFiles.map((f) => `"${f}"`).join(", ") || ""}]
|
|
28
|
+
- For newEndpoints and modifiedEndpoints: use the pre-parsed endpoint information provided by the tool as the primary source of truth.
|
|
29
|
+
- You may refine this list using only the metadata and code/context that is explicitly included in this prompt.
|
|
30
|
+
- Do NOT attempt to scan or open additional repository files beyond what has been provided here.
|
|
31
|
+
- It is acceptable if some indirect or transitive impacts are not fully captured; do not speculate beyond the available information.
|
|
32
|
+
- When you adjust endpoints, clearly explain your reasoning based on the pre-parsed data and any in-prompt context.
|
|
33
|
+
- affectedServices: treat the changed file paths as high-level hints about which services are likely impacted, but do not perform exhaustive dependency tracing beyond the information provided` : ""}
|
|
19
34
|
|
|
20
35
|
# Repository Analysis Guide
|
|
21
36
|
|
|
@@ -187,7 +202,8 @@ Return a JSON object with this exact structure:
|
|
|
187
202
|
"metadata": {
|
|
188
203
|
"repositoryName": "name-from-path",
|
|
189
204
|
"analysisDate": "2025-10-15T14:30:00Z",
|
|
190
|
-
"scanDepth": "full"
|
|
205
|
+
"scanDepth": "full",
|
|
206
|
+
"analysisScope": "${analysisScope}"
|
|
191
207
|
},
|
|
192
208
|
"projectClassification": {
|
|
193
209
|
"projectType": "rest-api | frontend | full-stack | microservices | library | cli | other",
|
|
@@ -270,7 +286,20 @@ Return a JSON object with this exact structure:
|
|
|
270
286
|
},
|
|
271
287
|
"hasCoverageReports": true,
|
|
272
288
|
"estimatedCoverage": 78
|
|
273
|
-
}
|
|
289
|
+
}${isDiffScope ? `,
|
|
290
|
+
"branchDiffContext": {
|
|
291
|
+
"currentBranch": "feature/my-feature",
|
|
292
|
+
"baseBranch": "main",
|
|
293
|
+
"changedFiles": ["src/routes/products.ts", "src/models/Product.ts"],
|
|
294
|
+
"newEndpoints": [
|
|
295
|
+
{ "path": "/api/v1/products/search", "method": "GET", "sourceFile": "src/routes/products.ts:45" }
|
|
296
|
+
],
|
|
297
|
+
"modifiedEndpoints": [
|
|
298
|
+
{ "path": "/api/v1/products", "method": "POST", "sourceFile": "src/routes/products.ts:12" }
|
|
299
|
+
],
|
|
300
|
+
"affectedServices": ["product-service"],
|
|
301
|
+
"summary": "Added product search endpoint and modified product creation validation"
|
|
302
|
+
}` : ""}
|
|
274
303
|
}
|
|
275
304
|
\`\`\`
|
|
276
305
|
|
|
@@ -280,14 +309,18 @@ VALIDATION CHECKLIST:
|
|
|
280
309
|
- [ ] API endpoints counted and categorized
|
|
281
310
|
- [ ] Authentication method determined
|
|
282
311
|
- [ ] Infrastructure flags verified
|
|
283
|
-
- [ ] Existing tests catalogued
|
|
312
|
+
- [ ] Existing tests catalogued${isDiffScope ? `
|
|
313
|
+
- [ ] branchDiffContext.changedFiles matches the list provided above
|
|
314
|
+
- [ ] branchDiffContext.modifiedEndpoints includes ALL endpoints whose behaviour changed — including those affected by model/schema/validator changes (trace: changed file → which models/types → which endpoints use them)
|
|
315
|
+
- [ ] branchDiffContext.newEndpoints includes any brand-new routes added in the diff
|
|
316
|
+
- [ ] branchDiffContext.summary describes what changed in plain English` : ""}
|
|
284
317
|
|
|
285
318
|
**CRITICAL INSTRUCTIONS**:
|
|
286
|
-
-
|
|
287
|
-
- DO NOT create
|
|
288
|
-
-
|
|
289
|
-
-
|
|
319
|
+
- Construct the complete RepositoryAnalysis JSON object.
|
|
320
|
+
- DO NOT create any .md or documentation files.
|
|
321
|
+
- Save the analysis JSON to the state file path provided in the tool response.
|
|
322
|
+
- Then call \`skyramp_map_tests\` with the \`stateFile\` parameter (NOT analysisReport) to avoid serialization issues.
|
|
290
323
|
|
|
291
|
-
Begin analysis now.
|
|
324
|
+
Begin analysis now.
|
|
292
325
|
`;
|
|
293
326
|
}
|
|
@@ -2,51 +2,90 @@
|
|
|
2
2
|
* Test Recommendation Prompt
|
|
3
3
|
* Prompt for generating actionable test recommendations
|
|
4
4
|
*/
|
|
5
|
-
export function getTestRecommendationPrompt(mapping, analysis, topN = 7) {
|
|
5
|
+
export function getTestRecommendationPrompt(mapping, analysis, topN = 7, analysisScope = "full_repo") {
|
|
6
|
+
const isDiffScope = analysisScope === "current_branch_diff";
|
|
7
|
+
const diffContext = analysis.branchDiffContext;
|
|
8
|
+
// --- Build a compact summary of priority scores (no full JSON dump) ---
|
|
9
|
+
const topScores = mapping.priorityScores
|
|
10
|
+
.filter((s) => s.feasibility !== "not-applicable")
|
|
11
|
+
.slice(0, topN);
|
|
12
|
+
const scoreLines = topScores
|
|
13
|
+
.map((s) => {
|
|
14
|
+
const priority = s._finalScore >= 100 ? "high" : s._finalScore >= 70 ? "medium" : "low";
|
|
15
|
+
const avail = s.requiredArtifacts.available.join(", ") || "none";
|
|
16
|
+
const miss = s.requiredArtifacts.missing.join(", ") || "none";
|
|
17
|
+
return (`- **${s.testType.toUpperCase()}** | priority: ${priority} | feasibility: ${s.feasibility}\n` +
|
|
18
|
+
` Reasoning: ${s.reasoning}\n` +
|
|
19
|
+
` Artifacts available: ${avail} | missing: ${miss}`);
|
|
20
|
+
})
|
|
21
|
+
.join("\n");
|
|
22
|
+
// --- Build compact repo context (no full JSON dump) ---
|
|
23
|
+
const endpoints = analysis.apiEndpoints.endpoints
|
|
24
|
+
.slice(0, 30) // cap at 30 to keep size sane
|
|
25
|
+
.map((e) => ` ${e.method} ${e.path}${e.authRequired ? " [auth]" : ""}`)
|
|
26
|
+
.join("\n");
|
|
27
|
+
const repoContext = `
|
|
28
|
+
Repository: ${analysis.metadata.repositoryName}
|
|
29
|
+
Framework: ${analysis.projectClassification.primaryFramework} (${analysis.projectClassification.primaryLanguage})
|
|
30
|
+
Project type: ${analysis.projectClassification.projectType}
|
|
31
|
+
Auth: ${analysis.authentication.method}
|
|
32
|
+
Base URL: ${analysis.apiEndpoints.baseUrl}
|
|
33
|
+
Endpoints (${analysis.apiEndpoints.totalCount} total):
|
|
34
|
+
${endpoints}
|
|
35
|
+
`.trim();
|
|
36
|
+
// --- Branch diff context (only the essentials) ---
|
|
37
|
+
let diffSection = "";
|
|
38
|
+
if (isDiffScope && diffContext) {
|
|
39
|
+
const newEps = diffContext.newEndpoints
|
|
40
|
+
.map((e) => ` ${e.method} ${e.path} (${e.sourceFile})`)
|
|
41
|
+
.join("\n") || " none";
|
|
42
|
+
const modEps = diffContext.modifiedEndpoints
|
|
43
|
+
.map((e) => ` ${e.method} ${e.path} (${e.sourceFile})`)
|
|
44
|
+
.join("\n") || " none";
|
|
45
|
+
diffSection = `
|
|
46
|
+
## Branch Diff Context
|
|
47
|
+
Branch: \`${diffContext.currentBranch}\` → base: \`${diffContext.baseBranch}\`
|
|
48
|
+
Changed files: ${diffContext.changedFiles.join(", ")}
|
|
49
|
+
New endpoints:
|
|
50
|
+
${newEps}
|
|
51
|
+
Modified endpoints:
|
|
52
|
+
${modEps}
|
|
53
|
+
Affected services: ${diffContext.affectedServices.join(", ") || "N/A"}
|
|
54
|
+
Summary: ${diffContext?.summary ?? "N/A"}
|
|
55
|
+
|
|
56
|
+
**CRITICAL**: Focus recommendations ONLY on tests that validate the branch changes above.
|
|
57
|
+
`;
|
|
58
|
+
}
|
|
59
|
+
const scopeNote = isDiffScope
|
|
60
|
+
? "Recommendations are scoped to the current branch changes only. All specific test suggestions MUST target the new/modified endpoints listed in the Branch Diff Context."
|
|
61
|
+
: "Recommendations cover the full repository.";
|
|
6
62
|
return `
|
|
7
|
-
Generate actionable test recommendations based on priority and repository analysis.
|
|
8
|
-
|
|
9
|
-
**MANDATORY RULES**:
|
|
10
|
-
1. **Priority Scores Must Remain Unchanged**: When a test type is missing required inputs (e.g., Playwright recordings, traces), do NOT:
|
|
11
|
-
- Mark the test as "blocked" in the output
|
|
12
|
-
- Adjust or reduce the priority score
|
|
13
|
-
- Exclude it from recommendations if it ranks in the top ${topN}
|
|
14
|
-
|
|
15
|
-
# Priority
|
|
16
|
-
|
|
17
|
-
\`\`\`json
|
|
18
|
-
${JSON.stringify(mapping, null, 2)}
|
|
19
|
-
\`\`\`
|
|
63
|
+
Generate actionable test recommendations based on the priority scores and repository analysis below.
|
|
20
64
|
|
|
21
|
-
|
|
65
|
+
Scope: ${scopeNote}
|
|
66
|
+
${diffSection}
|
|
67
|
+
## Repository Context
|
|
22
68
|
|
|
23
|
-
|
|
24
|
-
${JSON.stringify(analysis, null, 2)}
|
|
25
|
-
\`\`\`
|
|
69
|
+
${repoContext}
|
|
26
70
|
|
|
27
|
-
|
|
71
|
+
## Prioritized Test Types (top ${topN})
|
|
28
72
|
|
|
29
|
-
|
|
73
|
+
${scoreLines}
|
|
30
74
|
|
|
31
|
-
##
|
|
32
|
-
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
* Score >= 100: priority = "high"
|
|
36
|
-
* Score 70-99: priority = "medium"
|
|
37
|
-
* Score < 70: priority = "low"
|
|
38
|
-
- **DO NOT include the "score" field** in the JSON output (it has been removed from the schema).
|
|
39
|
-
- DO NOT mention or display numerical scores anywhere in user-facing text.
|
|
40
|
-
- Order test recommendations by their internal priority scores (highest to lowest).
|
|
75
|
+
## Priority Mapping Summary
|
|
76
|
+
- High priority: ${mapping.summary.highPriority.join(", ") || "none"}
|
|
77
|
+
- Medium priority: ${mapping.summary.mediumPriority.join(", ") || "none"}
|
|
78
|
+
- Low priority: ${mapping.summary.lowPriority.join(", ") || "none"}
|
|
41
79
|
|
|
42
|
-
##
|
|
80
|
+
## Your Task
|
|
43
81
|
|
|
44
|
-
|
|
45
|
-
1. **Provide rationale for prioritization based on repository attributes.
|
|
46
|
-
2. **Suggest 3-4 ACTUAL tests** that should be created based on individual user flows and API endpoints.
|
|
47
|
-
3. **Include available artifacts** with specific file paths
|
|
48
|
-
4. **Provide guidance** for creating missing artifacts
|
|
82
|
+
Generate specific, actionable test recommendations for the top ${topN} test types listed above.
|
|
49
83
|
|
|
84
|
+
**MANDATORY RULES**:
|
|
85
|
+
1. **NO numeric scores** in output — use only: "high", "medium", "low".
|
|
86
|
+
2. **DO NOT mark any test "blocked"** even if artifacts are missing.
|
|
87
|
+
3. **Branch scope**: All specific test suggestions MUST target endpoints/code that changed in the branch diff.
|
|
88
|
+
4. **DO NOT create or save any files** — output everything inline in your response.
|
|
50
89
|
|
|
51
90
|
## Output Structure
|
|
52
91
|
|
|
@@ -58,74 +97,41 @@ Return a JSON object with this structure:
|
|
|
58
97
|
"totalRecommended": 3,
|
|
59
98
|
"highPriorityCount": 2,
|
|
60
99
|
"estimatedEffort": "4-6 hours for top 3 tests",
|
|
61
|
-
"quickWins": [
|
|
62
|
-
"Smoke tests (OpenAPI available, 30 min)",
|
|
63
|
-
"Contract tests (OpenAPI available, 1 hour)"
|
|
64
|
-
]
|
|
100
|
+
"quickWins": ["Smoke tests (OpenAPI available, 30 min)"]
|
|
65
101
|
},
|
|
66
102
|
"recommendations": [
|
|
67
103
|
{
|
|
68
104
|
"priority": "high",
|
|
69
|
-
"testType": "
|
|
70
|
-
"rationale": "
|
|
105
|
+
"testType": "contract",
|
|
106
|
+
"rationale": "Why this is high priority (DO NOT mention numeric scores)",
|
|
71
107
|
"specificTests": [
|
|
72
108
|
{
|
|
73
|
-
"testName": "
|
|
74
|
-
"description": "
|
|
75
|
-
"targetEndpoint": "
|
|
76
|
-
"targetFlow": "
|
|
109
|
+
"testName": "Order quantity validation contract test",
|
|
110
|
+
"description": "Verify POST /api/v1/orders rejects quantity < 1",
|
|
111
|
+
"targetEndpoint": "POST /api/v1/orders",
|
|
112
|
+
"targetFlow": "Order creation validation",
|
|
77
113
|
"requiredInputs": {
|
|
78
|
-
"available": [
|
|
79
|
-
|
|
80
|
-
"name": "openApiSpec",
|
|
81
|
-
"path": "./docs/openapi.yaml"
|
|
82
|
-
}
|
|
83
|
-
],
|
|
84
|
-
"missing": [
|
|
85
|
-
{
|
|
86
|
-
"name": "traceFile",
|
|
87
|
-
"guidance": "Use Skyramp trace collection: Run 'skyramp_start_trace_collection'..."
|
|
88
|
-
}
|
|
89
|
-
]
|
|
114
|
+
"available": [{"name": "openApiSpec", "path": "./openapi.json"}],
|
|
115
|
+
"missing": []
|
|
90
116
|
},
|
|
91
|
-
"estimatedValue": "High - catches
|
|
117
|
+
"estimatedValue": "High - catches regression in validation rule change"
|
|
92
118
|
}
|
|
93
119
|
],
|
|
94
120
|
"gettingStarted": {
|
|
95
|
-
"prerequisites": [
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
"JWT_SECRET environment variable set"
|
|
99
|
-
],
|
|
100
|
-
"quickStartCommand": "npm install && docker-compose up -d && npm run test:integration",
|
|
101
|
-
"documentationUrl": "https://www.skyramp.dev/docs/integration-tests"
|
|
121
|
+
"prerequisites": ["Service running at localhost:8000"],
|
|
122
|
+
"quickStartCommand": "Use Skyramp MCP generate tools",
|
|
123
|
+
"documentationUrl": "https://www.skyramp.dev/docs"
|
|
102
124
|
}
|
|
103
125
|
}
|
|
104
126
|
],
|
|
105
127
|
"nextSteps": [
|
|
106
|
-
"1. Start with
|
|
107
|
-
"2.
|
|
108
|
-
"3.
|
|
109
|
-
"4. Execute generated tests: npm run test:integration",
|
|
110
|
-
"5. Once integration tests pass, proceed with Fuzz Tests for security"
|
|
128
|
+
"1. Start with highest priority test type",
|
|
129
|
+
"2. Use Skyramp MCP generate tools to create each test",
|
|
130
|
+
"3. Execute and validate results"
|
|
111
131
|
]
|
|
112
132
|
}
|
|
113
133
|
\`\`\`
|
|
114
134
|
|
|
115
|
-
## Important Guidelines
|
|
116
|
-
|
|
117
|
-
1. **Be Specific**: Use actual endpoint paths, file paths, and flow names from the repository
|
|
118
|
-
2. **Be Actionable**: Prompts should work without modification
|
|
119
|
-
3. **Highlight Quick Wins**: Highlight tests with all artifacts available
|
|
120
|
-
4. **Provide Guidance**: Clear instructions for creating missing artifacts
|
|
121
|
-
5. **Include Evidence**: Reference specific repository characteristics in rationale
|
|
122
|
-
|
|
123
|
-
**CRITICAL INSTRUCTIONS**:
|
|
124
|
-
- Return the JSON object directly in your response text.
|
|
125
|
-
- DO NOT create or save any files (no test_recommendations.md, .json, or other files).
|
|
126
|
-
- DO NOT use file write tools or save recommendations to disk.
|
|
127
|
-
- Output the complete recommendations JSON inline in your response.
|
|
128
|
-
|
|
129
135
|
Generate recommendations now. Return the JSON object directly in your response text.
|
|
130
136
|
`;
|
|
131
137
|
}
|
|
@@ -1,40 +1,71 @@
|
|
|
1
|
+
import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import { logger } from "../../utils/logger.js";
|
|
3
|
-
|
|
4
|
+
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
5
|
+
function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath, baseBranch) {
|
|
4
6
|
return `<TITLE>${prTitle}</TITLE>
|
|
5
7
|
<DESCRIPTION>${prDescription}</DESCRIPTION>
|
|
6
8
|
<CODE CHANGES>${diffFile}</CODE CHANGES>
|
|
7
9
|
<TEST DIRECTORY>${testDirectory}</TEST DIRECTORY>
|
|
10
|
+
<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
|
|
8
11
|
|
|
9
12
|
For all the following work, use the tools offered by Skyramp MCP server.
|
|
10
13
|
|
|
11
14
|
First analyze the pull request title, description, and code changes to determine a business case
|
|
12
15
|
justification for this code change.
|
|
13
16
|
|
|
14
|
-
Then perform the following tasks based on your
|
|
17
|
+
Then perform ALL of the following tasks. Every task is MANDATORY — do NOT skip any task based on your own judgment unless the task itself gives you an explicit condition to skip.
|
|
15
18
|
|
|
16
|
-
## Task 1: New
|
|
19
|
+
## Task 1: Recommend New Tests (MANDATORY)
|
|
17
20
|
|
|
18
|
-
|
|
19
|
-
for new contract, e2e, smoke, load tests specific to the code changes
|
|
20
|
-
and business case analysis. DO NOT use the whole repository for generating recommendations.
|
|
21
|
-
2. If, and only if, there are recommendations for new tests, then use Skyramp MCP generate
|
|
22
|
-
tools to generate contract, e2e, smoke and load tests.
|
|
23
|
-
3. Use Skyramp MCP to execute the generated tests and validate the results.
|
|
21
|
+
Read the diff at \`${diffFile}\`. Classify each changed file. A file is application source code if it is any of: a route/controller/handler, a model/schema/validator/serializer/DTO, business logic, middleware, service, utility, test helper, or has a source extension (.py, .ts, .js, .java, .go, .rb, .cs, .kt, .swift, etc.). When in doubt, treat the file as application source code.
|
|
24
22
|
|
|
25
|
-
|
|
23
|
+
**DEFAULT: You MUST run steps 1–5 below.** The only exception is if you can confirm that EVERY changed file is exclusively a CI workflow YAML, markdown documentation, README, CHANGELOG, or a dependency lock file — and nothing else.
|
|
26
24
|
|
|
27
|
-
1.
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
1. Call \`skyramp_analyze_repository\` with:
|
|
26
|
+
- \`repositoryPath\`: "${repositoryPath}"
|
|
27
|
+
- \`analysisScope\`: "current_branch_diff"${baseBranch ? `\n - \`baseBranch\`: "${baseBranch}"` : ''}
|
|
28
|
+
2. MANDATORY: Call \`skyramp_map_tests\` with \`stateFile\` (the state file path returned above) and \`analysisScope: "current_branch_diff"\`.
|
|
29
|
+
3. MANDATORY: Call \`skyramp_recommend_tests\` with the \`stateFile\` returned by \`skyramp_map_tests\`. Use the priority summary and the specific endpoints/files that changed to determine exactly what to test.
|
|
30
|
+
4. Generate tests using the Skyramp MCP generate tools, in priority order (minimum 3 test types).
|
|
31
|
+
5. Use Skyramp MCP to execute the generated tests and validate the results.
|
|
30
32
|
|
|
31
|
-
|
|
33
|
+
**IMPORTANT — Endpoint Renames:** If the diff shows an endpoint path was renamed (e.g. \`/products\` changed to \`/items\`) and existing tests already cover that endpoint under the old name, do NOT generate new tests for the renamed endpoint. The existing tests will be updated with the new path in Task 2 (Test Maintenance). Only generate new tests for genuinely new endpoints that have no existing test coverage under any name.
|
|
34
|
+
|
|
35
|
+
## Task 2: Existing Test Maintenance (MANDATORY)
|
|
36
|
+
|
|
37
|
+
You MUST always run steps 1–4 below. Do NOT skip this task based on your own assessment of whether tests exist or are relevant — use the tools to determine that.
|
|
38
|
+
|
|
39
|
+
1. Call \`skyramp_discover_tests\` with \`repositoryPath\`: "${repositoryPath}" to find all existing Skyramp-generated tests.
|
|
40
|
+
2. Call \`skyramp_analyze_test_drift\` with the \`stateFile\` returned by \`skyramp_discover_tests\`.
|
|
41
|
+
3. Call \`skyramp_calculate_health_scores\` with the \`stateFile\` from the previous step.
|
|
42
|
+
4. Call \`skyramp_actions\` with the updated \`stateFile\` to apply recommended updates.
|
|
43
|
+
- If \`skyramp_actions\` returns endpoint rename mappings (old path → new path), apply them as simple find-and-replace on the test file URLs. Do NOT regenerate or restructure the test — only update the paths.
|
|
44
|
+
- If \`skyramp_actions\` suggests file renames (e.g. \`products_smoke_test.py\` → \`items_smoke_test.py\`), rename the files using \`git mv\` after updating their content.
|
|
45
|
+
5. Execute any updated or affected tests using Skyramp MCP and validate the results.
|
|
46
|
+
6. You may skip this task ONLY if \`skyramp_discover_tests\` explicitly returns zero Skyramp-generated tests.
|
|
47
|
+
|
|
48
|
+
## Task 3: Submit Report (MANDATORY)
|
|
32
49
|
|
|
33
50
|
After completing Tasks 1 and 2, you MUST call the Skyramp MCP tool "skyramp_submit_report" to submit your report.
|
|
34
51
|
Pass '${summaryOutputFile}' as the summaryOutputFile parameter.
|
|
35
52
|
|
|
53
|
+
For the commitMessage parameter, write a succinct summary (under 72 chars) of what you did, without any prefix. Examples:
|
|
54
|
+
- "add contract tests for /products endpoint"
|
|
55
|
+
- "update smoke tests for order API changes"
|
|
56
|
+
- "add smoke and e2e tests for new /reviews endpoint"
|
|
57
|
+
|
|
36
58
|
Do NOT write the report to a file yourself. Do NOT skip this step. The skyramp_submit_report tool is the ONLY way to submit the report.
|
|
37
59
|
|
|
60
|
+
## Report Guidelines
|
|
61
|
+
|
|
62
|
+
When reporting test results, if you chose to skip executing a test, you MUST explain WHY you skipped it.
|
|
63
|
+
NEVER use the phrase "CI timeout" or imply a timeout occurred unless a tool call actually timed out.
|
|
64
|
+
Instead, set the status to "Skipped" and provide an honest reason in the details, for example:
|
|
65
|
+
- "Skipped: no code changes affect this endpoint"
|
|
66
|
+
- "Skipped: skyramp_discover_tests found no existing Skyramp tests"
|
|
67
|
+
- "Skipped: only CI/config changes in this PR, no API changes"
|
|
68
|
+
|
|
38
69
|
Reminder: Use the Skyramp MCP tools available to you for test analysis, generation, and execution.`;
|
|
39
70
|
}
|
|
40
71
|
export function registerTestbotPrompt(server) {
|
|
@@ -43,9 +74,7 @@ export function registerTestbotPrompt(server) {
|
|
|
43
74
|
description: "Run Skyramp TestBot to generate test recommendations and perform test maintenance for a pull request.",
|
|
44
75
|
argsSchema: {
|
|
45
76
|
prTitle: z.string().describe("Pull request title"),
|
|
46
|
-
prDescription: z
|
|
47
|
-
.string()
|
|
48
|
-
.describe("Pull request description/body"),
|
|
77
|
+
prDescription: z.string().describe("Pull request description/body"),
|
|
49
78
|
diffFile: z.string().describe("Path to the git diff file"),
|
|
50
79
|
testDirectory: z
|
|
51
80
|
.string()
|
|
@@ -54,9 +83,18 @@ export function registerTestbotPrompt(server) {
|
|
|
54
83
|
summaryOutputFile: z
|
|
55
84
|
.string()
|
|
56
85
|
.describe("File path where the agent should write the testbot summary report"),
|
|
86
|
+
repositoryPath: z
|
|
87
|
+
.string()
|
|
88
|
+
.default(".")
|
|
89
|
+
.describe("Absolute path to the repository being analyzed"),
|
|
90
|
+
baseBranch: z
|
|
91
|
+
.string()
|
|
92
|
+
.optional()
|
|
93
|
+
.describe("PR base branch name (e.g. 'main' or 'develop'). When provided, analyzeRepository diffs against this branch instead of auto-detecting."),
|
|
57
94
|
},
|
|
58
95
|
}, (args) => {
|
|
59
|
-
const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile);
|
|
96
|
+
const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath, args.baseBranch);
|
|
97
|
+
AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
|
|
60
98
|
return {
|
|
61
99
|
messages: [
|
|
62
100
|
{
|
|
@@ -70,3 +108,32 @@ export function registerTestbotPrompt(server) {
|
|
|
70
108
|
};
|
|
71
109
|
});
|
|
72
110
|
}
|
|
111
|
+
export function registerTestbotResource(server) {
|
|
112
|
+
logger.info("Registering testbot resource");
|
|
113
|
+
// RFC 6570 {+rest} (reserved expansion) captures the entire query string
|
|
114
|
+
// including the leading "?". This avoids the SDK's per-param regex which
|
|
115
|
+
// fails on empty query-param values (e.g. prDescription=).
|
|
116
|
+
// We then parse query params from the URL object which handles URL-decoding
|
|
117
|
+
// and empty values correctly.
|
|
118
|
+
const template = new ResourceTemplate("skyramp://prompts/testbot{+rest}", {
|
|
119
|
+
list: undefined,
|
|
120
|
+
});
|
|
121
|
+
server.registerResource("skyramp_testbot", template, {
|
|
122
|
+
title: "Skyramp TestBot Prompt",
|
|
123
|
+
description: "Returns task instructions for PR test analysis, generation, and maintenance.",
|
|
124
|
+
mimeType: "text/plain",
|
|
125
|
+
}, (uri) => {
|
|
126
|
+
const param = (name, fallback) => uri.searchParams.get(name) ?? fallback;
|
|
127
|
+
const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."), uri.searchParams.get("baseBranch") || undefined);
|
|
128
|
+
AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
|
|
129
|
+
return {
|
|
130
|
+
contents: [
|
|
131
|
+
{
|
|
132
|
+
uri: uri.toString(),
|
|
133
|
+
mimeType: "text/plain",
|
|
134
|
+
text: prompt,
|
|
135
|
+
},
|
|
136
|
+
],
|
|
137
|
+
};
|
|
138
|
+
});
|
|
139
|
+
}
|