@skyramp/mcp 0.0.56 → 0.0.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/build/index.js +56 -9
  2. package/build/prompts/startTraceCollectionPrompts.js +20 -2
  3. package/build/prompts/test-recommendation/repository-analysis-prompt.js +42 -9
  4. package/build/prompts/test-recommendation/test-recommendation-prompt.js +91 -85
  5. package/build/prompts/testbot/testbot-prompts.js +85 -18
  6. package/build/services/AnalyticsService.js +31 -12
  7. package/build/services/DriftAnalysisService.js +139 -13
  8. package/build/services/DriftAnalysisService.test.js +168 -0
  9. package/build/services/TestExecutionService.js +1 -1
  10. package/build/services/TestGenerationService.js +7 -2
  11. package/build/services/TestHealthService.js +38 -3
  12. package/build/services/TestHealthService.test.js +211 -0
  13. package/build/tools/submitReportTool.js +12 -3
  14. package/build/tools/submitReportTool.test.js +59 -1
  15. package/build/tools/test-maintenance/actionsTool.js +115 -9
  16. package/build/tools/test-maintenance/actionsTool.test.js +93 -0
  17. package/build/tools/test-recommendation/analyzeRepositoryTool.js +221 -14
  18. package/build/tools/test-recommendation/mapTestsTool.js +44 -22
  19. package/build/tools/test-recommendation/recommendTestsTool.js +87 -33
  20. package/build/tools/trace/startTraceCollectionTool.js +5 -2
  21. package/build/tools/trace/stopTraceCollectionTool.js +2 -2
  22. package/build/tools/workspace/initializeWorkspaceTool.js +222 -0
  23. package/build/types/RepositoryAnalysis.js +20 -0
  24. package/build/types/TestTypes.js +6 -2
  25. package/build/utils/initAgent.js +2 -1
  26. package/build/utils/scoring-engine.js +136 -18
  27. package/build/utils/telemetry.js +23 -0
  28. package/build/utils/utils.js +0 -1
  29. package/package.json +2 -2
package/build/index.js CHANGED
@@ -30,9 +30,10 @@ import { registerExecuteBatchTestsTool } from "./tools/test-maintenance/executeB
30
30
  import { registerCalculateHealthScoresTool } from "./tools/test-maintenance/calculateHealthScoresTool.js";
31
31
  import { registerActionsTool } from "./tools/test-maintenance/actionsTool.js";
32
32
  import { registerStateCleanupTool } from "./tools/test-maintenance/stateCleanupTool.js";
33
- import { registerTestbotPrompt } from "./prompts/testbot/testbot-prompts.js";
33
+ import { registerTestbotPrompt, registerTestbotResource, } from "./prompts/testbot/testbot-prompts.js";
34
34
  import { registerInitTestbotTool } from "./tools/initTestbotTool.js";
35
35
  import { registerSubmitReportTool } from "./tools/submitReportTool.js";
36
+ import { registerInitializeWorkspaceTool } from "./tools/workspace/initializeWorkspaceTool.js";
36
37
  import { AnalyticsService } from "./services/AnalyticsService.js";
37
38
  import { initCheck } from "./utils/initAgent.js";
38
39
  const server = new McpServer({
@@ -47,19 +48,62 @@ const server = new McpServer({
47
48
  listChanged: true,
48
49
  },
49
50
  },
51
+ instructions: `Skyramp MCP Server — generates and executes API tests (smoke, fuzz, contract, load, integration, E2E, UI).
52
+
53
+ ## Rules
54
+ - NEVER show CLI commands. ALWAYS use the MCP tools provided.
55
+ - For UI and E2E tests, use the trace collection start/stop tools.
56
+
57
+ ## Workspace Initialization (before ANY other Skyramp tool)
58
+ Follow this flow EVERY time before calling any Skyramp tool:
59
+
60
+ 1. **Check**: Does .skyramp/workspace.yml exist at the workspace root?
61
+ 2. **If YES** → proceed with the requested tool.
62
+ 3. **If NO** → you MUST call \`skyramp_initialize_workspace\` BEFORE doing anything else.
63
+ - Do NOT skip this step. Do NOT proceed to the requested tool first.
64
+ - Scan the repo for ALL services (see the tool description for detailed steps).
65
+ - A fullstack or monorepo MUST produce multiple services — never just one.
66
+ - After workspace init completes, THEN proceed with the originally requested tool.
67
+ 4. **ONLY skip init if the user EXPLICITLY says** "no", "skip", "don't create workspace", or similar.
68
+ - A request like "execute tests" or "generate tests" is NOT a signal to skip init.
69
+ - If the user does decline, respect it — do NOT ask again, and proceed with the requested tool.
70
+
71
+ ## Workspace Defaults for Test Generation (MANDATORY)
72
+ Before calling ANY test generation tool, you MUST follow this flow:
73
+
74
+ 1. **Read** the .skyramp/workspace.yml file to get the configured defaults.
75
+ 2. **Extract** the \`language\`, \`framework\`, \`outputDir\`, and \`api.baseUrl\` from the services section.
76
+ 3. **Use those values** as defaults for the test generation tool call. Do NOT ask the user for these values if they are already configured in the workspace file.
77
+ 4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
78
+ 5. **If the workspace file does not exist**, or the needed values (language, framework, outputDir) are missing from the workspace config, ASK the user which language and framework they want before calling the tool.
79
+ 6. The user can always override workspace defaults by explicitly specifying values in their request.
80
+ `,
50
81
  });
51
82
  // Check for first-time invocation after version update (runs in background, doesn't block)
52
- let initCheckCalled = false;
83
+ let initCheckInFlight = false;
84
+ let initCheckDone = false;
53
85
  const originalRegisterTool = server.registerTool.bind(server);
54
86
  server.registerTool = function (name, definition, handler) {
55
87
  const wrappedHandler = async (...args) => {
56
- if (!initCheckCalled) {
57
- initCheck()
58
- .then(() => {
59
- initCheckCalled = true; // Only set to true if initCheck succeeds, allowing retry on failure
60
- })
61
- .catch((err) => {
62
- logger.error("Background initialization check failed", { error: err });
88
+ if (!initCheckDone && !initCheckInFlight) {
89
+ // Guard with inFlight so concurrent tool calls don't each spawn a new initCheck(),
90
+ // but allow retry on failure (initCheckInFlight is reset to false on error).
91
+ // SkyrampClient constructor calls checkForUpdate("npm") via synchronous koffi FFI,
92
+ // which can block the event loop for up to 60 s if the update-check server is
93
+ // unreachable. Deferring via setImmediate ensures the tool response is written to
94
+ // stdout (and acknowledged by the MCP client) before any blocking FFI call runs.
95
+ initCheckInFlight = true;
96
+ setImmediate(() => {
97
+ initCheck()
98
+ .then(() => {
99
+ initCheckDone = true;
100
+ })
101
+ .catch((err) => {
102
+ logger.error("Background initialization check failed", { error: err });
103
+ })
104
+ .finally(() => {
105
+ initCheckInFlight = false;
106
+ });
63
107
  });
64
108
  }
65
109
  return handler(...args);
@@ -75,6 +119,7 @@ const prompts = [
75
119
  ];
76
120
  if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
77
121
  prompts.push(registerTestbotPrompt);
122
+ registerTestbotResource(server);
78
123
  logger.info("TestBot prompt enabled via SKYRAMP_FEATURE_TESTBOT");
79
124
  }
80
125
  prompts.forEach((registerPrompt) => registerPrompt(server));
@@ -109,6 +154,8 @@ registerExecuteBatchTestsTool(server);
109
154
  registerCalculateHealthScoresTool(server);
110
155
  registerActionsTool(server);
111
156
  registerStateCleanupTool(server);
157
+ // Register workspace management tools
158
+ registerInitializeWorkspaceTool(server);
112
159
  // Register other Skyramp tools
113
160
  const infrastructureTools = [
114
161
  registerLoginTool,
@@ -30,13 +30,25 @@ When playwright is enabled for trace collection, you can optionally configure:
30
30
  * 'webkit' - Safari/WebKit browser
31
31
  - Use firefox or webkit when you need to test cross-browser compatibility or specific browser behaviors
32
32
 
33
- 2. **Playwright Storage Path** (playwrightStoragePath):
33
+ 2. **Device** (device):
34
+ - Device to emulate for mobile/tablet testing
35
+ - Supported devices include:
36
+ * 'iPhone 13' - Apple iPhone 13
37
+ * 'iPhone 13 Pro' - Apple iPhone 13 Pro
38
+ * 'Pixel 5' - Google Pixel 5
39
+ * 'Galaxy S9+' - Samsung Galaxy S9 Plus
40
+ * 'iPad Pro 11' - Apple iPad Pro
41
+ * And many more Playwright device profiles
42
+ - Leave empty (default) for desktop/no device emulation
43
+ - Use specific device names when testing mobile-responsive applications or generating mobile UI tests
44
+
45
+ 3. **Playwright Storage Path** (playwrightStoragePath):
34
46
  - Path to a playwright session storage file containing authentication data (cookies, localStorage, sessionStorage, etc.)
35
47
  - MUST be an absolute path like /path/to/storage.json
36
48
  - Use this when you have manually created a session from the login flow and want to reuse it for future trace collections to avoid manual login every time
37
49
  - The session file should be created beforehand using Playwright's storageState feature during the login flow
38
50
 
39
- 3. **Playwright Viewport Size** (playwrightViewportSize):
51
+ 4. **Playwright Viewport Size** (playwrightViewportSize):
40
52
  - Defines the browser window size for trace collection
41
53
  - Supported formats:
42
54
  * 'hd' - 1280x720
@@ -52,6 +64,12 @@ When playwright is enabled for trace collection, you can optionally configure:
52
64
  * To start a playwright trace collection session using agent, run the following command:
53
65
  Start playwright trace collection with default settings.
54
66
 
67
+ **Example usage prompt for trace collection with device emulation:**
68
+ * To start a trace collection session with mobile device emulation:
69
+ Start trace collection with iPhone 13 device
70
+ Collect trace using Pixel 5 device emulation
71
+ Start playwright trace with iPad Pro device
72
+
55
73
  **Example usage prompt for trace collection with playwright storage and viewport:**
56
74
  * Start playwright trace collection with storage path /Users/dev/session-storage.json and viewport size full-hd
57
75
 
@@ -2,20 +2,35 @@
2
2
  * Repository Analysis Prompt
3
3
  * Comprehensive prompt for analyzing code repositories
4
4
  */
5
- export function getRepositoryAnalysisPrompt(repositoryPath) {
5
+ export function getRepositoryAnalysisPrompt(repositoryPath, analysisScope = "full_repo", parsedDiff) {
6
+ const isDiffScope = analysisScope === "current_branch_diff";
6
7
  return `
7
8
  Analyze this repository systematically using the guide below.
8
9
 
9
10
  REPOSITORY PATH: ${repositoryPath}
11
+ ANALYSIS SCOPE: ${analysisScope}${isDiffScope ? " (focus on branch changes only)" : " (full repository)"}
10
12
 
11
13
  IMPORTANT OUTPUT FORMAT:
12
14
  - Return analysis as valid JSON matching the RepositoryAnalysis interface
13
15
  - Use empty arrays [] for unavailable data rather than omitting fields
14
16
  - Use "unknown" for unknown string values
15
17
  - Provide evidence (file paths, line numbers) where possible
18
+ ${isDiffScope ? '- MUST include the "branchDiffContext" field in the output JSON' : ""}
16
19
 
17
20
  ## CRITICAL RULES
18
21
  - If user flows are defined in documentation explicitly, follow them strictly and do not generate new ones.
22
+ ${isDiffScope ? `- Do NOT run git commands — the diff has already been parsed by the tool
23
+ - Still gather full repo context (tech stack, infra, auth, existing tests) for accurate scoring
24
+ - For branchDiffContext, use the following pre-parsed metadata:
25
+ - currentBranch: "${parsedDiff?.currentBranch || "unknown"}"
26
+ - baseBranch: "${parsedDiff?.baseBranch || "main"}"
27
+ - changedFiles: [${parsedDiff?.changedFiles.map((f) => `"${f}"`).join(", ") || ""}]
28
+ - For newEndpoints and modifiedEndpoints: use the pre-parsed endpoint information provided by the tool as the primary source of truth.
29
+ - You may refine this list using only the metadata and code/context that is explicitly included in this prompt.
30
+ - Do NOT attempt to scan or open additional repository files beyond what has been provided here.
31
+ - It is acceptable if some indirect or transitive impacts are not fully captured; do not speculate beyond the available information.
32
+ - When you adjust endpoints, clearly explain your reasoning based on the pre-parsed data and any in-prompt context.
33
+ - affectedServices: treat the changed file paths as high-level hints about which services are likely impacted, but do not perform exhaustive dependency tracing beyond the information provided` : ""}
19
34
 
20
35
  # Repository Analysis Guide
21
36
 
@@ -187,7 +202,8 @@ Return a JSON object with this exact structure:
187
202
  "metadata": {
188
203
  "repositoryName": "name-from-path",
189
204
  "analysisDate": "2025-10-15T14:30:00Z",
190
- "scanDepth": "full"
205
+ "scanDepth": "full",
206
+ "analysisScope": "${analysisScope}"
191
207
  },
192
208
  "projectClassification": {
193
209
  "projectType": "rest-api | frontend | full-stack | microservices | library | cli | other",
@@ -270,7 +286,20 @@ Return a JSON object with this exact structure:
270
286
  },
271
287
  "hasCoverageReports": true,
272
288
  "estimatedCoverage": 78
273
- }
289
+ }${isDiffScope ? `,
290
+ "branchDiffContext": {
291
+ "currentBranch": "feature/my-feature",
292
+ "baseBranch": "main",
293
+ "changedFiles": ["src/routes/products.ts", "src/models/Product.ts"],
294
+ "newEndpoints": [
295
+ { "path": "/api/v1/products/search", "method": "GET", "sourceFile": "src/routes/products.ts:45" }
296
+ ],
297
+ "modifiedEndpoints": [
298
+ { "path": "/api/v1/products", "method": "POST", "sourceFile": "src/routes/products.ts:12" }
299
+ ],
300
+ "affectedServices": ["product-service"],
301
+ "summary": "Added product search endpoint and modified product creation validation"
302
+ }` : ""}
274
303
  }
275
304
  \`\`\`
276
305
 
@@ -280,14 +309,18 @@ VALIDATION CHECKLIST:
280
309
  - [ ] API endpoints counted and categorized
281
310
  - [ ] Authentication method determined
282
311
  - [ ] Infrastructure flags verified
283
- - [ ] Existing tests catalogued
312
+ - [ ] Existing tests catalogued${isDiffScope ? `
313
+ - [ ] branchDiffContext.changedFiles matches the list provided above
314
+ - [ ] branchDiffContext.modifiedEndpoints includes ALL endpoints whose behaviour changed — including those affected by model/schema/validator changes (trace: changed file → which models/types → which endpoints use them)
315
+ - [ ] branchDiffContext.newEndpoints includes any brand-new routes added in the diff
316
+ - [ ] branchDiffContext.summary describes what changed in plain English` : ""}
284
317
 
285
318
  **CRITICAL INSTRUCTIONS**:
286
- - Return the JSON object directly in your response text.
287
- - DO NOT create or save any files (no repository_analysis.json or .md files).
288
- - DO NOT use file write tools or save analysis to disk.
289
- - Output the complete analysis JSON inline in your response.
319
+ - Construct the complete RepositoryAnalysis JSON object.
320
+ - DO NOT create any .md or documentation files.
321
+ - Save the analysis JSON to the state file path provided in the tool response.
322
+ - Then call \`skyramp_map_tests\` with the \`stateFile\` parameter (NOT analysisReport) to avoid serialization issues.
290
323
 
291
- Begin analysis now. Return the JSON object directly in your response text.
324
+ Begin analysis now.
292
325
  `;
293
326
  }
@@ -2,51 +2,90 @@
2
2
  * Test Recommendation Prompt
3
3
  * Prompt for generating actionable test recommendations
4
4
  */
5
- export function getTestRecommendationPrompt(mapping, analysis, topN = 7) {
5
+ export function getTestRecommendationPrompt(mapping, analysis, topN = 7, analysisScope = "full_repo") {
6
+ const isDiffScope = analysisScope === "current_branch_diff";
7
+ const diffContext = analysis.branchDiffContext;
8
+ // --- Build a compact summary of priority scores (no full JSON dump) ---
9
+ const topScores = mapping.priorityScores
10
+ .filter((s) => s.feasibility !== "not-applicable")
11
+ .slice(0, topN);
12
+ const scoreLines = topScores
13
+ .map((s) => {
14
+ const priority = s._finalScore >= 100 ? "high" : s._finalScore >= 70 ? "medium" : "low";
15
+ const avail = s.requiredArtifacts.available.join(", ") || "none";
16
+ const miss = s.requiredArtifacts.missing.join(", ") || "none";
17
+ return (`- **${s.testType.toUpperCase()}** | priority: ${priority} | feasibility: ${s.feasibility}\n` +
18
+ ` Reasoning: ${s.reasoning}\n` +
19
+ ` Artifacts available: ${avail} | missing: ${miss}`);
20
+ })
21
+ .join("\n");
22
+ // --- Build compact repo context (no full JSON dump) ---
23
+ const endpoints = analysis.apiEndpoints.endpoints
24
+ .slice(0, 30) // cap at 30 to keep size sane
25
+ .map((e) => ` ${e.method} ${e.path}${e.authRequired ? " [auth]" : ""}`)
26
+ .join("\n");
27
+ const repoContext = `
28
+ Repository: ${analysis.metadata.repositoryName}
29
+ Framework: ${analysis.projectClassification.primaryFramework} (${analysis.projectClassification.primaryLanguage})
30
+ Project type: ${analysis.projectClassification.projectType}
31
+ Auth: ${analysis.authentication.method}
32
+ Base URL: ${analysis.apiEndpoints.baseUrl}
33
+ Endpoints (${analysis.apiEndpoints.totalCount} total):
34
+ ${endpoints}
35
+ `.trim();
36
+ // --- Branch diff context (only the essentials) ---
37
+ let diffSection = "";
38
+ if (isDiffScope && diffContext) {
39
+ const newEps = diffContext.newEndpoints
40
+ .map((e) => ` ${e.method} ${e.path} (${e.sourceFile})`)
41
+ .join("\n") || " none";
42
+ const modEps = diffContext.modifiedEndpoints
43
+ .map((e) => ` ${e.method} ${e.path} (${e.sourceFile})`)
44
+ .join("\n") || " none";
45
+ diffSection = `
46
+ ## Branch Diff Context
47
+ Branch: \`${diffContext.currentBranch}\` → base: \`${diffContext.baseBranch}\`
48
+ Changed files: ${diffContext.changedFiles.join(", ")}
49
+ New endpoints:
50
+ ${newEps}
51
+ Modified endpoints:
52
+ ${modEps}
53
+ Affected services: ${diffContext.affectedServices.join(", ") || "N/A"}
54
+ Summary: ${diffContext?.summary ?? "N/A"}
55
+
56
+ **CRITICAL**: Focus recommendations ONLY on tests that validate the branch changes above.
57
+ `;
58
+ }
59
+ const scopeNote = isDiffScope
60
+ ? "Recommendations are scoped to the current branch changes only. All specific test suggestions MUST target the new/modified endpoints listed in the Branch Diff Context."
61
+ : "Recommendations cover the full repository.";
6
62
  return `
7
- Generate actionable test recommendations based on priority and repository analysis.
8
-
9
- **MANDATORY RULES**:
10
- 1. **Priority Scores Must Remain Unchanged**: When a test type is missing required inputs (e.g., Playwright recordings, traces), do NOT:
11
- - Mark the test as "blocked" in the output
12
- - Adjust or reduce the priority score
13
- - Exclude it from recommendations if it ranks in the top ${topN}
14
-
15
- # Priority
16
-
17
- \`\`\`json
18
- ${JSON.stringify(mapping, null, 2)}
19
- \`\`\`
63
+ Generate actionable test recommendations based on the priority scores and repository analysis below.
20
64
 
21
- # Repository Context
65
+ Scope: ${scopeNote}
66
+ ${diffSection}
67
+ ## Repository Context
22
68
 
23
- \`\`\`json
24
- ${JSON.stringify(analysis, null, 2)}
25
- \`\`\`
69
+ ${repoContext}
26
70
 
27
- # Your Task
71
+ ## Prioritized Test Types (top ${topN})
28
72
 
29
- Generate specific, actionable test recommendations for the top ${topN} highest-scoring test types.
73
+ ${scoreLines}
30
74
 
31
- ## CRITICAL RULES
32
- - **SCORING**: Use the numeric scores ONLY internally to:
33
- 1. Select the top ${topN} test types by score (highest to lowest)
34
- 2. Assign priority levels based on score ranges:
35
- * Score >= 100: priority = "high"
36
- * Score 70-99: priority = "medium"
37
- * Score < 70: priority = "low"
38
- - **DO NOT include the "score" field** in the JSON output (it has been removed from the schema).
39
- - DO NOT mention or display numerical scores anywhere in user-facing text.
40
- - Order test recommendations by their internal priority scores (highest to lowest).
75
+ ## Priority Mapping Summary
76
+ - High priority: ${mapping.summary.highPriority.join(", ") || "none"}
77
+ - Medium priority: ${mapping.summary.mediumPriority.join(", ") || "none"}
78
+ - Low priority: ${mapping.summary.lowPriority.join(", ") || "none"}
41
79
 
42
- ## Requirements
80
+ ## Your Task
43
81
 
44
- For each recommended test type:
45
- 1. **Provide rationale for prioritization based on repository attributes.
46
- 2. **Suggest 3-4 ACTUAL tests** that should be created based on individual user flows and API endpoints.
47
- 3. **Include available artifacts** with specific file paths
48
- 4. **Provide guidance** for creating missing artifacts
82
+ Generate specific, actionable test recommendations for the top ${topN} test types listed above.
49
83
 
84
+ **MANDATORY RULES**:
85
+ 1. **NO numeric scores** in output — use only: "high", "medium", "low".
86
+ 2. **DO NOT mark any test "blocked"** even if artifacts are missing.
87
+ 3. **Branch scope**: All specific test suggestions MUST target endpoints/code that changed in the branch diff.
88
+ 4. **DO NOT create or save any files** — output everything inline in your response.
50
89
 
51
90
  ## Output Structure
52
91
 
@@ -58,74 +97,41 @@ Return a JSON object with this structure:
58
97
  "totalRecommended": 3,
59
98
  "highPriorityCount": 2,
60
99
  "estimatedEffort": "4-6 hours for top 3 tests",
61
- "quickWins": [
62
- "Smoke tests (OpenAPI available, 30 min)",
63
- "Contract tests (OpenAPI available, 1 hour)"
64
- ]
100
+ "quickWins": ["Smoke tests (OpenAPI available, 30 min)"]
65
101
  },
66
102
  "recommendations": [
67
103
  {
68
104
  "priority": "high",
69
- "testType": "integration",
70
- "rationale": "Detailed explanation of why this is high priority based on repository characteristics (CRITICAL: DO NOT MENTION THE NUMERICAL SCORE HERE)",
105
+ "testType": "contract",
106
+ "rationale": "Why this is high priority (DO NOT mention numeric scores)",
71
107
  "specificTests": [
72
108
  {
73
- "testName": "User Registration and Authentication Flow",
74
- "description": "Test complete user lifecycle: register verify → login → access protected resource",
75
- "targetEndpoint": "/api/v1/auth/register, /api/v1/auth/login",
76
- "targetFlow": "New user onboarding",
109
+ "testName": "Order quantity validation contract test",
110
+ "description": "Verify POST /api/v1/orders rejects quantity < 1",
111
+ "targetEndpoint": "POST /api/v1/orders",
112
+ "targetFlow": "Order creation validation",
77
113
  "requiredInputs": {
78
- "available": [
79
- {
80
- "name": "openApiSpec",
81
- "path": "./docs/openapi.yaml"
82
- }
83
- ],
84
- "missing": [
85
- {
86
- "name": "traceFile",
87
- "guidance": "Use Skyramp trace collection: Run 'skyramp_start_trace_collection'..."
88
- }
89
- ]
114
+ "available": [{"name": "openApiSpec", "path": "./openapi.json"}],
115
+ "missing": []
90
116
  },
91
- "estimatedValue": "High - catches integration bugs between auth and user services"
117
+ "estimatedValue": "High - catches regression in validation rule change"
92
118
  }
93
119
  ],
94
120
  "gettingStarted": {
95
- "prerequisites": [
96
- "Docker and Docker Compose installed",
97
- "MongoDB running (via docker-compose up)",
98
- "JWT_SECRET environment variable set"
99
- ],
100
- "quickStartCommand": "npm install && docker-compose up -d && npm run test:integration",
101
- "documentationUrl": "https://www.skyramp.dev/docs/integration-tests"
121
+ "prerequisites": ["Service running at localhost:8000"],
122
+ "quickStartCommand": "Use Skyramp MCP generate tools",
123
+ "documentationUrl": "https://www.skyramp.dev/docs"
102
124
  }
103
125
  }
104
126
  ],
105
127
  "nextSteps": [
106
- "1. Start with Integration Tests (highest priority, fills critical gap)",
107
- "2. Copy the 'User Registration and Authentication Flow' prompt above",
108
- "3. Run: skyramp_integration_test_generation with the prompt",
109
- "4. Execute generated tests: npm run test:integration",
110
- "5. Once integration tests pass, proceed with Fuzz Tests for security"
128
+ "1. Start with highest priority test type",
129
+ "2. Use Skyramp MCP generate tools to create each test",
130
+ "3. Execute and validate results"
111
131
  ]
112
132
  }
113
133
  \`\`\`
114
134
 
115
- ## Important Guidelines
116
-
117
- 1. **Be Specific**: Use actual endpoint paths, file paths, and flow names from the repository
118
- 2. **Be Actionable**: Prompts should work without modification
119
- 3. **Highlight Quick Wins**: Highlight tests with all artifacts available
120
- 4. **Provide Guidance**: Clear instructions for creating missing artifacts
121
- 5. **Include Evidence**: Reference specific repository characteristics in rationale
122
-
123
- **CRITICAL INSTRUCTIONS**:
124
- - Return the JSON object directly in your response text.
125
- - DO NOT create or save any files (no test_recommendations.md, .json, or other files).
126
- - DO NOT use file write tools or save recommendations to disk.
127
- - Output the complete recommendations JSON inline in your response.
128
-
129
135
  Generate recommendations now. Return the JSON object directly in your response text.
130
136
  `;
131
137
  }
@@ -1,40 +1,71 @@
1
+ import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
1
2
  import { z } from "zod";
2
3
  import { logger } from "../../utils/logger.js";
3
- function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile) {
4
+ import { AnalyticsService } from "../../services/AnalyticsService.js";
5
+ function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath, baseBranch) {
4
6
  return `<TITLE>${prTitle}</TITLE>
5
7
  <DESCRIPTION>${prDescription}</DESCRIPTION>
6
8
  <CODE CHANGES>${diffFile}</CODE CHANGES>
7
9
  <TEST DIRECTORY>${testDirectory}</TEST DIRECTORY>
10
+ <REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
8
11
 
9
12
  For all the following work, use the tools offered by Skyramp MCP server.
10
13
 
11
14
  First analyze the pull request title, description, and code changes to determine a business case
12
15
  justification for this code change.
13
16
 
14
- Then perform the following tasks based on your analysis:
17
+ Then perform ALL of the following tasks. Every task is MANDATORY — do NOT skip any task based on your own judgment unless the task itself gives you an explicit condition to skip.
15
18
 
16
- ## Task 1: New Test Recommendations
19
+ ## Task 1: Recommend New Tests (MANDATORY)
17
20
 
18
- 1. If, and only if, there are new endpoints in the code changes, get recommendations
19
- for new contract, e2e, smoke, load tests specific to the code changes
20
- and business case analysis. DO NOT use the whole repository for generating recommendations.
21
- 2. If, and only if, there are recommendations for new tests, then use Skyramp MCP generate
22
- tools to generate contract, e2e, smoke and load tests.
23
- 3. Use Skyramp MCP to execute the generated tests and validate the results.
21
+ Read the diff at \`${diffFile}\`. Classify each changed file. A file is application source code if it is any of: a route/controller/handler, a model/schema/validator/serializer/DTO, business logic, middleware, service, utility, test helper, or has a source extension (.py, .ts, .js, .java, .go, .rb, .cs, .kt, .swift, etc.). When in doubt, treat the file as application source code.
24
22
 
25
- ## Task 2: Existing Test Maintenance
23
+ **DEFAULT: You MUST run steps 1–5 below.** The only exception is if you can confirm that EVERY changed file is exclusively a CI workflow YAML, markdown documentation, README, CHANGELOG, or a dependency lock file — and nothing else.
26
24
 
27
- 1. Check if there are any relevant Skyramp tests in the '${testDirectory}' directory that correspond to the code changes.
28
- 2. If there are relevant tests, use Skyramp MCP tools to analyze and modify them to ensure they pass with the new changes.
29
- 3. Use Skyramp MCP to execute the updated tests and validate the results.
25
+ 1. Call \`skyramp_analyze_repository\` with:
26
+ - \`repositoryPath\`: "${repositoryPath}"
27
+ - \`analysisScope\`: "current_branch_diff"${baseBranch ? `\n - \`baseBranch\`: "${baseBranch}"` : ''}
28
+ 2. MANDATORY: Call \`skyramp_map_tests\` with \`stateFile\` (the state file path returned above) and \`analysisScope: "current_branch_diff"\`.
29
+ 3. MANDATORY: Call \`skyramp_recommend_tests\` with the \`stateFile\` returned by \`skyramp_map_tests\`. Use the priority summary and the specific endpoints/files that changed to determine exactly what to test.
30
+ 4. Generate tests using the Skyramp MCP generate tools, in priority order (minimum 3 test types).
31
+ 5. Use Skyramp MCP to execute the generated tests and validate the results.
30
32
 
31
- ## Task 3: Submit Report
33
+ **IMPORTANT — Endpoint Renames:** If the diff shows an endpoint path was renamed (e.g. \`/products\` changed to \`/items\`) and existing tests already cover that endpoint under the old name, do NOT generate new tests for the renamed endpoint. The existing tests will be updated with the new path in Task 2 (Test Maintenance). Only generate new tests for genuinely new endpoints that have no existing test coverage under any name.
34
+
35
+ ## Task 2: Existing Test Maintenance (MANDATORY)
36
+
37
+ You MUST always run steps 1–4 below. Do NOT skip this task based on your own assessment of whether tests exist or are relevant — use the tools to determine that.
38
+
39
+ 1. Call \`skyramp_discover_tests\` with \`repositoryPath\`: "${repositoryPath}" to find all existing Skyramp-generated tests.
40
+ 2. Call \`skyramp_analyze_test_drift\` with the \`stateFile\` returned by \`skyramp_discover_tests\`.
41
+ 3. Call \`skyramp_calculate_health_scores\` with the \`stateFile\` from the previous step.
42
+ 4. Call \`skyramp_actions\` with the updated \`stateFile\` to apply recommended updates.
43
+ - If \`skyramp_actions\` returns endpoint rename mappings (old path → new path), apply them as simple find-and-replace on the test file URLs. Do NOT regenerate or restructure the test — only update the paths.
44
+ - If \`skyramp_actions\` suggests file renames (e.g. \`products_smoke_test.py\` → \`items_smoke_test.py\`), rename the files using \`git mv\` after updating their content.
45
+ 5. Execute any updated or affected tests using Skyramp MCP and validate the results.
46
+ 6. You may skip this task ONLY if \`skyramp_discover_tests\` explicitly returns zero Skyramp-generated tests.
47
+
48
+ ## Task 3: Submit Report (MANDATORY)
32
49
 
33
50
  After completing Tasks 1 and 2, you MUST call the Skyramp MCP tool "skyramp_submit_report" to submit your report.
34
51
  Pass '${summaryOutputFile}' as the summaryOutputFile parameter.
35
52
 
53
+ For the commitMessage parameter, write a succinct summary (under 72 chars) of what you did, without any prefix. Examples:
54
+ - "add contract tests for /products endpoint"
55
+ - "update smoke tests for order API changes"
56
+ - "add smoke and e2e tests for new /reviews endpoint"
57
+
36
58
  Do NOT write the report to a file yourself. Do NOT skip this step. The skyramp_submit_report tool is the ONLY way to submit the report.
37
59
 
60
+ ## Report Guidelines
61
+
62
+ When reporting test results, if you chose to skip executing a test, you MUST explain WHY you skipped it.
63
+ NEVER use the phrase "CI timeout" or imply a timeout occurred unless a tool call actually timed out.
64
+ Instead, set the status to "Skipped" and provide an honest reason in the details, for example:
65
+ - "Skipped: no code changes affect this endpoint"
66
+ - "Skipped: skyramp_discover_tests found no existing Skyramp tests"
67
+ - "Skipped: only CI/config changes in this PR, no API changes"
68
+
38
69
  Reminder: Use the Skyramp MCP tools available to you for test analysis, generation, and execution.`;
39
70
  }
40
71
  export function registerTestbotPrompt(server) {
@@ -43,9 +74,7 @@ export function registerTestbotPrompt(server) {
43
74
  description: "Run Skyramp TestBot to generate test recommendations and perform test maintenance for a pull request.",
44
75
  argsSchema: {
45
76
  prTitle: z.string().describe("Pull request title"),
46
- prDescription: z
47
- .string()
48
- .describe("Pull request description/body"),
77
+ prDescription: z.string().describe("Pull request description/body"),
49
78
  diffFile: z.string().describe("Path to the git diff file"),
50
79
  testDirectory: z
51
80
  .string()
@@ -54,9 +83,18 @@ export function registerTestbotPrompt(server) {
54
83
  summaryOutputFile: z
55
84
  .string()
56
85
  .describe("File path where the agent should write the testbot summary report"),
86
+ repositoryPath: z
87
+ .string()
88
+ .default(".")
89
+ .describe("Absolute path to the repository being analyzed"),
90
+ baseBranch: z
91
+ .string()
92
+ .optional()
93
+ .describe("PR base branch name (e.g. 'main' or 'develop'). When provided, analyzeRepository diffs against this branch instead of auto-detecting."),
57
94
  },
58
95
  }, (args) => {
59
- const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile);
96
+ const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath, args.baseBranch);
97
+ AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
60
98
  return {
61
99
  messages: [
62
100
  {
@@ -70,3 +108,32 @@ export function registerTestbotPrompt(server) {
70
108
  };
71
109
  });
72
110
  }
111
+ export function registerTestbotResource(server) {
112
+ logger.info("Registering testbot resource");
113
+ // RFC 6570 {+rest} (reserved expansion) captures the entire query string
114
+ // including the leading "?". This avoids the SDK's per-param regex which
115
+ // fails on empty query-param values (e.g. prDescription=).
116
+ // We then parse query params from the URL object which handles URL-decoding
117
+ // and empty values correctly.
118
+ const template = new ResourceTemplate("skyramp://prompts/testbot{+rest}", {
119
+ list: undefined,
120
+ });
121
+ server.registerResource("skyramp_testbot", template, {
122
+ title: "Skyramp TestBot Prompt",
123
+ description: "Returns task instructions for PR test analysis, generation, and maintenance.",
124
+ mimeType: "text/plain",
125
+ }, (uri) => {
126
+ const param = (name, fallback) => uri.searchParams.get(name) ?? fallback;
127
+ const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."), uri.searchParams.get("baseBranch") || undefined);
128
+ AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
129
+ return {
130
+ contents: [
131
+ {
132
+ uri: uri.toString(),
133
+ mimeType: "text/plain",
134
+ text: prompt,
135
+ },
136
+ ],
137
+ };
138
+ });
139
+ }