@skyramp/mcp 0.0.61 → 0.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +44 -6
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +101 -0
- package/build/prompts/test-recommendation/recommendationSections.js +193 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +65 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +165 -99
- package/build/prompts/testGenerationPrompt.js +2 -3
- package/build/prompts/testbot/testbot-prompts.js +116 -96
- package/build/resources/analysisResources.js +248 -0
- package/build/services/ScenarioGenerationService.js +38 -40
- package/build/services/TestExecutionService.js +10 -1
- package/build/tools/generate-tests/generateScenarioRestTool.js +18 -6
- package/build/tools/submitReportTool.js +28 -0
- package/build/tools/test-maintenance/stateCleanupTool.js +8 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +386 -217
- package/build/tools/test-recommendation/recommendTestsTool.js +162 -163
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/types/RepositoryAnalysis.js +100 -12
- package/build/utils/AnalysisStateManager.js +56 -23
- package/build/utils/branchDiff.js +47 -0
- package/build/utils/initAgent.js +62 -26
- package/build/utils/pr-comment-parser.js +124 -0
- package/build/utils/projectMetadata.js +188 -0
- package/build/utils/projectMetadata.test.js +81 -0
- package/build/utils/repoScanner.js +425 -0
- package/build/utils/routeParsers.js +213 -0
- package/build/utils/routeParsers.test.js +87 -0
- package/build/utils/scenarioDrafting.js +119 -0
- package/build/utils/scenarioDrafting.test.js +66 -0
- package/build/utils/skyrampMdContent.js +100 -0
- package/build/utils/trace-parser.js +166 -0
- package/build/utils/workspaceAuth.js +16 -0
- package/package.json +2 -2
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +0 -326
- package/build/prompts/test-recommendation/test-mapping-prompt.js +0 -266
- package/build/tools/test-recommendation/mapTestsTool.js +0 -243
- package/build/types/TestMapping.js +0 -173
- package/build/utils/scoring-engine.js +0 -380
package/build/index.js
CHANGED
|
@@ -19,8 +19,8 @@ import { registerLoginTool } from "./tools/auth/loginTool.js";
|
|
|
19
19
|
import { registerLogoutTool } from "./tools/auth/logoutTool.js";
|
|
20
20
|
import { registerFixErrorTool } from "./tools/fixErrorTool.js";
|
|
21
21
|
import { registerAnalyzeRepositoryTool } from "./tools/test-recommendation/analyzeRepositoryTool.js";
|
|
22
|
-
import { registerMapTestsTool } from "./tools/test-recommendation/mapTestsTool.js";
|
|
23
22
|
import { registerRecommendTestsTool } from "./tools/test-recommendation/recommendTestsTool.js";
|
|
23
|
+
import { registerRecommendTestsPrompt } from "./prompts/test-recommendation/registerRecommendTestsPrompt.js";
|
|
24
24
|
import { registerModularizationTool } from "./tools/code-refactor/modularizationTool.js";
|
|
25
25
|
import { registerCodeReuseTool } from "./tools/code-refactor/codeReuseTool.js";
|
|
26
26
|
import { registerScenarioTestTool } from "./tools/generate-tests/generateScenarioRestTool.js";
|
|
@@ -34,6 +34,7 @@ import { registerTestbotPrompt, registerTestbotResource, } from "./prompts/testb
|
|
|
34
34
|
import { registerInitTestbotTool } from "./tools/initTestbotTool.js";
|
|
35
35
|
import { registerSubmitReportTool } from "./tools/submitReportTool.js";
|
|
36
36
|
import { registerInitializeWorkspaceTool } from "./tools/workspace/initializeWorkspaceTool.js";
|
|
37
|
+
import { registerAnalysisResources } from "./resources/analysisResources.js";
|
|
37
38
|
import { AnalyticsService } from "./services/AnalyticsService.js";
|
|
38
39
|
import { initCheck } from "./utils/initAgent.js";
|
|
39
40
|
const server = new McpServer({
|
|
@@ -47,13 +48,32 @@ const server = new McpServer({
|
|
|
47
48
|
prompts: {
|
|
48
49
|
listChanged: true,
|
|
49
50
|
},
|
|
51
|
+
resources: {
|
|
52
|
+
listChanged: true,
|
|
53
|
+
},
|
|
50
54
|
},
|
|
51
|
-
instructions: `Skyramp MCP Server — generates and executes API tests (
|
|
55
|
+
instructions: `Skyramp MCP Server — generates and executes API tests (fuzz, contract, integration, E2E, UI).
|
|
52
56
|
|
|
53
57
|
## Rules
|
|
54
58
|
- NEVER show CLI commands. ALWAYS use the MCP tools provided.
|
|
55
59
|
- For UI and E2E tests, use the trace collection start/stop tools.
|
|
56
60
|
|
|
61
|
+
## Test Recommendation Flow (2-step)
|
|
62
|
+
1. Call \`skyramp_analyze_repository\` → returns a \`sessionId\`.
|
|
63
|
+
The analysis scans source code (code-first) to build enriched endpoints
|
|
64
|
+
(Path → Method → Interaction with request/response bodies, headers, cookies)
|
|
65
|
+
and draft user-flow scenarios for integration/E2E tests.
|
|
66
|
+
2. Call \`skyramp_recommend_tests\` with \`sessionId\` → the LLM reasons over the
|
|
67
|
+
enriched data to recommend tests, referencing specific interactions and scenarios.
|
|
68
|
+
|
|
69
|
+
After analysis, you can also inspect data via MCP Resources:
|
|
70
|
+
- \`skyramp://analysis/{sessionId}/summary\` — high-level overview
|
|
71
|
+
- \`skyramp://analysis/{sessionId}/endpoints\` — compact endpoint listing
|
|
72
|
+
- \`skyramp://analysis/{sessionId}/endpoints/{path}\` — full path detail
|
|
73
|
+
- \`skyramp://analysis/{sessionId}/endpoints/{path}/{method}\` — single method detail
|
|
74
|
+
- \`skyramp://analysis/{sessionId}/scenarios\` — drafted scenarios
|
|
75
|
+
- \`skyramp://analysis/{sessionId}/diff\` — branch diff context
|
|
76
|
+
|
|
57
77
|
## Workspace Initialization (before ANY other Skyramp tool)
|
|
58
78
|
Follow this flow EVERY time before calling any Skyramp tool:
|
|
59
79
|
|
|
@@ -78,16 +98,23 @@ Before calling ANY test generation tool, you MUST follow this flow:
|
|
|
78
98
|
2. **Extract** the \`language\`, \`framework\`, \`outputDir\`, and \`api.baseUrl\` from the services section.
|
|
79
99
|
3. **Use those values** as defaults for the test generation tool call. Do NOT ask the user for these values if they are already configured in the workspace file.
|
|
80
100
|
4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
|
|
81
|
-
5. **
|
|
82
|
-
|
|
101
|
+
5. **CRITICAL — scenario generation**: When calling \`skyramp_scenario_test_generation\`, ALWAYS pass:
|
|
102
|
+
- \`baseURL\`: The full base URL from \`api.baseUrl\` (e.g., \`http://localhost:3000\`). This determines the scheme, host, and port in the generated trace. Without it, the trace defaults to https:443 which is almost always wrong for local development.
|
|
103
|
+
- \`authHeader\`: The auth header name from \`api.authHeader\` in the workspace config. Use \`Cookie\` for cookie/session-based auth (NextAuth, etc.), \`Authorization\` for Bearer tokens, \`X-API-Key\` for API keys. Without it, the trace defaults to \`Authorization: Bearer\` which breaks cookie-based apps.
|
|
104
|
+
- \`apiSchema\` is OPTIONAL — omit it for code-first apps without OpenAPI specs.
|
|
105
|
+
6. **CRITICAL — integration test from scenario**: When calling \`skyramp_integration_test_generation\` with a \`scenarioFile\`, ALSO pass \`authHeader\` (same value as used in scenario generation). This tells the CLI which header to parameterize with the auth token. Without it, the generated test defaults to \`Authorization: Bearer\` regardless of what's in the trace.
|
|
106
|
+
7. **If the workspace file does not exist**, or the needed values (language, framework, outputDir) are missing from the workspace config, ASK the user which language and framework they want before calling the tool.
|
|
107
|
+
8. The user can always override workspace defaults by explicitly specifying values in their request.
|
|
83
108
|
`,
|
|
84
109
|
});
|
|
85
110
|
// Check for first-time invocation after version update (runs in background, doesn't block)
|
|
86
111
|
let initCheckInFlight = false;
|
|
87
112
|
let initCheckDone = false;
|
|
113
|
+
const INIT_MESSAGE = "Skyramp init: Triggering pull of Skyramp worker and executor images if not present locally.";
|
|
88
114
|
const originalRegisterTool = server.registerTool.bind(server);
|
|
89
115
|
server.registerTool = function (name, definition, handler) {
|
|
90
116
|
const wrappedHandler = async (...args) => {
|
|
117
|
+
let triggeredInitThisCall = false;
|
|
91
118
|
if (!initCheckDone && !initCheckInFlight) {
|
|
92
119
|
// Guard with inFlight so concurrent tool calls don't each spawn a new initCheck(),
|
|
93
120
|
// but allow retry on failure (initCheckInFlight is reset to false on error).
|
|
@@ -96,6 +123,7 @@ server.registerTool = function (name, definition, handler) {
|
|
|
96
123
|
// unreachable. Deferring via setImmediate ensures the tool response is written to
|
|
97
124
|
// stdout (and acknowledged by the MCP client) before any blocking FFI call runs.
|
|
98
125
|
initCheckInFlight = true;
|
|
126
|
+
triggeredInitThisCall = true;
|
|
99
127
|
setImmediate(() => {
|
|
100
128
|
initCheck()
|
|
101
129
|
.then(() => {
|
|
@@ -109,7 +137,15 @@ server.registerTool = function (name, definition, handler) {
|
|
|
109
137
|
});
|
|
110
138
|
});
|
|
111
139
|
}
|
|
112
|
-
|
|
140
|
+
const result = await handler(...args);
|
|
141
|
+
if (triggeredInitThisCall && result) {
|
|
142
|
+
const content = result.content ?? [];
|
|
143
|
+
result.content = [
|
|
144
|
+
{ type: "text", text: INIT_MESSAGE },
|
|
145
|
+
...content,
|
|
146
|
+
];
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
113
149
|
};
|
|
114
150
|
return originalRegisterTool(name, definition, wrappedHandler);
|
|
115
151
|
};
|
|
@@ -119,6 +155,7 @@ const prompts = [
|
|
|
119
155
|
registerTestGenerationPrompt,
|
|
120
156
|
registerStartTraceCollectionPrompt,
|
|
121
157
|
registerTestHealthPrompt,
|
|
158
|
+
registerRecommendTestsPrompt,
|
|
122
159
|
];
|
|
123
160
|
if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
|
|
124
161
|
prompts.push(registerTestbotPrompt);
|
|
@@ -148,8 +185,9 @@ const codeQualityTools = [
|
|
|
148
185
|
codeQualityTools.forEach((registerTool) => registerTool(server));
|
|
149
186
|
// Register test recommendation tools
|
|
150
187
|
registerAnalyzeRepositoryTool(server);
|
|
151
|
-
registerMapTestsTool(server);
|
|
152
188
|
registerRecommendTestsTool(server);
|
|
189
|
+
// Register analysis resources (MCP Resources for enriched data access)
|
|
190
|
+
registerAnalysisResources(server);
|
|
153
191
|
// Register test maintenance tools
|
|
154
192
|
registerDiscoverTestsTool(server);
|
|
155
193
|
registerAnalyzeTestDriftTool(server);
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
function buildEnrichmentInstructions(p) {
|
|
2
|
+
const isDiffScope = p.analysisScope === "current_branch_diff";
|
|
3
|
+
if (!isDiffScope) {
|
|
4
|
+
return `## Your Task — Enrich & Recommend (full repo)
|
|
5
|
+
|
|
6
|
+
### Step 1: Read key files
|
|
7
|
+
Read \`package.json\` / \`requirements.txt\`, \`docker-compose.yml\`, route/controller files,
|
|
8
|
+
and model/schema files (Zod schemas, Pydantic models, TypeScript interfaces, DTOs)
|
|
9
|
+
to understand the tech stack, endpoint shapes, auth mechanisms, and request/response schemas.
|
|
10
|
+
|
|
11
|
+
### Step 2: Identify resource relationships
|
|
12
|
+
Map how endpoints relate to each other — which POST creates resources consumed by other endpoints?
|
|
13
|
+
**Resolve nested/sub-router paths** from the Router Mounting section above.
|
|
14
|
+
|
|
15
|
+
### Step 3: Call recommend tests
|
|
16
|
+
Call \`skyramp_recommend_tests\` with \`sessionId: "${p.sessionId}"\``;
|
|
17
|
+
}
|
|
18
|
+
const changedFiles = p.parsedDiff?.changedFiles.join(", ") ?? "";
|
|
19
|
+
const hasApiEndpoints = p.parsedDiff && (p.parsedDiff.newEndpoints.length > 0 || p.parsedDiff.modifiedEndpoints.length > 0);
|
|
20
|
+
const isUIOnly = !hasApiEndpoints && (p.parsedDiff?.changedFiles.every(f => !f.match(/\/(api|routes?|controllers?|routers?|handlers?|endpoints?)\//)) ?? false);
|
|
21
|
+
const step2 = hasApiEndpoints
|
|
22
|
+
? `### Step 2: Discover related endpoints
|
|
23
|
+
Read handler code for the changed endpoints and their model/schema files (Zod schemas,
|
|
24
|
+
Pydantic models, DTOs) to understand request/response shapes. Find related endpoints via
|
|
25
|
+
imports, shared models, adjacent route files. Resolve nested/sub-router paths from Router
|
|
26
|
+
Mounting context.`
|
|
27
|
+
: isUIOnly
|
|
28
|
+
? `### Step 2: Identify consumed API endpoints
|
|
29
|
+
UI-only PR — read changed components to find API calls (fetch, axios, hooks).`
|
|
30
|
+
: `### Step 2: Identify affected endpoints
|
|
31
|
+
No API route changes detected — read changed files to identify affected endpoints.`;
|
|
32
|
+
return `## Your Task — Enrich & Recommend (PR-scoped)
|
|
33
|
+
|
|
34
|
+
### Step 1: Read the changed files
|
|
35
|
+
${changedFiles}
|
|
36
|
+
|
|
37
|
+
${step2}
|
|
38
|
+
|
|
39
|
+
### Step 3: Draft integration scenarios
|
|
40
|
+
Draft multi-step scenarios simulating realistic user workflows:
|
|
41
|
+
- **Cross-resource data flow**: Foreign key relationships, parent→child creation, verification
|
|
42
|
+
- **Search/filter verification**: Create data, search, verify results
|
|
43
|
+
- **Negative/error paths**: Invalid references → appropriate errors
|
|
44
|
+
- **UI user journeys**: Concrete browser steps for frontend changes
|
|
45
|
+
|
|
46
|
+
**Quality:** Realistic request bodies, response data verification, actual field names for chaining.
|
|
47
|
+
|
|
48
|
+
### Step 4: Call recommend tests
|
|
49
|
+
Call \`skyramp_recommend_tests\` with \`sessionId: "${p.sessionId}"\``;
|
|
50
|
+
}
|
|
51
|
+
export function buildAnalysisOutputText(p) {
|
|
52
|
+
const isDiffScope = p.analysisScope === "current_branch_diff";
|
|
53
|
+
const diffSection = p.parsedDiff
|
|
54
|
+
? `
|
|
55
|
+
## Branch Diff Context
|
|
56
|
+
**Branch**: \`${p.parsedDiff.currentBranch}\` → base: \`${p.parsedDiff.baseBranch}\`
|
|
57
|
+
**Changed Files** (${p.parsedDiff.changedFiles.length}): ${p.parsedDiff.changedFiles.join(", ")}
|
|
58
|
+
**New Endpoints** (${p.parsedDiff.newEndpoints.length}): ${p.parsedDiff.newEndpoints.map((e) => `${e.method} ${e.path} (${e.sourceFile})`).join(", ") || "none"}
|
|
59
|
+
**Modified Endpoints** (${p.parsedDiff.modifiedEndpoints.length}): ${p.parsedDiff.modifiedEndpoints.map((e) => `${e.method} ${e.path} (${e.sourceFile})`).join(", ") || "none"}
|
|
60
|
+
**Affected Services**: ${p.parsedDiff.affectedServices.join(", ") || "none"}
|
|
61
|
+
`
|
|
62
|
+
: "";
|
|
63
|
+
const endpointCatalog = p.scannedEndpoints.length > 0
|
|
64
|
+
? `
|
|
65
|
+
## Pre-Scanned Endpoint Catalog (${p.scannedEndpoints.length} routes)
|
|
66
|
+
${p.scannedEndpoints.map((ep) => ` ${ep.methods.join("|")} ${ep.path} (${ep.sourceFile})`).join("\n")}
|
|
67
|
+
`
|
|
68
|
+
: "";
|
|
69
|
+
const wsLine = p.wsBaseUrl
|
|
70
|
+
? `**Base URL**: \`${p.wsBaseUrl}\` | **Auth header**: \`${p.wsAuthHeader || "Authorization"}\``
|
|
71
|
+
: "";
|
|
72
|
+
const specSection = p.wsSchemaPath
|
|
73
|
+
? `
|
|
74
|
+
## OpenAPI Spec Available
|
|
75
|
+
Spec at \`${p.wsSchemaPath}\`. **Read it** for authoritative paths and schemas.
|
|
76
|
+
Pass \`apiSchema: "${p.wsSchemaPath}"\` to ALL test generation tool calls.`
|
|
77
|
+
: p.routerMountContext
|
|
78
|
+
? `
|
|
79
|
+
## Router Mounting / Nesting
|
|
80
|
+
\`\`\`
|
|
81
|
+
${p.routerMountContext}
|
|
82
|
+
\`\`\`
|
|
83
|
+
Use this to resolve full URL paths for nested endpoints.`
|
|
84
|
+
: "";
|
|
85
|
+
const enrichment = buildEnrichmentInstructions(p);
|
|
86
|
+
return `# Repository Analysis
|
|
87
|
+
|
|
88
|
+
**Session ID**: \`${p.sessionId}\`
|
|
89
|
+
**Repository**: \`${p.repositoryPath}\`
|
|
90
|
+
**Analysis Scope**: \`${p.analysisScope}\`
|
|
91
|
+
${isDiffScope ? `**Diff endpoints**: ${(p.parsedDiff?.newEndpoints.length ?? 0) + (p.parsedDiff?.modifiedEndpoints.length ?? 0)}` : `**Pre-scanned endpoints**: ${p.scannedEndpoints.length}`}
|
|
92
|
+
${wsLine}
|
|
93
|
+
${p.wsSchemaPath ? `**OpenAPI Spec**: \`${p.wsSchemaPath}\` (spec-based flow)` : "**Flow**: Code-scanning (may miss nesting)"}
|
|
94
|
+
|
|
95
|
+
${diffSection}
|
|
96
|
+
${endpointCatalog}
|
|
97
|
+
${specSection}
|
|
98
|
+
${enrichment}
|
|
99
|
+
|
|
100
|
+
**CRITICAL**: No .json/.md file creation. Prioritize cross-resource workflows.`;
|
|
101
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
export const MAX_TESTS_TO_GENERATE = 4;
|
|
2
|
+
export const MAX_RECOMMENDATIONS = 10;
|
|
3
|
+
export function buildPrioritizationDimensions() {
|
|
4
|
+
return `## Prioritization Dimensions (evaluate each candidate test)
|
|
5
|
+
|
|
6
|
+
For each candidate test, assess these dimensions using your judgment:
|
|
7
|
+
|
|
8
|
+
| Dimension | What to assess |
|
|
9
|
+
|-----------|---------------|
|
|
10
|
+
| **Sophistication** | Does it test a multi-step workflow or non-obvious scenario? Or is it a simple request→response check? |
|
|
11
|
+
| **Bug-Finding Potential** | Does it target known failure modes (race conditions, data consistency, state transitions, cascade effects)? |
|
|
12
|
+
| **User Journey Relevance** | Does it reflect how real users interact with the system (from traces, business flows, or critical paths)? |
|
|
13
|
+
| **Coverage Gap** | Does it address an area with zero existing test coverage? Or does it duplicate what\'s already tested? |
|
|
14
|
+
| **Code Insight** | Is it derived from actual implementation analysis (e.g., spotted a middleware pattern, found an N+1 risk) rather than just API shape? |
|
|
15
|
+
|
|
16
|
+
Candidates scoring well across MULTIPLE dimensions should be recommended first.
|
|
17
|
+
Candidates satisfying only ONE dimension (e.g., covers a gap but is trivially simple) should be deprioritized.
|
|
18
|
+
|
|
19
|
+
**Quality Gate:** For each candidate, ask: "Would a senior engineer be impressed by this test?"
|
|
20
|
+
If the answer is no — deprioritize it. Impressive tests catch real bugs, exercise real workflows,
|
|
21
|
+
and demonstrate understanding of the system\'s behavior. Trivial tests do not.`;
|
|
22
|
+
}
|
|
23
|
+
export function buildTestExamples() {
|
|
24
|
+
return `## Test Examples (calibrate your judgment)
|
|
25
|
+
|
|
26
|
+
**Impressive tests (recommend these):**
|
|
27
|
+
1. "Register user → login → create order → verify order appears in user\'s order list"
|
|
28
|
+
Cross-resource workflow with auth chaining and data verification across users + orders.
|
|
29
|
+
2. "Create product with inventory=10 → place order for qty=10 → verify inventory=0 →
|
|
30
|
+
place another order → verify 409 out-of-stock error"
|
|
31
|
+
Cross-resource state machine + business rule validation (products + orders + inventory).
|
|
32
|
+
3. "POST /users with duplicate email → verify 409 Conflict → verify original user unchanged"
|
|
33
|
+
Error handling with side-effect verification — not just status code check.
|
|
34
|
+
|
|
35
|
+
**Non-impressive tests (deprioritize or skip):**
|
|
36
|
+
1. "GET /products → 200" — trivial health check, no assertions beyond status code.
|
|
37
|
+
2. "POST /products → GET /products/{id} → PUT /products/{id} → DELETE /products/{id}"
|
|
38
|
+
Single-resource CRUD — baseline, not impressive by itself.
|
|
39
|
+
3. "POST /products with missing name → 422" — obvious validation, already covered by contract/fuzz.`;
|
|
40
|
+
}
|
|
41
|
+
export function buildTestPatternGuidelines() {
|
|
42
|
+
return `## Test Pattern Guidelines (reference, not rigid rules)
|
|
43
|
+
|
|
44
|
+
### Tier 1 — Base Patterns
|
|
45
|
+
- CRUD lifecycle per resource group (Create → Read → Update → Delete)
|
|
46
|
+
- Auth flow (Register → Login → Access protected → Token refresh → Logout)
|
|
47
|
+
- Pagination & filtering (boundary values, empty results, large page sizes)
|
|
48
|
+
- Error responses (400, 401, 403, 404, 409, 422 — each with specific trigger)
|
|
49
|
+
|
|
50
|
+
### Tier 2 — Code-Informed Patterns (higher value — look for these in the codebase)
|
|
51
|
+
- **Middleware chains**: If auth/rate-limit/logging middleware exists, test the chain
|
|
52
|
+
(e.g., rate limit hit → auth still checked → correct error returned)
|
|
53
|
+
- **N+1 query risk**: If list endpoints join related data (e.g., orders with products),
|
|
54
|
+
test with large datasets under load
|
|
55
|
+
- **State machines**: If resources have status transitions (draft→published→archived),
|
|
56
|
+
test invalid transitions (e.g., archived→draft should fail)
|
|
57
|
+
- **Cascade deletes**: If deleting a parent removes children, verify cascade AND verify
|
|
58
|
+
orphan prevention (delete product → orders referencing it get error or cascade)
|
|
59
|
+
- **Race conditions**: If concurrent writes are possible (inventory deduction, counter
|
|
60
|
+
increment), test concurrent requests under load
|
|
61
|
+
- **Computed fields**: If response contains derived values (total, average, count),
|
|
62
|
+
verify computation with known inputs
|
|
63
|
+
- **Webhook/event side effects**: If endpoints trigger async operations, test that side
|
|
64
|
+
effects occur (e.g., POST /orders triggers email notification)`;
|
|
65
|
+
}
|
|
66
|
+
export function buildTestQualityCriteria() {
|
|
67
|
+
return `## What Makes a Good Test
|
|
68
|
+
|
|
69
|
+
**Integration tests** should demonstrate cross-resource data flow — step A creates data
|
|
70
|
+
that step B depends on (e.g., create product \u2192 create order referencing that product's ID \u2192
|
|
71
|
+
verify order contains correct product). Single-resource CRUD alone is not an integration test.
|
|
72
|
+
Use realistic request bodies from source code schemas and verify response data, not just
|
|
73
|
+
status codes.
|
|
74
|
+
|
|
75
|
+
**E2E tests** should follow realistic user journeys end-to-end: browse products \u2192 search \u2192
|
|
76
|
+
add to cart \u2192 checkout. Verify that frontend actions trigger the correct API calls and
|
|
77
|
+
that the UI reflects backend state.
|
|
78
|
+
|
|
79
|
+
**UI tests** should exercise component behavior and interaction flows: fill form \u2192 validate
|
|
80
|
+
inputs \u2192 submit \u2192 see confirmation. Include visual state changes (loading, error, empty)
|
|
81
|
+
and accessibility checks.`;
|
|
82
|
+
}
|
|
83
|
+
export function buildGenerationRules(isUIOnlyPR) {
|
|
84
|
+
return `## Generation Guidelines
|
|
85
|
+
|
|
86
|
+
**Scenario fidelity:** Every workflow scenario should reflect the actual resource
|
|
87
|
+
relationships in the code. If the pre-drafted scenarios don't match the real data model,
|
|
88
|
+
replace them with accurate ones.
|
|
89
|
+
|
|
90
|
+
**Available test types:**
|
|
91
|
+
- **Integration** — multi-endpoint workflows that chain data across resources
|
|
92
|
+
- **Fuzz** — boundary/invalid input testing for POST/PUT endpoints
|
|
93
|
+
- **Contract** — response schema validation for new/changed endpoints
|
|
94
|
+
- **E2E** — user journeys spanning frontend to backend (needs Playwright traces)
|
|
95
|
+
- **UI** — frontend component interaction flows (needs Playwright traces)
|
|
96
|
+
${isUIOnlyPR ? `
|
|
97
|
+
This is a **UI-only PR** — no backend changes. UI and E2E tests are most relevant.
|
|
98
|
+
Without Playwright traces, recommend them with trace recording instructions
|
|
99
|
+
(\`skyramp_start_trace_collection\` with \`playwright: true\`).
|
|
100
|
+
` : `
|
|
101
|
+
When no Playwright trace exists, still recommend E2E/UI tests with instructions for
|
|
102
|
+
recording a trace using \`skyramp_start_trace_collection\` with \`playwright: true\`.
|
|
103
|
+
`}
|
|
104
|
+
**No duplicate coverage.** If an existing test already covers an endpoint + test type,
|
|
105
|
+
recommend a different test that adds new coverage.
|
|
106
|
+
|
|
107
|
+
Choose the test types and distribution that maximize coverage for this specific PR.
|
|
108
|
+
No smoke tests.`;
|
|
109
|
+
}
|
|
110
|
+
export function buildToolWorkflows(authHeaderValue) {
|
|
111
|
+
return `## How to Generate Tests — Tool Workflows
|
|
112
|
+
|
|
113
|
+
**Auth Header:** \`authHeader: "${authHeaderValue}"\` — pass to EVERY tool call below.
|
|
114
|
+
|
|
115
|
+
**For multi-endpoint workflows (integration tests) — Scenario → Integration pipeline:**
|
|
116
|
+
1. Call \`skyramp_scenario_test_generation\` once per step: \`scenarioName\`, \`destination\`,
|
|
117
|
+
\`baseURL\`, \`method\`, \`path\`, \`requestBody\`, \`responseBody\`, \`authHeader: "${authHeaderValue}"\`.
|
|
118
|
+
\`statusCode\` is optional — defaults: POST→201, DELETE→204, GET/PUT/PATCH→200. Only override for non-standard codes.
|
|
119
|
+
**OpenAPI spec is NOT required.** \`apiSchema\` is OPTIONAL — omit it if no spec exists.
|
|
120
|
+
\`requestBody\` should use realistic field values from source code schemas (Zod, Pydantic, DTOs).
|
|
121
|
+
\`responseBody\` should match the actual API response shape from source code (including all fields
|
|
122
|
+
returned by the controller — e.g., \`id\`, \`ownerId\`, \`createdAt\`, included relations like \`collection\`, \`tags\`).
|
|
123
|
+
Wrap in \`{"response": ...}\` if the API uses an envelope pattern. If omitted, a synthetic response is generated.
|
|
124
|
+
Inspect the source code to determine the correct request AND response body shapes — avoid sending \`{}\`.
|
|
125
|
+
Use unique names with timestamp suffix to avoid conflicts on re-runs.
|
|
126
|
+
For GET/PUT/DELETE with path IDs, use a placeholder — chaining resolves the real ID.
|
|
127
|
+
2. Produces a \`scenario_<name>.json\` in the same \`outputDir\` as the test files (not \`.skyramp/\`).
|
|
128
|
+
3. Call \`skyramp_integration_test_generation\` with \`scenarioFile\` AND \`authHeader: "${authHeaderValue}"\`.
|
|
129
|
+
Do NOT pass \`chainingKey\` — defaults to \`response.id\`. After generation, the testbot
|
|
130
|
+
will verify and fix path param chaining in the generated test.
|
|
131
|
+
|
|
132
|
+
**For single-endpoint tests (contract/fuzz):**
|
|
133
|
+
\`skyramp_{type}_test_generation\` with \`endpointURL\` (full URL incl. base + path), \`method\`,
|
|
134
|
+
\`authHeader: "${authHeaderValue}"\`, and \`requestData\` from source code schemas.
|
|
135
|
+
If an OpenAPI spec exists, ALSO pass \`apiSchema\` — it enables schema-aware validation
|
|
136
|
+
(contract tests verify response structure, fuzz tests generate smarter boundary values).
|
|
137
|
+
Without a spec, \`endpointURL\` alone is sufficient.
|
|
138
|
+
|
|
139
|
+
**For UI tests (no Playwright recording):**
|
|
140
|
+
1. \`skyramp_start_trace_collection\` (playwright: true)
|
|
141
|
+
2. Perform browser steps
|
|
142
|
+
3. \`skyramp_stop_trace_collection\`
|
|
143
|
+
4. \`skyramp_ui_test_generation\` with playwright zip
|
|
144
|
+
|
|
145
|
+
**For E2E tests:**
|
|
146
|
+
Same trace flow, pass both trace file and playwright zip to \`skyramp_e2e_test_generation\`.`;
|
|
147
|
+
}
|
|
148
|
+
export function buildCoverageChecklist(openApiSpec, isUIOnlyPR, topN, maxGenerate = MAX_TESTS_TO_GENERATE) {
|
|
149
|
+
const specNote = openApiSpec
|
|
150
|
+
? `\n**OpenAPI Spec available**: \`${openApiSpec.path}\`
|
|
151
|
+
Use it actively:
|
|
152
|
+
- **Contract tests**: pass \`apiSchema: "${openApiSpec.path}"\` — the CLI validates response schemas against the spec.
|
|
153
|
+
- **Fuzz tests**: pass \`apiSchema: "${openApiSpec.path}"\` — the CLI generates boundary values from schema constraints.
|
|
154
|
+
- **Integration tests**: pass \`apiSchema\` to \`skyramp_scenario_test_generation\` — it extracts destination and request/response shapes.
|
|
155
|
+
- **Single-endpoint tests**: pass both \`endpointURL\` AND \`apiSchema\` for schema-aware generation.
|
|
156
|
+
\n`
|
|
157
|
+
: "";
|
|
158
|
+
return `## Coverage Checklist
|
|
159
|
+
${specNote}
|
|
160
|
+
${isUIOnlyPR ? `**UI-only PR** — no backend changes.
|
|
161
|
+
Without Playwright traces, the testbot skips generation entirely — all recommendations
|
|
162
|
+
become additionalRecommendations in the report.
|
|
163
|
+
` : `**Available test types:** integration, fuzz, contract, E2E, UI. No smoke tests.
|
|
164
|
+
Choose based on what adds the most value for this PR's changes.
|
|
165
|
+
`}
|
|
166
|
+
## For Each Recommendation Include:
|
|
167
|
+
1. Test type 2. Priority (high/medium/low) 3. Target endpoint/scenario
|
|
168
|
+
4. What it validates (business logic, not just "tests the endpoint")
|
|
169
|
+
5. Skyramp tool call details — exact tool + key params for zero-editing execution
|
|
170
|
+
6. For integration/E2E: reference draftedScenario by scenarioName
|
|
171
|
+
|
|
172
|
+
## When Artifacts Are Missing
|
|
173
|
+
Recommend the test anyway — never mark it "blocked":
|
|
174
|
+
- **No OpenAPI spec** \u2192 use \`endpointURL\` and \`requestBody\` from source code
|
|
175
|
+
- **No Playwright recording** \u2192 provide trace recording instructions
|
|
176
|
+
- **No backend trace** \u2192 use the scenario generation pipeline
|
|
177
|
+
|
|
178
|
+
## Select the Top ${topN}
|
|
179
|
+
Consider all possible tests (endpoints \u00d7 interaction types + scenarios), then select the
|
|
180
|
+
top ${topN} most valuable. Include \`totalConsidered\` count in your output. The top ${Math.min(maxGenerate, topN)} will
|
|
181
|
+
be generated; ${topN > maxGenerate ? `recommendations #${Math.min(maxGenerate, topN) + 1}\u2013${topN} go to additionalRecommendations in the report,
|
|
182
|
+
so ensure the top ${Math.min(maxGenerate, topN)} are the highest-impact tests.` : `all will be generated.`}
|
|
183
|
+
|
|
184
|
+
- Each integration scenario's step sequence should be logically valid — preconditions
|
|
185
|
+
met by prior steps.
|
|
186
|
+
|
|
187
|
+
Each recommendation should include enough detail for direct tool invocation.
|
|
188
|
+
Reference draftedScenarios by name and interactions by description.
|
|
189
|
+
Use "high"/"medium"/"low" for priority — no numeric scores.
|
|
190
|
+
Total candidates should be \u2265 ${topN}.
|
|
191
|
+
|
|
192
|
+
Generate recommendations now.`;
|
|
193
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { StateManager, getSessionFilePath, hasSessionData, getSessionData, } from "../../utils/AnalysisStateManager.js";
|
|
3
|
+
import { logger } from "../../utils/logger.js";
|
|
4
|
+
import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
|
|
5
|
+
import { getWorkspaceAuthHeader } from "../../utils/workspaceAuth.js";
|
|
6
|
+
export function registerRecommendTestsPrompt(server) {
|
|
7
|
+
server.registerPrompt("skyramp_recommend_tests", {
|
|
8
|
+
description: "Generate test recommendations from enriched repository analysis. " +
|
|
9
|
+
"Provide a sessionId from skyramp_analyze_repository.",
|
|
10
|
+
argsSchema: {
|
|
11
|
+
sessionId: z
|
|
12
|
+
.string()
|
|
13
|
+
.describe("Session ID from skyramp_analyze_repository"),
|
|
14
|
+
scope: z
|
|
15
|
+
.enum(["full_repo", "current_branch_diff"])
|
|
16
|
+
.default("full_repo")
|
|
17
|
+
.optional()
|
|
18
|
+
.describe("Analysis scope (defaults to the scope used during analysis)"),
|
|
19
|
+
},
|
|
20
|
+
}, async (args) => {
|
|
21
|
+
const sessionId = args.sessionId;
|
|
22
|
+
if (!sessionId) {
|
|
23
|
+
throw new Error("sessionId is required");
|
|
24
|
+
}
|
|
25
|
+
// Try process memory first, then fall back to state file
|
|
26
|
+
let data = null;
|
|
27
|
+
if (hasSessionData(sessionId)) {
|
|
28
|
+
data = getSessionData(sessionId);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
const registeredPath = getSessionFilePath(sessionId);
|
|
32
|
+
const mgr = registeredPath
|
|
33
|
+
? StateManager.fromStatePath(registeredPath)
|
|
34
|
+
: StateManager.fromSessionId(sessionId);
|
|
35
|
+
if (!mgr.exists()) {
|
|
36
|
+
throw new Error(`Analysis session "${sessionId}" not found. Run skyramp_analyze_repository first.`);
|
|
37
|
+
}
|
|
38
|
+
data = await mgr.readData();
|
|
39
|
+
}
|
|
40
|
+
if (!data?.analysis) {
|
|
41
|
+
throw new Error(`Session "${sessionId}" has no analysis data.`);
|
|
42
|
+
}
|
|
43
|
+
const scope = args.scope || data.analysisScope || "full_repo";
|
|
44
|
+
const effectiveTopN = scope === "current_branch_diff" ? 7 : 10;
|
|
45
|
+
const workspaceAuthHeader = data.repositoryPath
|
|
46
|
+
? await getWorkspaceAuthHeader(data.repositoryPath)
|
|
47
|
+
: undefined;
|
|
48
|
+
const prompt = buildRecommendationPrompt(data.analysis, scope, effectiveTopN, data.prContext, workspaceAuthHeader);
|
|
49
|
+
logger.info("Serving recommendation prompt via MCP Prompt", {
|
|
50
|
+
sessionId,
|
|
51
|
+
scope,
|
|
52
|
+
});
|
|
53
|
+
return {
|
|
54
|
+
messages: [
|
|
55
|
+
{
|
|
56
|
+
role: "user",
|
|
57
|
+
content: {
|
|
58
|
+
type: "text",
|
|
59
|
+
text: `Session: ${sessionId}\nRepository: ${data.repositoryPath}\nScope: ${scope}\n\nAvailable MCP Resources:\n- skyramp://analysis/${sessionId}/summary\n- skyramp://analysis/${sessionId}/endpoints\n- skyramp://analysis/${sessionId}/scenarios\n- skyramp://analysis/${sessionId}/diff\n\n${prompt}`,
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
}
|