@skyramp/mcp 0.0.61 → 0.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +44 -6
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +101 -0
- package/build/prompts/test-recommendation/recommendationSections.js +193 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +65 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +165 -99
- package/build/prompts/testGenerationPrompt.js +2 -3
- package/build/prompts/testbot/testbot-prompts.js +116 -96
- package/build/resources/analysisResources.js +248 -0
- package/build/services/ScenarioGenerationService.js +38 -40
- package/build/services/TestExecutionService.js +10 -1
- package/build/tools/generate-tests/generateScenarioRestTool.js +18 -6
- package/build/tools/submitReportTool.js +28 -0
- package/build/tools/test-maintenance/stateCleanupTool.js +8 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +386 -217
- package/build/tools/test-recommendation/recommendTestsTool.js +162 -163
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/types/RepositoryAnalysis.js +100 -12
- package/build/utils/AnalysisStateManager.js +56 -23
- package/build/utils/branchDiff.js +47 -0
- package/build/utils/initAgent.js +62 -26
- package/build/utils/pr-comment-parser.js +124 -0
- package/build/utils/projectMetadata.js +188 -0
- package/build/utils/projectMetadata.test.js +81 -0
- package/build/utils/repoScanner.js +425 -0
- package/build/utils/routeParsers.js +213 -0
- package/build/utils/routeParsers.test.js +87 -0
- package/build/utils/scenarioDrafting.js +119 -0
- package/build/utils/scenarioDrafting.test.js +66 -0
- package/build/utils/skyrampMdContent.js +100 -0
- package/build/utils/trace-parser.js +166 -0
- package/build/utils/workspaceAuth.js +16 -0
- package/package.json +2 -2
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +0 -326
- package/build/prompts/test-recommendation/test-mapping-prompt.js +0 -266
- package/build/tools/test-recommendation/mapTestsTool.js +0 -243
- package/build/types/TestMapping.js +0 -173
- package/build/utils/scoring-engine.js +0 -380
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { StateManager, getSessionFilePath, getRegisteredSessions, hasSessionData, getSessionData, normalizeRecommendationState, } from "../utils/AnalysisStateManager.js";
|
|
4
|
+
import { logger } from "../utils/logger.js";
|
|
5
|
+
export const ANALYSIS_URI_PREFIX = "skyramp://analysis";
|
|
6
|
+
/**
|
|
7
|
+
* Register MCP Resources for analysis data access.
|
|
8
|
+
*
|
|
9
|
+
* Resources provide read-only, token-efficient access to analysis data
|
|
10
|
+
* via sessionId. The state file is an internal backing store.
|
|
11
|
+
*
|
|
12
|
+
* URI scheme:
|
|
13
|
+
* skyramp://analysis/{sessionId}/summary
|
|
14
|
+
* skyramp://analysis/{sessionId}/endpoints
|
|
15
|
+
* skyramp://analysis/{sessionId}/endpoints/{+path}
|
|
16
|
+
* skyramp://analysis/{sessionId}/endpoints/{+path}/{method}
|
|
17
|
+
* skyramp://analysis/{sessionId}/scenarios
|
|
18
|
+
* skyramp://analysis/{sessionId}/diff
|
|
19
|
+
*/
|
|
20
|
+
export function registerAnalysisResources(server) {
|
|
21
|
+
logger.info("Registering analysis resources");
|
|
22
|
+
async function loadAnalysis(sessionId) {
|
|
23
|
+
// Try process memory first (new flow — no state file needed)
|
|
24
|
+
if (hasSessionData(sessionId)) {
|
|
25
|
+
const memData = getSessionData(sessionId);
|
|
26
|
+
if (memData?.analysis) {
|
|
27
|
+
logger.debug("Loaded analysis from process memory for resource", { sessionId });
|
|
28
|
+
return memData;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
// Fall back to state file for backward compatibility
|
|
32
|
+
const registeredPath = getSessionFilePath(sessionId);
|
|
33
|
+
const mgr = registeredPath
|
|
34
|
+
? StateManager.fromStatePath(registeredPath)
|
|
35
|
+
: StateManager.fromSessionId(sessionId);
|
|
36
|
+
if (!mgr.exists()) {
|
|
37
|
+
throw new Error(`Analysis session "${sessionId}" not found or expired.`);
|
|
38
|
+
}
|
|
39
|
+
const data = await mgr.readData();
|
|
40
|
+
if (!data) {
|
|
41
|
+
throw new Error(`Analysis session "${sessionId}" has no data.`);
|
|
42
|
+
}
|
|
43
|
+
const normalized = normalizeRecommendationState(data);
|
|
44
|
+
if (!normalized.analysis) {
|
|
45
|
+
throw new Error(`Analysis session "${sessionId}" has no analysis data.`);
|
|
46
|
+
}
|
|
47
|
+
return normalized;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Shared list callback: returns resource URIs for sessions created by
|
|
51
|
+
* this process only (via the in-memory registry). No filesystem scan —
|
|
52
|
+
* eliminates cross-process leakage when multiple clients share /tmp.
|
|
53
|
+
*/
|
|
54
|
+
function makeListCallback(suffix) {
|
|
55
|
+
return async () => {
|
|
56
|
+
const sessions = getRegisteredSessions();
|
|
57
|
+
const available = Array.from(sessions.entries()).filter(([sessionId, filePath]) => hasSessionData(sessionId) || fs.existsSync(filePath));
|
|
58
|
+
return {
|
|
59
|
+
resources: available.map(([sessionId]) => ({
|
|
60
|
+
uri: `${ANALYSIS_URI_PREFIX}/${sessionId}/${suffix}`,
|
|
61
|
+
name: `Analysis ${suffix} (${sessionId})`,
|
|
62
|
+
mimeType: "application/json",
|
|
63
|
+
})),
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
// ── Summary ──
|
|
68
|
+
server.registerResource("analysis_summary", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/summary`, {
|
|
69
|
+
list: makeListCallback("summary"),
|
|
70
|
+
}), {
|
|
71
|
+
title: "Analysis Summary",
|
|
72
|
+
description: "High-level overview: project type, tech stack, endpoint/scenario counts, coverage gaps.",
|
|
73
|
+
mimeType: "application/json",
|
|
74
|
+
}, async (uri, params) => {
|
|
75
|
+
const sessionId = params.sessionId;
|
|
76
|
+
const { analysis, repositoryPath, analysisScope } = await loadAnalysis(sessionId);
|
|
77
|
+
const summary = {
|
|
78
|
+
sessionId,
|
|
79
|
+
repositoryPath,
|
|
80
|
+
analysisScope: analysisScope || "full_repo",
|
|
81
|
+
metadata: analysis.metadata,
|
|
82
|
+
projectClassification: analysis.projectClassification,
|
|
83
|
+
technologyStack: {
|
|
84
|
+
languages: analysis.technologyStack.languages,
|
|
85
|
+
frameworks: analysis.technologyStack.frameworks,
|
|
86
|
+
runtime: analysis.technologyStack.runtime,
|
|
87
|
+
},
|
|
88
|
+
authentication: analysis.authentication,
|
|
89
|
+
infrastructure: analysis.infrastructure,
|
|
90
|
+
endpointStats: {
|
|
91
|
+
totalPaths: analysis.apiEndpoints.endpoints.length,
|
|
92
|
+
totalMethods: analysis.apiEndpoints.endpoints.reduce((sum, ep) => sum + ep.methods.length, 0),
|
|
93
|
+
totalInteractions: analysis.apiEndpoints.endpoints.reduce((sum, ep) => sum +
|
|
94
|
+
ep.methods.reduce((msum, m) => msum + m.interactions.length, 0), 0),
|
|
95
|
+
baseUrl: analysis.apiEndpoints.baseUrl,
|
|
96
|
+
},
|
|
97
|
+
scenarioCount: analysis.businessContext.draftedScenarios.length,
|
|
98
|
+
existingTests: analysis.existingTests,
|
|
99
|
+
};
|
|
100
|
+
return {
|
|
101
|
+
contents: [
|
|
102
|
+
{
|
|
103
|
+
uri: uri.href,
|
|
104
|
+
mimeType: "application/json",
|
|
105
|
+
text: JSON.stringify(summary, null, 2),
|
|
106
|
+
},
|
|
107
|
+
],
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
// ── Endpoints (compact listing) ──
|
|
111
|
+
server.registerResource("analysis_endpoints", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/endpoints`, {
|
|
112
|
+
list: makeListCallback("endpoints"),
|
|
113
|
+
}), {
|
|
114
|
+
title: "Endpoint Listing",
|
|
115
|
+
description: "Compact listing of all endpoints (path, methods, interaction counts). Use path-specific resources for full detail.",
|
|
116
|
+
mimeType: "application/json",
|
|
117
|
+
}, async (uri, params) => {
|
|
118
|
+
const sessionId = params.sessionId;
|
|
119
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
120
|
+
const compact = analysis.apiEndpoints.endpoints.map((ep) => ({
|
|
121
|
+
path: ep.path,
|
|
122
|
+
resourceGroup: ep.resourceGroup,
|
|
123
|
+
pathParams: ep.pathParams.map((p) => p.name),
|
|
124
|
+
methods: ep.methods.map((m) => ({
|
|
125
|
+
method: m.method,
|
|
126
|
+
authRequired: m.authRequired,
|
|
127
|
+
interactionCount: m.interactions.length,
|
|
128
|
+
interactionTypes: m.interactions.map((i) => i.type),
|
|
129
|
+
hasCookies: m.interactions.some((i) => (i.response.cookies?.length ?? 0) > 0),
|
|
130
|
+
hasResponseHeaders: m.interactions.some((i) => Object.keys(i.response.headers ?? {}).length > 0),
|
|
131
|
+
createsResource: m.createsResource,
|
|
132
|
+
})),
|
|
133
|
+
}));
|
|
134
|
+
return {
|
|
135
|
+
contents: [
|
|
136
|
+
{
|
|
137
|
+
uri: uri.href,
|
|
138
|
+
mimeType: "application/json",
|
|
139
|
+
text: JSON.stringify({ baseUrl: analysis.apiEndpoints.baseUrl, endpoints: compact }, null, 2),
|
|
140
|
+
},
|
|
141
|
+
],
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
// ── Single Path Detail ──
|
|
145
|
+
// No list callback — these are parameterized drilldowns, not top-level resources.
|
|
146
|
+
server.registerResource("analysis_endpoint_path", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/endpoints/{+path}`, { list: undefined }), {
|
|
147
|
+
title: "Endpoint Path Detail",
|
|
148
|
+
description: "Full detail for a specific path: all methods, interactions, params.",
|
|
149
|
+
mimeType: "application/json",
|
|
150
|
+
}, async (uri, params) => {
|
|
151
|
+
const sessionId = params.sessionId;
|
|
152
|
+
const endpointPath = "/" + params.path;
|
|
153
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
154
|
+
const ep = analysis.apiEndpoints.endpoints.find((e) => e.path === endpointPath);
|
|
155
|
+
if (!ep) {
|
|
156
|
+
throw new Error(`Endpoint path "${endpointPath}" not found in session "${sessionId}".`);
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
contents: [
|
|
160
|
+
{
|
|
161
|
+
uri: uri.href,
|
|
162
|
+
mimeType: "application/json",
|
|
163
|
+
text: JSON.stringify(ep, null, 2),
|
|
164
|
+
},
|
|
165
|
+
],
|
|
166
|
+
};
|
|
167
|
+
});
|
|
168
|
+
// ── Single Method Detail ──
|
|
169
|
+
server.registerResource("analysis_endpoint_method", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/endpoints/{+path}/{method}`, { list: undefined }), {
|
|
170
|
+
title: "Endpoint Method Detail",
|
|
171
|
+
description: "Full detail for a specific method on a path: interactions, params, auth.",
|
|
172
|
+
mimeType: "application/json",
|
|
173
|
+
}, async (uri, params) => {
|
|
174
|
+
const sessionId = params.sessionId;
|
|
175
|
+
const endpointPath = "/" + params.path;
|
|
176
|
+
const method = params.method.toUpperCase();
|
|
177
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
178
|
+
const ep = analysis.apiEndpoints.endpoints.find((e) => e.path === endpointPath);
|
|
179
|
+
if (!ep) {
|
|
180
|
+
throw new Error(`Endpoint path "${endpointPath}" not found in session "${sessionId}".`);
|
|
181
|
+
}
|
|
182
|
+
const m = ep.methods.find((em) => em.method.toUpperCase() === method);
|
|
183
|
+
if (!m) {
|
|
184
|
+
throw new Error(`Method "${method}" not found on "${endpointPath}" in session "${sessionId}".`);
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
contents: [
|
|
188
|
+
{
|
|
189
|
+
uri: uri.href,
|
|
190
|
+
mimeType: "application/json",
|
|
191
|
+
text: JSON.stringify({ path: ep.path, resourceGroup: ep.resourceGroup, pathParams: ep.pathParams, ...m }, null, 2),
|
|
192
|
+
},
|
|
193
|
+
],
|
|
194
|
+
};
|
|
195
|
+
});
|
|
196
|
+
// ── Scenarios ──
|
|
197
|
+
server.registerResource("analysis_scenarios", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/scenarios`, {
|
|
198
|
+
list: makeListCallback("scenarios"),
|
|
199
|
+
}), {
|
|
200
|
+
title: "Drafted Scenarios",
|
|
201
|
+
description: "All drafted user-flow scenarios with steps, chaining, and priority.",
|
|
202
|
+
mimeType: "application/json",
|
|
203
|
+
}, async (uri, params) => {
|
|
204
|
+
const sessionId = params.sessionId;
|
|
205
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
206
|
+
return {
|
|
207
|
+
contents: [
|
|
208
|
+
{
|
|
209
|
+
uri: uri.href,
|
|
210
|
+
mimeType: "application/json",
|
|
211
|
+
text: JSON.stringify(analysis.businessContext.draftedScenarios, null, 2),
|
|
212
|
+
},
|
|
213
|
+
],
|
|
214
|
+
};
|
|
215
|
+
});
|
|
216
|
+
// ── Branch Diff ──
|
|
217
|
+
server.registerResource("analysis_diff", new ResourceTemplate(`${ANALYSIS_URI_PREFIX}/{sessionId}/diff`, {
|
|
218
|
+
list: makeListCallback("diff"),
|
|
219
|
+
}), {
|
|
220
|
+
title: "Branch Diff Context",
|
|
221
|
+
description: "Branch diff context with new/modified endpoints (only present for current_branch_diff scope).",
|
|
222
|
+
mimeType: "application/json",
|
|
223
|
+
}, async (uri, params) => {
|
|
224
|
+
const sessionId = params.sessionId;
|
|
225
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
226
|
+
if (!analysis.branchDiffContext) {
|
|
227
|
+
return {
|
|
228
|
+
contents: [
|
|
229
|
+
{
|
|
230
|
+
uri: uri.href,
|
|
231
|
+
mimeType: "application/json",
|
|
232
|
+
text: JSON.stringify({ message: "No branch diff context (analysis scope was full_repo)." }, null, 2),
|
|
233
|
+
},
|
|
234
|
+
],
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
contents: [
|
|
239
|
+
{
|
|
240
|
+
uri: uri.href,
|
|
241
|
+
mimeType: "application/json",
|
|
242
|
+
text: JSON.stringify(analysis.branchDiffContext, null, 2),
|
|
243
|
+
},
|
|
244
|
+
],
|
|
245
|
+
};
|
|
246
|
+
});
|
|
247
|
+
logger.info("Analysis resources registered successfully");
|
|
248
|
+
}
|
|
@@ -7,19 +7,6 @@ export class ScenarioGenerationService {
|
|
|
7
7
|
logger.info("Parsing scenario into API requests", {
|
|
8
8
|
scenarioName: params.scenarioName,
|
|
9
9
|
});
|
|
10
|
-
// Check if we have API schema to work with
|
|
11
|
-
if (!params.apiSchema) {
|
|
12
|
-
return {
|
|
13
|
-
content: [
|
|
14
|
-
{
|
|
15
|
-
type: "text",
|
|
16
|
-
text: "Please provide an API schema so that I can parse the scenario and map it to specific API endpoints.",
|
|
17
|
-
},
|
|
18
|
-
],
|
|
19
|
-
isError: true,
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
// Generate a single trace request from the scenario
|
|
23
10
|
const traceRequest = this.generateTraceRequestFromInput(params);
|
|
24
11
|
if (!traceRequest) {
|
|
25
12
|
return {
|
|
@@ -32,14 +19,10 @@ export class ScenarioGenerationService {
|
|
|
32
19
|
isError: true,
|
|
33
20
|
};
|
|
34
21
|
}
|
|
35
|
-
// Handle file writing
|
|
36
|
-
//add hyphen to the scenario name
|
|
37
|
-
//make file in tmp directory
|
|
38
22
|
const scenarioName = params.scenarioName.replace(/ /g, "-").toLowerCase();
|
|
39
23
|
const fileName = `scenario_${scenarioName}.json`;
|
|
40
24
|
const filePath = path.join(params.outputDir, fileName);
|
|
41
25
|
try {
|
|
42
|
-
// Check if file exists to determine if we should append or create new
|
|
43
26
|
let existingRequests = [];
|
|
44
27
|
if (fs.existsSync(filePath)) {
|
|
45
28
|
try {
|
|
@@ -49,14 +32,11 @@ export class ScenarioGenerationService {
|
|
|
49
32
|
existingRequests = [];
|
|
50
33
|
}
|
|
51
34
|
}
|
|
52
|
-
catch
|
|
53
|
-
// If file exists but can't be parsed, start fresh
|
|
35
|
+
catch {
|
|
54
36
|
existingRequests = [];
|
|
55
37
|
}
|
|
56
38
|
}
|
|
57
|
-
// Add the new request to the array
|
|
58
39
|
existingRequests.push(traceRequest);
|
|
59
|
-
// Write the updated array to the file
|
|
60
40
|
fs.writeFileSync(filePath, JSON.stringify(existingRequests, null, 2), "utf8");
|
|
61
41
|
logger.info("Trace request added to file", {
|
|
62
42
|
filePath,
|
|
@@ -118,37 +98,55 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
118
98
|
}
|
|
119
99
|
}
|
|
120
100
|
generateTraceRequestFromInput(params) {
|
|
121
|
-
|
|
122
|
-
|
|
101
|
+
let destination = params.destination;
|
|
102
|
+
let scheme = "https";
|
|
103
|
+
let port = 443;
|
|
104
|
+
if (params.baseURL) {
|
|
105
|
+
try {
|
|
106
|
+
const parsed = new URL(params.baseURL);
|
|
107
|
+
scheme = parsed.protocol.replace(":", "");
|
|
108
|
+
destination = parsed.hostname;
|
|
109
|
+
port = parsed.port
|
|
110
|
+
? parseInt(parsed.port, 10)
|
|
111
|
+
: scheme === "https"
|
|
112
|
+
? 443
|
|
113
|
+
: 80;
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
logger.warning("Could not parse baseURL, using destination param", {
|
|
117
|
+
baseURL: params.baseURL,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
123
121
|
const timestamp = new Date().toISOString();
|
|
124
122
|
const method = params.method;
|
|
125
|
-
const
|
|
126
|
-
const statusCode = params.statusCode ||
|
|
127
|
-
(method === "POST" ? 201 : method === "DELETE" ? 204 : 200);
|
|
123
|
+
const statusCode = params.statusCode ?? (method === "POST" ? 201 : method === "DELETE" ? 204 : 200);
|
|
128
124
|
const requestBody = params.requestBody ||
|
|
129
125
|
(method === "GET" || method === "DELETE" ? "" : "{}");
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
const
|
|
126
|
+
const responseHeaders = params.responseHeaders
|
|
127
|
+
|| { "Content-Type": ["application/json"] };
|
|
128
|
+
const isJsonResponse = (responseHeaders["Content-Type"] || [])
|
|
129
|
+
.some(v => v.includes("application/json"));
|
|
130
|
+
const responseBody = params.responseBody || (isJsonResponse ? "{}" : "");
|
|
131
|
+
const authHeaderName = params.authHeader || "Authorization";
|
|
132
|
+
const requestHeaders = {
|
|
133
|
+
"Content-Type": ["application/json"],
|
|
134
|
+
[authHeaderName]: [params.authToken ?? ""],
|
|
135
|
+
};
|
|
133
136
|
return {
|
|
134
|
-
Source:
|
|
137
|
+
Source: "192.168.65.1:39998",
|
|
135
138
|
Destination: destination,
|
|
136
139
|
RequestBody: requestBody,
|
|
137
140
|
ResponseBody: responseBody,
|
|
138
|
-
RequestHeaders:
|
|
139
|
-
|
|
140
|
-
Authorization: ["Bearer demo-token"],
|
|
141
|
-
},
|
|
142
|
-
ResponseHeaders: {
|
|
143
|
-
"Content-Type": ["application/json"],
|
|
144
|
-
},
|
|
141
|
+
RequestHeaders: requestHeaders,
|
|
142
|
+
ResponseHeaders: responseHeaders,
|
|
145
143
|
Method: method,
|
|
146
|
-
Path: path,
|
|
144
|
+
Path: params.path,
|
|
147
145
|
QueryParams: {},
|
|
148
146
|
StatusCode: statusCode,
|
|
149
|
-
Port:
|
|
147
|
+
Port: port,
|
|
150
148
|
Timestamp: timestamp,
|
|
151
|
-
Scheme:
|
|
149
|
+
Scheme: scheme,
|
|
152
150
|
};
|
|
153
151
|
}
|
|
154
152
|
}
|
|
@@ -6,7 +6,7 @@ import { stripVTControlCharacters } from "util";
|
|
|
6
6
|
import { logger } from "../utils/logger.js";
|
|
7
7
|
const DEFAULT_TIMEOUT = 300000; // 5 minutes
|
|
8
8
|
const MAX_CONCURRENT_EXECUTIONS = 5;
|
|
9
|
-
export const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.
|
|
9
|
+
export const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.13";
|
|
10
10
|
const DOCKER_PLATFORM = "linux/amd64";
|
|
11
11
|
const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
|
|
12
12
|
// Files and directories to exclude when mounting workspace to Docker container
|
|
@@ -384,6 +384,15 @@ export class TestExecutionService {
|
|
|
384
384
|
`SKYRAMP_TEST_TOKEN=${options.token || ""}`,
|
|
385
385
|
"SKYRAMP_IN_DOCKER=true",
|
|
386
386
|
];
|
|
387
|
+
// Skyramp-generated tests are standalone HTTP tests that never need host repo
|
|
388
|
+
// conftest.py files or pytest configuration. --noconftest prevents loading any
|
|
389
|
+
// conftest in the test directory tree (avoids missing deps like boto3, django).
|
|
390
|
+
// -c /dev/null overrides all config file discovery (pyproject.toml, pytest.ini,
|
|
391
|
+
// setup.cfg, tox.ini) so user-repo plugins (e.g. pytest-timeout) not installed
|
|
392
|
+
// in the executor container don't cause INTERNALERROR at collection time.
|
|
393
|
+
if (options.language === "python") {
|
|
394
|
+
env.push(`PYTEST_ADDOPTS=--noconftest -c /dev/null`);
|
|
395
|
+
}
|
|
387
396
|
// Add save storage path to environment if provided
|
|
388
397
|
if (saveStorageTargetPath) {
|
|
389
398
|
env.push(`PLAYWRIGHT_SAVE_STORAGE_PATH=${saveStorageTargetPath}`);
|
|
@@ -11,7 +11,8 @@ const scenarioTestSchema = {
|
|
|
11
11
|
.describe("Destination hostname or IP address for the test (e.g., api.example.com, localhost, 192.168.1.1). Do NOT include port numbers."),
|
|
12
12
|
apiSchema: z
|
|
13
13
|
.string()
|
|
14
|
-
.
|
|
14
|
+
.optional()
|
|
15
|
+
.describe("Optional. Absolute path (/path/to/openapi.json) to the OpenAPI/Swagger schema file or a URL (e.g., https://demoshop.skyramp.dev/openapi.json). DO NOT TRY TO ASSUME THE OPENAPI SCHEMA IF NOT PROVIDED. NOTE TO AI ASSISTANTS: You do not need to read the contents of this file - simply pass the file path as the backend will handle reading and processing it."),
|
|
15
16
|
baseURL: z
|
|
16
17
|
.string()
|
|
17
18
|
.optional()
|
|
@@ -23,7 +24,7 @@ const scenarioTestSchema = {
|
|
|
23
24
|
.describe("HTTP method (GET, POST, PUT, DELETE, etc.) parsed by AI from the scenario"),
|
|
24
25
|
path: z
|
|
25
26
|
.string()
|
|
26
|
-
.describe("API path (e.g
|
|
27
|
+
.describe("API path parsed by AI from the scenario. CRITICAL: For requests that reference an ID created by a prior step (e.g. GET/PUT/DELETE after a POST), use the ACTUAL ID value from the prior step's responseBody in the path, NOT a template variable. Example: use '/api/v1/products/70885' instead of '/api/v1/products/{product_id}'. The CLI detects chaining by matching concrete values across requests."),
|
|
27
28
|
// AI-parsed parameters (optional)
|
|
28
29
|
requestBody: z
|
|
29
30
|
.string()
|
|
@@ -36,8 +37,21 @@ const scenarioTestSchema = {
|
|
|
36
37
|
statusCode: z
|
|
37
38
|
.number()
|
|
38
39
|
.optional()
|
|
39
|
-
.describe("HTTP status code
|
|
40
|
+
.describe("Expected HTTP status code. Defaults: POST→201, DELETE→204, GET/PUT/PATCH→200. Only override for non-standard codes."),
|
|
40
41
|
outputDir: baseSchema.shape.outputDir,
|
|
42
|
+
authHeader: z
|
|
43
|
+
.string()
|
|
44
|
+
.optional()
|
|
45
|
+
.default("Authorization")
|
|
46
|
+
.describe("Name of the HTTP header to use for authorization. Use 'Cookie' for cookie-based auth (e.g., NextAuth), 'Authorization' for Bearer tokens, 'X-API-Key' for API keys. Defaults to 'Authorization'."),
|
|
47
|
+
authToken: z
|
|
48
|
+
.string()
|
|
49
|
+
.optional()
|
|
50
|
+
.describe("Full auth token value to include in the request header. For Authorization headers, include the scheme prefix (e.g., 'Bearer my-token'). For Cookie headers, use the cookie string (e.g., 'session=abc123'). For API key headers, use the raw key. If omitted, the header value is left empty (the CLI injects the real token at runtime)."),
|
|
51
|
+
responseHeaders: z
|
|
52
|
+
.record(z.array(z.string()))
|
|
53
|
+
.optional()
|
|
54
|
+
.describe('Response headers as a JSON object (e.g., {"Content-Type": ["application/json"]}). Defaults to Content-Type: application/json.'),
|
|
41
55
|
};
|
|
42
56
|
const TOOL_NAME = "skyramp_scenario_test_generation";
|
|
43
57
|
export function registerScenarioTestTool(server) {
|
|
@@ -65,7 +79,7 @@ Returns a single TraceRequest object with:
|
|
|
65
79
|
**AI Responsibilities:**
|
|
66
80
|
The AI should parse the natural language scenario and provide:
|
|
67
81
|
- HTTP method (POST, GET, PUT, DELETE)
|
|
68
|
-
- API path (e.g., /api/v1/products, /api/v1/products/{product_id})
|
|
82
|
+
- API path with CONCRETE ID values, not templates (e.g., /api/v1/products/70885, NOT /api/v1/products/{product_id})
|
|
69
83
|
- Request body (JSON string, if applicable)
|
|
70
84
|
- Response body (JSON string, if applicable)
|
|
71
85
|
- Status code (optional, defaults based on method)
|
|
@@ -77,8 +91,6 @@ The AI should parse the natural language scenario and provide:
|
|
|
77
91
|
- AI-parsed HTTP method and path (required)
|
|
78
92
|
- AI-parsed request/response bodies (optional)
|
|
79
93
|
|
|
80
|
-
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file.**
|
|
81
|
-
|
|
82
94
|
**Note:** This tool generates one request at a time. Call multiple times for multi-step scenarios.
|
|
83
95
|
|
|
84
96
|
**CRITICAL - Integration Test Generation After Scenario Creation:**
|
|
@@ -15,10 +15,32 @@ const newTestSchema = z.object({
|
|
|
15
15
|
testType: z.string().describe("Type of test created: Smoke, Contract, Integration, etc."),
|
|
16
16
|
endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
|
|
17
17
|
fileName: z.string().describe("Name of the generated test file"),
|
|
18
|
+
description: z.string().optional().describe("What the test scenario covers, e.g. 'Creates a collection, adds a link, then verifies the link exists'"),
|
|
19
|
+
scenarioFile: z.string().optional().describe("Path to the scenario JSON file if one was generated (e.g. 'tests/scenario_collections-links.json')"),
|
|
20
|
+
traceFile: z.string().optional().describe("Path to the backend trace file if used or created"),
|
|
21
|
+
frontendTrace: z.string().optional().describe("Path to the Playwright/UI trace file if used or created"),
|
|
18
22
|
});
|
|
19
23
|
const descriptionSchema = z.object({
|
|
20
24
|
description: z.string().describe("One-line description"),
|
|
21
25
|
});
|
|
26
|
+
const scenarioStepSchema = z.object({
|
|
27
|
+
method: z.string().optional().describe("HTTP method (e.g. 'POST', 'GET'). Required for API steps, omit for UI/E2E actions."),
|
|
28
|
+
path: z.string().optional().describe("Endpoint or page path (e.g. '/api/v1/products' or '/products'). Required for API steps, omit for UI actions."),
|
|
29
|
+
description: z.string().describe("What this step does, e.g. 'Create a product' or 'Click checkout button and verify confirmation'"),
|
|
30
|
+
expectedStatusCode: z.number().optional().describe("Expected HTTP status code, e.g. 200, 201, 404"),
|
|
31
|
+
requestBody: z.record(z.any()).optional().describe("Example request body with realistic field values"),
|
|
32
|
+
responseBody: z.record(z.any()).optional().describe("Key response fields to verify, e.g. { id: 'number', name: 'string', in_stock: 'boolean?' }"),
|
|
33
|
+
});
|
|
34
|
+
const additionalRecommendationSchema = z.object({
|
|
35
|
+
testType: z.string().describe("Type of test: Integration, E2E, Fuzz, Contract, UI, etc."),
|
|
36
|
+
scenarioName: z.string().describe("Name of the scenario, e.g. 'products_orders_workflow'"),
|
|
37
|
+
steps: z.array(scenarioStepSchema).describe("Ordered sequence of API/UI steps in this test scenario"),
|
|
38
|
+
description: z.string().describe("Why this test is valuable and what it would cover"),
|
|
39
|
+
priority: z.string().describe("Priority level: high, medium, or low"),
|
|
40
|
+
openApiSpec: z.string().optional().describe("Path to OpenAPI/Swagger spec file if available, e.g. 'openapi.yaml'"),
|
|
41
|
+
backendTrace: z.string().optional().describe("Path to backend trace file if available, e.g. 'tests/skyramp-traces.json'. Used by integration and E2E tests."),
|
|
42
|
+
frontendTrace: z.string().optional().describe("Path to Playwright/UI trace file if available, e.g. 'tests/skyramp-playwright.zip'. UI tests need this; E2E tests need both frontend and backend traces."),
|
|
43
|
+
});
|
|
22
44
|
const testMaintenanceSchema = z.object({
|
|
23
45
|
fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
|
|
24
46
|
description: z.string().describe("What was changed and why"),
|
|
@@ -47,6 +69,11 @@ export function registerSubmitReportTool(server) {
|
|
|
47
69
|
testResults: z
|
|
48
70
|
.array(testResultSchema)
|
|
49
71
|
.describe("List of ALL test execution results. One entry per test executed."),
|
|
72
|
+
additionalRecommendations: z
|
|
73
|
+
.array(additionalRecommendationSchema)
|
|
74
|
+
.optional()
|
|
75
|
+
.default([])
|
|
76
|
+
.describe("Recommended tests that were not generated (lower priority). Include the remaining recommendations from skyramp_recommend_tests that were not implemented."),
|
|
50
77
|
issuesFound: z
|
|
51
78
|
.array(descriptionSchema)
|
|
52
79
|
.describe("List of issues, failures, or bugs found. Use empty array [] if none."),
|
|
@@ -68,6 +95,7 @@ export function registerSubmitReportTool(server) {
|
|
|
68
95
|
newTestsCreated: params.newTestsCreated,
|
|
69
96
|
testMaintenance: params.testMaintenance,
|
|
70
97
|
testResults: params.testResults,
|
|
98
|
+
additionalRecommendations: params.additionalRecommendations ?? [],
|
|
71
99
|
issuesFound: params.issuesFound,
|
|
72
100
|
commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim().slice(0, 72) || DEFAULT_COMMIT_MESSAGE,
|
|
73
101
|
}, null, 2);
|
|
@@ -105,6 +105,14 @@ Information about state files and cleanup results.`,
|
|
|
105
105
|
const maxAgeHours = args.maxAgeHours || 24;
|
|
106
106
|
const deletedCount = await StateManager.cleanupOldStateFiles(maxAgeHours);
|
|
107
107
|
logger.info(`Cleaned up ${deletedCount} state files older than ${maxAgeHours} hours`);
|
|
108
|
+
if (deletedCount > 0) {
|
|
109
|
+
try {
|
|
110
|
+
await server.sendResourceListChanged();
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
logger.warning("Unable to update MCP clients with new resource list");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
108
116
|
// Get remaining files
|
|
109
117
|
const remainingFiles = await StateManager.listStateFiles();
|
|
110
118
|
return {
|