@skyramp/mcp 0.0.59 → 0.0.60-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +32 -5
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +98 -0
- package/build/prompts/test-recommendation/recommendationSections.js +226 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +71 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +166 -104
- package/build/prompts/testGenerationPrompt.js +2 -3
- package/build/prompts/testbot/testbot-prompts.js +96 -93
- package/build/resources/analysisResources.js +254 -0
- package/build/services/ScenarioGenerationService.js +70 -26
- package/build/tools/generate-tests/generateIntegrationRestTool.js +54 -1
- package/build/tools/generate-tests/generateScenarioRestTool.js +8 -5
- package/build/tools/submitReportTool.js +28 -0
- package/build/tools/test-maintenance/stateCleanupTool.js +8 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +349 -217
- package/build/tools/test-recommendation/recommendTestsTool.js +163 -159
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/types/RepositoryAnalysis.js +99 -12
- package/build/utils/AnalysisStateManager.js +40 -23
- package/build/utils/branchDiff.js +47 -0
- package/build/utils/pr-comment-parser.js +124 -0
- package/build/utils/projectMetadata.js +188 -0
- package/build/utils/projectMetadata.test.js +81 -0
- package/build/utils/repoScanner.js +378 -0
- package/build/utils/routeParsers.js +213 -0
- package/build/utils/routeParsers.test.js +87 -0
- package/build/utils/scenarioDrafting.js +119 -0
- package/build/utils/scenarioDrafting.test.js +66 -0
- package/build/utils/trace-parser.js +166 -0
- package/build/utils/workspaceAuth.js +16 -0
- package/package.json +1 -1
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +0 -326
- package/build/prompts/test-recommendation/test-mapping-prompt.js +0 -266
- package/build/tools/test-recommendation/mapTestsTool.js +0 -243
- package/build/types/TestMapping.js +0 -173
- package/build/utils/scoring-engine.js +0 -380
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { StateManager, getSessionFilePath, getRegisteredSessions, hasSessionData, getSessionData, } from "../utils/AnalysisStateManager.js";
|
|
4
|
+
import { logger } from "../utils/logger.js";
|
|
5
|
+
/**
|
|
6
|
+
* Register MCP Resources for analysis data access.
|
|
7
|
+
*
|
|
8
|
+
* Resources provide read-only, token-efficient access to analysis data
|
|
9
|
+
* via sessionId. The state file is an internal backing store.
|
|
10
|
+
*
|
|
11
|
+
* URI scheme:
|
|
12
|
+
* skyramp://analysis/{sessionId}/summary
|
|
13
|
+
* skyramp://analysis/{sessionId}/endpoints
|
|
14
|
+
* skyramp://analysis/{sessionId}/endpoints/{+path}
|
|
15
|
+
* skyramp://analysis/{sessionId}/endpoints/{+path}/{method}
|
|
16
|
+
* skyramp://analysis/{sessionId}/scenarios
|
|
17
|
+
* skyramp://analysis/{sessionId}/diff
|
|
18
|
+
*/
|
|
19
|
+
export function registerAnalysisResources(server) {
|
|
20
|
+
logger.info("Registering analysis resources");
|
|
21
|
+
async function loadAnalysis(sessionId) {
|
|
22
|
+
// Try process memory first (new flow — no state file needed)
|
|
23
|
+
if (hasSessionData(sessionId)) {
|
|
24
|
+
const memData = getSessionData(sessionId);
|
|
25
|
+
if (memData?.analysis) {
|
|
26
|
+
logger.debug("Loaded analysis from process memory for resource", { sessionId });
|
|
27
|
+
return memData;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
// Fall back to state file for backward compatibility
|
|
31
|
+
const registeredPath = getSessionFilePath(sessionId);
|
|
32
|
+
const mgr = registeredPath
|
|
33
|
+
? StateManager.fromStatePath(registeredPath)
|
|
34
|
+
: StateManager.fromSessionId(sessionId);
|
|
35
|
+
if (!mgr.exists()) {
|
|
36
|
+
throw new Error(`Analysis session "${sessionId}" not found or expired.`);
|
|
37
|
+
}
|
|
38
|
+
const data = await mgr.readData();
|
|
39
|
+
if (!data) {
|
|
40
|
+
throw new Error(`Analysis session "${sessionId}" has no data.`);
|
|
41
|
+
}
|
|
42
|
+
if (!data.analysis && data.apiEndpoints) {
|
|
43
|
+
logger.info("Detected unwrapped RepositoryAnalysis in resource handler — adapting");
|
|
44
|
+
return {
|
|
45
|
+
repositoryPath: data.metadata?.repositoryName || "unknown",
|
|
46
|
+
analysisScope: data.metadata?.analysisScope,
|
|
47
|
+
analysis: data,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
if (!data.analysis) {
|
|
51
|
+
throw new Error(`Analysis session "${sessionId}" has no analysis data.`);
|
|
52
|
+
}
|
|
53
|
+
return data;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Shared list callback: returns resource URIs for sessions created by
|
|
57
|
+
* this process only (via the in-memory registry). No filesystem scan —
|
|
58
|
+
* eliminates cross-process leakage when multiple clients share /tmp.
|
|
59
|
+
*/
|
|
60
|
+
function makeListCallback(suffix) {
|
|
61
|
+
return async () => {
|
|
62
|
+
const sessions = getRegisteredSessions();
|
|
63
|
+
const available = Array.from(sessions.entries()).filter(([sessionId, filePath]) => hasSessionData(sessionId) || fs.existsSync(filePath));
|
|
64
|
+
return {
|
|
65
|
+
resources: available.map(([sessionId]) => ({
|
|
66
|
+
uri: `skyramp://analysis/${sessionId}/${suffix}`,
|
|
67
|
+
name: `Analysis ${suffix} (${sessionId})`,
|
|
68
|
+
mimeType: "application/json",
|
|
69
|
+
})),
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
// ── Summary ──
|
|
74
|
+
server.registerResource("analysis_summary", new ResourceTemplate("skyramp://analysis/{sessionId}/summary", {
|
|
75
|
+
list: makeListCallback("summary"),
|
|
76
|
+
}), {
|
|
77
|
+
title: "Analysis Summary",
|
|
78
|
+
description: "High-level overview: project type, tech stack, endpoint/scenario counts, coverage gaps.",
|
|
79
|
+
mimeType: "application/json",
|
|
80
|
+
}, async (uri, params) => {
|
|
81
|
+
const sessionId = params.sessionId;
|
|
82
|
+
const { analysis, repositoryPath, analysisScope } = await loadAnalysis(sessionId);
|
|
83
|
+
const summary = {
|
|
84
|
+
sessionId,
|
|
85
|
+
repositoryPath,
|
|
86
|
+
analysisScope: analysisScope || "full_repo",
|
|
87
|
+
metadata: analysis.metadata,
|
|
88
|
+
projectClassification: analysis.projectClassification,
|
|
89
|
+
technologyStack: {
|
|
90
|
+
languages: analysis.technologyStack.languages,
|
|
91
|
+
frameworks: analysis.technologyStack.frameworks,
|
|
92
|
+
runtime: analysis.technologyStack.runtime,
|
|
93
|
+
},
|
|
94
|
+
authentication: analysis.authentication,
|
|
95
|
+
infrastructure: analysis.infrastructure,
|
|
96
|
+
endpointStats: {
|
|
97
|
+
totalPaths: analysis.apiEndpoints.endpoints.length,
|
|
98
|
+
totalMethods: analysis.apiEndpoints.endpoints.reduce((sum, ep) => sum + ep.methods.length, 0),
|
|
99
|
+
totalInteractions: analysis.apiEndpoints.endpoints.reduce((sum, ep) => sum +
|
|
100
|
+
ep.methods.reduce((msum, m) => msum + m.interactions.length, 0), 0),
|
|
101
|
+
baseUrl: analysis.apiEndpoints.baseUrl,
|
|
102
|
+
},
|
|
103
|
+
scenarioCount: analysis.businessContext.draftedScenarios.length,
|
|
104
|
+
existingTests: analysis.existingTests,
|
|
105
|
+
};
|
|
106
|
+
return {
|
|
107
|
+
contents: [
|
|
108
|
+
{
|
|
109
|
+
uri: uri.href,
|
|
110
|
+
mimeType: "application/json",
|
|
111
|
+
text: JSON.stringify(summary, null, 2),
|
|
112
|
+
},
|
|
113
|
+
],
|
|
114
|
+
};
|
|
115
|
+
});
|
|
116
|
+
// ── Endpoints (compact listing) ──
|
|
117
|
+
server.registerResource("analysis_endpoints", new ResourceTemplate("skyramp://analysis/{sessionId}/endpoints", {
|
|
118
|
+
list: makeListCallback("endpoints"),
|
|
119
|
+
}), {
|
|
120
|
+
title: "Endpoint Listing",
|
|
121
|
+
description: "Compact listing of all endpoints (path, methods, interaction counts). Use path-specific resources for full detail.",
|
|
122
|
+
mimeType: "application/json",
|
|
123
|
+
}, async (uri, params) => {
|
|
124
|
+
const sessionId = params.sessionId;
|
|
125
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
126
|
+
const compact = analysis.apiEndpoints.endpoints.map((ep) => ({
|
|
127
|
+
path: ep.path,
|
|
128
|
+
resourceGroup: ep.resourceGroup,
|
|
129
|
+
pathParams: ep.pathParams.map((p) => p.name),
|
|
130
|
+
methods: ep.methods.map((m) => ({
|
|
131
|
+
method: m.method,
|
|
132
|
+
authRequired: m.authRequired,
|
|
133
|
+
interactionCount: m.interactions.length,
|
|
134
|
+
interactionTypes: m.interactions.map((i) => i.type),
|
|
135
|
+
hasCookies: m.interactions.some((i) => (i.response.cookies?.length ?? 0) > 0),
|
|
136
|
+
hasResponseHeaders: m.interactions.some((i) => Object.keys(i.response.headers ?? {}).length > 0),
|
|
137
|
+
createsResource: m.createsResource,
|
|
138
|
+
})),
|
|
139
|
+
}));
|
|
140
|
+
return {
|
|
141
|
+
contents: [
|
|
142
|
+
{
|
|
143
|
+
uri: uri.href,
|
|
144
|
+
mimeType: "application/json",
|
|
145
|
+
text: JSON.stringify({ baseUrl: analysis.apiEndpoints.baseUrl, endpoints: compact }, null, 2),
|
|
146
|
+
},
|
|
147
|
+
],
|
|
148
|
+
};
|
|
149
|
+
});
|
|
150
|
+
// ── Single Path Detail ──
|
|
151
|
+
// No list callback — these are parameterized drilldowns, not top-level resources.
|
|
152
|
+
server.registerResource("analysis_endpoint_path", new ResourceTemplate("skyramp://analysis/{sessionId}/endpoints/{+path}", { list: undefined }), {
|
|
153
|
+
title: "Endpoint Path Detail",
|
|
154
|
+
description: "Full detail for a specific path: all methods, interactions, params.",
|
|
155
|
+
mimeType: "application/json",
|
|
156
|
+
}, async (uri, params) => {
|
|
157
|
+
const sessionId = params.sessionId;
|
|
158
|
+
const endpointPath = "/" + params.path;
|
|
159
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
160
|
+
const ep = analysis.apiEndpoints.endpoints.find((e) => e.path === endpointPath);
|
|
161
|
+
if (!ep) {
|
|
162
|
+
throw new Error(`Endpoint path "${endpointPath}" not found in session "${sessionId}".`);
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
contents: [
|
|
166
|
+
{
|
|
167
|
+
uri: uri.href,
|
|
168
|
+
mimeType: "application/json",
|
|
169
|
+
text: JSON.stringify(ep, null, 2),
|
|
170
|
+
},
|
|
171
|
+
],
|
|
172
|
+
};
|
|
173
|
+
});
|
|
174
|
+
// ── Single Method Detail ──
|
|
175
|
+
server.registerResource("analysis_endpoint_method", new ResourceTemplate("skyramp://analysis/{sessionId}/endpoints/{+path}/{method}", { list: undefined }), {
|
|
176
|
+
title: "Endpoint Method Detail",
|
|
177
|
+
description: "Full detail for a specific method on a path: interactions, params, auth.",
|
|
178
|
+
mimeType: "application/json",
|
|
179
|
+
}, async (uri, params) => {
|
|
180
|
+
const sessionId = params.sessionId;
|
|
181
|
+
const endpointPath = "/" + params.path;
|
|
182
|
+
const method = params.method.toUpperCase();
|
|
183
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
184
|
+
const ep = analysis.apiEndpoints.endpoints.find((e) => e.path === endpointPath);
|
|
185
|
+
if (!ep) {
|
|
186
|
+
throw new Error(`Endpoint path "${endpointPath}" not found in session "${sessionId}".`);
|
|
187
|
+
}
|
|
188
|
+
const m = ep.methods.find((em) => em.method.toUpperCase() === method);
|
|
189
|
+
if (!m) {
|
|
190
|
+
throw new Error(`Method "${method}" not found on "${endpointPath}" in session "${sessionId}".`);
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
contents: [
|
|
194
|
+
{
|
|
195
|
+
uri: uri.href,
|
|
196
|
+
mimeType: "application/json",
|
|
197
|
+
text: JSON.stringify({ path: ep.path, resourceGroup: ep.resourceGroup, pathParams: ep.pathParams, ...m }, null, 2),
|
|
198
|
+
},
|
|
199
|
+
],
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
// ── Scenarios ──
|
|
203
|
+
server.registerResource("analysis_scenarios", new ResourceTemplate("skyramp://analysis/{sessionId}/scenarios", {
|
|
204
|
+
list: makeListCallback("scenarios"),
|
|
205
|
+
}), {
|
|
206
|
+
title: "Drafted Scenarios",
|
|
207
|
+
description: "All drafted user-flow scenarios with steps, chaining, and priority.",
|
|
208
|
+
mimeType: "application/json",
|
|
209
|
+
}, async (uri, params) => {
|
|
210
|
+
const sessionId = params.sessionId;
|
|
211
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
212
|
+
return {
|
|
213
|
+
contents: [
|
|
214
|
+
{
|
|
215
|
+
uri: uri.href,
|
|
216
|
+
mimeType: "application/json",
|
|
217
|
+
text: JSON.stringify(analysis.businessContext.draftedScenarios, null, 2),
|
|
218
|
+
},
|
|
219
|
+
],
|
|
220
|
+
};
|
|
221
|
+
});
|
|
222
|
+
// ── Branch Diff ──
|
|
223
|
+
server.registerResource("analysis_diff", new ResourceTemplate("skyramp://analysis/{sessionId}/diff", {
|
|
224
|
+
list: makeListCallback("diff"),
|
|
225
|
+
}), {
|
|
226
|
+
title: "Branch Diff Context",
|
|
227
|
+
description: "Branch diff context with new/modified endpoints (only present for current_branch_diff scope).",
|
|
228
|
+
mimeType: "application/json",
|
|
229
|
+
}, async (uri, params) => {
|
|
230
|
+
const sessionId = params.sessionId;
|
|
231
|
+
const { analysis } = await loadAnalysis(sessionId);
|
|
232
|
+
if (!analysis.branchDiffContext) {
|
|
233
|
+
return {
|
|
234
|
+
contents: [
|
|
235
|
+
{
|
|
236
|
+
uri: uri.href,
|
|
237
|
+
mimeType: "application/json",
|
|
238
|
+
text: JSON.stringify({ message: "No branch diff context (analysis scope was full_repo)." }, null, 2),
|
|
239
|
+
},
|
|
240
|
+
],
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
contents: [
|
|
245
|
+
{
|
|
246
|
+
uri: uri.href,
|
|
247
|
+
mimeType: "application/json",
|
|
248
|
+
text: JSON.stringify(analysis.branchDiffContext, null, 2),
|
|
249
|
+
},
|
|
250
|
+
],
|
|
251
|
+
};
|
|
252
|
+
});
|
|
253
|
+
logger.info("Analysis resources registered successfully");
|
|
254
|
+
}
|
|
@@ -7,18 +7,6 @@ export class ScenarioGenerationService {
|
|
|
7
7
|
logger.info("Parsing scenario into API requests", {
|
|
8
8
|
scenarioName: params.scenarioName,
|
|
9
9
|
});
|
|
10
|
-
// Check if we have API schema to work with
|
|
11
|
-
if (!params.apiSchema) {
|
|
12
|
-
return {
|
|
13
|
-
content: [
|
|
14
|
-
{
|
|
15
|
-
type: "text",
|
|
16
|
-
text: "Please provide an API schema so that I can parse the scenario and map it to specific API endpoints.",
|
|
17
|
-
},
|
|
18
|
-
],
|
|
19
|
-
isError: true,
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
10
|
// Generate a single trace request from the scenario
|
|
23
11
|
const traceRequest = this.generateTraceRequestFromInput(params);
|
|
24
12
|
if (!traceRequest) {
|
|
@@ -33,8 +21,6 @@ export class ScenarioGenerationService {
|
|
|
33
21
|
};
|
|
34
22
|
}
|
|
35
23
|
// Handle file writing
|
|
36
|
-
//add hyphen to the scenario name
|
|
37
|
-
//make file in tmp directory
|
|
38
24
|
const scenarioName = params.scenarioName.replace(/ /g, "-").toLowerCase();
|
|
39
25
|
const fileName = `scenario_${scenarioName}.json`;
|
|
40
26
|
const filePath = path.join(params.outputDir, fileName);
|
|
@@ -118,27 +104,63 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
118
104
|
}
|
|
119
105
|
}
|
|
120
106
|
generateTraceRequestFromInput(params) {
|
|
121
|
-
|
|
122
|
-
|
|
107
|
+
let destination = params.destination;
|
|
108
|
+
let scheme = "https";
|
|
109
|
+
let port = 443;
|
|
110
|
+
// Derive scheme, host, and port from baseURL when available
|
|
111
|
+
if (params.baseURL) {
|
|
112
|
+
try {
|
|
113
|
+
const parsed = new URL(params.baseURL);
|
|
114
|
+
scheme = parsed.protocol.replace(":", "");
|
|
115
|
+
destination = parsed.hostname;
|
|
116
|
+
port = parsed.port
|
|
117
|
+
? parseInt(parsed.port, 10)
|
|
118
|
+
: scheme === "https"
|
|
119
|
+
? 443
|
|
120
|
+
: 80;
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
logger.warning("Could not parse baseURL, using destination param", {
|
|
124
|
+
baseURL: params.baseURL,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
123
128
|
const timestamp = new Date().toISOString();
|
|
124
129
|
const method = params.method;
|
|
125
130
|
const path = params.path;
|
|
126
|
-
const statusCode = params.statusCode
|
|
127
|
-
(method === "POST" ? 201 : method === "DELETE" ? 204 : 200);
|
|
131
|
+
const statusCode = params.statusCode;
|
|
128
132
|
const requestBody = params.requestBody ||
|
|
129
133
|
(method === "GET" || method === "DELETE" ? "" : "{}");
|
|
130
|
-
|
|
131
|
-
|
|
134
|
+
let responseBody = params.responseBody;
|
|
135
|
+
if (!responseBody) {
|
|
136
|
+
if (method === "DELETE") {
|
|
137
|
+
responseBody = "";
|
|
138
|
+
}
|
|
139
|
+
else if (method === "POST" && requestBody && requestBody !== "{}") {
|
|
140
|
+
responseBody = this.synthesizePostResponse(requestBody);
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
responseBody = "{}";
|
|
144
|
+
}
|
|
145
|
+
}
|
|
132
146
|
const source = "192.168.65.1:39998";
|
|
147
|
+
// Build auth header from param or default to Bearer
|
|
148
|
+
const authHeaderName = params.authHeader || "Authorization";
|
|
149
|
+
const requestHeaders = {
|
|
150
|
+
"Content-Type": ["application/json"],
|
|
151
|
+
};
|
|
152
|
+
if (authHeaderName === "Cookie") {
|
|
153
|
+
requestHeaders["Cookie"] = ["session-token=demo-token"];
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
requestHeaders[authHeaderName] = ["Bearer demo-token"];
|
|
157
|
+
}
|
|
133
158
|
return {
|
|
134
159
|
Source: source,
|
|
135
160
|
Destination: destination,
|
|
136
161
|
RequestBody: requestBody,
|
|
137
162
|
ResponseBody: responseBody,
|
|
138
|
-
RequestHeaders:
|
|
139
|
-
"Content-Type": ["application/json"],
|
|
140
|
-
Authorization: ["Bearer demo-token"],
|
|
141
|
-
},
|
|
163
|
+
RequestHeaders: requestHeaders,
|
|
142
164
|
ResponseHeaders: {
|
|
143
165
|
"Content-Type": ["application/json"],
|
|
144
166
|
},
|
|
@@ -146,9 +168,31 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
146
168
|
Path: path,
|
|
147
169
|
QueryParams: {},
|
|
148
170
|
StatusCode: statusCode,
|
|
149
|
-
Port:
|
|
171
|
+
Port: port,
|
|
150
172
|
Timestamp: timestamp,
|
|
151
|
-
Scheme:
|
|
173
|
+
Scheme: scheme,
|
|
152
174
|
};
|
|
153
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Generate a synthetic POST response body that echoes the request body
|
|
178
|
+
* wrapped in a "response" envelope with an auto-incrementing "id".
|
|
179
|
+
* This gives the CLI response data to chain from (e.g., response.id).
|
|
180
|
+
*/
|
|
181
|
+
synthesizePostResponse(requestBody) {
|
|
182
|
+
try {
|
|
183
|
+
const parsed = JSON.parse(requestBody);
|
|
184
|
+
const id = Date.now() % 100000;
|
|
185
|
+
if (Array.isArray(parsed)) {
|
|
186
|
+
const withIds = parsed.map((item, idx) => ({
|
|
187
|
+
id: id + idx,
|
|
188
|
+
...item,
|
|
189
|
+
}));
|
|
190
|
+
return JSON.stringify({ response: withIds });
|
|
191
|
+
}
|
|
192
|
+
return JSON.stringify({ response: { id, ...parsed } });
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
return "{}";
|
|
196
|
+
}
|
|
197
|
+
}
|
|
154
198
|
}
|
|
@@ -17,7 +17,6 @@ const integrationTestSchema = z
|
|
|
17
17
|
.describe("Path to the scenario file to be used for test generation. This file is generated by the skyramp_scenario_test_generation tool.")
|
|
18
18
|
.optional(),
|
|
19
19
|
...codeRefactoringSchema.shape,
|
|
20
|
-
...baseTestSchema,
|
|
21
20
|
endpointURL: baseTestSchema.endpointURL.default(""),
|
|
22
21
|
})
|
|
23
22
|
.omit({ method: true }).shape;
|
|
@@ -26,6 +25,10 @@ export class IntegrationTestService extends TestGenerationService {
|
|
|
26
25
|
return TestType.INTEGRATION;
|
|
27
26
|
}
|
|
28
27
|
buildGenerationOptions(params) {
|
|
28
|
+
if ((params.scenarioFile || params.trace) &&
|
|
29
|
+
!params.chainingKey) {
|
|
30
|
+
params.chainingKey = "response.id";
|
|
31
|
+
}
|
|
29
32
|
return {
|
|
30
33
|
...super.buildBaseGenerationOptions(params),
|
|
31
34
|
responseData: params.responseData,
|
|
@@ -33,6 +36,56 @@ export class IntegrationTestService extends TestGenerationService {
|
|
|
33
36
|
scenarioFile: params.scenarioFile,
|
|
34
37
|
};
|
|
35
38
|
}
|
|
39
|
+
async generateTest(params) {
|
|
40
|
+
const result = await super.generateTest(params);
|
|
41
|
+
if (!result.isError && params.scenarioFile) {
|
|
42
|
+
const chainingInstructions = `
|
|
43
|
+
|
|
44
|
+
⏭️ **CRITICAL NEXT STEP — Verify & Fix Chaining in Generated Test**
|
|
45
|
+
|
|
46
|
+
The generated integration test uses \`chainingKey: "response.id"\` for all steps, but
|
|
47
|
+
multi-resource workflows often need **different chaining per step**. You MUST:
|
|
48
|
+
|
|
49
|
+
1. **Read the generated test file** in \`${params.outputDir}\`
|
|
50
|
+
2. **Identify each API call** in the test (POST, GET, PUT, DELETE)
|
|
51
|
+
3. **Verify chaining is correct** for each step:
|
|
52
|
+
- After each POST, the response ID must be extracted into a **uniquely named variable**
|
|
53
|
+
(e.g., \`product_id\`, \`order_id\`, \`review_id\` — NOT all named \`id\`)
|
|
54
|
+
- GET/PUT/DELETE calls that reference a resource must use the **correct** variable in their path
|
|
55
|
+
(e.g., \`/products/{product_id}\` uses the product's ID, not the order's ID)
|
|
56
|
+
- POST calls that create a child resource must include the **parent's ID** in the request body
|
|
57
|
+
or path (e.g., \`POST /orders\` body should include \`product_id\` from the products POST response)
|
|
58
|
+
4. **Fix ONLY chaining** (path params AND request body ID references) — nothing else:
|
|
59
|
+
- Replace hardcoded path IDs (like \`/products/1\`) with the dynamic variable from the POST response
|
|
60
|
+
- Replace hardcoded IDs in request bodies (like \`"product_id": 1\`) with the dynamic variable
|
|
61
|
+
(use \`data_override\`/\`dataOverride\` or direct variable substitution)
|
|
62
|
+
- Rename duplicate variable names so each resource has its own ID variable
|
|
63
|
+
- For Python: use \`skyramp.get_response_value(response_N, "id")\` to extract and f-strings for paths
|
|
64
|
+
- For TypeScript: use \`getResponseValue(response, "response.id")\` or the appropriate accessor
|
|
65
|
+
⚠️ **Preserve everything else exactly as generated** — do not add, remove, or modify
|
|
66
|
+
auth headers, cookies, tokens, env vars, imports, assertions, or non-chaining request body fields.
|
|
67
|
+
The CLI output for auth/headers is intentional.
|
|
68
|
+
|
|
69
|
+
**Example fix for a products → orders workflow:**
|
|
70
|
+
\`\`\`
|
|
71
|
+
# Step 1: POST /products → extract product_id
|
|
72
|
+
# Step 2: POST /orders with product_id in body → extract order_id
|
|
73
|
+
# Step 3: GET /orders/{order_id} → uses order_id (NOT product_id)
|
|
74
|
+
# Step 4: DELETE /products/{product_id} → uses product_id (NOT order_id)
|
|
75
|
+
\`\`\``;
|
|
76
|
+
const existingText = result.content[0] && "text" in result.content[0] ? result.content[0].text : "";
|
|
77
|
+
return {
|
|
78
|
+
content: [
|
|
79
|
+
{
|
|
80
|
+
type: "text",
|
|
81
|
+
text: existingText + chainingInstructions,
|
|
82
|
+
},
|
|
83
|
+
],
|
|
84
|
+
isError: false,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
36
89
|
async handleApiAnalysis(params, generateOptions) {
|
|
37
90
|
return null;
|
|
38
91
|
}
|
|
@@ -11,7 +11,8 @@ const scenarioTestSchema = {
|
|
|
11
11
|
.describe("Destination hostname or IP address for the test (e.g., api.example.com, localhost, 192.168.1.1). Do NOT include port numbers."),
|
|
12
12
|
apiSchema: z
|
|
13
13
|
.string()
|
|
14
|
-
.
|
|
14
|
+
.optional()
|
|
15
|
+
.describe("MUST be absolute path(/path/to/openapi.json) to the OpenAPI/Swagger schema file or a URL to the OpenAPI/Swagger schema file(e.g. https://demoshop.skyramp.dev/openapi.json). DO NOT TRY TO ASSUME THE OPENAPI SCHEMA IF NOT PROVIDED. NOTE TO AI ASSISTANTS: You do not need to read the contents of this file - simply pass the file path as the backend will handle reading and processing it."),
|
|
15
16
|
baseURL: z
|
|
16
17
|
.string()
|
|
17
18
|
.optional()
|
|
@@ -35,9 +36,13 @@ const scenarioTestSchema = {
|
|
|
35
36
|
.describe("JSON string of the response body parsed by AI from the scenario"),
|
|
36
37
|
statusCode: z
|
|
37
38
|
.number()
|
|
38
|
-
.
|
|
39
|
-
.describe("HTTP status code (e.g., 200, 201, 204) parsed by AI from the scenario"),
|
|
39
|
+
.describe("Expected HTTP status code. Read status codes from the API schema file else defaults: POST→201, DELETE→204, GET/PUT/PATCH→200."),
|
|
40
40
|
outputDir: baseSchema.shape.outputDir,
|
|
41
|
+
authHeader: z
|
|
42
|
+
.string()
|
|
43
|
+
.optional()
|
|
44
|
+
.default("Authorization")
|
|
45
|
+
.describe("Name of the HTTP header to use for authorization. Use 'Cookie' for cookie-based auth (e.g., NextAuth), 'Authorization' for Bearer tokens, 'X-API-Key' for API keys. Defaults to 'Authorization'."),
|
|
41
46
|
};
|
|
42
47
|
const TOOL_NAME = "skyramp_scenario_test_generation";
|
|
43
48
|
export function registerScenarioTestTool(server) {
|
|
@@ -77,8 +82,6 @@ The AI should parse the natural language scenario and provide:
|
|
|
77
82
|
- AI-parsed HTTP method and path (required)
|
|
78
83
|
- AI-parsed request/response bodies (optional)
|
|
79
84
|
|
|
80
|
-
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file.**
|
|
81
|
-
|
|
82
85
|
**Note:** This tool generates one request at a time. Call multiple times for multi-step scenarios.
|
|
83
86
|
|
|
84
87
|
**CRITICAL - Integration Test Generation After Scenario Creation:**
|
|
@@ -15,10 +15,32 @@ const newTestSchema = z.object({
|
|
|
15
15
|
testType: z.string().describe("Type of test created: Smoke, Contract, Integration, etc."),
|
|
16
16
|
endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
|
|
17
17
|
fileName: z.string().describe("Name of the generated test file"),
|
|
18
|
+
description: z.string().optional().describe("What the test scenario covers, e.g. 'Creates a collection, adds a link, then verifies the link exists'"),
|
|
19
|
+
scenarioFile: z.string().optional().describe("Path to the scenario JSON file if one was generated (e.g. 'tests/scenario_collections-links.json')"),
|
|
20
|
+
traceFile: z.string().optional().describe("Path to the backend trace file if used or created"),
|
|
21
|
+
frontendTrace: z.string().optional().describe("Path to the Playwright/UI trace file if used or created"),
|
|
18
22
|
});
|
|
19
23
|
const descriptionSchema = z.object({
|
|
20
24
|
description: z.string().describe("One-line description"),
|
|
21
25
|
});
|
|
26
|
+
const scenarioStepSchema = z.object({
|
|
27
|
+
method: z.string().optional().describe("HTTP method (e.g. 'POST', 'GET'). Required for API steps, omit for UI/E2E actions."),
|
|
28
|
+
path: z.string().optional().describe("Endpoint or page path (e.g. '/api/v1/products' or '/products'). Required for API steps, omit for UI actions."),
|
|
29
|
+
description: z.string().describe("What this step does, e.g. 'Create a product' or 'Click checkout button and verify confirmation'"),
|
|
30
|
+
expectedStatusCode: z.number().optional().describe("Expected HTTP status code, e.g. 200, 201, 404"),
|
|
31
|
+
requestBody: z.record(z.any()).optional().describe("Example request body with realistic field values"),
|
|
32
|
+
responseBody: z.record(z.any()).optional().describe("Key response fields to verify, e.g. { id: 'number', name: 'string', in_stock: 'boolean?' }"),
|
|
33
|
+
});
|
|
34
|
+
const additionalRecommendationSchema = z.object({
|
|
35
|
+
testType: z.string().describe("Type of test: Integration, E2E, Fuzz, Contract, UI, etc."),
|
|
36
|
+
scenarioName: z.string().describe("Name of the scenario, e.g. 'products_orders_workflow'"),
|
|
37
|
+
steps: z.array(scenarioStepSchema).describe("Ordered sequence of API/UI steps in this test scenario"),
|
|
38
|
+
description: z.string().describe("Why this test is valuable and what it would cover"),
|
|
39
|
+
priority: z.string().describe("Priority level: high, medium, or low"),
|
|
40
|
+
openApiSpec: z.string().optional().describe("Path to OpenAPI/Swagger spec file if available, e.g. 'openapi.yaml'"),
|
|
41
|
+
backendTrace: z.string().optional().describe("Path to backend trace file if available, e.g. 'tests/skyramp-traces.json'. Used by integration and E2E tests."),
|
|
42
|
+
frontendTrace: z.string().optional().describe("Path to Playwright/UI trace file if available, e.g. 'tests/skyramp-playwright.zip'. UI tests need this; E2E tests need both frontend and backend traces."),
|
|
43
|
+
});
|
|
22
44
|
const testMaintenanceSchema = z.object({
|
|
23
45
|
fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
|
|
24
46
|
description: z.string().describe("What was changed and why"),
|
|
@@ -47,6 +69,11 @@ export function registerSubmitReportTool(server) {
|
|
|
47
69
|
testResults: z
|
|
48
70
|
.array(testResultSchema)
|
|
49
71
|
.describe("List of ALL test execution results. One entry per test executed."),
|
|
72
|
+
additionalRecommendations: z
|
|
73
|
+
.array(additionalRecommendationSchema)
|
|
74
|
+
.optional()
|
|
75
|
+
.default([])
|
|
76
|
+
.describe("Recommended tests that were not generated (lower priority). Include the remaining recommendations from skyramp_recommend_tests that were not implemented."),
|
|
50
77
|
issuesFound: z
|
|
51
78
|
.array(descriptionSchema)
|
|
52
79
|
.describe("List of issues, failures, or bugs found. Use empty array [] if none."),
|
|
@@ -68,6 +95,7 @@ export function registerSubmitReportTool(server) {
|
|
|
68
95
|
newTestsCreated: params.newTestsCreated,
|
|
69
96
|
testMaintenance: params.testMaintenance,
|
|
70
97
|
testResults: params.testResults,
|
|
98
|
+
additionalRecommendations: params.additionalRecommendations ?? [],
|
|
71
99
|
issuesFound: params.issuesFound,
|
|
72
100
|
commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim().slice(0, 72) || DEFAULT_COMMIT_MESSAGE,
|
|
73
101
|
}, null, 2);
|
|
@@ -105,6 +105,14 @@ Information about state files and cleanup results.`,
|
|
|
105
105
|
const maxAgeHours = args.maxAgeHours || 24;
|
|
106
106
|
const deletedCount = await StateManager.cleanupOldStateFiles(maxAgeHours);
|
|
107
107
|
logger.info(`Cleaned up ${deletedCount} state files older than ${maxAgeHours} hours`);
|
|
108
|
+
if (deletedCount > 0) {
|
|
109
|
+
try {
|
|
110
|
+
await server.sendResourceListChanged();
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
logger.debug("sendResourceListChanged not available");
|
|
114
|
+
}
|
|
115
|
+
}
|
|
108
116
|
// Get remaining files
|
|
109
117
|
const remainingFiles = await StateManager.listStateFiles();
|
|
110
118
|
return {
|