@skyramp/mcp 0.0.60-rc.2 → 0.0.60-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/test-recommendation/recommendationSections.js +29 -64
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +9 -5
- package/build/prompts/testbot/testbot-prompts.js +11 -5
- package/build/services/ScenarioGenerationService.js +10 -56
- package/build/tools/generate-tests/generateIntegrationRestTool.js +1 -35
- package/build/tools/generate-tests/generateScenarioRestTool.js +10 -2
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +2 -0
- package/build/tools/test-recommendation/recommendTestsTool.js +6 -7
- package/build/types/RepositoryAnalysis.js +1 -0
- package/package.json +1 -1
|
@@ -85,37 +85,25 @@ export function buildGenerationRules(isUIOnlyPR) {
|
|
|
85
85
|
relationships in the code. If the pre-drafted scenarios don't match the real data model,
|
|
86
86
|
replace them with accurate ones.
|
|
87
87
|
|
|
88
|
-
**
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
3. **Contract tests** — per endpoint with new/changed response schemas. Validates the response
|
|
103
|
-
structure matches expectations (field types, required fields, nested objects).
|
|
104
|
-
4. **E2E tests** — per distinct user flow if the API serves a frontend or client
|
|
105
|
-
5. **CRUD lifecycle integration tests** — only for resources with new/changed endpoints
|
|
106
|
-
where multi-resource tests don't already cover them.
|
|
107
|
-
`}When no Playwright trace exists, still recommend the test with instructions for recording
|
|
108
|
-
a trace using \`skyramp_start_trace_collection\` with \`playwright: true\`.
|
|
109
|
-
|
|
110
|
-
**Mixed PRs with frontend changes:** Include at least 1 E2E or UI test in the top 7,
|
|
111
|
-
ranked by value regardless of trace availability. If traces exist, place it in the top 4.
|
|
112
|
-
If no traces, it can still rank highly — the testbot will handle trace-dependent generation.
|
|
113
|
-
|
|
114
|
-
**Before finalizing:** Check that the top 4 aren't filled with CRUD tests for unchanged
|
|
115
|
-
resources when PR-relevant tests exist lower in the ranking. Swap if needed.
|
|
116
|
-
|
|
88
|
+
**Available test types:**
|
|
89
|
+
- **Integration** — multi-endpoint workflows that chain data across resources
|
|
90
|
+
- **Fuzz** — boundary/invalid input testing for POST/PUT endpoints
|
|
91
|
+
- **Contract** — response schema validation for new/changed endpoints
|
|
92
|
+
- **E2E** — user journeys spanning frontend to backend (needs Playwright traces)
|
|
93
|
+
- **UI** — frontend component interaction flows (needs Playwright traces)
|
|
94
|
+
${isUIOnlyPR ? `
|
|
95
|
+
This is a **UI-only PR** — no backend changes. UI and E2E tests are most relevant.
|
|
96
|
+
Without Playwright traces, recommend them with trace recording instructions
|
|
97
|
+
(\`skyramp_start_trace_collection\` with \`playwright: true\`).
|
|
98
|
+
` : `
|
|
99
|
+
When no Playwright trace exists, still recommend E2E/UI tests with instructions for
|
|
100
|
+
recording a trace using \`skyramp_start_trace_collection\` with \`playwright: true\`.
|
|
101
|
+
`}
|
|
117
102
|
**No duplicate coverage.** If an existing test already covers an endpoint + test type,
|
|
118
|
-
recommend a
|
|
103
|
+
recommend a different test that adds new coverage.
|
|
104
|
+
|
|
105
|
+
Choose the test types and distribution that maximize coverage for this specific PR.
|
|
106
|
+
No smoke tests.`;
|
|
119
107
|
}
|
|
120
108
|
export function buildToolWorkflows(authHeaderValue) {
|
|
121
109
|
return `## How to Generate Tests — Tool Workflows
|
|
@@ -124,11 +112,14 @@ export function buildToolWorkflows(authHeaderValue) {
|
|
|
124
112
|
|
|
125
113
|
**For multi-endpoint workflows (integration tests) — Scenario → Integration pipeline:**
|
|
126
114
|
1. Call \`skyramp_scenario_test_generation\` once per step: \`scenarioName\`, \`destination\`,
|
|
127
|
-
\`baseURL\`, \`method\`, \`path\`, \`requestBody\`, \`authHeader: "${authHeaderValue}"\`.
|
|
115
|
+
\`baseURL\`, \`method\`, \`path\`, \`requestBody\`, \`responseBody\`, \`authHeader: "${authHeaderValue}"\`.
|
|
128
116
|
\`statusCode\` is optional — defaults: POST→201, DELETE→204, GET/PUT/PATCH→200. Only override for non-standard codes.
|
|
129
117
|
**OpenAPI spec is NOT required.** \`apiSchema\` is OPTIONAL — omit it if no spec exists.
|
|
130
118
|
\`requestBody\` should use realistic field values from source code schemas (Zod, Pydantic, DTOs).
|
|
131
|
-
|
|
119
|
+
\`responseBody\` should match the actual API response shape from source code (including all fields
|
|
120
|
+
returned by the controller — e.g., \`id\`, \`ownerId\`, \`createdAt\`, included relations like \`collection\`, \`tags\`).
|
|
121
|
+
Wrap in \`{"response": ...}\` if the API uses an envelope pattern. If omitted, a synthetic response is generated.
|
|
122
|
+
Inspect the source code to determine the correct request AND response body shapes — avoid sending \`{}\`.
|
|
132
123
|
Use unique names with timestamp suffix to avoid conflicts on re-runs.
|
|
133
124
|
For GET/PUT/DELETE with path IDs, use a placeholder — chaining resolves the real ID.
|
|
134
125
|
2. Produces a \`scenario_<name>.json\` in the same \`outputDir\` as the test files (not \`.skyramp/\`).
|
|
@@ -162,32 +153,14 @@ Use it actively:
|
|
|
162
153
|
- **Single-endpoint tests**: pass both \`endpointURL\` AND \`apiSchema\` for schema-aware generation.
|
|
163
154
|
\n`
|
|
164
155
|
: "";
|
|
165
|
-
const distribution = isUIOnlyPR
|
|
166
|
-
? `- Prioritize UI tests (≥3), then E2E tests (≥2), then integration only if UI calls APIs. 0% smoke.`
|
|
167
|
-
: hasFrontendChanges
|
|
168
|
-
? `- Mix: integration (2-3), E2E (1-2), UI (1-2), fuzz or contract (1). 0% smoke.`
|
|
169
|
-
: `- Mix: integration (2-3, multi-resource first), fuzz (1-2), contract (1-2), E2E (1 if user-facing flows exist). 0% smoke.`;
|
|
170
156
|
return `## Coverage Checklist
|
|
171
157
|
${specNote}
|
|
172
|
-
${isUIOnlyPR ? `**UI-only PR** —
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
Without traces: recommend UI/E2E tests with scenario steps and trace recording instructions
|
|
178
|
-
(\`skyramp_start_trace_collection\` with \`playwright: true\`). The testbot will skip generation
|
|
179
|
-
entirely for frontend-only PRs without traces — all recommendations become additional
|
|
180
|
-
recommendations in the report. Skip fuzz, contract, and smoke tests.
|
|
181
|
-
` : `For each endpoint, recommend the most valuable test types — aim for variety:
|
|
182
|
-
1. **Integration** — multi-resource workflows (not just single-resource CRUD)
|
|
183
|
-
2. **Fuzz** — POST/PUT endpoints with request bodies (validates edge cases, type safety)
|
|
184
|
-
3. **Contract** — endpoints with new/changed response schemas (validates structure)
|
|
185
|
-
4. **E2E** — user flows spanning frontend to backend${hasFrontendChanges ? " (include at least 1 for this PR)" : ""}
|
|
186
|
-
5. **UI** — changed frontend components${hasFrontendChanges ? " (include at least 1)" : ""}
|
|
187
|
-
6. No smoke tests.
|
|
188
|
-
Do NOT recommend 7 integration tests — diversify across test types.
|
|
158
|
+
${isUIOnlyPR ? `**UI-only PR** — no backend changes.
|
|
159
|
+
Without Playwright traces, the testbot skips generation entirely — all recommendations
|
|
160
|
+
become additionalRecommendations in the report.
|
|
161
|
+
` : `**Available test types:** integration, fuzz, contract, E2E, UI. No smoke tests.
|
|
162
|
+
Choose based on what adds the most value for this PR's changes.
|
|
189
163
|
`}
|
|
190
|
-
|
|
191
164
|
## For Each Recommendation Include:
|
|
192
165
|
1. Test type 2. Priority (high/medium/low) 3. Target endpoint/scenario
|
|
193
166
|
4. What it validates (business logic, not just "tests the endpoint")
|
|
@@ -203,20 +176,12 @@ Recommend the test anyway — never mark it "blocked":
|
|
|
203
176
|
## Select the Top ${topN}
|
|
204
177
|
Consider all possible tests (endpoints \u00d7 interaction types + scenarios), then select the
|
|
205
178
|
top ${topN} most valuable. Include \`totalConsidered\` count in your output. The top 4 will
|
|
206
|
-
be generated; recommendations #5
|
|
179
|
+
be generated; recommendations #5\u2013${topN} go to additionalRecommendations in the report,
|
|
207
180
|
so ensure the top 4 are the highest-impact tests.
|
|
208
181
|
|
|
209
|
-
**Before outputting, verify:**
|
|
210
|
-
${isUIOnlyPR ? `- If traces exist, at least 2 of the top 4 should be UI/E2E tests.
|
|
211
|
-
- Without traces, all 7 become additionalRecommendations (no generation). Rank UI/E2E highest.
|
|
212
|
-
- Avoid CRUD tests for unchanged resources the UI doesn't call.` : `- If the PR includes frontend changes, include at least 1 E2E/UI test in the top 4.
|
|
213
|
-
- CRUD tests for unchanged resources should not displace PR-relevant tests in the top 4.`}
|
|
214
182
|
- Each integration scenario's step sequence should be logically valid — preconditions
|
|
215
183
|
met by prior steps.
|
|
216
184
|
|
|
217
|
-
Preferred ordering: ${isUIOnlyPR ? "UI \u2192 E2E \u2192 integration (if UI calls APIs)." : "integration \u2192 fuzz \u2192 contract \u2192 E2E \u2192 UI."}
|
|
218
|
-
${distribution}
|
|
219
|
-
|
|
220
185
|
Each recommendation should include enough detail for direct tool invocation.
|
|
221
186
|
Reference draftedScenarios by name and interactions by description.
|
|
222
187
|
Use "high"/"medium"/"low" for priority — no numeric scores.
|
|
@@ -26,10 +26,8 @@ export function buildRecommendationPrompt(analysis, analysisScope = "full_repo",
|
|
|
26
26
|
const modePreamble = isDiffScope
|
|
27
27
|
? `You are in **PR mode**. Maximize test coverage for the branch changes.
|
|
28
28
|
Focus on tests that validate the changed fields, endpoints, and their interactions.
|
|
29
|
-
**
|
|
30
|
-
|
|
31
|
-
${isUIOnlyPR ? `\n**UI-only PR.** Prioritize UI tests and E2E. Skip fuzz/contract.`
|
|
32
|
-
: hasFrontendChanges ? `\n**Mixed PR (frontend + API).** Integration > E2E > UI > fuzz/contract.`
|
|
29
|
+
${isUIOnlyPR ? `\n**UI-only PR** — no backend changes. UI and E2E tests are most relevant.`
|
|
30
|
+
: hasFrontendChanges ? `\n**Mixed PR** — both frontend and backend changes detected.`
|
|
33
31
|
: ``}
|
|
34
32
|
Output should be concise and immediately actionable.`
|
|
35
33
|
: `You are in **Repo mode**. Comprehensive test strategy across all endpoints.`;
|
|
@@ -44,6 +42,10 @@ Output should be concise and immediately actionable.`
|
|
|
44
42
|
: /session|cookie|nextauth/i.test(authMethod) ? "Cookie"
|
|
45
43
|
: /api[_-]?key/i.test(authMethod) ? "X-API-Key"
|
|
46
44
|
: "Authorization";
|
|
45
|
+
const sourcePriority = `
|
|
46
|
+
## Source Priority
|
|
47
|
+
When information conflicts, prefer: **Traces** (actual behavior) > **Code** (implemented behavior) > **Spec/Docs** (documented behavior).
|
|
48
|
+
`;
|
|
47
49
|
const repoContext = `
|
|
48
50
|
Repository: ${analysis.metadata.repositoryName}
|
|
49
51
|
Framework: ${analysis.projectClassification.primaryFramework} (${analysis.projectClassification.primaryLanguage})
|
|
@@ -122,7 +124,7 @@ ${detailBlocks}
|
|
|
122
124
|
})
|
|
123
125
|
.join("\n\n");
|
|
124
126
|
scenarioSection = `
|
|
125
|
-
## Drafted Scenarios
|
|
127
|
+
## Drafted Scenarios
|
|
126
128
|
**Base URL:** \`${baseUrl}\` | **Auth:** \`${authHeaderValue}\`
|
|
127
129
|
|
|
128
130
|
Only use scenarios where resources are ACTUALLY related in the codebase. Replace any
|
|
@@ -172,6 +174,8 @@ Scope: ${scopeNote}
|
|
|
172
174
|
|
|
173
175
|
${buildTestQualityCriteria()}
|
|
174
176
|
|
|
177
|
+
${sourcePriority}
|
|
178
|
+
|
|
175
179
|
${buildPrioritizationDimensions()}
|
|
176
180
|
|
|
177
181
|
${buildTestExamples()}
|
|
@@ -20,15 +20,21 @@ Read the diff at \`${diffFile}\`. Skip Task 1 if all changed files are non-appli
|
|
|
20
20
|
|
|
21
21
|
1. Call \`skyramp_analyze_repository\` with \`repositoryPath\`: "${repositoryPath}", \`analysisScope\`: "current_branch_diff"${baseBranch ? `\n , \`baseBranch\`: "${baseBranch}"` : ''}
|
|
22
22
|
2. Call \`skyramp_recommend_tests\` with the returned \`sessionId\`.
|
|
23
|
-
It returns
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
It returns 10 ranked recommendations. Walk through them in rank order and generate
|
|
24
|
+
up to 4 tests. Any recommendation you skip or cannot generate goes to
|
|
25
|
+
\`additionalRecommendations\`.
|
|
26
|
+
|
|
27
|
+
3. **Generate** up to 4 tests by walking the ranked list top-to-bottom:
|
|
28
|
+
- If a recommendation can be generated, generate it and count it.
|
|
29
|
+
- If it cannot (e.g. E2E/UI without traces), move it to \`additionalRecommendations\`
|
|
30
|
+
and continue to the next recommendation.
|
|
31
|
+
- Stop once you have 4 generated tests OR exhaust all 10 recommendations.
|
|
32
|
+
- All remaining (ungenerated) recommendations go to \`additionalRecommendations\`.
|
|
27
33
|
Keep a list of every file the CLI creates (test files AND scenario JSON files).
|
|
28
34
|
|
|
29
35
|
**Frontend-only PRs** (no backend/API changes): only generate tests if relevant
|
|
30
36
|
Playwright traces exist. If no traces are available, skip generation entirely and
|
|
31
|
-
move all
|
|
37
|
+
move all 10 recommendations to \`additionalRecommendations\` with scenario steps and
|
|
32
38
|
trace recording instructions. Do not generate integration tests for unchanged backend
|
|
33
39
|
APIs just to fill the quota — those tests don't validate the PR's changes.
|
|
34
40
|
|
|
@@ -7,7 +7,6 @@ export class ScenarioGenerationService {
|
|
|
7
7
|
logger.info("Parsing scenario into API requests", {
|
|
8
8
|
scenarioName: params.scenarioName,
|
|
9
9
|
});
|
|
10
|
-
// Generate a single trace request from the scenario
|
|
11
10
|
const traceRequest = this.generateTraceRequestFromInput(params);
|
|
12
11
|
if (!traceRequest) {
|
|
13
12
|
return {
|
|
@@ -20,12 +19,10 @@ export class ScenarioGenerationService {
|
|
|
20
19
|
isError: true,
|
|
21
20
|
};
|
|
22
21
|
}
|
|
23
|
-
// Handle file writing
|
|
24
22
|
const scenarioName = params.scenarioName.replace(/ /g, "-").toLowerCase();
|
|
25
23
|
const fileName = `scenario_${scenarioName}.json`;
|
|
26
24
|
const filePath = path.join(params.outputDir, fileName);
|
|
27
25
|
try {
|
|
28
|
-
// Check if file exists to determine if we should append or create new
|
|
29
26
|
let existingRequests = [];
|
|
30
27
|
if (fs.existsSync(filePath)) {
|
|
31
28
|
try {
|
|
@@ -35,14 +32,11 @@ export class ScenarioGenerationService {
|
|
|
35
32
|
existingRequests = [];
|
|
36
33
|
}
|
|
37
34
|
}
|
|
38
|
-
catch
|
|
39
|
-
// If file exists but can't be parsed, start fresh
|
|
35
|
+
catch {
|
|
40
36
|
existingRequests = [];
|
|
41
37
|
}
|
|
42
38
|
}
|
|
43
|
-
// Add the new request to the array
|
|
44
39
|
existingRequests.push(traceRequest);
|
|
45
|
-
// Write the updated array to the file
|
|
46
40
|
fs.writeFileSync(filePath, JSON.stringify(existingRequests, null, 2), "utf8");
|
|
47
41
|
logger.info("Trace request added to file", {
|
|
48
42
|
filePath,
|
|
@@ -107,7 +101,6 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
107
101
|
let destination = params.destination;
|
|
108
102
|
let scheme = "https";
|
|
109
103
|
let port = 443;
|
|
110
|
-
// Derive scheme, host, and port from baseURL when available
|
|
111
104
|
if (params.baseURL) {
|
|
112
105
|
try {
|
|
113
106
|
const parsed = new URL(params.baseURL);
|
|
@@ -127,45 +120,28 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
127
120
|
}
|
|
128
121
|
const timestamp = new Date().toISOString();
|
|
129
122
|
const method = params.method;
|
|
130
|
-
const path = params.path;
|
|
131
123
|
const statusCode = params.statusCode;
|
|
132
124
|
const requestBody = params.requestBody ||
|
|
133
125
|
(method === "GET" || method === "DELETE" ? "" : "{}");
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
else if (method === "POST" && requestBody && requestBody !== "{}") {
|
|
140
|
-
responseBody = this.synthesizePostResponse(requestBody);
|
|
141
|
-
}
|
|
142
|
-
else {
|
|
143
|
-
responseBody = "{}";
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
const source = "192.168.65.1:39998";
|
|
147
|
-
// Build auth header from param or default to Bearer
|
|
126
|
+
const responseHeaders = params.responseHeaders
|
|
127
|
+
|| { "Content-Type": ["application/json"] };
|
|
128
|
+
const isJsonResponse = (responseHeaders["Content-Type"] || [])
|
|
129
|
+
.some(v => v.includes("application/json"));
|
|
130
|
+
const responseBody = params.responseBody || (isJsonResponse ? "{}" : "");
|
|
148
131
|
const authHeaderName = params.authHeader || "Authorization";
|
|
149
132
|
const requestHeaders = {
|
|
150
133
|
"Content-Type": ["application/json"],
|
|
134
|
+
[authHeaderName]: [params.authToken ?? ""],
|
|
151
135
|
};
|
|
152
|
-
if (authHeaderName === "Cookie") {
|
|
153
|
-
requestHeaders["Cookie"] = ["session-token=demo-token"];
|
|
154
|
-
}
|
|
155
|
-
else {
|
|
156
|
-
requestHeaders[authHeaderName] = ["Bearer demo-token"];
|
|
157
|
-
}
|
|
158
136
|
return {
|
|
159
|
-
Source:
|
|
137
|
+
Source: "192.168.65.1:39998",
|
|
160
138
|
Destination: destination,
|
|
161
139
|
RequestBody: requestBody,
|
|
162
140
|
ResponseBody: responseBody,
|
|
163
141
|
RequestHeaders: requestHeaders,
|
|
164
|
-
ResponseHeaders:
|
|
165
|
-
"Content-Type": ["application/json"],
|
|
166
|
-
},
|
|
142
|
+
ResponseHeaders: responseHeaders,
|
|
167
143
|
Method: method,
|
|
168
|
-
Path: path,
|
|
144
|
+
Path: params.path,
|
|
169
145
|
QueryParams: {},
|
|
170
146
|
StatusCode: statusCode,
|
|
171
147
|
Port: port,
|
|
@@ -173,26 +149,4 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
173
149
|
Scheme: scheme,
|
|
174
150
|
};
|
|
175
151
|
}
|
|
176
|
-
/**
|
|
177
|
-
* Generate a synthetic POST response body that echoes the request body
|
|
178
|
-
* wrapped in a "response" envelope with an auto-incrementing "id".
|
|
179
|
-
* This gives the CLI response data to chain from (e.g., response.id).
|
|
180
|
-
*/
|
|
181
|
-
synthesizePostResponse(requestBody) {
|
|
182
|
-
try {
|
|
183
|
-
const parsed = JSON.parse(requestBody);
|
|
184
|
-
const id = Date.now() % 100000;
|
|
185
|
-
if (Array.isArray(parsed)) {
|
|
186
|
-
const withIds = parsed.map((item, idx) => ({
|
|
187
|
-
id: id + idx,
|
|
188
|
-
...item,
|
|
189
|
-
}));
|
|
190
|
-
return JSON.stringify({ response: withIds });
|
|
191
|
-
}
|
|
192
|
-
return JSON.stringify({ response: { id, ...parsed } });
|
|
193
|
-
}
|
|
194
|
-
catch {
|
|
195
|
-
return "{}";
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
152
|
}
|
|
@@ -39,46 +39,12 @@ export class IntegrationTestService extends TestGenerationService {
|
|
|
39
39
|
async generateTest(params) {
|
|
40
40
|
const result = await super.generateTest(params);
|
|
41
41
|
if (!result.isError && params.scenarioFile) {
|
|
42
|
-
const chainingInstructions = `
|
|
43
|
-
|
|
44
|
-
⏭️ **CRITICAL NEXT STEP — Verify & Fix Chaining in Generated Test**
|
|
45
|
-
|
|
46
|
-
The generated integration test uses \`chainingKey: "response.id"\` for all steps, but
|
|
47
|
-
multi-resource workflows often need **different chaining per step**. You MUST:
|
|
48
|
-
|
|
49
|
-
1. **Read the generated test file** in \`${params.outputDir}\`
|
|
50
|
-
2. **Identify each API call** in the test (POST, GET, PUT, DELETE)
|
|
51
|
-
3. **Verify chaining is correct** for each step:
|
|
52
|
-
- After each POST, the response ID must be extracted into a **uniquely named variable**
|
|
53
|
-
(e.g., \`product_id\`, \`order_id\`, \`review_id\` — NOT all named \`id\`)
|
|
54
|
-
- GET/PUT/DELETE calls that reference a resource must use the **correct** variable in their path
|
|
55
|
-
(e.g., \`/products/{product_id}\` uses the product's ID, not the order's ID)
|
|
56
|
-
- POST calls that create a child resource must include the **parent's ID** in the request body
|
|
57
|
-
or path (e.g., \`POST /orders\` body should include \`product_id\` from the products POST response)
|
|
58
|
-
4. **Fix ONLY chaining** (path params AND request body ID references) — nothing else:
|
|
59
|
-
- Replace hardcoded path IDs (like \`/products/1\`) with the dynamic variable from the POST response
|
|
60
|
-
- Replace hardcoded IDs in request bodies (like \`"product_id": 1\`) with the dynamic variable
|
|
61
|
-
(use \`data_override\`/\`dataOverride\` or direct variable substitution)
|
|
62
|
-
- Rename duplicate variable names so each resource has its own ID variable
|
|
63
|
-
- For Python: use \`skyramp.get_response_value(response_N, "id")\` to extract and f-strings for paths
|
|
64
|
-
- For TypeScript: use \`getResponseValue(response, "response.id")\` or the appropriate accessor
|
|
65
|
-
⚠️ **Preserve everything else exactly as generated** — do not add, remove, or modify
|
|
66
|
-
auth headers, cookies, tokens, env vars, imports, assertions, or non-chaining request body fields.
|
|
67
|
-
The CLI output for auth/headers is intentional.
|
|
68
|
-
|
|
69
|
-
**Example fix for a products → orders workflow:**
|
|
70
|
-
\`\`\`
|
|
71
|
-
# Step 1: POST /products → extract product_id
|
|
72
|
-
# Step 2: POST /orders with product_id in body → extract order_id
|
|
73
|
-
# Step 3: GET /orders/{order_id} → uses order_id (NOT product_id)
|
|
74
|
-
# Step 4: DELETE /products/{product_id} → uses product_id (NOT order_id)
|
|
75
|
-
\`\`\``;
|
|
76
42
|
const existingText = result.content[0] && "text" in result.content[0] ? result.content[0].text : "";
|
|
77
43
|
return {
|
|
78
44
|
content: [
|
|
79
45
|
{
|
|
80
46
|
type: "text",
|
|
81
|
-
text: existingText
|
|
47
|
+
text: existingText,
|
|
82
48
|
},
|
|
83
49
|
],
|
|
84
50
|
isError: false,
|
|
@@ -24,7 +24,7 @@ const scenarioTestSchema = {
|
|
|
24
24
|
.describe("HTTP method (GET, POST, PUT, DELETE, etc.) parsed by AI from the scenario"),
|
|
25
25
|
path: z
|
|
26
26
|
.string()
|
|
27
|
-
.describe("API path (e.g
|
|
27
|
+
.describe("API path parsed by AI from the scenario. CRITICAL: For requests that reference an ID created by a prior step (e.g. GET/PUT/DELETE after a POST), use the ACTUAL ID value from the prior step's responseBody in the path, NOT a template variable. Example: use '/api/v1/products/70885' instead of '/api/v1/products/{product_id}'. The CLI detects chaining by matching concrete values across requests."),
|
|
28
28
|
// AI-parsed parameters (optional)
|
|
29
29
|
requestBody: z
|
|
30
30
|
.string()
|
|
@@ -43,6 +43,14 @@ const scenarioTestSchema = {
|
|
|
43
43
|
.optional()
|
|
44
44
|
.default("Authorization")
|
|
45
45
|
.describe("Name of the HTTP header to use for authorization. Use 'Cookie' for cookie-based auth (e.g., NextAuth), 'Authorization' for Bearer tokens, 'X-API-Key' for API keys. Defaults to 'Authorization'."),
|
|
46
|
+
authToken: z
|
|
47
|
+
.string()
|
|
48
|
+
.optional()
|
|
49
|
+
.describe("Auth token value to include in the request header. If omitted, the header value is left empty (the CLI injects the real token at runtime)."),
|
|
50
|
+
responseHeaders: z
|
|
51
|
+
.record(z.array(z.string()))
|
|
52
|
+
.optional()
|
|
53
|
+
.describe('Response headers as a JSON object (e.g., {"Content-Type": ["application/json"]}). Defaults to Content-Type: application/json.'),
|
|
46
54
|
};
|
|
47
55
|
const TOOL_NAME = "skyramp_scenario_test_generation";
|
|
48
56
|
export function registerScenarioTestTool(server) {
|
|
@@ -70,7 +78,7 @@ Returns a single TraceRequest object with:
|
|
|
70
78
|
**AI Responsibilities:**
|
|
71
79
|
The AI should parse the natural language scenario and provide:
|
|
72
80
|
- HTTP method (POST, GET, PUT, DELETE)
|
|
73
|
-
- API path (e.g., /api/v1/products, /api/v1/products/{product_id})
|
|
81
|
+
- API path with CONCRETE ID values, not templates (e.g., /api/v1/products/70885, NOT /api/v1/products/{product_id})
|
|
74
82
|
- Request body (JSON string, if applicable)
|
|
75
83
|
- Response body (JSON string, if applicable)
|
|
76
84
|
- Status code (optional, defaults based on method)
|
|
@@ -310,6 +310,7 @@ Output: Detailed RepositoryAnalysis JSON object with all repository characterist
|
|
|
310
310
|
response: {
|
|
311
311
|
statusCode: entry.statusCode,
|
|
312
312
|
description: `Observed in trace (${traceResult.format})`,
|
|
313
|
+
...(entry.responseBody ? { body: entry.responseBody } : {}),
|
|
313
314
|
},
|
|
314
315
|
});
|
|
315
316
|
}
|
|
@@ -333,6 +334,7 @@ Output: Detailed RepositoryAnalysis JSON object with all repository characterist
|
|
|
333
334
|
description: `${e.method} ${e.path} \u2192 ${e.statusCode}`,
|
|
334
335
|
interactionType: e.statusCode < 400 ? "success" : "error",
|
|
335
336
|
requestBody: e.requestBody,
|
|
337
|
+
responseBody: e.responseBody,
|
|
336
338
|
expectedStatusCode: e.statusCode,
|
|
337
339
|
})),
|
|
338
340
|
chainingKeys: [],
|
|
@@ -62,12 +62,11 @@ For each recommended test, you'll get:
|
|
|
62
62
|
- For integration/E2E: references to draftedScenarios by scenarioName
|
|
63
63
|
- Which Skyramp tool to call for generation
|
|
64
64
|
|
|
65
|
-
**
|
|
66
|
-
-
|
|
67
|
-
-
|
|
68
|
-
-
|
|
69
|
-
- Reference
|
|
70
|
-
- Reference specific draftedScenarios by scenarioName when recommending integration/E2E tests.
|
|
65
|
+
**Output guidelines:**
|
|
66
|
+
- Use "high", "medium", or "low" for priority.
|
|
67
|
+
- Never mark a test as blocked — recommend it with instructions for missing artifacts.
|
|
68
|
+
- Reference specific interactions by description for contract/fuzz tests.
|
|
69
|
+
- Reference draftedScenarios by scenarioName for integration/E2E tests.
|
|
71
70
|
|
|
72
71
|
** This tool is currently in Early Preview stage. Please verify the results. **`,
|
|
73
72
|
inputSchema: recommendTestsSchema,
|
|
@@ -144,7 +143,7 @@ For each recommended test, you'll get:
|
|
|
144
143
|
}
|
|
145
144
|
const scope = analysisScope || "full_repo";
|
|
146
145
|
const sessionId = params.sessionId;
|
|
147
|
-
const effectiveTopN = params.topN ??
|
|
146
|
+
const effectiveTopN = params.topN ?? 10;
|
|
148
147
|
// Fetch PR context when prNumber is provided and in diff scope
|
|
149
148
|
let prContext = stateData.prContext;
|
|
150
149
|
if (!prContext && params.prNumber && scope === "current_branch_diff") {
|
|
@@ -64,6 +64,7 @@ export const scenarioStepSchema = z.object({
|
|
|
64
64
|
description: z.string(),
|
|
65
65
|
interactionType: z.enum(["success", "error", "edge-case"]),
|
|
66
66
|
requestBody: z.record(z.any()).optional(),
|
|
67
|
+
responseBody: z.record(z.any()).optional(),
|
|
67
68
|
expectedStatusCode: z.number(),
|
|
68
69
|
expectedResponseFields: z.array(z.string()).optional(),
|
|
69
70
|
chainsFrom: z
|