@skyramp/mcp 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +6 -5
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +150 -149
- package/build/prompts/personas.js +2 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +2 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +28 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +72 -14
- package/build/prompts/test-recommendation/analysisOutputPrompt.test.js +154 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +290 -0
- package/build/prompts/test-recommendation/fullRepoCatalog.js +271 -0
- package/build/prompts/test-recommendation/recommendationSections.js +4 -2
- package/build/prompts/test-recommendation/recommendationShared.js +68 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +20 -4
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -640
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +6 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -7
- package/build/prompts/testbot/testbot-prompts.test.js +22 -5
- package/build/resources/analysisResources.js +1 -0
- package/build/services/ScenarioGenerationService.js +5 -1
- package/build/services/TestGenerationService.js +3 -0
- package/build/tools/code-refactor/codeReuseTool.js +3 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +5 -1
- package/build/tools/code-refactor/modularizationTool.js +3 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +123 -1
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +205 -9
- package/build/tools/generate-tests/generateContractRestTool.js +19 -19
- package/build/tools/generate-tests/generateIntegrationRestTool.js +9 -2
- package/build/tools/generate-tests/generateUIRestTool.js +23 -8
- package/build/tools/test-management/analyzeChangesTool.js +218 -2
- package/build/tools/test-management/analyzeChangesTool.test.js +233 -1
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/utils/docker.test.js +1 -1
- package/build/utils/featureFlags.js +7 -0
- package/build/utils/featureFlags.test.js +81 -0
- package/build/utils/gitStaging.js +18 -0
- package/build/utils/gitStaging.test.js +87 -0
- package/build/utils/httpDefaults.js +17 -0
- package/build/utils/httpDefaults.test.js +21 -0
- package/build/utils/scenarioDrafting.js +37 -15
- package/build/utils/scenarioDrafting.test.js +66 -0
- package/build/utils/telemetry.js +2 -1
- package/build/utils/utils.js +23 -0
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/browser/context.js +2 -0
- package/node_modules/playwright/lib/mcp/browser/tools/keyboard.js +2 -2
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +17 -26
- package/package.json +2 -2
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
import { logger } from "../../utils/logger.js";
|
|
2
|
+
import { buildTestQualityCriteria } from "./recommendationSections.js";
|
|
3
|
+
import { externalDedupKey } from "./recommendationShared.js";
|
|
4
|
+
export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, isFrontendProject = false, isFrontendOnlyProject = false, externalCoverage = new Set()) {
|
|
5
|
+
// Full-repo mode only — percentage-based UI/E2E slot targets (15% each, floor 1).
|
|
6
|
+
const rawE2E = isFrontendProject ? Math.max(1, Math.round(topN * 0.15)) : 0;
|
|
7
|
+
const rawUI = isFrontendProject ? Math.max(1, Math.round(topN * 0.15)) : 0;
|
|
8
|
+
const slotsFloor = Math.floor(topN / 2);
|
|
9
|
+
const minE2ESlots = Math.min(rawE2E, slotsFloor);
|
|
10
|
+
const minUISlots = Math.min(rawUI, Math.max(0, topN - minE2ESlots));
|
|
11
|
+
const authRef = authHeaderValue
|
|
12
|
+
? `, authHeader: "${authHeaderValue}"${authSchemeSnippet}`
|
|
13
|
+
: `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
|
|
14
|
+
const hasWorkspaceAuthType = !!authTypeValue && authTypeValue !== "none";
|
|
15
|
+
const authHeaderOnlyRef = hasWorkspaceAuthType
|
|
16
|
+
? ""
|
|
17
|
+
: authHeaderValue
|
|
18
|
+
? `, authHeader: "${authHeaderValue}"`
|
|
19
|
+
: `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
|
|
20
|
+
// Supplement count for full-repo mode
|
|
21
|
+
const supplementCount = topN - Math.min(scored.length, topN);
|
|
22
|
+
const toTitle = (name) => name.replace(/-/g, " ").replace(/\b\w/g, c => c.toUpperCase());
|
|
23
|
+
const TYPE_ORDER = ["e2e", "ui", "integration", "contract"];
|
|
24
|
+
const TYPE_LABEL = {
|
|
25
|
+
e2e: "E2E", ui: "UI", integration: "Integration", contract: "Contract",
|
|
26
|
+
};
|
|
27
|
+
// Filter out scenarios already covered by external tests before slicing.
|
|
28
|
+
const scoredFiltered = externalCoverage.size > 0
|
|
29
|
+
? scored.filter(item => {
|
|
30
|
+
const key = externalDedupKey(item.scenario);
|
|
31
|
+
if (externalCoverage.has(key)) {
|
|
32
|
+
logger.info(`External dedup (full-repo): skipping "${item.scenario.scenarioName}" (${key})`);
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
})
|
|
37
|
+
: scored;
|
|
38
|
+
// For full-stack repos, carve out E2E and UI slots before filling with backend tests.
|
|
39
|
+
const backendSlotCount = isFrontendProject
|
|
40
|
+
? Math.max(0, topN - minE2ESlots - minUISlots)
|
|
41
|
+
: topN;
|
|
42
|
+
const allItems = scoredFiltered.slice(0, backendSlotCount);
|
|
43
|
+
const byType = new Map();
|
|
44
|
+
for (const t of TYPE_ORDER)
|
|
45
|
+
byType.set(t, []);
|
|
46
|
+
for (const item of allItems) {
|
|
47
|
+
const t = item.scenario.testType ?? (item.scenario.steps.length === 1 ? "contract" : "integration");
|
|
48
|
+
if (!byType.has(t))
|
|
49
|
+
byType.set(t, []);
|
|
50
|
+
byType.get(t).push(item);
|
|
51
|
+
}
|
|
52
|
+
const renderItem = (item, rank) => {
|
|
53
|
+
const s = item.scenario;
|
|
54
|
+
const testType = s.testType ?? (s.steps.length === 1 ? "contract" : "integration");
|
|
55
|
+
const title = toTitle(s.scenarioName);
|
|
56
|
+
if (testType === "contract") {
|
|
57
|
+
const step = s.steps[0];
|
|
58
|
+
const endpointURL = `${baseUrl}${step.path}`;
|
|
59
|
+
const isBodyMethod = ["POST", "PUT", "PATCH"].includes(step.method);
|
|
60
|
+
const dataParam = isBodyMethod
|
|
61
|
+
? `, requestData: <${step.method} ${step.path} required fields from source code>`
|
|
62
|
+
: "";
|
|
63
|
+
return [
|
|
64
|
+
`**${rank}. ${title}**`,
|
|
65
|
+
` ${s.description}`,
|
|
66
|
+
` ${step.method} ${step.path} \u2192 ${step.expectedStatusCode}`,
|
|
67
|
+
` Tool: \`skyramp_contract_test_generation({ endpointURL: "${endpointURL}", method: "${step.method}"${authRef}${dataParam} })\``,
|
|
68
|
+
` From source: fill in requestData field names and the specific production boundary this validates`,
|
|
69
|
+
].join("\n");
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
const stepLines = s.steps.map(st => {
|
|
73
|
+
const isBody = ["POST", "PUT", "PATCH"].includes(st.method);
|
|
74
|
+
const bodyHint = isBody ? ` \u2014 body: <${st.method} ${st.path} required fields from source>` : "";
|
|
75
|
+
return ` ${st.order}. ${st.method} ${st.path} \u2192 ${st.expectedStatusCode}: ${st.description}${bodyHint}`;
|
|
76
|
+
}).join("\n");
|
|
77
|
+
const isTraceBased = testType === "e2e" || testType === "ui";
|
|
78
|
+
let toolCallsBlock;
|
|
79
|
+
if (isTraceBased) {
|
|
80
|
+
// E2E and UI need browser recording first, then generation
|
|
81
|
+
const frontendUrl = "<frontend_url>";
|
|
82
|
+
const zipPath = `<repositoryPath>/.skyramp/${s.scenarioName}_trace.zip`;
|
|
83
|
+
if (testType === "ui") {
|
|
84
|
+
toolCallsBlock = [
|
|
85
|
+
` 1. browser_navigate({ url: "${frontendUrl}" })`,
|
|
86
|
+
` 2. Interact with the changed components (browser_click, browser_type, browser_fill_form, etc.)`,
|
|
87
|
+
` 3. browser_snapshot() after each key interaction`,
|
|
88
|
+
` 4. skyramp_export_zip({ outputPath: "${zipPath}" }) — use absolute path`,
|
|
89
|
+
` 5. skyramp_ui_test_generation({ playwrightInput: "${zipPath}"${authHeaderOnlyRef} })`,
|
|
90
|
+
].join("\n");
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
toolCallsBlock = [
|
|
94
|
+
` 1. browser_navigate({ url: "${frontendUrl}" }) — record frontend trace`,
|
|
95
|
+
` 2. Interact with the user journey described above`,
|
|
96
|
+
` 3. skyramp_export_zip({ outputPath: "${zipPath}" }) — use absolute path`,
|
|
97
|
+
` 4. Capture backend trace JSON separately (skyramp_start_trace_collection / skyramp_stop_trace_collection)`,
|
|
98
|
+
` 5. skyramp_e2e_test_generation({ playwrightInput: "${zipPath}", trace: "<backend trace path>"${authHeaderOnlyRef} })`,
|
|
99
|
+
].join("\n");
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
// Integration: use batch scenario tool (all steps in one call)
|
|
104
|
+
let destinationHost = s.scenarioName;
|
|
105
|
+
try {
|
|
106
|
+
destinationHost = new URL(baseUrl).hostname;
|
|
107
|
+
}
|
|
108
|
+
catch { /* keep fallback */ }
|
|
109
|
+
const batchSteps = s.steps.map(st => {
|
|
110
|
+
const isBody = ["POST", "PUT", "PATCH"].includes(st.method);
|
|
111
|
+
let dataParam = "";
|
|
112
|
+
if (isBody) {
|
|
113
|
+
if (st.requestBody && Object.keys(st.requestBody).length > 0) {
|
|
114
|
+
const bodyJson = JSON.stringify(st.requestBody).replace(/"/g, '\\"');
|
|
115
|
+
dataParam = `, requestBody: "${bodyJson}"`;
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
dataParam = `, requestBody: <${st.method} ${st.path} required fields from source code>`;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return ` { method: "${st.method}", path: "${st.path}", statusCode: ${st.expectedStatusCode}${dataParam} }`;
|
|
122
|
+
}).join(",\n");
|
|
123
|
+
toolCallsBlock = [
|
|
124
|
+
` skyramp_batch_scenario_test_generation({ scenarioName: "${s.scenarioName}", destination: "${destinationHost}", baseURL: "${baseUrl}"${authRef}, steps: [\n${batchSteps}\n ] })`,
|
|
125
|
+
` skyramp_integration_test_generation({ scenarioFile: <filePath returned by skyramp_batch_scenario_test_generation above>${authHeaderOnlyRef} })`,
|
|
126
|
+
].join("\n");
|
|
127
|
+
}
|
|
128
|
+
return [
|
|
129
|
+
`**${rank}. ${title}**`,
|
|
130
|
+
` ${s.description}`,
|
|
131
|
+
` Steps:`,
|
|
132
|
+
stepLines,
|
|
133
|
+
` Tool calls:`,
|
|
134
|
+
toolCallsBlock,
|
|
135
|
+
` From source: fill in requestBody field values and assert all computed response fields`,
|
|
136
|
+
].join("\n");
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
const backendSections = TYPE_ORDER
|
|
140
|
+
.filter(t => (byType.get(t) ?? []).length > 0)
|
|
141
|
+
.map(t => {
|
|
142
|
+
const items = byType.get(t);
|
|
143
|
+
const label = TYPE_LABEL[t];
|
|
144
|
+
let globalRank = 0;
|
|
145
|
+
for (const prev of TYPE_ORDER) {
|
|
146
|
+
if (prev === t)
|
|
147
|
+
break;
|
|
148
|
+
globalRank += (byType.get(prev) ?? []).length;
|
|
149
|
+
}
|
|
150
|
+
const entries = items.map((item, i) => renderItem(item, globalRank + i + 1)).join("\n\n");
|
|
151
|
+
return `### ${label} (${items.length})\n\n${entries}`;
|
|
152
|
+
});
|
|
153
|
+
// Pre-allocate E2E and UI placeholder sections for full-stack repos.
|
|
154
|
+
const e2eSectionParts = [];
|
|
155
|
+
const uiSectionParts = [];
|
|
156
|
+
if (isFrontendProject) {
|
|
157
|
+
for (let i = 0; i < minE2ESlots; i++) {
|
|
158
|
+
const rank = i + 1;
|
|
159
|
+
e2eSectionParts.push(`**${rank}. E2E User Journey ${i + 1}**\n` +
|
|
160
|
+
` End-to-end test covering a complete user journey through the frontend and backend.\n` +
|
|
161
|
+
` To generate: record a browser trace, then call the generation tool.\n` +
|
|
162
|
+
` browser_navigate({ url: "${baseUrl}" }) \u2192 exercise key user flow \u2192 skyramp_export_zip({ outputPath: "<repo>/.skyramp/e2e_journey_${i + 1}.zip" })\n` +
|
|
163
|
+
` Tool: \`skyramp_e2e_test_generation({ playwrightInput: "<repo>/.skyramp/e2e_journey_${i + 1}.zip"${authHeaderOnlyRef} })\`\n` +
|
|
164
|
+
` From source: read frontend components and their API calls to identify the highest-value user journey`);
|
|
165
|
+
}
|
|
166
|
+
for (let i = 0; i < minUISlots; i++) {
|
|
167
|
+
const rank = minE2ESlots + i + 1;
|
|
168
|
+
uiSectionParts.push(`**${rank}. UI Component Test ${i + 1}**\n` +
|
|
169
|
+
` Test key UI component interactions and state changes.\n` +
|
|
170
|
+
` To generate: record a browser trace, then call the generation tool.\n` +
|
|
171
|
+
` browser_navigate({ url: "${baseUrl}" }) \u2192 interact with UI components \u2192 skyramp_export_zip({ outputPath: "<repo>/.skyramp/ui_component_${i + 1}.zip" })\n` +
|
|
172
|
+
` Tool: \`skyramp_ui_test_generation({ playwrightInput: "<repo>/.skyramp/ui_component_${i + 1}.zip"${authHeaderOnlyRef} })\`\n` +
|
|
173
|
+
` From source: read frontend component files to identify interactions, form submissions, and state transitions`);
|
|
174
|
+
}
|
|
175
|
+
// Offset backend section ranks by the number of E2E + UI placeholders
|
|
176
|
+
const offset = minE2ESlots + minUISlots;
|
|
177
|
+
backendSections.forEach((_, idx) => {
|
|
178
|
+
const t = TYPE_ORDER.filter(t => (byType.get(t) ?? []).length > 0)[idx];
|
|
179
|
+
if (!t)
|
|
180
|
+
return;
|
|
181
|
+
const items = byType.get(t);
|
|
182
|
+
const label = TYPE_LABEL[t];
|
|
183
|
+
let globalRank = offset;
|
|
184
|
+
for (const prev of TYPE_ORDER) {
|
|
185
|
+
if (prev === t)
|
|
186
|
+
break;
|
|
187
|
+
globalRank += (byType.get(prev) ?? []).length;
|
|
188
|
+
}
|
|
189
|
+
backendSections[idx] = `### ${label} (${items.length})\n\n${items.map((item, i) => renderItem(item, globalRank + i + 1)).join("\n\n")}`;
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
const allSections = [
|
|
193
|
+
...(e2eSectionParts.length > 0 ? [`### E2E (${e2eSectionParts.length})\n\n${e2eSectionParts.join("\n\n")}`] : []),
|
|
194
|
+
...(uiSectionParts.length > 0 ? [`### UI (${uiSectionParts.length})\n\n${uiSectionParts.join("\n\n")}`] : []),
|
|
195
|
+
...backendSections,
|
|
196
|
+
];
|
|
197
|
+
const sections = allSections.join("\n\n");
|
|
198
|
+
const frontendTierNote = isFrontendOnlyProject
|
|
199
|
+
? `\n\n**Frontend repo:** supplement MUST include at least ${minE2ESlots} E2E test${minE2ESlots > 1 ? "s" : ""} (\`skyramp_e2e_test_generation\`) and at least ${minUISlots} UI test${minUISlots > 1 ? "s" : ""} (\`skyramp_ui_test_generation\`). Do NOT add integration or contract tests.`
|
|
200
|
+
: isFrontendProject
|
|
201
|
+
? `\n\n**Full-stack repo:** supplement MUST include at least ${minE2ESlots} E2E test${minE2ESlots > 1 ? "s" : ""} (\`skyramp_e2e_test_generation\`) and at least ${minUISlots} UI test${minUISlots > 1 ? "s" : ""} (\`skyramp_ui_test_generation\`). Add these before exhausting backend tiers.`
|
|
202
|
+
: "";
|
|
203
|
+
const repoSupplementNote = supplementCount > 0
|
|
204
|
+
? `
|
|
205
|
+
<supplement_guidance>
|
|
206
|
+
**When to use:** The pre-ranked sections above contain fewer than ${topN} items. Add exactly ${supplementCount} more using the tiers below — exhaust each tier before moving to the next.
|
|
207
|
+
|
|
208
|
+
**Tier 1 — Error paths for endpoints already in the list** (highest value, do first):
|
|
209
|
+
• Auth boundary (no Authorization header → 403/401) → \`testType: contract, category: security_boundary\`
|
|
210
|
+
• Invalid/non-existent IDs (→ 404) → \`testType: contract, category: error_handling\`
|
|
211
|
+
• Missing required fields (→ 422) → \`testType: contract, category: data_validation\`
|
|
212
|
+
• Boundary values for numeric fields → \`testType: integration, category: data_validation\`
|
|
213
|
+
Note: DISCARD unique-constraint scenarios if the storage backend is Redis, MongoDB, or schema-less.
|
|
214
|
+
|
|
215
|
+
**Tier 2 — Auth coverage for any endpoint not yet covered by Tier 1:**
|
|
216
|
+
→ \`testType: contract, category: security_boundary\`
|
|
217
|
+
|
|
218
|
+
**Tier 3 — Cross-resource integration** (only when one resource's POST body contains another's \`_id\` field):
|
|
219
|
+
→ \`testType: integration, category: workflow\`
|
|
220
|
+
|
|
221
|
+
**Tier 4 — CRUD lifecycle** for any resource not yet covered:
|
|
222
|
+
→ \`testType: integration, category: crud\`
|
|
223
|
+
|
|
224
|
+
**How to fill each item:** Use path parameters in \`{param}\` format. Use real field names from the analysis or handler source — no generic placeholders. Describe behavior in API terms (HTTP method, path, status code), not storage internals.${frontendTierNote}
|
|
225
|
+
</supplement_guidance>`
|
|
226
|
+
: "";
|
|
227
|
+
const typeMixText = isFrontendOnlyProject
|
|
228
|
+
? `This is a frontend repo. Focus on E2E and UI tests only. Include at least ${minE2ESlots} E2E test${minE2ESlots > 1 ? "s" : ""} (\`skyramp_e2e_test_generation\`) and at least ${minUISlots} UI test${minUISlots > 1 ? "s" : ""} (\`skyramp_ui_test_generation\`). Do NOT add integration or contract tests.`
|
|
229
|
+
: isFrontendProject
|
|
230
|
+
? `This is a full-stack repo. Coverage ranking: E2E > UI > Integration > Contract. Include at least ${minE2ESlots} E2E test${minE2ESlots > 1 ? "s" : ""} (\`skyramp_e2e_test_generation\`) and at least ${minUISlots} UI test${minUISlots > 1 ? "s" : ""} (\`skyramp_ui_test_generation\`), in addition to backend integration and contract tests.`
|
|
231
|
+
: `Focus on integration and contract tests for all API endpoints.`;
|
|
232
|
+
return `## Test Recommendations — ${topN} total (grouped by test type)
|
|
233
|
+
|
|
234
|
+
> Repo mode — no tests are executed. Ranked by risk within each type.
|
|
235
|
+
> To generate any item: read the handler source, fill \`<…from source>\` placeholders with real values, then call the tool.
|
|
236
|
+
|
|
237
|
+
${sections}
|
|
238
|
+
|
|
239
|
+
**Test type mix — MANDATORY. No smoke tests. No fuzz tests. Only: integration, contract, E2E, UI.**
|
|
240
|
+
${typeMixText}
|
|
241
|
+
|
|
242
|
+
${repoSupplementNote}
|
|
243
|
+
|
|
244
|
+
**Present up to ${topN} recommendations.** Prioritize quality — only include a recommendation if it adds genuine new coverage. If fewer than ${topN} high-value tests exist for this codebase, stop at the last useful item rather than padding with trivial ones.
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
<enrichment_notes>
|
|
248
|
+
**Path resolution (do this before filling in any tool call):**
|
|
249
|
+
Cross-check every endpoint path against the Router Mounting / Nesting section in the analysis above. Sub-routers may be mounted at nested prefixes — e.g. a reviews router with \`@router.get("/")\` may actually be \`GET /api/v1/products/{product_id}/reviews\` if mounted under that prefix. Always use the fully-qualified nested path in tool calls, not the path as it appears in the route file alone.
|
|
250
|
+
|
|
251
|
+
**Existing test files (check before assigning output filenames):**
|
|
252
|
+
See the Existing Tests section above. If a recommendation's primary resource already has a \`[skyramp]\` test file listed there, prefer passing an explicit \`output\` filename (e.g. \`output: "orders_integration_test.py"\`) to update the existing file rather than creating a duplicate. Do NOT update \`[external]\` test files — they are user-maintained.
|
|
253
|
+
|
|
254
|
+
Before filling in tool call parameters for each item, use the analysis data already provided above (endpoint interactions, source context) first. Only read the route handler source code directly when the analysis data does not contain the specific value you need:
|
|
255
|
+
- Required request body fields (POST/PUT/PATCH) — use field names from the analysis interactions; read source only if they show \`{}\` or are missing
|
|
256
|
+
- Computed/derived response fields and their formulas — assert exact values; read source for formula details not captured in the analysis
|
|
257
|
+
- Auth middleware — set authHeader/authScheme from the repository context above; FastAPI HTTPBearer → 403 not 401
|
|
258
|
+
- Storage backend — if Redis or schema-less, discard unique-constraint and cascade-delete scenarios
|
|
259
|
+
- Delete behavior — hard-delete → 204; soft-delete/cancel → 200
|
|
260
|
+
|
|
261
|
+
${buildTestQualityCriteria()}
|
|
262
|
+
|
|
263
|
+
**5-dimension rubric — use to assign priority for supplement items:**
|
|
264
|
+
| Dimension | What to assess |
|
|
265
|
+
| Production Safety | Guards a critical boundary (auth, unique constraint, cascade delete, data integrity, breaking migration)? → HIGH |
|
|
266
|
+
| Bug-Finding Potential | Targets a known failure mode (race condition, data consistency, state transition, cascade effect)? → HIGH |
|
|
267
|
+
| User Journey Relevance | Reflects how real users interact (from traces, business flows, critical paths)? → HIGH or MEDIUM |
|
|
268
|
+
| Coverage Gap | Addresses an area with zero existing test coverage? → bump up one tier |
|
|
269
|
+
| Code Insight | Derived from actual implementation (spotted middleware pattern, N+1 risk, unique constraint)? → bump up one tier |
|
|
270
|
+
</enrichment_notes>`;
|
|
271
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
|
|
2
|
+
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
2
3
|
import { WorkspaceAuthType, getAuthScheme, isAuthorizationHeaderName, AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
3
|
-
// Cached at module-load —
|
|
4
|
+
// Cached at module-load — flags are process-wide and cannot change per call.
|
|
4
5
|
const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
|
|
6
|
+
const SERVICE_REFS = resolveServiceDetailsRef();
|
|
5
7
|
export const MAX_TESTS_TO_GENERATE = 3;
|
|
6
8
|
export const MAX_RECOMMENDATIONS = 20;
|
|
7
9
|
export const MAX_CRITICAL_TESTS = 3;
|
|
@@ -356,7 +358,7 @@ Only provider-side contract tests are supported. Pass \`providerMode: true\` for
|
|
|
356
358
|
3. Interact using \`browser_click\`, \`browser_type\`, \`browser_fill_form\`, etc.
|
|
357
359
|
4. \`browser_snapshot\` after each interaction that changes the page
|
|
358
360
|
5. \`skyramp_export_zip\` with an **absolute** output path: \`<repositoryPath>/.skyramp/<test_name>_trace.zip\`
|
|
359
|
-
6. \`skyramp_ui_test_generation\` with \`playwrightInput\` = the **absolute** path of the exported zip, and \`outputDir\` =
|
|
361
|
+
6. \`skyramp_ui_test_generation\` with \`playwrightInput\` = the **absolute** path of the exported zip, and \`outputDir\` = ${SERVICE_REFS.frontendTestDirRef} (e.g. \`frontend/tests\`). Do NOT use the backend service's testDirectory — UI tests must go in the frontend service's test directory.
|
|
360
362
|
|
|
361
363
|
Tips: For custom dropdowns (Radix, MUI): click combobox → snapshot → click option (NOT \`browser_select_option\`).
|
|
362
364
|
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { extractResourceFromPath } from "../../utils/routeParsers.js";
|
|
2
|
+
import { logger } from "../../utils/logger.js";
|
|
3
|
+
/** Resolve the primary step and inferred test type for a scenario. */
|
|
4
|
+
function resolvePrimaryStep(scenario) {
|
|
5
|
+
const testType = scenario.testType ?? (scenario.steps.length === 1 ? "contract" : "integration");
|
|
6
|
+
const mutatingSteps = scenario.steps.filter(st => ["POST", "PUT", "PATCH", "DELETE"].includes(st.method));
|
|
7
|
+
// Use the last mutating step — earlier steps are typically prerequisite setup
|
|
8
|
+
// (e.g. POST /products before PATCH /orders), while the final mutation is the
|
|
9
|
+
// primary action under test.
|
|
10
|
+
const primaryStep = mutatingSteps[mutatingSteps.length - 1] ?? scenario.steps[scenario.steps.length - 1];
|
|
11
|
+
return { primaryStep, testType };
|
|
12
|
+
}
|
|
13
|
+
export function scenarioCoverageKey(scenario) {
|
|
14
|
+
const { primaryStep, testType } = resolvePrimaryStep(scenario);
|
|
15
|
+
const resource = extractResourceFromPath(primaryStep?.path ?? "");
|
|
16
|
+
return `${resource}::${testType}`;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Method-aware coverage key for external test dedup.
|
|
20
|
+
* Unlike scenarioCoverageKey (resource::testType), this includes the HTTP method
|
|
21
|
+
* so that e.g. an external test covering "GET /orders" doesn't block generating
|
|
22
|
+
* a test for "PUT /orders" — a different operation on the same resource.
|
|
23
|
+
*/
|
|
24
|
+
export function externalDedupKey(scenario) {
|
|
25
|
+
const { primaryStep, testType } = resolvePrimaryStep(scenario);
|
|
26
|
+
const method = primaryStep?.method ?? "GET";
|
|
27
|
+
const resource = extractResourceFromPath(primaryStep?.path ?? "");
|
|
28
|
+
return `${method}::${resource}::${testType}`;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Build a set of coverage keys from external (non-Skyramp) tests.
|
|
32
|
+
* Parses `testLocations` entries tagged with `[external]` to extract the
|
|
33
|
+
* method-aware `METHOD::resource::testType` keys they cover.
|
|
34
|
+
*/
|
|
35
|
+
export function buildExternalCoverageSet(testLocations) {
|
|
36
|
+
const coverage = new Set();
|
|
37
|
+
let externalWithoutCoverage = 0;
|
|
38
|
+
for (const [testType, fileList] of Object.entries(testLocations)) {
|
|
39
|
+
const externalCount = (fileList.match(/\[external\]/g) || []).length;
|
|
40
|
+
const coveredCount = (fileList.match(/\[external\]\s*\(covers:/g) || []).length;
|
|
41
|
+
externalWithoutCoverage += externalCount - coveredCount;
|
|
42
|
+
for (const m of fileList.matchAll(/\[external\]\s*\(covers:\s*([^)]+)\)/g)) {
|
|
43
|
+
const endpoints = m[1].split(",").map(e => e.trim());
|
|
44
|
+
for (const ep of endpoints) {
|
|
45
|
+
const spaceIdx = ep.indexOf(" ");
|
|
46
|
+
if (spaceIdx < 0)
|
|
47
|
+
continue;
|
|
48
|
+
const method = ep.slice(0, spaceIdx).toUpperCase();
|
|
49
|
+
const epPath = ep.slice(spaceIdx + 1);
|
|
50
|
+
const resource = extractResourceFromPath(epPath);
|
|
51
|
+
if (resource !== "unknown") {
|
|
52
|
+
if (testType === "unknown") {
|
|
53
|
+
coverage.add(`${method}::${resource}::integration`);
|
|
54
|
+
coverage.add(`${method}::${resource}::contract`);
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
coverage.add(`${method}::${resource}::${testType}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (externalWithoutCoverage > 0) {
|
|
64
|
+
logger.info(`${externalWithoutCoverage} external test file(s) have no extractable endpoint coverage — ` +
|
|
65
|
+
`programmatic dedup skipped for these; Step 0 semantic check is the fallback.`);
|
|
66
|
+
}
|
|
67
|
+
return coverage;
|
|
68
|
+
}
|
|
@@ -4,6 +4,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
4
4
|
import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
|
|
5
5
|
import { ScenarioSource, AnalysisScope } from "../../types/RepositoryAnalysis.js";
|
|
6
6
|
import { SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
|
|
7
|
+
import { inferExpectedStatus } from "../../utils/httpDefaults.js";
|
|
7
8
|
export function mergeEnrichedScenarios(serverScenarios, raw) {
|
|
8
9
|
const rejectionNotes = [];
|
|
9
10
|
let parsed;
|
|
@@ -55,10 +56,7 @@ export function mergeEnrichedScenarios(serverScenarios, raw) {
|
|
|
55
56
|
queryParams: st.queryParams,
|
|
56
57
|
responseBody: st.responseBody,
|
|
57
58
|
// Default status code by method if omitted to avoid `statusCode: undefined` in tool calls
|
|
58
|
-
expectedStatusCode: st.expectedStatusCode ??
|
|
59
|
-
(String(st.method ?? "").toUpperCase() === "POST" ? 201
|
|
60
|
-
: String(st.method ?? "").toUpperCase() === "DELETE" ? 204
|
|
61
|
-
: 200),
|
|
59
|
+
expectedStatusCode: st.expectedStatusCode ?? inferExpectedStatus(String(st.method ?? "GET")),
|
|
62
60
|
expectedResponseFields: st.expectedResponseFields,
|
|
63
61
|
bodyMustInclude: st.bodyMustInclude,
|
|
64
62
|
chainsFrom: st.chainsFrom,
|
|
@@ -153,11 +151,29 @@ export function registerRecommendTestsPrompt(server) {
|
|
|
153
151
|
}
|
|
154
152
|
}
|
|
155
153
|
if (!fullAnalysis) {
|
|
154
|
+
if (sessionId) {
|
|
155
|
+
logger.warning(`Session not found in memory (sessionId=${sessionId}) — server may have restarted; falling back to state file`);
|
|
156
|
+
}
|
|
156
157
|
fullAnalysis = state.repositoryAnalysis.fullAnalysis;
|
|
157
158
|
}
|
|
158
159
|
if (!fullAnalysis) {
|
|
159
160
|
throw new Error(`Analysis data for session not found in memory or on disk. Re-run skyramp_analyze_changes.`);
|
|
160
161
|
}
|
|
162
|
+
// Hydrate testLocations from the disk-persisted field when fullAnalysis came from disk
|
|
163
|
+
// (after a server restart, fullAnalysis is loaded from state.repositoryAnalysis.fullAnalysis
|
|
164
|
+
// but testLocations was persisted separately under state.repositoryAnalysis.testLocations)
|
|
165
|
+
if (fullAnalysis.existingTests &&
|
|
166
|
+
!fullAnalysis.existingTests.testLocations &&
|
|
167
|
+
state.repositoryAnalysis.testLocations) {
|
|
168
|
+
fullAnalysis = {
|
|
169
|
+
...fullAnalysis,
|
|
170
|
+
existingTests: {
|
|
171
|
+
...fullAnalysis.existingTests,
|
|
172
|
+
testLocations: state.repositoryAnalysis.testLocations,
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
logger.debug("Hydrated existingTests.testLocations from disk-persisted state", { sessionId });
|
|
176
|
+
}
|
|
161
177
|
// Normalize legacy state files: before AnalysisScope enum normalization, state stored
|
|
162
178
|
// the user-facing param value "branch_diff". Map it explicitly so diff-mode detection
|
|
163
179
|
// works correctly on state created before this deployment (2-hour TTL window).
|