@skyramp/mcp 0.2.3 → 0.2.5-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +21 -25
- package/build/playwright/traceRecordingPrompt.js +2 -2
- package/build/prompts/test-maintenance/actionsInstructions.js +60 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +18 -101
- package/build/prompts/test-maintenance/driftAnalysisSections.js +210 -171
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
- package/build/prompts/test-recommendation/diffExecutionPlan.js +4 -3
- package/build/prompts/test-recommendation/recommendationSections.js +6 -6
- package/build/prompts/test-recommendation/scopeAssessment.js +3 -1
- package/build/prompts/test-recommendation/scopeAssessment.test.js +13 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +21 -17
- package/build/prompts/testbot/testbot-prompts.test.js +21 -17
- package/build/services/TestDiscoveryService.js +11 -43
- package/build/tools/submitReportTool.js +9 -12
- package/build/tools/submitReportTool.test.js +4 -5
- package/build/tools/test-management/actionsTool.js +160 -240
- package/build/tools/test-management/analyzeChangesTool.js +43 -18
- package/build/tools/test-management/analyzeTestHealthTool.js +17 -29
- package/build/utils/docker.test.js +1 -1
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/common/visualSnapshot.js +95 -0
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +150 -2
- package/node_modules/playwright/lib/mcp/skyramp/visualSnapshotTool.js +63 -0
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +36 -0
- package/package.json +2 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +0 -116
|
@@ -2,11 +2,11 @@ import { z } from "zod";
|
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
4
4
|
import { TestSource, DriftAction, RecommendationPriority, EstimatedWork } from "../../types/TestAnalysis.js";
|
|
5
|
-
import { TestType } from "../../types/TestTypes.js";
|
|
6
5
|
import * as fs from "fs";
|
|
7
6
|
import * as path from "path";
|
|
8
7
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
9
8
|
import { toolError } from "../../utils/utils.js";
|
|
9
|
+
import { buildRenameStrategy, buildFileRenameStrategy, buildUpdateStrategy, buildRegenerateStrategy, buildUpdateFileInstruction, buildRegenerateFileInstruction, } from "../../prompts/test-maintenance/actionsInstructions.js";
|
|
10
10
|
/**
|
|
11
11
|
* Compute a suggested new filename when an endpoint is renamed.
|
|
12
12
|
*/
|
|
@@ -44,22 +44,6 @@ export function computeRenamedTestFile(testFile, renames) {
|
|
|
44
44
|
}
|
|
45
45
|
return newFilePath;
|
|
46
46
|
}
|
|
47
|
-
/**
|
|
48
|
-
* Select test types to generate based on HTTP method.
|
|
49
|
-
*/
|
|
50
|
-
function selectTestTypesForEndpoint(method) {
|
|
51
|
-
switch (method.toUpperCase()) {
|
|
52
|
-
case "POST":
|
|
53
|
-
case "PUT":
|
|
54
|
-
case "PATCH":
|
|
55
|
-
return [TestType.INTEGRATION, TestType.CONTRACT];
|
|
56
|
-
case "DELETE":
|
|
57
|
-
return [TestType.INTEGRATION, TestType.SMOKE];
|
|
58
|
-
case "GET":
|
|
59
|
-
default:
|
|
60
|
-
return [TestType.CONTRACT, TestType.SMOKE];
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
47
|
const recommendationSchema = z.object({
|
|
64
48
|
testFile: z
|
|
65
49
|
.string()
|
|
@@ -106,7 +90,7 @@ const actionsSchema = {
|
|
|
106
90
|
recommendations: z
|
|
107
91
|
.array(recommendationSchema)
|
|
108
92
|
.optional()
|
|
109
|
-
.describe("LLM
|
|
93
|
+
.describe("LLM Drift assessment produced by skyramp_analyze_test_health — one entry per test assessed, including IGNORE and VERIFY entries. Required for UPDATE instructions to be emitted; omitting results in no maintenance actions. For [external] tests: UPDATE edits are applied automatically; REGENERATE and DELETE are surfaced as report-only findings and never rewrite or delete a user-authored file."),
|
|
110
94
|
};
|
|
111
95
|
const TOOL_NAME = "skyramp_actions";
|
|
112
96
|
export function registerActionsTool(server) {
|
|
@@ -117,21 +101,7 @@ export function registerActionsTool(server) {
|
|
|
117
101
|
idempotentHint: false,
|
|
118
102
|
openWorldHint: true,
|
|
119
103
|
},
|
|
120
|
-
description: `Execute test maintenance
|
|
121
|
-
|
|
122
|
-
**PREREQUISITE:** Call \`skyramp_analyze_changes\` (produces the stateFile), then \`skyramp_analyze_test_health\` (runs the drift assessment). This tool reads the stateFile from \`skyramp_analyze_changes\`.
|
|
123
|
-
|
|
124
|
-
Call this tool after completing the drift assessment. It executes maintenance actions automatically from the stateFile — no user confirmation required.
|
|
125
|
-
|
|
126
|
-
**EXECUTING ACTIONS:**
|
|
127
|
-
- UPDATE: Tests with drift — emits targeted per-file edit instructions driven by updateInstructions and renamedEndpoints
|
|
128
|
-
- REGENERATE: Emits file-level summary; follow up by calling the appropriate generation tool (e.g. skyramp_integration_test_generation) with the same filename to overwrite
|
|
129
|
-
- VERIFY: Emits file-level summary for human review — no automated edits
|
|
130
|
-
- ADD: Auto-generates tests for new endpoints via LLM instructions
|
|
131
|
-
|
|
132
|
-
**OUTPUT:**
|
|
133
|
-
Comprehensive report with executed actions, summary, and instructions for ADD recommendations
|
|
134
|
-
`,
|
|
104
|
+
description: `Execute test maintenance actions — final step of the unified Test Health Analysis Flow.`,
|
|
135
105
|
inputSchema: actionsSchema,
|
|
136
106
|
}, async (args) => {
|
|
137
107
|
let errorResult;
|
|
@@ -146,17 +116,22 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
146
116
|
errorResult = toolError(`State file is empty or invalid: ${args.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
|
|
147
117
|
return errorResult;
|
|
148
118
|
}
|
|
149
|
-
// External tests must not be candidates for UPDATE/REGENERATE/DELETE actions.
|
|
150
|
-
// Default source to Skyramp for backwards compat with state files created before the source field existed.
|
|
151
|
-
const testAnalysisResults = (stateData.existingTests || []).filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External);
|
|
152
|
-
const newEndpoints = stateData.newEndpoints || [];
|
|
153
119
|
// Resolve repo root for path normalization and security checks.
|
|
154
120
|
const repoRoot = repositoryPath ? path.resolve(repositoryPath) : "";
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
|
|
159
|
-
|
|
121
|
+
// Relevant external (user-written) tests: UPDATE is permitted; REGENERATE/DELETE
|
|
122
|
+
// are report-only (the LLM may recommend them but this tool will not apply them).
|
|
123
|
+
// Paths are stored relative to repositoryPath in the state file — re-absolutize.
|
|
124
|
+
const relevantExternalPaths = new Set((stateData.repositoryAnalysis?.relevantExternalTestPaths ?? []).map((p) => path.isAbsolute(p) ? p : path.resolve(repoRoot, p)));
|
|
125
|
+
// Allowlist: Skyramp-generated tests + relevant external tests.
|
|
126
|
+
// Using an allowlist (not a blocklist) catches hallucinated paths the LLM
|
|
127
|
+
// may supply that are not in the scanned catalog at all.
|
|
128
|
+
const testAnalysisResults = (stateData.existingTests || []);
|
|
129
|
+
const skyrampTestFiles = new Set(testAnalysisResults
|
|
130
|
+
.filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External)
|
|
131
|
+
.map((t) => t.testFile));
|
|
132
|
+
const externalTestFiles = new Set(testAnalysisResults
|
|
133
|
+
.filter((t) => (t.source ?? TestSource.Skyramp) === TestSource.External)
|
|
134
|
+
.map((t) => t.testFile));
|
|
160
135
|
// ── Build recommendations from LLM-supplied drift assessment ──
|
|
161
136
|
// The LLM performs the drift assessment in context after skyramp_analyze_test_health
|
|
162
137
|
// and passes results here directly — analyzeTestHealthTool never writes assessment
|
|
@@ -165,23 +140,55 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
165
140
|
(args.recommendations ?? []).forEach((rec) => {
|
|
166
141
|
// Schema requires absolute paths; resolve any relative paths defensively
|
|
167
142
|
// against repoRoot in case the LLM sends a relative path despite the schema.
|
|
168
|
-
|
|
143
|
+
// Normalize via path.resolve to collapse any `..` segments before the
|
|
144
|
+
// traversal guard — otherwise "/repo/../etc/passwd" would pass startsWith.
|
|
145
|
+
const rawFile = path.isAbsolute(rec.testFile)
|
|
169
146
|
? rec.testFile
|
|
170
147
|
: repoRoot
|
|
171
148
|
? path.resolve(repoRoot, rec.testFile)
|
|
172
149
|
: rec.testFile;
|
|
150
|
+
const resolvedFile = path.resolve(rawFile);
|
|
173
151
|
// Reject files outside the repo root (path-traversal guard).
|
|
174
|
-
|
|
152
|
+
// Exception: files already in the scanned test catalog (externalTestFiles / skyrampTestFiles)
|
|
153
|
+
// may legitimately live in a separate testsRepoDir outside repositoryPath — catalog
|
|
154
|
+
// membership is a sufficient provenance check for those paths.
|
|
155
|
+
const isInCatalog = skyrampTestFiles.has(resolvedFile) || skyrampTestFiles.has(rec.testFile)
|
|
156
|
+
|| externalTestFiles.has(resolvedFile) || externalTestFiles.has(rec.testFile);
|
|
157
|
+
if (repoRoot && !isInCatalog && !resolvedFile.startsWith(repoRoot + path.sep) && resolvedFile !== repoRoot) {
|
|
175
158
|
logger.warning(`Skipping recommendation for path outside repo root: ${rec.testFile}`);
|
|
176
159
|
return;
|
|
177
160
|
}
|
|
178
|
-
// Guard: only
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
//
|
|
161
|
+
// Guard: only files present in the scanned test catalog may receive any
|
|
162
|
+
// recommendation. Hallucinated paths (not in either set) are rejected for
|
|
163
|
+
// all actions, including VERIFY and IGNORE, to keep the report consistent
|
|
164
|
+
// with what was actually discovered.
|
|
165
|
+
const isSkyramp = skyrampTestFiles.has(resolvedFile) || skyrampTestFiles.has(rec.testFile);
|
|
166
|
+
const isRelevantExternal = (externalTestFiles.has(resolvedFile) || externalTestFiles.has(rec.testFile)) &&
|
|
167
|
+
(relevantExternalPaths.has(resolvedFile) || relevantExternalPaths.has(rec.testFile));
|
|
168
|
+
const isInAnyKnownCatalog = isSkyramp || isRelevantExternal
|
|
169
|
+
|| externalTestFiles.has(resolvedFile) || externalTestFiles.has(rec.testFile);
|
|
170
|
+
if (!isInAnyKnownCatalog) {
|
|
171
|
+
logger.warning(`Skipping ${rec.action} for unknown test (not in scanned catalog): ${rec.testFile}`);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
182
174
|
const isActionable = [DriftAction.Update, DriftAction.Regenerate, DriftAction.Delete].includes(rec.action);
|
|
183
|
-
if (isActionable && !
|
|
184
|
-
logger.warning(`Skipping ${rec.action} for
|
|
175
|
+
if (isActionable && !isSkyramp && !isRelevantExternal) {
|
|
176
|
+
logger.warning(`Skipping ${rec.action} for irrelevant external test: ${rec.testFile}`);
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
// REGENERATE and DELETE on external tests are report-only — convert to VERIFY so
|
|
180
|
+
// the finding surfaces to the developer without touching the file.
|
|
181
|
+
if (isRelevantExternal && !isSkyramp &&
|
|
182
|
+
(rec.action === DriftAction.Regenerate || rec.action === DriftAction.Delete)) {
|
|
183
|
+
recommendations.push({
|
|
184
|
+
testFile: resolvedFile,
|
|
185
|
+
action: DriftAction.Verify,
|
|
186
|
+
priority: rec.priority ?? RecommendationPriority.Medium,
|
|
187
|
+
rationale: `[external test — needs manual review] ${rec.rationale ?? ""}`.trimEnd(),
|
|
188
|
+
estimatedWork: rec.estimatedWork ?? EstimatedWork.Small,
|
|
189
|
+
updateInstructions: "",
|
|
190
|
+
renamedEndpoints: [],
|
|
191
|
+
});
|
|
185
192
|
return;
|
|
186
193
|
}
|
|
187
194
|
recommendations.push({
|
|
@@ -194,7 +201,7 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
194
201
|
renamedEndpoints: rec.renamedEndpoints ?? [],
|
|
195
202
|
});
|
|
196
203
|
});
|
|
197
|
-
// ── Process UPDATE recommendations ──
|
|
204
|
+
// ── Process UPDATE and REGENERATE recommendations ──
|
|
198
205
|
// Deduplicate by testFile — keep the highest-priority entry when the LLM
|
|
199
206
|
// repeats a file. Priority order: high > medium > low.
|
|
200
207
|
const priorityRank = {
|
|
@@ -202,32 +209,46 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
202
209
|
[RecommendationPriority.Medium]: 1,
|
|
203
210
|
[RecommendationPriority.Low]: 0,
|
|
204
211
|
};
|
|
212
|
+
// Build per-file winner maps. REGENERATE beats UPDATE for the same file —
|
|
213
|
+
// if the LLM emits both, keep REGENERATE (higher severity) and drop UPDATE.
|
|
205
214
|
const updateByFile = new Map();
|
|
215
|
+
const regenerateByFile = new Map();
|
|
206
216
|
for (const rec of recommendations) {
|
|
207
|
-
if (rec.action
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
217
|
+
if (rec.action === DriftAction.Regenerate) {
|
|
218
|
+
const existing = regenerateByFile.get(rec.testFile);
|
|
219
|
+
if (!existing || priorityRank[rec.priority] > priorityRank[existing.priority]) {
|
|
220
|
+
regenerateByFile.set(rec.testFile, rec);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
else if (rec.action === DriftAction.Update) {
|
|
224
|
+
// Only add to updateByFile if no REGENERATE exists for this file.
|
|
225
|
+
if (!regenerateByFile.has(rec.testFile)) {
|
|
226
|
+
const existing = updateByFile.get(rec.testFile);
|
|
227
|
+
if (!existing || priorityRank[rec.priority] > priorityRank[existing.priority]) {
|
|
228
|
+
updateByFile.set(rec.testFile, rec);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
212
231
|
}
|
|
213
232
|
}
|
|
233
|
+
// Second pass: drop any UPDATE entries for files that ended up with REGENERATE
|
|
234
|
+
// (handles ordering where UPDATE was inserted before REGENERATE was seen).
|
|
235
|
+
for (const file of regenerateByFile.keys()) {
|
|
236
|
+
updateByFile.delete(file);
|
|
237
|
+
}
|
|
214
238
|
const updateRecommendations = Array.from(updateByFile.values());
|
|
239
|
+
const regenerateRecommendations = Array.from(regenerateByFile.values());
|
|
215
240
|
const fileInstructions = [];
|
|
216
241
|
const testFilesToUpdate = [];
|
|
217
242
|
const testFileContentMap = new Map();
|
|
243
|
+
// ── UPDATE: read file, emit targeted edit instructions ──
|
|
218
244
|
for (const rec of updateRecommendations) {
|
|
219
245
|
if (!rec.testFile) {
|
|
220
246
|
logger.warning("Recommendation missing testFile", rec);
|
|
221
247
|
continue;
|
|
222
248
|
}
|
|
223
249
|
testFilesToUpdate.push(rec.testFile);
|
|
224
|
-
const testData = testAnalysisResults.find((t) => t.testFile === rec.testFile);
|
|
225
|
-
const driftData = testData?.drift;
|
|
226
|
-
const driftChanges = driftData?.changes || [];
|
|
227
|
-
let testFileContent = "";
|
|
228
250
|
try {
|
|
229
|
-
|
|
230
|
-
testFileContentMap.set(rec.testFile, testFileContent);
|
|
251
|
+
testFileContentMap.set(rec.testFile, fs.readFileSync(rec.testFile, "utf-8"));
|
|
231
252
|
}
|
|
232
253
|
catch (error) {
|
|
233
254
|
logger.error(`Failed to read test file ${rec.testFile}: ${error.message}`);
|
|
@@ -235,173 +256,83 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
235
256
|
}
|
|
236
257
|
const renames = rec.renamedEndpoints || [];
|
|
237
258
|
const isRenameUpdate = renames.length > 0;
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
}
|
|
260
|
-
const recUpdateInstructions = rec.updateInstructions ?? "";
|
|
261
|
-
if (recUpdateInstructions) {
|
|
262
|
-
instruction += `**What to change:**\n\n${recUpdateInstructions}\n\n`;
|
|
263
|
-
instruction += `Match the assertion style already used in the file. `;
|
|
264
|
-
instruction += `Preserve all existing test logic — only add or adjust what is described above.\n\n`;
|
|
265
|
-
}
|
|
266
|
-
else if (!isRenameUpdate) {
|
|
267
|
-
instruction += `**Action:** Update this test file per the rationale above. `;
|
|
268
|
-
instruction += `Match the assertion style already used in the file. `;
|
|
269
|
-
instruction += `Preserve all existing test logic — only add or adjust the minimum required assertions.\n\n`;
|
|
270
|
-
}
|
|
271
|
-
if (driftData) {
|
|
272
|
-
instruction += `**Analysis:**\n`;
|
|
273
|
-
instruction += `- Changes Detected: ${driftData.changes?.length || 0}\n`;
|
|
274
|
-
instruction += `- Affected Files: ${driftData.affectedFiles.files?.length || 0}\n\n`;
|
|
259
|
+
const suggestedNewFile = isRenameUpdate
|
|
260
|
+
? computeRenamedTestFile(rec.testFile, renames) ?? undefined
|
|
261
|
+
: undefined;
|
|
262
|
+
if (suggestedNewFile)
|
|
263
|
+
rec._suggestedNewFile = suggestedNewFile;
|
|
264
|
+
fileInstructions.push(buildUpdateFileInstruction({
|
|
265
|
+
testFile: rec.testFile,
|
|
266
|
+
renames,
|
|
267
|
+
suggestedNewFile,
|
|
268
|
+
updateInstructions: rec.updateInstructions,
|
|
269
|
+
rationale: rec.rationale,
|
|
270
|
+
}));
|
|
271
|
+
}
|
|
272
|
+
// ── REGENERATE: read file for context, emit overwrite instructions ──
|
|
273
|
+
const regenerateInstructions = [];
|
|
274
|
+
const testFilesToRegenerate = [];
|
|
275
|
+
const regenerateContentMap = new Map();
|
|
276
|
+
for (const rec of regenerateRecommendations) {
|
|
277
|
+
if (!rec.testFile) {
|
|
278
|
+
logger.warning("Recommendation missing testFile", rec);
|
|
279
|
+
continue;
|
|
275
280
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
instruction += `**${change.type}** (Severity: ${change.severity}): ${change.description}\n`;
|
|
280
|
-
if (change.details) {
|
|
281
|
-
instruction += ` └─ ${change.details}\n`;
|
|
282
|
-
}
|
|
283
|
-
if (change.file) {
|
|
284
|
-
instruction += ` └─ In: \`${change.file}\`\n`;
|
|
285
|
-
}
|
|
286
|
-
});
|
|
287
|
-
instruction += `\n`;
|
|
281
|
+
testFilesToRegenerate.push(rec.testFile);
|
|
282
|
+
try {
|
|
283
|
+
regenerateContentMap.set(rec.testFile, fs.readFileSync(rec.testFile, "utf-8"));
|
|
288
284
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
}
|
|
292
|
-
// ── Build ADD section for new endpoints ──
|
|
293
|
-
const wsBaseUrl = stateData.repositoryAnalysis?.wsBaseUrl || "";
|
|
294
|
-
const wsSchemaPath = stateData.repositoryAnalysis?.wsSchemaPath || "";
|
|
295
|
-
const primaryLanguage = stateData.repositoryAnalysis?.projectMeta?.primaryLanguage ||
|
|
296
|
-
"python";
|
|
297
|
-
const primaryFramework = stateData.repositoryAnalysis?.projectMeta?.primaryFramework ||
|
|
298
|
-
"pytest";
|
|
299
|
-
// Determine output directory from workspace config or repo path
|
|
300
|
-
const outputDir = repositoryPath
|
|
301
|
-
? path.join(repositoryPath, "tests", "skyramp")
|
|
302
|
-
: "./tests/skyramp";
|
|
303
|
-
const addSummaryLines = [];
|
|
304
|
-
const llmToolCalls = [];
|
|
305
|
-
for (const ep of newEndpoints) {
|
|
306
|
-
const testTypes = selectTestTypesForEndpoint(ep.method);
|
|
307
|
-
const endpointURL = wsBaseUrl
|
|
308
|
-
? wsBaseUrl.replace(/\/$/, "") + ep.path
|
|
309
|
-
: ep.path;
|
|
310
|
-
addSummaryLines.push(`- ${ep.method} ${ep.path} → ${testTypes.join(", ")} tests`);
|
|
311
|
-
for (const testType of testTypes) {
|
|
312
|
-
let toolName = "";
|
|
313
|
-
switch (testType) {
|
|
314
|
-
case TestType.CONTRACT:
|
|
315
|
-
toolName = "skyramp_contract_test_generation";
|
|
316
|
-
break;
|
|
317
|
-
case TestType.INTEGRATION:
|
|
318
|
-
toolName = "skyramp_integration_test_generation";
|
|
319
|
-
break;
|
|
320
|
-
case TestType.SMOKE:
|
|
321
|
-
toolName = "skyramp_smoke_test_generation";
|
|
322
|
-
break;
|
|
323
|
-
default:
|
|
324
|
-
toolName = "skyramp_contract_test_generation";
|
|
325
|
-
}
|
|
326
|
-
llmToolCalls.push({
|
|
327
|
-
tool: toolName,
|
|
328
|
-
params: {
|
|
329
|
-
endpointURL,
|
|
330
|
-
method: ep.method,
|
|
331
|
-
language: primaryLanguage,
|
|
332
|
-
framework: primaryFramework,
|
|
333
|
-
outputDir,
|
|
334
|
-
...(wsSchemaPath ? { apiSchema: wsSchemaPath } : {}),
|
|
335
|
-
},
|
|
336
|
-
endpoint: `${ep.method} ${ep.path}`,
|
|
337
|
-
testType: testType,
|
|
338
|
-
});
|
|
285
|
+
catch (error) {
|
|
286
|
+
logger.warning(`Could not read file for REGENERATE context ${rec.testFile}: ${error.message}`);
|
|
339
287
|
}
|
|
288
|
+
regenerateInstructions.push(buildRegenerateFileInstruction({
|
|
289
|
+
testFile: rec.testFile,
|
|
290
|
+
updateInstructions: rec.updateInstructions,
|
|
291
|
+
outputDir: path.dirname(rec.testFile),
|
|
292
|
+
outputFile: path.basename(rec.testFile),
|
|
293
|
+
}));
|
|
340
294
|
}
|
|
341
295
|
// ── Build response text ──
|
|
342
|
-
|
|
296
|
+
const otherRecs = recommendations.filter((rec) => rec.action !== DriftAction.Update && rec.action !== DriftAction.Regenerate);
|
|
297
|
+
const sections = [`# Test Actions Report`];
|
|
343
298
|
if (updateRecommendations.length > 0) {
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
responseText += `\n---\n`;
|
|
349
|
-
responseText += fileInstructions.join("\n---\n");
|
|
350
|
-
}
|
|
351
|
-
if (newEndpoints.length > 0) {
|
|
352
|
-
responseText += `\n## New Endpoint Tests to Generate (${newEndpoints.length} endpoints)\n\n`;
|
|
353
|
-
addSummaryLines.forEach((line) => {
|
|
354
|
-
responseText += `${line}\n`;
|
|
355
|
-
});
|
|
356
|
-
responseText += `\nThe following tests will be generated automatically.\n`;
|
|
357
|
-
}
|
|
358
|
-
if (updateRecommendations.length === 0 && newEndpoints.length === 0) {
|
|
359
|
-
const otherRecs = recommendations.filter((rec) => rec.action !== DriftAction.Update);
|
|
360
|
-
if (otherRecs.length > 0) {
|
|
361
|
-
responseText += `## Recommendations (${otherRecs.length})\n\n`;
|
|
362
|
-
otherRecs.forEach((rec) => {
|
|
363
|
-
responseText += `- **${rec.testFile}** — Action: ${rec.action}, Priority: ${rec.priority}\n`;
|
|
364
|
-
responseText += ` ${rec.rationale}\n`;
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
else {
|
|
368
|
-
responseText += `No action required. All existing tests appear healthy.\n`;
|
|
369
|
-
}
|
|
299
|
+
sections.push(`## Tests Requiring Updates (${updateRecommendations.length})\n\n` +
|
|
300
|
+
testFilesToUpdate.map((f, i) => `${i + 1}. \`${f}\``).join("\n") +
|
|
301
|
+
`\n\n---\n` +
|
|
302
|
+
fileInstructions.join("\n---\n"));
|
|
370
303
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
responseText += `${stepNumber++}. Update test files to fix compatibility issues\n`;
|
|
377
|
-
responseText += `${stepNumber++}. Preserve original test logic and structure\n`;
|
|
378
|
-
responseText += `${stepNumber++}. Show you the changes made\n`;
|
|
304
|
+
if (regenerateRecommendations.length > 0) {
|
|
305
|
+
sections.push(`## Tests Requiring Regeneration (${regenerateRecommendations.length})\n\n` +
|
|
306
|
+
testFilesToRegenerate.map((f, i) => `${i + 1}. \`${f}\``).join("\n") +
|
|
307
|
+
`\n\n---\n` +
|
|
308
|
+
regenerateInstructions.join("\n---\n"));
|
|
379
309
|
}
|
|
380
|
-
if (
|
|
381
|
-
|
|
310
|
+
if (otherRecs.length > 0) {
|
|
311
|
+
sections.push(`## Other Findings (${otherRecs.length})\n\n` +
|
|
312
|
+
otherRecs.map((rec) => `- **${rec.testFile}** — Action: ${rec.action}, Priority: ${rec.priority}` +
|
|
313
|
+
(rec.rationale ? ` — ${rec.rationale}` : "")).join("\n"));
|
|
382
314
|
}
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
const allRenames = [];
|
|
386
|
-
for (const rec of updateRecommendations) {
|
|
387
|
-
if (rec.renamedEndpoints && rec.renamedEndpoints.length > 0) {
|
|
388
|
-
allRenames.push(...rec.renamedEndpoints);
|
|
389
|
-
}
|
|
315
|
+
else if (updateRecommendations.length === 0 && regenerateRecommendations.length === 0) {
|
|
316
|
+
sections.push(`No action required. All existing tests appear healthy.`);
|
|
390
317
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
318
|
+
sections.push(`**This tool is currently in Early Preview stage. Please verify the results.**`);
|
|
319
|
+
const responseText = sections.join("\n\n");
|
|
320
|
+
// ── Build LLM instructions ──
|
|
321
|
+
const uniqueRenames = updateRecommendations
|
|
322
|
+
.flatMap((rec) => rec.renamedEndpoints ?? [])
|
|
323
|
+
.filter((r, i, arr) => arr.findIndex((x) => x.oldPath === r.oldPath && x.newPath === r.newPath && x.method === r.method) === i);
|
|
394
324
|
const llmInstructionsObj = {
|
|
395
325
|
workflow: "test_maintenance",
|
|
396
326
|
action: "execute_updates",
|
|
397
327
|
auto_proceed: true,
|
|
398
328
|
files_to_update: testFilesToUpdate,
|
|
399
329
|
update_count: updateRecommendations.length,
|
|
330
|
+
files_to_regenerate: testFilesToRegenerate,
|
|
331
|
+
regenerate_count: regenerateRecommendations.length,
|
|
400
332
|
};
|
|
401
333
|
if (uniqueRenames.length > 0) {
|
|
402
334
|
llmInstructionsObj.endpoint_renames = uniqueRenames;
|
|
403
|
-
llmInstructionsObj.rename_strategy =
|
|
404
|
-
"For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.";
|
|
335
|
+
llmInstructionsObj.rename_strategy = buildRenameStrategy();
|
|
405
336
|
const fileRenames = [];
|
|
406
337
|
for (const rec of updateRecommendations) {
|
|
407
338
|
if (rec._suggestedNewFile) {
|
|
@@ -410,25 +341,31 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
410
341
|
}
|
|
411
342
|
if (fileRenames.length > 0) {
|
|
412
343
|
llmInstructionsObj.file_renames = fileRenames;
|
|
413
|
-
llmInstructionsObj.file_rename_strategy =
|
|
414
|
-
"After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.";
|
|
344
|
+
llmInstructionsObj.file_rename_strategy = buildFileRenameStrategy();
|
|
415
345
|
}
|
|
416
346
|
}
|
|
417
347
|
// Update context: per-file guidance + current content for the downstream LLM.
|
|
418
348
|
// Including file content avoids re-reads on each Edit turn, reducing token usage.
|
|
419
|
-
const
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
349
|
+
const updateContext = updateRecommendations
|
|
350
|
+
.filter((rec) => !!rec.updateInstructions)
|
|
351
|
+
.map((rec) => {
|
|
352
|
+
const current_content = testFileContentMap.get(rec.testFile);
|
|
353
|
+
return { file: rec.testFile, context: rec.updateInstructions, ...(current_content !== undefined && { current_content }) };
|
|
354
|
+
});
|
|
355
|
+
if (updateContext.length > 0) {
|
|
356
|
+
llmInstructionsObj.update_context = updateContext;
|
|
357
|
+
llmInstructionsObj.update_strategy = buildUpdateStrategy();
|
|
425
358
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
359
|
+
// REGENERATE context: existing file content gives the generation tool the
|
|
360
|
+
// endpoint URL, auth pattern, test type, and language to replicate.
|
|
361
|
+
if (regenerateRecommendations.length > 0) {
|
|
362
|
+
llmInstructionsObj.regenerate_context = regenerateRecommendations.map((rec) => {
|
|
363
|
+
const existing_content = regenerateContentMap.get(rec.testFile);
|
|
364
|
+
return { file: rec.testFile, rationale: rec.updateInstructions || rec.rationale, ...(existing_content !== undefined && { existing_content }) };
|
|
365
|
+
});
|
|
366
|
+
llmInstructionsObj.regenerate_strategy = buildRegenerateStrategy();
|
|
430
367
|
}
|
|
431
|
-
const llmInstructions =
|
|
368
|
+
const llmInstructions = JSON.stringify(llmInstructionsObj, null, 2);
|
|
432
369
|
const contentBlocks = [
|
|
433
370
|
{
|
|
434
371
|
type: "text",
|
|
@@ -445,23 +382,6 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
|
|
|
445
382
|
},
|
|
446
383
|
},
|
|
447
384
|
];
|
|
448
|
-
// ── Build ADD instructions for new endpoints ──
|
|
449
|
-
if (newEndpoints.length > 0 && llmToolCalls.length > 0) {
|
|
450
|
-
const addInstructionsObj = {
|
|
451
|
-
workflow: "add_tests_for_new_endpoints",
|
|
452
|
-
auto_proceed: true,
|
|
453
|
-
instruction: "Call each tool in tool_calls immediately and in order. Do NOT ask for confirmation. If endpointURL is a bare path, read .skyramp/workspace.yml for baseUrl and prepend it.",
|
|
454
|
-
tool_calls: llmToolCalls,
|
|
455
|
-
};
|
|
456
|
-
const addInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(addInstructionsObj, null, 2)}\n-->\n`;
|
|
457
|
-
contentBlocks.push({
|
|
458
|
-
type: "text",
|
|
459
|
-
text: addInstructions,
|
|
460
|
-
annotations: {
|
|
461
|
-
audience: ["assistant"],
|
|
462
|
-
},
|
|
463
|
-
});
|
|
464
|
-
}
|
|
465
385
|
return {
|
|
466
386
|
content: contentBlocks,
|
|
467
387
|
};
|