@skyramp/mcp 0.1.8 → 0.2.0-rc.2

package/build/tools/test-management/actionsTool.js

@@ -1,11 +1,12 @@
 import { z } from "zod";
 import { logger } from "../../utils/logger.js";
 import { StateManager, } from "../../utils/AnalysisStateManager.js";
-import { TestSource } from "../../types/TestAnalysis.js";
+import { TestSource, DriftAction, RecommendationPriority, EstimatedWork } from "../../types/TestAnalysis.js";
 import { TestType } from "../../types/TestTypes.js";
 import * as fs from "fs";
 import * as path from "path";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
+import { toolError } from "../../utils/utils.js";
 /**
  * Compute a suggested new filename when an endpoint is renamed.
  */
@@ -59,25 +60,74 @@ function selectTestTypesForEndpoint(method) {
             return [TestType.CONTRACT, TestType.SMOKE];
     }
 }
+const recommendationSchema = z.object({
+    testFile: z
+        .string()
+        .refine((p) => path.isAbsolute(p), { message: "testFile must be an absolute path" })
+        .describe("Absolute path to the test file — use the path as reported by skyramp_analyze_changes or skyramp_analyze_test_health"),
+    action: z
+        .nativeEnum(DriftAction)
+        .describe("Drift action assigned by the LLM health assessment"),
+    priority: z
+        .nativeEnum(RecommendationPriority)
+        .optional()
+        .describe("Update priority"),
+    rationale: z
+        .string()
+        .optional()
+        .describe("1-2 sentence explanation of why this action is needed"),
+    estimatedWork: z
+        .nativeEnum(EstimatedWork)
+        .optional()
+        .describe("Estimated effort to apply the update"),
+    updateInstructions: z
+        .string()
+        .optional()
+        .describe("Free-form summary of what this test must change — written for the downstream LLM that will edit the file. " +
+        "Specificity prevents incomplete or mismatched edits. Include diff-specific details: " +
+        "new response fields to assert, constraint details (types, ranges, defaults), " +
+        "auth changes, new request params, removed fields, or other drift-related changes. " +
+        "Example: 'Added stock_count: int (ge=0, default=0) to ProductBase. " +
+        "Test hits GET /products — assert stock_count is present and non-negative.'"),
+    renamedEndpoints: z
+        .array(z.object({
+        oldPath: z.string().describe("Previous endpoint path"),
+        newPath: z.string().describe("New endpoint path after rename"),
+        method: z.string().describe("HTTP method, e.g. GET"),
+    }))
+        .optional()
+        .describe("Renamed endpoints — supply this array when action is UPDATE and the endpoint path has changed. Omit if action is not UPDATE."),
+});
 const actionsSchema = {
     stateFile: z
         .string()
-        .describe("Path to state file from skyramp_analyze_test_health"),
+        .refine((p) => path.isAbsolute(p), { message: "stateFile must be an absolute path" })
+        .describe("Path to state file from skyramp_analyze_changes"),
+    recommendations: z
+        .array(recommendationSchema)
+        .optional()
+        .describe("LLM drift assessment — one entry per test assessed. Required for UPDATE instructions to be emitted; omitting results in no maintenance actions."),
 };
 const TOOL_NAME = "skyramp_actions";
 export function registerActionsTool(server) {
     server.registerTool(TOOL_NAME, {
+        annotations: {
+            destructiveHint: true,
+            readOnlyHint: false,
+            idempotentHint: false,
+            openWorldHint: true,
+        },
         description: `Execute test maintenance and generation actions — final step of the unified Test Health Analysis Flow.
 
-**PREREQUISITE:** Call \`skyramp_analyze_test_health\`.
+**PREREQUISITE:** Call \`skyramp_analyze_changes\` (produces the stateFile), then \`skyramp_analyze_test_health\` (runs the drift assessment). This tool reads the stateFile from \`skyramp_analyze_changes\`.
 
-**CRITICAL:** This tool MUST be called automatically after the LLM completes the drift assessment. Do NOT wait for user confirmation.
+Call this tool after completing the drift assessment. It executes maintenance actions automatically from the stateFile — no user confirmation required.
 
 **EXECUTING ACTIONS:**
-- UPDATE: Apply changes to test files (path renames, field updates) using the write tool
-- REGENERATE: Provide summary for human review / tool re-invocation
-- VERIFY: Provide summary for human review
-- ADD: Auto-generate tests for new endpoints via LLM instructions
+- UPDATE: Tests with drift — emits targeted per-file edit instructions driven by updateInstructions and renamedEndpoints
+- REGENERATE: Emits file-level summary; follow up by calling the appropriate generation tool (e.g. skyramp_integration_test_generation) with the same filename to overwrite
+- VERIFY: Emits file-level summary for human review — no automated edits
+- ADD: Auto-generates tests for new endpoints via LLM instructions
 
 **OUTPUT:**
 Comprehensive report with executed actions, summary, and instructions for ADD recommendations
@@ -93,43 +143,78 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
             const fullState = await stateManager.readFullState();
             const repositoryPath = fullState?.metadata.repositoryPath || "";
             if (!stateData) {
-                errorResult = {
-                    content: [
-                        {
-                            type: "text",
-                            text: JSON.stringify({
-                                error: "State file is empty or invalid",
-                                stateFile: args.stateFile,
-                            }, null, 2),
-                        },
-                    ],
-                    isError: true,
-                };
+                errorResult = toolError(`State file is empty or invalid: ${args.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
                 return errorResult;
             }
             // External tests must not be candidates for UPDATE/REGENERATE/DELETE actions.
             // Default source to Skyramp for backwards compat with state files created before the source field existed.
             const testAnalysisResults = (stateData.existingTests || []).filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External);
             const newEndpoints = stateData.newEndpoints || [];
-            // ── Build recommendations from existing tests ──
+            // Resolve repo root for path normalization and security checks.
+            const repoRoot = repositoryPath ? path.resolve(repositoryPath) : "";
+            // Set of non-external (Skyramp-generated) test file paths — the only files
+            // that may receive UPDATE/REGENERATE/DELETE actions. Using the allowlist rather
+            // than a blocklist catches both external tests AND hallucinated paths the LLM
+            // may supply that are not present in the scanned catalog at all.
+            const skyrampTestFiles = new Set(testAnalysisResults.map((t) => t.testFile));
+            // ── Build recommendations from LLM-supplied drift assessment ──
+            // The LLM performs the drift assessment in context after skyramp_analyze_test_health
+            // and passes results here directly — analyzeTestHealthTool never writes assessment
+            // data back to the state file.
             const recommendations = [];
-            testAnalysisResults.forEach((test) => {
-                if (test.healthScore !== undefined && test.recommendation) {
-                    recommendations.push({
-                        testFile: test.testFile,
-                        action: test.recommendation.action,
-                        priority: test.recommendation.priority,
-                        rationale: test.recommendation.rationale,
-                        estimatedWork: test.recommendation.estimatedWork,
-                        issues: test.issues || [],
-                        renamedEndpoints: test.recommendation.details?.renamedEndpoints || [],
-                    });
+            (args.recommendations ?? []).forEach((rec) => {
+                // Schema requires absolute paths; resolve any relative paths defensively
+                // against repoRoot in case the LLM sends a relative path despite the schema.
+                const resolvedFile = path.isAbsolute(rec.testFile)
+                    ? rec.testFile
+                    : repoRoot
+                        ? path.resolve(repoRoot, rec.testFile)
+                        : rec.testFile;
+                // Reject files outside the repo root (path-traversal guard).
+                if (repoRoot && !resolvedFile.startsWith(repoRoot + path.sep) && resolvedFile !== repoRoot) {
+                    logger.warning(`Skipping recommendation for path outside repo root: ${rec.testFile}`);
+                    return;
                 }
+                // Guard: only Skyramp-generated tests may receive UPDATE/REGENERATE/DELETE.
+                // Using an allowlist (skyrampTestFiles) rather than a blocklist catches both
+                // external tests and hallucinated paths the LLM may supply that are not in
+                // the scanned catalog. IGNORE/VERIFY are informational and pass through.
+                const isActionable = [DriftAction.Update, DriftAction.Regenerate, DriftAction.Delete].includes(rec.action);
+                if (isActionable && !skyrampTestFiles.has(resolvedFile) && !skyrampTestFiles.has(rec.testFile)) {
+                    logger.warning(`Skipping ${rec.action} for non-Skyramp or unknown test: ${rec.testFile}`);
+                    return;
+                }
+                recommendations.push({
+                    testFile: resolvedFile,
+                    action: rec.action,
+                    priority: rec.priority ?? RecommendationPriority.Medium,
+                    rationale: rec.rationale ?? "",
+                    estimatedWork: rec.estimatedWork ?? EstimatedWork.Small,
+                    updateInstructions: rec.updateInstructions ?? "",
+                    renamedEndpoints: rec.renamedEndpoints ?? [],
+                });
             });
             // ── Process UPDATE recommendations ──
-            const updateRecommendations = (recommendations || []).filter((rec) => rec.action === "UPDATE");
-            const updateInstructions = [];
+            // Deduplicate by testFile — keep the highest-priority entry when the LLM
+            // repeats a file. Priority order: high > medium > low.
+            const priorityRank = {
+                [RecommendationPriority.High]: 2,
+                [RecommendationPriority.Medium]: 1,
+                [RecommendationPriority.Low]: 0,
+            };
+            const updateByFile = new Map();
+            for (const rec of recommendations) {
+                if (rec.action !== DriftAction.Update)
+                    continue;
+                const existing = updateByFile.get(rec.testFile);
+                if (!existing || priorityRank[rec.priority] > priorityRank[existing.priority]) {
+                    updateByFile.set(rec.testFile, rec);
+                }
+            }
+            const updateRecommendations = Array.from(updateByFile.values());
+            const fileInstructions = [];
             const testFilesToUpdate = [];
+            const testFileContentMap = new Map();
             for (const rec of updateRecommendations) {
                 if (!rec.testFile) {
                     logger.warning("Recommendation missing testFile", rec);
@@ -138,11 +223,11 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                 testFilesToUpdate.push(rec.testFile);
                 const testData = testAnalysisResults.find((t) => t.testFile === rec.testFile);
                 const driftData = testData?.drift;
-                const issues = rec.issues || [];
                 const driftChanges = driftData?.changes || [];
                 let testFileContent = "";
                 try {
                     testFileContent = fs.readFileSync(rec.testFile, "utf-8");
+                    testFileContentMap.set(rec.testFile, testFileContent);
                 }
                 catch (error) {
                     logger.error(`Failed to read test file ${rec.testFile}: ${error.message}`);
@@ -152,7 +237,7 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                 const isRenameUpdate = renames.length > 0;
                 let instruction = `\n### ${rec.testFile}\n\n`;
                 instruction += `**Priority:** ${rec.priority} | `;
-                instruction += `**Estimated Effort:** ${rec.estimatedWork || "Small"}\n\n`;
+                instruction += `**Estimated Effort:** ${rec.estimatedWork || EstimatedWork.Small}\n\n`;
                 instruction += `**Why Update Needed:** ${rec.rationale}\n\n`;
                 if (isRenameUpdate) {
                     instruction += `**Endpoint Rename Detected — Path Substitution Required:**\n\n`;
@@ -172,11 +257,21 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                         rec._suggestedNewFile = suggestedNewFile;
                     }
                 }
+                const recUpdateInstructions = rec.updateInstructions ?? "";
+                if (recUpdateInstructions) {
+                    instruction += `**What to change:**\n\n${recUpdateInstructions}\n\n`;
+                    instruction += `Match the assertion style already used in the file. `;
+                    instruction += `Preserve all existing test logic — only add or adjust what is described above.\n\n`;
+                }
+                else if (!isRenameUpdate) {
+                    instruction += `**Action:** Update this test file per the rationale above. `;
+                    instruction += `Match the assertion style already used in the file. `;
+                    instruction += `Preserve all existing test logic — only add or adjust the minimum required assertions.\n\n`;
+                }
                 if (driftData) {
                     instruction += `**Analysis:**\n`;
-                    instruction += `- Drift Score: ${driftData.driftScore ?? "N/A"}\n`;
                     instruction += `- Changes Detected: ${driftData.changes?.length || 0}\n`;
-                    instruction += `- Affected Files: ${driftData.affectedFiles.files || 0}\n\n`;
+                    instruction += `- Affected Files: ${driftData.affectedFiles.files?.length || 0}\n\n`;
                 }
                 if (driftChanges.length > 0) {
                     instruction += `**Changes Detected:**\n`;
@@ -191,18 +286,8 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                     });
                     instruction += `\n`;
                 }
-                if (issues.length > 0) {
-                    instruction += `**Issues Found:**\n`;
-                    issues.forEach((issue) => {
-                        instruction += `**${issue.type}** (Severity: ${issue.severity}): ${issue.description}\n`;
-                        if (issue.details) {
-                            instruction += `   └─ ${issue.details}\n`;
-                        }
-                    });
-                    instruction += `\n`;
-                }
-                instruction += `**Test File Content:**\n\`\`\`\n${testFileContent}\n\`\`\`\n\n`;
-                updateInstructions.push(instruction);
+                // File content is provided in LLM_INSTRUCTIONS.update_context.current_content — omit here to avoid duplication.
+                fileInstructions.push(instruction);
             }
             // ── Build ADD section for new endpoints ──
             const wsBaseUrl = stateData.repositoryAnalysis?.wsBaseUrl || "";
@@ -261,7 +346,7 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                     responseText += `${idx + 1}. \`${file}\`\n`;
                 });
                 responseText += `\n---\n`;
-                responseText += updateInstructions.join("\n---\n");
+                responseText += fileInstructions.join("\n---\n");
             }
             if (newEndpoints.length > 0) {
                 responseText += `\n## New Endpoint Tests to Generate (${newEndpoints.length} endpoints)\n\n`;
@@ -271,7 +356,7 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                 responseText += `\nThe following tests will be generated automatically.\n`;
             }
             if (updateRecommendations.length === 0 && newEndpoints.length === 0) {
-                const otherRecs = recommendations.filter((rec) => rec.action !== "UPDATE");
+                const otherRecs = recommendations.filter((rec) => rec.action !== DriftAction.Update);
                 if (otherRecs.length > 0) {
                     responseText += `## Recommendations (${otherRecs.length})\n\n`;
                     otherRecs.forEach((rec) => {
@@ -329,6 +414,20 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
                         "After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.";
                 }
             }
+            // Update context: per-file guidance + current content for the downstream LLM.
+            // Including file content avoids re-reads on each Edit turn, reducing token usage.
+            const updateInstructionsFiles = [];
+            for (const rec of updateRecommendations) {
+                if (rec.updateInstructions) {
+                    const current_content = testFileContentMap.get(rec.testFile);
+                    updateInstructionsFiles.push({ file: rec.testFile, context: rec.updateInstructions, ...(current_content !== undefined && { current_content }) });
+                }
+            }
+            if (updateInstructionsFiles.length > 0) {
+                llmInstructionsObj.update_context = updateInstructionsFiles;
+                llmInstructionsObj.update_strategy =
+                    "For each file in update_context, apply the changes described in context to the provided current_content. Write the result using the Edit tool. Do NOT re-read the file first. Match the assertion style already used in the file. Preserve all existing test logic. After applying all edits, call skyramp_enhance_assertions with each updated file path to strengthen the assertions.";
+            }
             const llmInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(llmInstructionsObj, null, 2)}\n-->\n`;
             const contentBlocks = [
                 {
@@ -369,17 +468,7 @@ Comprehensive report with executed actions, summary, and instructions for ADD re
         }
         catch (error) {
             logger.error(`Actions tool failed: ${error.message}`, error);
-            errorResult = {
-                content: [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            error: error.message,
-                        }, null, 2),
-                    },
-                ],
-                isError: true,
-            };
+            errorResult = toolError(`Actions tool failed: ${error.message}`);
             return errorResult;
         }
         finally {