@skyramp/mcp 0.0.62 → 0.0.63-rc.2

package/build/tools/test-management/actionsTool.js

@@ -0,0 +1,396 @@
+import { z } from "zod";
+import { logger } from "../../utils/logger.js";
+import { StateManager, } from "../../utils/AnalysisStateManager.js";
+import { TestType } from "../../types/TestTypes.js";
+import * as fs from "fs";
+import * as path from "path";
+import { AnalyticsService } from "../../services/AnalyticsService.js";
+/**
+ * Compute a suggested new filename when an endpoint is renamed.
+ */
+export function computeRenamedTestFile(testFile, renames) {
+    const basename = path.basename(testFile);
+    let newBasename = basename;
+    for (const rename of renames) {
+        const oldSegments = rename.oldPath.split("/").filter((s) => s.length > 0);
+        const newSegments = rename.newPath.split("/").filter((s) => s.length > 0);
+        if (oldSegments.length !== newSegments.length)
+            continue;
+        const paramPattern = /^\{[^}]+\}$/;
+        for (let i = 0; i < oldSegments.length; i++) {
+            if (paramPattern.test(oldSegments[i]))
+                continue;
+            if (oldSegments[i] !== newSegments[i]) {
+                const oldName = oldSegments[i].toLowerCase();
+                const newName = newSegments[i].toLowerCase();
+                if (newBasename.toLowerCase().includes(oldName)) {
+                    const idx = newBasename.toLowerCase().indexOf(oldName);
+                    newBasename =
+                        newBasename.substring(0, idx) +
+                            newName +
+                            newBasename.substring(idx + oldName.length);
+                }
+            }
+        }
+    }
+    if (newBasename === basename)
+        return null;
+    const newFilePath = path.join(path.dirname(testFile), newBasename);
+    if (fs.existsSync(newFilePath)) {
+        logger.info(`Skipping file rename suggestion: ${newFilePath} already exists`);
+        return null;
+    }
+    return newFilePath;
+}
+/**
+ * Select test types to generate based on HTTP method.
+ */
+function selectTestTypesForEndpoint(method) {
+    switch (method.toUpperCase()) {
+        case "POST":
+        case "PUT":
+        case "PATCH":
+            return [TestType.INTEGRATION, TestType.CONTRACT];
+        case "DELETE":
+            return [TestType.INTEGRATION, TestType.SMOKE];
+        case "GET":
+        default:
+            return [TestType.CONTRACT, TestType.SMOKE];
+    }
+}
+const actionsSchema = {
+    stateFile: z
+        .string()
+        .describe("Path to state file from skyramp_analyze_test_health (or skyramp_execute_tests)"),
+};
+const TOOL_NAME = "skyramp_actions";
+export function registerActionsTool(server) {
+    server.registerTool(TOOL_NAME, {
+        description: `Execute test maintenance and generation actions — final step of the unified Test Health Analysis Flow.
+
+**PREREQUISITE:** Call \`skyramp_analyze_test_health\` (and optionally \`skyramp_execute_tests\`).
+
+**CRITICAL:** This tool MUST be called automatically after the LLM completes the drift assessment. Do NOT wait for user confirmation.
+
+**EXECUTING ACTIONS:**
+- UPDATE: Apply changes to test files (path renames, field updates) using the write tool
+- REGENERATE: Provide summary for human review / tool re-invocation
+- VERIFY: Provide summary for human review
+- ADD: Auto-generate tests for new endpoints via LLM instructions
+
+**OUTPUT:**
+Comprehensive report with executed actions, summary, and instructions for ADD recommendations
+`,
+        inputSchema: actionsSchema,
+    }, async (args) => {
+        let errorResult;
+        try {
+            logger.info("Performing Actions (unified test-management)");
+            // Load UnifiedAnalysisState from state file
+            const stateManager = StateManager.fromStatePath(args.stateFile);
+            const stateData = await stateManager.readData();
+            const fullState = await stateManager.readFullState();
+            const repositoryPath = fullState?.metadata.repositoryPath || "";
+            if (!stateData) {
+                errorResult = {
+                    content: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                error: "State file is empty or invalid",
+                                stateFile: args.stateFile,
+                            }, null, 2),
+                        },
+                    ],
+                    isError: true,
+                };
+                return errorResult;
+            }
+            const testAnalysisResults = stateData.existingTests || [];
+            const newEndpoints = stateData.newEndpoints || [];
+            // ── Build recommendations from existing tests ──
+            const recommendations = [];
+            testAnalysisResults.forEach((test) => {
+                if (test.healthScore !== undefined && test.recommendation) {
+                    recommendations.push({
+                        testFile: test.testFile,
+                        action: test.recommendation.action,
+                        priority: test.recommendation.priority,
+                        rationale: test.recommendation.rationale,
+                        estimatedWork: test.recommendation.estimatedWork,
+                        issues: test.issues || [],
+                        renamedEndpoints: test.recommendation.details?.renamedEndpoints || [],
+                    });
+                }
+            });
+            // ── Process UPDATE recommendations ──
+            const updateRecommendations = (recommendations || []).filter((rec) => rec.action === "UPDATE");
+            const updateInstructions = [];
+            const testFilesToUpdate = [];
+            for (const rec of updateRecommendations) {
+                if (!rec.testFile) {
+                    logger.warning("Recommendation missing testFile", rec);
+                    continue;
+                }
+                testFilesToUpdate.push(rec.testFile);
+                const testData = testAnalysisResults.find((t) => t.testFile === rec.testFile);
+                const driftData = testData?.drift;
+                const issues = rec.issues || [];
+                const driftChanges = driftData?.changes || [];
+                let testFileContent = "";
+                try {
+                    testFileContent = fs.readFileSync(rec.testFile, "utf-8");
+                }
+                catch (error) {
+                    logger.error(`Failed to read test file ${rec.testFile}: ${error.message}`);
+                    continue;
+                }
+                const renames = rec.renamedEndpoints || [];
+                const isRenameUpdate = renames.length > 0;
+                let instruction = `\n### ${rec.testFile}\n\n`;
+                instruction += `**Priority:** ${rec.priority} | `;
+                instruction += `**Estimated Effort:** ${rec.estimatedWork || "Small"}\n\n`;
+                instruction += `**Why Update Needed:** ${rec.rationale}\n\n`;
+                if (isRenameUpdate) {
+                    instruction += `**Endpoint Rename Detected — Path Substitution Required:**\n\n`;
+                    instruction += `| Old Path | New Path | Method |\n`;
+                    instruction += `|----------|----------|--------|\n`;
+                    for (const rename of renames) {
+                        instruction += `| \`${rename.oldPath}\` | \`${rename.newPath}\` | ${rename.method} |\n`;
+                    }
+                    instruction += `\n`;
+                    instruction += `**Action:** Find-and-replace all occurrences of the old path with the new path in this test file. `;
+                    instruction += `Do NOT change any test logic, assertions, or structure — only update the URL paths.\n\n`;
+                    const suggestedNewFile = computeRenamedTestFile(rec.testFile, renames);
+                    if (suggestedNewFile) {
+                        instruction += `**File Rename:** After updating the paths, rename this file:\n`;
+                        instruction += `- From: \`${path.basename(rec.testFile)}\`\n`;
+                        instruction += `- To: \`${path.basename(suggestedNewFile)}\`\n\n`;
+                        rec._suggestedNewFile = suggestedNewFile;
+                    }
+                }
+                if (driftData) {
+                    instruction += `**Analysis:**\n`;
+                    instruction += `- Drift Score: ${driftData.driftScore ?? "N/A"}\n`;
+                    instruction += `- Changes Detected: ${driftData.changes?.length || 0}\n`;
+                    instruction += `- Affected Files: ${driftData.affectedFiles.files || 0}\n\n`;
+                }
+                if (driftChanges.length > 0) {
+                    instruction += `**Changes Detected:**\n`;
+                    driftChanges.forEach((change) => {
+                        instruction += `**${change.type}** (Severity: ${change.severity}): ${change.description}\n`;
+                        if (change.details) {
+                            instruction += `   └─ ${change.details}\n`;
+                        }
+                        if (change.file) {
+                            instruction += `   └─ In: \`${change.file}\`\n`;
+                        }
+                    });
+                    instruction += `\n`;
+                }
+                if (issues.length > 0) {
+                    instruction += `**Issues Found:**\n`;
+                    issues.forEach((issue) => {
+                        instruction += `**${issue.type}** (Severity: ${issue.severity}): ${issue.description}\n`;
+                        if (issue.details) {
+                            instruction += `   └─ ${issue.details}\n`;
+                        }
+                    });
+                    instruction += `\n`;
+                }
+                instruction += `**Test File Content:**\n\`\`\`\n${testFileContent}\n\`\`\`\n\n`;
+                updateInstructions.push(instruction);
+            }
+            // ── Build ADD section for new endpoints ──
+            const wsBaseUrl = stateData.repositoryAnalysis?.wsBaseUrl || "";
+            const wsSchemaPath = stateData.repositoryAnalysis?.wsSchemaPath || "";
+            const primaryLanguage = stateData.repositoryAnalysis?.projectMeta?.primaryLanguage ||
+                "python";
+            const primaryFramework = stateData.repositoryAnalysis?.projectMeta?.primaryFramework ||
+                "pytest";
+            // Determine output directory from workspace config or repo path
+            const outputDir = repositoryPath
+                ? path.join(repositoryPath, "tests", "skyramp")
+                : "./tests/skyramp";
+            const addSummaryLines = [];
+            const llmToolCalls = [];
+            for (const ep of newEndpoints) {
+                const testTypes = selectTestTypesForEndpoint(ep.method);
+                const endpointURL = wsBaseUrl
+                    ? wsBaseUrl.replace(/\/$/, "") + ep.path
+                    : ep.path;
+                addSummaryLines.push(`- ${ep.method} ${ep.path} → ${testTypes.join(", ")} tests`);
+                for (const testType of testTypes) {
+                    let toolName = "";
+                    switch (testType) {
+                        case TestType.CONTRACT:
+                            toolName = "skyramp_contract_test_generation";
+                            break;
+                        case TestType.INTEGRATION:
+                            toolName = "skyramp_integration_test_generation";
+                            break;
+                        case TestType.SMOKE:
+                            toolName = "skyramp_smoke_test_generation";
+                            break;
+                        default:
+                            toolName = "skyramp_contract_test_generation";
+                    }
+                    llmToolCalls.push({
+                        tool: toolName,
+                        params: {
+                            endpointURL,
+                            method: ep.method,
+                            language: primaryLanguage,
+                            framework: primaryFramework,
+                            outputDir,
+                            ...(wsSchemaPath ? { apiSchema: wsSchemaPath } : {}),
+                        },
+                        endpoint: `${ep.method} ${ep.path}`,
+                        testType: testType,
+                    });
+                }
+            }
+            // ── Build response text ──
+            let responseText = `# Test Actions Report\n\n`;
+            if (updateRecommendations.length > 0) {
+                responseText += `## Tests Requiring Updates (${updateRecommendations.length})\n\n`;
+                testFilesToUpdate.forEach((file, idx) => {
+                    responseText += `${idx + 1}. \`${file}\`\n`;
+                });
+                responseText += `\n---\n`;
+                responseText += updateInstructions.join("\n---\n");
+            }
+            if (newEndpoints.length > 0) {
+                responseText += `\n## New Endpoint Tests to Generate (${newEndpoints.length} endpoints)\n\n`;
+                addSummaryLines.forEach((line) => {
+                    responseText += `${line}\n`;
+                });
+                responseText += `\nThe following tests will be generated automatically.\n`;
+            }
+            if (updateRecommendations.length === 0 && newEndpoints.length === 0) {
+                const otherRecs = recommendations.filter((rec) => rec.action !== "UPDATE");
+                if (otherRecs.length > 0) {
+                    responseText += `## Recommendations (${otherRecs.length})\n\n`;
+                    otherRecs.forEach((rec) => {
+                        responseText += `- **${rec.testFile}** — Action: ${rec.action}, Priority: ${rec.priority}\n`;
+                        responseText += `  ${rec.rationale}\n`;
+                    });
+                }
+                else {
+                    responseText += `No action required. All existing tests appear healthy.\n`;
+                }
+            }
+            responseText += `\n\n## Next Steps\n\n`;
+            responseText += `The AI assistant will:\n`;
+            let stepNumber = 1;
+            if (updateRecommendations.length > 0) {
+                responseText += `${stepNumber++}. Review the changes and issues for each test\n`;
+                responseText += `${stepNumber++}. Update test files to fix compatibility issues\n`;
+                responseText += `${stepNumber++}. Preserve original test logic and structure\n`;
+                responseText += `${stepNumber++}. Show you the changes made\n`;
+            }
+            if (newEndpoints.length > 0) {
+                responseText += `${stepNumber++}. Generate new tests for new endpoints\n`;
+            }
+            responseText += `\n**This tool is currently in Early Preview stage. Please verify the results.**\n`;
+            // ── Build LLM instructions for UPDATE ──
+            const allRenames = [];
+            for (const rec of updateRecommendations) {
+                if (rec.renamedEndpoints && rec.renamedEndpoints.length > 0) {
+                    allRenames.push(...rec.renamedEndpoints);
+                }
+            }
+            const uniqueRenames = allRenames.filter((r, i, arr) => arr.findIndex((x) => x.oldPath === r.oldPath &&
+                x.newPath === r.newPath &&
+                x.method === r.method) === i);
+            const llmInstructionsObj = {
+                workflow: "test_maintenance",
+                action: "execute_updates",
+                auto_proceed: true,
+                files_to_update: testFilesToUpdate,
+                update_count: updateRecommendations.length,
+            };
+            if (uniqueRenames.length > 0) {
+                llmInstructionsObj.endpoint_renames = uniqueRenames;
+                llmInstructionsObj.rename_strategy =
+                    "For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.";
+                const fileRenames = [];
+                for (const rec of updateRecommendations) {
+                    if (rec._suggestedNewFile) {
+                        fileRenames.push({ from: rec.testFile, to: rec._suggestedNewFile });
+                    }
+                }
+                if (fileRenames.length > 0) {
+                    llmInstructionsObj.file_renames = fileRenames;
+                    llmInstructionsObj.file_rename_strategy =
+                        "After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.";
+                }
+            }
+            const llmInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(llmInstructionsObj, null, 2)}\n-->\n`;
+            const contentBlocks = [
+                {
+                    type: "text",
+                    text: responseText,
+                    annotations: {
+                        audience: ["user"],
+                    },
+                },
+                {
+                    type: "text",
+                    text: llmInstructions,
+                    annotations: {
+                        audience: ["assistant"],
+                    },
+                },
+            ];
+            // ── Build ADD instructions for new endpoints ──
+            if (newEndpoints.length > 0 && llmToolCalls.length > 0) {
+                const addInstructionsObj = {
+                    workflow: "add_tests_for_new_endpoints",
+                    auto_proceed: true,
+                    instruction: "Call each tool in tool_calls immediately and in order. Do NOT ask for confirmation. If endpointURL is a bare path, read .skyramp/workspace.yml for baseUrl and prepend it.",
+                    tool_calls: llmToolCalls,
+                };
+                const addInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(addInstructionsObj, null, 2)}\n-->\n`;
+                contentBlocks.push({
+                    type: "text",
+                    text: addInstructions,
+                    annotations: {
+                        audience: ["assistant"],
+                    },
+                });
+            }
+            return {
+                content: contentBlocks,
+            };
+        }
+        catch (error) {
+            logger.error(`Actions tool failed: ${error.message}`, error);
+            errorResult = {
+                content: [
+                    {
+                        type: "text",
+                        text: JSON.stringify({
+                            error: error.message,
+                        }, null, 2),
+                    },
+                ],
+                isError: true,
+            };
+            return errorResult;
+        }
+        finally {
+            let repositoryPath = "";
+            try {
+                const fullState = await StateManager.fromStatePath(args.stateFile).readFullState();
+                repositoryPath = fullState?.metadata.repositoryPath || "";
+            }
+            catch (analyticsError) {
+                logger.error(`Failed to read state for analytics in actions tool: ${analyticsError?.message ?? analyticsError}`, analyticsError);
+            }
+            AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
+                repositoryPath,
+            });
+        }
+    });
+}