@skyramp/mcp 0.0.44 → 0.0.46

package/build/index.js

@@ -2,6 +2,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { registerStartTraceCollectionPrompt } from "./prompts/startTraceCollectionPrompts.js";
+import { registerTestHealthPrompt } from "./prompts/testHealthPrompt.js";
 import { registerTraceTool } from "./tools/trace/startTraceCollectionTool.js";
 import { registerTraceStopTool } from "./tools/trace/stopTraceCollectionTool.js";
 import { registerExecuteSkyrampTestTool } from "./tools/executeSkyrampTestTool.js";
@@ -23,9 +24,17 @@ import { registerRecommendTestsTool } from "./tools/test-recommendation/recommen
 import { registerModularizationTool } from "./tools/code-refactor/modularizationTool.js";
 import { registerCodeReuseTool } from "./tools/code-refactor/codeReuseTool.js";
 import { registerScenarioTestTool } from "./tools/generate-tests/generateScenarioRestTool.js";
+import { registerDiscoverTestsTool } from "./tools/test-maintenance/discoverTestsTool.js";
+import { registerAnalyzeTestDriftTool } from "./tools/test-maintenance/analyzeTestDriftTool.js";
+import { registerExecuteBatchTestsTool } from "./tools/test-maintenance/executeBatchTestsTool.js";
+import { registerCalculateHealthScoresTool } from "./tools/test-maintenance/calculateHealthScoresTool.js";
+import { registerActionsTool } from "./tools/test-maintenance/actionsTool.js";
+import { registerStateCleanupTool } from "./tools/test-maintenance/stateCleanupTool.js";
+import { AnalyticsService } from "./services/AnalyticsService.js";
 const server = new McpServer({
     name: "Skyramp MCP Server",
     version: "1.0.0",
+}, {
     capabilities: {
         tools: {
             listChanged: true,
@@ -40,6 +49,7 @@ logger.info("Starting prompt registration process");
 const prompts = [
     registerTestGenerationPrompt,
     registerStartTraceCollectionPrompt,
+    registerTestHealthPrompt,
 ];
 prompts.forEach((registerPrompt) => registerPrompt(server));
 logger.info("All prompts registered successfully");
@@ -62,17 +72,17 @@ const codeQualityTools = [
     registerCodeReuseTool,
 ];
 codeQualityTools.forEach((registerTool) => registerTool(server));
-// Register internal/test recommendation tools (controlled by environment variable)
-const enableInternalTools = process.env.SKYRAMP_ENABLE_INTERNAL_TOOLS === "true";
-if (enableInternalTools) {
-    logger.info("Internal tools enabled - registering test recommendation tools");
-    registerAnalyzeRepositoryTool(server);
-    registerMapTestsTool(server);
-    registerRecommendTestsTool(server);
-}
-else {
-    logger.info("Internal tools disabled. Set SKYRAMP_ENABLE_INTERNAL_TOOLS=true to enable.");
-}
+// Register test recommendation tools
+registerAnalyzeRepositoryTool(server);
+registerMapTestsTool(server);
+registerRecommendTestsTool(server);
+// Register test maintenance tools
+registerDiscoverTestsTool(server);
+registerAnalyzeTestDriftTool(server);
+registerExecuteBatchTestsTool(server);
+registerCalculateHealthScoresTool(server);
+registerActionsTool(server);
+registerStateCleanupTool(server);
 // Register other Skyramp tools
 const infrastructureTools = [
     registerLoginTool,
@@ -82,6 +92,35 @@ const infrastructureTools = [
     registerTraceStopTool,
 ];
 infrastructureTools.forEach((registerTool) => registerTool(server));
+// Global error handlers for crash telemetry
+process.on("uncaughtException", async (error) => {
+    logger.critical("Uncaught exception - MCP server crashing", {
+        error: error.message,
+        stack: error.stack,
+    });
+    try {
+        await AnalyticsService.pushServerCrashEvent("uncaughtException", error.message, error.stack);
+    }
+    catch (telemetryError) {
+        logger.error("Failed to send crash telemetry", { telemetryError });
+    }
+    process.exit(1);
+});
+process.on("unhandledRejection", async (reason, promise) => {
+    const errorMessage = reason instanceof Error ? reason.message : String(reason);
+    const errorStack = reason instanceof Error ? reason.stack : undefined;
+    logger.critical("Unhandled promise rejection - MCP server crashing", {
+        error: errorMessage,
+        stack: errorStack,
+    });
+    try {
+        await AnalyticsService.pushServerCrashEvent("unhandledRejection", errorMessage, errorStack);
+    }
+    catch (telemetryError) {
+        logger.error("Failed to send crash telemetry", { telemetryError });
+    }
+    process.exit(1);
+});
 // Start MCP server
 async function main() {
     const transport = new StdioServerTransport();
@@ -102,10 +141,16 @@ async function main() {
         process.exit(0);
     });
 }
-main().catch((error) => {
+main().catch(async (error) => {
     logger.critical("Fatal error in main()", {
         error: error.message,
         stack: error.stack,
     });
+    try {
+        await AnalyticsService.pushServerCrashEvent("mainFatalError", error.message, error.stack);
+    }
+    catch (telemetryError) {
+        logger.error("Failed to send crash telemetry", { telemetryError });
+    }
     process.exit(1);
 });

package/build/prompts/code-reuse.js

@@ -144,7 +144,7 @@ NOT A HELPER FUNCTION (do not extract):
 4. Found no other test files? → SKIP to Step 6, NO utils file
 5. Other test files identical to current? → SKIP to Step 6, NO utils file
 
-**MANDATORY**: After code reuse is complete, proceed to modularization by calling skyramp_modularization tool
+**MANDATORY**: After code reuse is complete, proceed to modularization by calling skyramp_modularization tool ONLY for UI, E2E, INTEGRATION or LOAD test types generated from traces.
 
 SUMMARIZE THE CODE REUSE PROCESS AND THE RESULTS.
 `;

package/build/prompts/driftAnalysisPrompt.js

@@ -0,0 +1,159 @@
+/**
+ * Build optimized drift analysis prompt - compares TEST expectations vs CURRENT CODE
+ */
+export function buildDriftAnalysisPrompt(testType, testFile, fileChanges, apiSchemaChanges, uiComponentChanges) {
+    const isUiTest = testType === "ui" || testType === "e2e";
+    return `# Test Drift Analysis
+
+You are analyzing drift for a **${testType.toUpperCase()}** test to find breaking changes.
+
+## Test File
+${testFile}
+
+## Changed Files (${fileChanges.length})
+${fileChanges.length > 0
+        ? fileChanges
+            .map((c) => `- ${c.file} (+${c.linesAdded}/-${c.linesRemoved})`)
+            .join("\n")
+        : "No changes detected"}
+
+${buildSchemaChangesSection(apiSchemaChanges, uiComponentChanges)}
+
+## Your Task
+
+${isUiTest ? buildUiAnalysisInstructions() : buildApiAnalysisInstructions()}
+
+## Scoring Guide
+- **85-100**: Critical breaking changes (type mismatch, missing endpoint/selector, auth change)
+- **60-84**: High impact (path changed, required field added, status code change)
+- **40-59**: Medium impact (optional field removed, component renamed)
+- **20-39**: Low impact (new optional fields, minor refactoring)
+- **0-19**: Minimal/no impact
+
+## Output Format (JSON)
+\`\`\`json
+{
+  "driftScore": 0-100,
+  "reasoning": "Concise explanation of key mismatches",
+  "breakingChanges": ["specific mismatch 1", "specific mismatch 2"],
+  "apiDependencyChanges": [{"file": "", "linesAdded": 0, "linesRemoved": 0, "updates": ""}],
+  "uiDependencyChanges": [{"file": "", "linesAdded": 0, "linesRemoved": 0, "updates": ""}],
+  "fieldTypeChanges": [{"file": "", "field": "", "testExpects": "", "codeHas": ""}],
+  "endpointChanges": [{"type": "removed|modified", "path": "", "method": "", "issue": ""}],
+  "selectorChanges": [{"file": "", "testExpects": "", "codeHas": ""}]
+}
+\`\`\``;
+}
+function buildSchemaChangesSection(apiSchemaChanges, uiComponentChanges) {
+    let section = "";
+    if (apiSchemaChanges) {
+        const removed = apiSchemaChanges.endpointsRemoved;
+        const modified = apiSchemaChanges.endpointsModified;
+        const authChanged = apiSchemaChanges.authenticationChanged;
+        if (removed.length > 0 || modified.length > 0 || authChanged) {
+            section += "\n## API Schema Changes\n";
+            if (removed.length > 0) {
+                section += `- **Removed**: ${removed
+                    .map((e) => `${e.method} ${e.path}`)
+                    .join(", ")}\n`;
+            }
+            if (modified.length > 0) {
+                section += `- **Modified**: ${modified
+                    .map((e) => `${e.method} ${e.path}`)
+                    .join(", ")}\n`;
+            }
+            if (authChanged) {
+                section += "- **Authentication**: Changed\n";
+            }
+        }
+    }
+    if (uiComponentChanges) {
+        const hasChanges = uiComponentChanges.componentFiles.length > 0 ||
+            uiComponentChanges.routeFiles.length > 0 ||
+            uiComponentChanges.hasSelectorsChanges ||
+            uiComponentChanges.hasStylingChanges;
+        if (hasChanges) {
+            section += "\n## UI Component Changes\n";
+            if (uiComponentChanges.componentFiles.length > 0) {
+                section += `- **Components**: ${uiComponentChanges.componentFiles.length} files\n`;
+            }
+            if (uiComponentChanges.routeFiles.length > 0) {
+                section += `- **Routes**: ${uiComponentChanges.routeFiles.length} files\n`;
+            }
+            if (uiComponentChanges.hasSelectorsChanges) {
+                section += "- **Selectors**: Likely changed\n";
+            }
+            if (uiComponentChanges.hasStylingChanges) {
+                section += "- **Styles**: Changed\n";
+            }
+        }
+    }
+    return section;
+}
+function buildUiAnalysisInstructions() {
+    return `### Analysis Steps
+
+1. **Extract Test Selectors**
+   - Find: \`getByTestId()\`, \`querySelector()\`, \`getElementById()\`, \`getByRole()\`
+   - Record: selector type and exact value
+
+2. **Analyze Git Diffs**
+   - Look for changes in: \`data-testid\`, \`className\`, \`id\`, \`role\`
+   - Check: Lines with "-" (removed) vs "+" (added)
+   - Pattern: \`- data-testid="old-value"\` → \`+ data-testid="new-value"\`
+
+3. **Compare & Detect Breaks**
+   - ❌ **CRITICAL**: Test selector not found in current code
+   - ❌ **CRITICAL**: Selector value changed (\`submit-button\` → \`submit-btn\`)
+   - ❌ **CRITICAL**: Element type changed (\`<button>\` → \`<div>\`)
+   - ❌ **HIGH**: Route changed (\`/products\` → \`/items\`)
+   - ❌ **HIGH**: Component renamed/removed
+
+4. **Categorize Changes**
+   - **uiDependencyChanges**: Components, pages, templates (.tsx, .jsx, .vue)
+   - **apiDependencyChanges**: API calls, services (if E2E test)
+
+### Key Patterns
+- Selector mismatch → Score 85+
+- Route change → Score 75+
+- Styling only → Score 20-39`;
+}
+function buildApiAnalysisInstructions() {
+    return `### Analysis Steps
+
+1. **Extract Test Request Data**
+   - Find: Request bodies, payloads, field names
+   - Infer types from values:
+     * \`"text"\` = string
+     * \`123\` = integer
+     * \`12.5\` = decimal/float
+     * \`true\`/\`false\` = boolean
+     * \`[]\` = array
+     * \`null\` = optional
+
+2. **Analyze Git Diffs**
+   - Look for field type changes: \`- field: type_A\` → \`+ field: type_B\`
+   - Look for endpoint changes: \`- @app.post("/old")\` → \`+ @app.post("/new")\`
+   - Look for auth changes: \`- def endpoint()\` → \`+ @require_auth def endpoint()\`
+   - Check status codes: \`- return 200\` → \`+ return 201\`
+
+3. **Compare & Detect Breaks**
+   - ❌ **CRITICAL**: Field type changed (\`int\` → \`string\`)
+   - ❌ **CRITICAL**: Test value type ≠ code field type
+   - ❌ **CRITICAL**: Endpoint removed or path changed
+   - ❌ **HIGH**: Required field added (test doesn't send it)
+   - ❌ **HIGH**: Auth added/removed
+   - ❌ **HIGH**: Status code changed
+   - ⚠️ **MEDIUM**: Response field removed (test expects it)
+
+4. **Categorize Changes**
+   - **apiDependencyChanges**: Routes, controllers, models, schemas
+   - **uiDependencyChanges**: Only if E2E test with UI components
+
+### Key Patterns
+- Type mismatch in diff → Score 85+
+- Endpoint removed → Score 85+
+- Auth change → Score 75+
+- New required field → Score 75+
+- Response structure change → Score 60-74`;
+}

package/build/prompts/modularization/ui-test-modularization.js

@@ -137,6 +137,8 @@ Use the EXACT same values from the original test when calling helpers.
 ## STEP 5: VERIFY - CRITICAL CHECKS
 
 **BUGS TO AVOID:**
+- [ ] **NO TYPE ANNOTATIONS FOR \`page\` IN FUNCTION SIGNATURES** - Parameters must be untyped (e.g., \`page\`, not \`page: Page\`)
+- [ ] **NO RETURN TYPES** - Do not add \`: Promise<void>\` or any return type annotations
 - [ ] **ALL field mappings verified** - EVERY \`.fill()\` uses the correct parameter. Check field gets quantity parameter, NOT hardcoded 
 - [ ] **No duplicate helpers** - Consolidate similar helpers (e.g., one createOrder vs createSimpleOrder + createOrderWithItems)
 - [ ] **No nested helpers** - Helpers don't call other helpers unnecessarily

package/build/prompts/testGenerationPrompt.js

@@ -15,8 +15,8 @@ export function registerTestGenerationPrompt(mcpServer) {
 - Provide clear execution instructions for generated tests
 - Answer questions about Skyramp capabilities and testing concepts
 - ALWAYS SHOW STEPS TO GENERATE TEST USING MCP TOOLS AND NEVER SHOW THE CLI COMMANDS.
-- CRITICAL: UI, INTEGRATION, E2E TESTS MUST BE MODULARIZED USING skyramp_modularization TOOL. ALWAYS ADD A TASK TO MODULARIZE THE TEST USING skyramp_modularization TOOL AFTER GENERATING THESE(UI, INTEGRATION, E2E) TESTS.
-- **CRITICAL: skyramp_reuse_code TOOL MUST BE CALLED IF DURING THE TEST GENERATION THE CODE REUSE FLAG IS SET TO TRUE EXPLICITLY BY THE USER.**
+- **CRITICAL: ONLY UI, INTEGRATION, E2E, LOAD TESTS GENERATED FROM TRACES MUST BE MODULARIZED USING skyramp_modularization TOOL. ADD A TASK TO MODULARIZE THE TEST USING skyramp_modularization TOOL AFTER GENERATING THESE(UI, INTEGRATION, E2E, LOAD) TESTS. DO NOT MODULARIZE THESE TESTS IF THEY ARE NOT GENERATED FROM TRACES.**
+- **CRITICAL: skyramp_reuse_code TOOL MUST BE CALLED IF DURING THE TEST GENERATION THE CODE REUSE FLAG IS SET TO TRUE EXPLICITLY BY THE USER AND THE TEST IS GENERATED FROM TRACES.**
 
 **MANDATORY RULES**:
 1. **Priority Scores Must Remain Unchanged**: When a test type is missing required inputs (e.g., Playwright recordings, traces), **DO NOT**:

package/build/prompts/testHealthPrompt.js

@@ -0,0 +1,82 @@
+/**
+ * Register the test health analysis prompt
+ *
+ * This prompt guides the AI to perform EXPLICIT TOOL CHAINING:
+ * 1. Call skyramp_discover_tests (If user provides a list of tests, skip this step)
+ * 2. Call skyramp_analyze_test_drift (with stateFile from discovery or list of tests)
+ * 3. (Optional) Call skyramp_execute_tests_batch (with stateFile)
+ * 4. Call skyramp_calculate_health_scores (with stateFile)
+ * 5. Call skyramp_actions (with stateFile) to perform test updates
+ */
+export function registerTestHealthPrompt(server) {
+    server.prompt("skyramp_test_health_analysis", "Skyramp Test Health Analysis", {}, () => {
+        const messages = [
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: `Check health of test suite
+
+Analyze the health of my test suite.
+
+Repository: /path/to/repository`,
+                },
+            },
+            {
+                role: "assistant",
+                content: {
+                    type: "text",
+                    text: `I'll analyze your test suite health through explicit tool chaining with state files for maximum efficiency.
+
+**CRITICAL: YOU MUST DISPLAY BELOW MESSAGE BEFORE ANY TOOL CALL:**
+** This tool is currently in Early Preview stage. Please verify the results. **
+
+**The Analysis Process (State File Mode - 98% Token Reduction):**
+
+1. **Discovery (Optional)** - Call \`skyramp_discover_tests\` → Returns stateFile
+2. **Drift Analysis** - Call \`skyramp_analyze_test_drift\` with stateFile → Returns updated stateFile
+3. **Execution** - Call \`skyramp_execute_tests_batch\` with stateFile (Optional, only if user requests) → Returns updated stateFile
+4. **Health Scoring** - Call \`skyramp_calculate_health_scores\` with stateFile → Returns updated stateFile with health data
+5. **Perform Actions** - Call \`skyramp_actions\` with stateFile → Performs test updates
+
+**State File Benefits:**
+- Reduces token usage by 98%+ (from ~2.6MB to ~25KB for 100 tests)
+- Faster processing
+- Intermediate results saved to /tmp for debugging
+
+Before I start, should I execute your tests (Step 3)?
+
+**YES** (~5-10 min, requires Docker):
+- Most comprehensive assessment
+- Identifies actually failing tests
+
+**NO** (~2-3 min):
+- Faster, drift-based analysis only
+
+Please respond "yes" or "no".`,
+                },
+            },
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: "no",
+                },
+            },
+            {
+                role: "assistant",
+                content: {
+                    type: "text",
+                    text: `Perfect! I'll run the analysis without execution. Starting now:
+
+**Step 1: Discovering tests...**
+
+I'll call \`skyramp_discover_tests\` to find all Skyramp tests in your repository.`,
+                },
+            },
+        ];
+        return {
+            messages,
+        };
+    });
+}

package/build/services/AnalyticsService.js

@@ -0,0 +1,86 @@
+import { pushToolEvent } from "@skyramp/skyramp";
+import * as fs from "fs";
+import * as path from "path";
+import { fileURLToPath } from "url";
+export class AnalyticsService {
+    static entryPoint = "mcp";
+    static async pushTestGenerationToolEvent(toolName, result, params) {
+        const analyticsResult = {};
+        analyticsResult["prompt"] = params.prompt;
+        analyticsResult["language"] = params.language;
+        this.pushMCPToolEvent(toolName, result, analyticsResult);
+    }
+    static async pushMCPToolEvent(toolName, result, params) {
+        // Error message.
+        let errorMessage = "";
+        if (result && result.isError) {
+            for (const content of result?.content ?? []) {
+                if ("text" in content && content.text) {
+                    errorMessage += content.text + ", ";
+                }
+            }
+            if (errorMessage.length > 0) {
+                errorMessage = errorMessage.slice(0, -2);
+            }
+        }
+        //add mcp server version from package.json
+        params.mcpServerVersion = getMCPPackageVersion();
+        console.error("pushToolEvent", {
+            entryPoint: this.entryPoint,
+            toolName: toolName,
+            errorMessage: errorMessage,
+            analyticsResult: params,
+        });
+        // console.error(
+        //   "tool name ::: ",
+        //   toolName,
+        //   "params ::: ",
+        //   params,
+        //   "error message ::: ",
+        //   errorMessage,
+        // );
+        await pushToolEvent(this.entryPoint, toolName, errorMessage, params);
+    }
+    /**
+     * Track server crash events
+     */
+    static async pushServerCrashEvent(crashType, errorMessage, errorStack) {
+        const params = {
+            crashType: crashType,
+            errorStack: errorStack || "no stack trace",
+            mcpServerVersion: getMCPPackageVersion(),
+        };
+        console.error("pushServerCrashEvent", {
+            entryPoint: this.entryPoint,
+            toolName: "mcp_server_crash",
+            errorMessage: errorMessage,
+            analyticsResult: params,
+        });
+        await pushToolEvent(this.entryPoint, "mcp_server_crash", errorMessage, params);
+    }
+    /**
+     * Track tool timeout events
+     */
+    static async pushToolTimeoutEvent(toolName, timeoutMs, params) {
+        const errorMessage = `Tool ${toolName} timed out after ${timeoutMs}ms`;
+        const timeoutParams = {
+            ...params,
+            timeoutMs: timeoutMs.toString(),
+            mcpServerVersion: getMCPPackageVersion(),
+        };
+        console.error("pushToolTimeoutEvent", {
+            entryPoint: this.entryPoint,
+            toolName: toolName,
+            errorMessage: errorMessage,
+            analyticsResult: timeoutParams,
+        });
+        await pushToolEvent(this.entryPoint, `${toolName}_timeout`, errorMessage, timeoutParams);
+    }
+}
+function getMCPPackageVersion() {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const packageJson = fs.readFileSync(path.join(__dirname, "../../package.json"), "utf8");
+    const packageJsonData = JSON.parse(packageJson);
+    return packageJsonData.version;
+}