@skyramp/mcp 0.0.44 → 0.0.45

package/build/services/TestHealthService.js

@@ -0,0 +1,653 @@
+import * as fs from "fs";
+import { logger } from "../utils/logger.js";
+export class TestHealthService {
+    /**
+     * Generate comprehensive health report for tests
+     */
+    async generateHealthReport(tests, driftResults) {
+        logger.info(`Generating health report for ${tests.length} tests`);
+        const healthAnalyses = [];
+        const recommendations = [];
+        // Analyze each test
+        for (const test of tests) {
+            const driftData = driftResults?.find((d) => d.testFile === test.testFile);
+            const analysis = await this.analyzeTestHealth(test.testFile, test.execution, driftData, test.apiSchema);
+            healthAnalyses.push(analysis);
+            recommendations.push(analysis.recommendation);
+        }
+        // Calculate summary statistics
+        const summary = {
+            totalTests: tests.length,
+            healthy: healthAnalyses.filter((a) => a.healthScore.status === "healthy")
+                .length,
+            atRisk: healthAnalyses.filter((a) => a.healthScore.status === "at_risk")
+                .length,
+            broken: healthAnalyses.filter((a) => a.healthScore.status === "broken")
+                .length,
+            unknown: healthAnalyses.filter((a) => a.healthScore.status === "unknown")
+                .length,
+            averageHealthScore: healthAnalyses.length > 0
+                ? Math.round(healthAnalyses.reduce((sum, a) => sum + a.healthScore.overall, 0) / healthAnalyses.length)
+                : 0,
+        };
+        // Analyze coverage if API schemas are available
+        let coverage;
+        const apiSchemas = tests
+            .map((t) => t.apiSchema)
+            .filter((s) => s);
+        if (apiSchemas.length > 0) {
+            try {
+                coverage = await this.analyzeCoverage(tests, apiSchemas[0]);
+            }
+            catch (error) {
+                logger.error(`Failed to analyze coverage: ${error.message}`);
+            }
+        }
+        return {
+            summary,
+            tests: healthAnalyses,
+            recommendations: this.prioritizeRecommendations(recommendations),
+            coverage,
+            generatedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Analyze health of a single test
+     */
+    async analyzeTestHealth(testFile, execution, drift, apiSchema) {
+        // Calculate execution score
+        const executionScore = execution
+            ? this.calculateExecutionScore(execution)
+            : undefined;
+        // Calculate health score
+        const healthScore = this.calculateHealthScore(executionScore?.score, drift?.driftScore);
+        // Identify issues
+        const issues = this.identifyIssues(execution, drift);
+        // Extract API endpoint info from test
+        const apiEndpoint = apiSchema
+            ? await this.extractEndpointFromTest(testFile, apiSchema)
+            : undefined;
+        // Generate recommendation
+        const recommendation = this.generateRecommendation(testFile, healthScore, drift?.driftScore, execution, issues, apiEndpoint);
+        return {
+            testFile,
+            healthScore,
+            issues,
+            recommendation,
+            executionData: execution
+                ? {
+                    passed: execution.passed,
+                    duration: execution.duration,
+                    errors: execution.errors,
+                    warnings: execution.warnings,
+                }
+                : undefined,
+            driftData: drift
+                ? {
+                    driftScore: drift.driftScore,
+                    changes: drift.changes?.length || 0,
+                    affectedFiles: drift.affectedFiles?.files?.length || 0,
+                }
+                : undefined,
+            apiEndpoint,
+        };
+    }
+    /**
+     * Calculate execution score (0-100)
+     */
+    calculateExecutionScore(execution) {
+        let score;
+        let status;
+        if (execution.crashed) {
+            score = 0;
+            status = "crashed";
+        }
+        else if (execution.passed) {
+            if (execution.warnings.length > 0) {
+                // Passed with warnings: 50-100 based on warning count
+                score = Math.max(50, 100 - execution.warnings.length * 10);
+                status = "passed_with_warnings";
+            }
+            else {
+                score = 100;
+                status = "passed";
+            }
+        }
+        else if (execution.errors.length > 0) {
+            // Failed with errors: 30-50 based on error count
+            score = Math.max(30, 50 - execution.errors.length * 5);
+            status = "failed";
+        }
+        else if (execution.duration > 280000) {
+            // Near timeout (5 min = 300000ms)
+            score = 20;
+            status = "timeout";
+        }
+        else {
+            score = 30;
+            status = "failed";
+        }
+        return {
+            score,
+            status,
+            hasWarnings: execution.warnings.length > 0,
+            hasErrors: execution.errors.length > 0,
+            crashed: execution.crashed,
+        };
+    }
+    /**
+     * Calculate overall health score
+     *
+     * Health status is primarily drift-based with optional execution refinement:
+     * - Healthy: drift < 20 (and execution >= 80 if available)
+     * - At Risk: drift 20-40 (and execution >= 60 if available)
+     * - Broken: drift >= 40 (or execution < 60 if available)
+     */
+    calculateHealthScore(executionScore, driftScore) {
+        let overall;
+        let calculationMethod;
+        if (executionScore !== undefined && driftScore !== undefined) {
+            // Combined: 60% execution, 40% drift (inverted)
+            overall = Math.round(0.6 * executionScore + 0.4 * (100 - driftScore));
+            calculationMethod = "combined";
+        }
+        else if (executionScore !== undefined) {
+            overall = executionScore;
+            calculationMethod = "execution_only";
+        }
+        else if (driftScore !== undefined) {
+            overall = 100 - driftScore;
+            calculationMethod = "drift_only";
+        }
+        else {
+            overall = 0;
+            calculationMethod = "drift_only";
+        }
+        // Determine health status (primarily drift-based, execution is optional refinement)
+        // User requirements interpretation:
+        // - Healthy: drift < 20 AND (no execution OR execution >= 80)
+        // - At Risk: drift 20-40 AND (no execution OR execution >= 60)
+        // - Broken: drift >= 40 OR execution < 60
+        let status;
+        const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
+        const exec = executionScore !== undefined ? executionScore : -1; // -1 means no execution data
+        if (drift === -1 && exec === -1) {
+            // No data available
+            status = "unknown";
+        }
+        else if (drift !== -1) {
+            // Drift-based status (primary)
+            if (drift >= 40) {
+                // High drift (>= 40): Broken regardless of execution
+                status = "broken";
+            }
+            else if (drift >= 20 && drift < 40) {
+                // Medium drift (20-40): At risk, unless execution is very poor
+                status = exec !== -1 && exec < 60 ? "broken" : "at_risk";
+            }
+            else {
+                // Low drift (< 20): Healthy, but execution can downgrade to at_risk
+                // Since code hasn't changed much, even failing tests are "at_risk" not "broken"
+                if (exec !== -1 && exec < 80) {
+                    status = "at_risk"; // Any execution issues with low drift → at_risk
+                }
+                else {
+                    status = "healthy";
+                }
+            }
+        }
+        else if (exec !== -1) {
+            // Only execution data available (no drift)
+            if (exec >= 80) {
+                status = "healthy";
+            }
+            else if (exec >= 60) {
+                status = "at_risk";
+            }
+            else {
+                status = "broken";
+            }
+        }
+        else {
+            status = "unknown";
+        }
+        return {
+            overall,
+            executionScore,
+            driftScore,
+            status,
+            calculationMethod,
+        };
+    }
+    /**
+     * Identify specific issues with a test
+     */
+    identifyIssues(execution, drift) {
+        const issues = [];
+        // Check execution issues
+        if (execution) {
+            if (execution.crashed) {
+                issues.push({
+                    type: "crash",
+                    severity: "critical",
+                    description: "Test crashed during execution",
+                    details: execution.errors.join("; "),
+                });
+            }
+            else if (execution.errors.length > 0) {
+                issues.push({
+                    type: "test_failures",
+                    severity: "high",
+                    description: `Test failed with ${execution.errors.length} error(s)`,
+                    details: execution.errors.slice(0, 3).join("; "),
+                });
+            }
+            if (execution.duration > 280000) {
+                issues.push({
+                    type: "timeout",
+                    severity: "medium",
+                    description: "Test approaching timeout threshold",
+                    details: `Duration: ${(execution.duration / 1000).toFixed(1)}s`,
+                });
+            }
+        }
+        // Check drift issues
+        if (drift &&
+            drift.changes &&
+            Array.isArray(drift.changes) &&
+            drift.changes.length > 0) {
+            const hasCodeChanges = drift.changes.some((c) => ["code_change", "function_changed", "class_changed"].includes(c.type));
+            if (hasCodeChanges) {
+                issues.push({
+                    type: "code_changes",
+                    severity: "medium",
+                    description: "Code changes detected in dependencies",
+                    details: `${drift.affectedFiles?.files.length || 0} file(s) changed`,
+                });
+            }
+            const endpointsRemoved = drift.changes.filter((c) => c.type === "endpoint_removed");
+            if (endpointsRemoved.length > 0) {
+                issues.push({
+                    type: "endpoints_removed",
+                    severity: "high",
+                    description: `${endpointsRemoved.length} API endpoint(s) removed`,
+                    details: endpointsRemoved.map((c) => c.description).join("; "),
+                });
+            }
+            const schemaChanges = drift.changes.filter((c) => ["endpoint_modified", "authentication_changed"].includes(c.type));
+            if (schemaChanges.length > 0) {
+                issues.push({
+                    type: "schema_changes",
+                    severity: "high",
+                    description: "API schema changes detected",
+                    details: schemaChanges.map((c) => c.description).join("; "),
+                });
+            }
+            const authChanged = drift.changes.some((c) => c.type === "authentication_changed");
+            if (authChanged) {
+                issues.push({
+                    type: "authentication_changed",
+                    severity: "critical",
+                    description: "Authentication mechanism changed",
+                });
+            }
+        }
+        return issues;
+    }
+    /**
+     * Generate recommendation for a test
+     *
+     * Recommendation logic (primarily drift-based):
+     * 1. IF drift > 70: REGENERATE (HIGH priority)
+     * 2. ELSE IF endpoint missing: DELETE (HIGH priority)
+     * 3. ELSE IF 30 < drift <= 70: UPDATE (MEDIUM priority)
+     * 4. ELSE IF drift > 10: VERIFY (LOW priority)
+     * 5. ELSE: VERIFY (LOW priority, healthy test)
+     *
+     * Execution failures enhance rationale but don't change primary action
+     */
+    generateRecommendation(testFile, healthScore, driftScore, execution, issues, apiEndpoint) {
+        const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
+        let action;
+        let priority;
+        let rationale;
+        let estimatedWork = "SMALL";
+        // Handle missing drift data first
+        if (drift === -1) {
+            // No drift data available - base recommendation on health status and execution
+            if (healthScore.status === "unknown") {
+                action = "VERIFY";
+                priority = "MEDIUM";
+                rationale = "Unable to analyze drift - manual verification recommended";
+                estimatedWork = "SMALL";
+                // If execution data shows failure, escalate
+                if (execution && !execution.passed) {
+                    priority = "HIGH";
+                    rationale = "Drift analysis unavailable and test is failing - investigate immediately";
+                    estimatedWork = "MEDIUM";
+                }
+            }
+            else if (execution && !execution.passed) {
+                // No drift data but test is failing
+                action = "UPDATE";
+                priority = "HIGH";
+                rationale = "Test is failing but drift analysis unavailable - review test logic and dependencies";
+                estimatedWork = "MEDIUM";
+            }
+            else if (execution && execution.passed) {
+                // No drift data but test is passing
+                action = "VERIFY";
+                priority = "LOW";
+                rationale = "Drift analysis unavailable but test is passing - periodic verification recommended";
+                estimatedWork = "SMALL";
+            }
+            else {
+                // No drift data, no execution data
+                action = "VERIFY";
+                priority = "MEDIUM";
+                rationale = "No drift or execution data available - analysis needed";
+                estimatedWork = "SMALL";
+            }
+        }
+        else if (drift > 70) {
+            // High drift -> REGENERATE
+            action = "REGENERATE";
+            priority = "HIGH";
+            rationale =
+                "High drift detected - significant code changes since test creation";
+            estimatedWork = "MEDIUM";
+            // Enhance rationale with test failures if present
+            if (execution && !execution.passed) {
+                rationale += ". Test is also failing";
+            }
+            // Add specific issues
+            if (issues && issues.length > 0) {
+                const criticalIssues = issues.filter((i) => ["critical", "high"].includes(i.severity));
+                if (criticalIssues.length > 0) {
+                    rationale += `. Critical issues: ${criticalIssues
+                        .map((i) => i.description)
+                        .join(", ")}`;
+                }
+            }
+        }
+        else if (apiEndpoint?.exists === false) {
+            // Endpoint removed from schema -> DELETE
+            action = "DELETE";
+            priority = "HIGH";
+            rationale = "Endpoint no longer exists in API schema";
+            estimatedWork = "SMALL";
+        }
+        else if (drift > 30 && drift <= 70) {
+            // Moderate drift -> UPDATE
+            action = "UPDATE";
+            priority = "MEDIUM";
+            rationale =
+                "Moderate drift detected - related code changes may affect test";
+            estimatedWork = "SMALL";
+            // Enhance with schema changes
+            const schemaChanges = issues?.filter((i) => [
+                "schema_changes",
+                "endpoints_removed",
+                "authentication_changed",
+            ].includes(i.type));
+            if (schemaChanges && schemaChanges.length > 0) {
+                rationale += `. Schema changes: ${schemaChanges
+                    .map((i) => i.description)
+                    .join(", ")}`;
+                estimatedWork = "MEDIUM";
+            }
+            // Note execution failures
+            if (execution && !execution.passed) {
+                rationale += ". Test is also failing - requires immediate attention";
+                priority = "HIGH"; // Escalate priority for failing tests
+            }
+        }
+        else if (drift > 10) {
+            // Low drift -> VERIFY
+            action = "VERIFY";
+            priority = "LOW";
+            rationale = "Minor changes detected - test likely still valid";
+            estimatedWork = "SMALL";
+            // Note execution status for context
+            if (execution) {
+                if (!execution.passed) {
+                    rationale += ". However, test is failing - review needed";
+                    priority = "MEDIUM"; // Escalate for failing tests
+                }
+                else if (execution.warnings.length > 0) {
+                    rationale += ". Test passed with warnings";
+                }
+            }
+        }
+        else {
+            // Minimal/no drift -> VERIFY
+            action = "VERIFY";
+            priority = "LOW";
+            rationale = "Test appears healthy - periodic verification recommended";
+            estimatedWork = "SMALL";
+            // Handle edge case: low drift but test is failing
+            if (execution && !execution.passed) {
+                action = "UPDATE";
+                priority = "HIGH";
+                rationale =
+                    "Test is failing despite low drift - may indicate environmental issues or test flakiness";
+                estimatedWork = "MEDIUM";
+            }
+        }
+        // Determine endpoint status
+        let endpointStatus;
+        if (apiEndpoint === undefined) {
+            endpointStatus = undefined;
+        }
+        else if (apiEndpoint.exists) {
+            endpointStatus = "exists";
+        }
+        else {
+            endpointStatus = "missing";
+        }
+        return {
+            testFile,
+            action,
+            priority,
+            rationale,
+            estimatedWork,
+            issues,
+            details: {
+                driftScore: drift,
+                executionPassed: execution?.passed,
+                endpointStatus,
+            },
+        };
+    }
+    /**
+     * Parse OpenAPI schema and extract endpoints
+     */
+    async parseApiSchema(schemaPath) {
+        let schema;
+        try {
+            if (schemaPath.startsWith("http://") ||
+                schemaPath.startsWith("https://")) {
+                // Fetch from URL
+                const response = await fetch(schemaPath);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                schema = await response.json();
+            }
+            else {
+                // Read from file
+                const content = fs.readFileSync(schemaPath, "utf-8");
+                schema = JSON.parse(content);
+            }
+        }
+        catch (error) {
+            logger.error(`Failed to parse API schema: ${error.message}`);
+            throw new Error(`Could not parse API schema at ${schemaPath}`);
+        }
+        const endpoints = [];
+        const paths = schema.paths || {};
+        for (const [pathStr, pathItem] of Object.entries(paths)) {
+            for (const [method, operation] of Object.entries(pathItem)) {
+                if (["get", "post", "put", "delete", "patch"].includes(method)) {
+                    const op = operation;
+                    const authRequired = this.checkAuthRequired(op, schema);
+                    endpoints.push({
+                        path: pathStr,
+                        method: method.toUpperCase(),
+                        operationId: op.operationId,
+                        authRequired,
+                        parameters: op.parameters || [],
+                        requestBody: op.requestBody,
+                        responses: op.responses,
+                    });
+                }
+            }
+        }
+        return endpoints;
+    }
+    /**
+     * Check if endpoint requires authentication
+     */
+    checkAuthRequired(operation, schema) {
+        // Check security at operation level
+        if (operation.security && operation.security.length > 0) {
+            return true;
+        }
+        // Check security at schema level
+        if (schema.security && schema.security.length > 0) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Analyze test coverage
+     */
+    async analyzeCoverage(tests, apiSchema) {
+        const endpoints = await this.parseApiSchema(apiSchema);
+        const coverage = [];
+        for (const endpoint of endpoints) {
+            const endpointKey = `${endpoint.method} ${endpoint.path}`;
+            const coveredBy = tests.filter((t) => this.testCoversEndpoint(t.testFile, endpoint));
+            coverage.push({
+                endpoint: endpointKey,
+                method: endpoint.method,
+                covered: coveredBy.length > 0,
+                testFiles: coveredBy.map((t) => t.testFile),
+                endpointInfo: endpoint,
+            });
+        }
+        const coveredEndpoints = coverage.filter((c) => c.covered).length;
+        const uncoveredEndpoints = endpoints.filter((ep) => !coverage.find((c) => c.endpoint === `${ep.method} ${ep.path}`)
+            ?.covered);
+        return {
+            totalEndpoints: endpoints.length,
+            coveredEndpoints,
+            coveragePercentage: endpoints.length > 0
+                ? Math.round((coveredEndpoints / endpoints.length) * 100)
+                : 0,
+            uncoveredEndpoints,
+            coverage,
+        };
+    }
+    /**
+     * Check if test covers endpoint
+     */
+    testCoversEndpoint(testFile, endpoint) {
+        try {
+            const content = fs.readFileSync(testFile, "utf-8");
+            // Convert OpenAPI-style path to regex, e.g. /users/{id} -> /users/[^/]+
+            // Replace path parameters with regex pattern, escaping the path parts but not the regex itself
+            const pathRegexString = endpoint.path
+                .split(/(\{[^}]+\})/) // Split on path parameters, keeping them in the result
+                .map((part, index) => {
+                if (part.match(/^\{[^}]+\}$/)) {
+                    // This is a path parameter like {id}, replace with regex pattern
+                    return "[^/]+";
+                }
+                else {
+                    // This is a literal path part, escape special regex chars
+                    return part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                }
+            })
+                .join('');
+            // pathRegexString is already escaped, so we can use it directly
+            const pathRegex = new RegExp(pathRegexString);
+            const method = endpoint.method.toUpperCase();
+            const methodLower = endpoint.method.toLowerCase();
+            // Check if path matches
+            if (!pathRegex.test(content)) {
+                return false;
+            }
+            // More precise method detection patterns
+            const methodPatterns = [
+                // Python: method="PUT" or method='PUT'
+                new RegExp(`method\\s*=\\s*["']${method}["']`, "i"),
+                // JavaScript/TypeScript: method: 'PUT' or .put( or .PUT(
+                new RegExp(`method\\s*:\\s*["']${method}["']`, "i"),
+                new RegExp(`\\.${methodLower}\\s*\\(`, "i"),
+                // HTTP request patterns: PUT /path
+                new RegExp(`${method}\\s+${pathRegexString}`, "i"),
+                // Axios/fetch: { method: 'PUT' }
+                new RegExp(`["']method["']\\s*:\\s*["']${method}["']`, "i"),
+                // RestAssured/Java: .put()
+                new RegExp(`\\.${methodLower}\\(`, "i"),
+                // Go: http.MethodPut or "PUT"
+                new RegExp(`http\\.Method${method.charAt(0) + methodLower.slice(1)}`, "i"),
+                new RegExp(`["']${method}["']`, "i"),
+            ];
+            // Check if any pattern matches
+            return methodPatterns.some((pattern) => pattern.test(content));
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    /**
+     * Extract endpoint information from test file
+     */
+    async extractEndpointFromTest(testFile, apiSchema) {
+        try {
+            const content = fs.readFileSync(testFile, "utf-8");
+            const endpoints = await this.parseApiSchema(apiSchema);
+            // Find matching endpoint
+            for (const endpoint of endpoints) {
+                if (this.testCoversEndpoint(testFile, endpoint)) {
+                    return {
+                        path: endpoint.path,
+                        method: endpoint.method,
+                        exists: true,
+                    };
+                }
+            }
+            return { path: "unknown", method: "unknown", exists: false };
+        }
+        catch (error) {
+            return undefined;
+        }
+    }
+    /**
+     * Prioritize and sort recommendations
+     */
+    prioritizeRecommendations(recommendations) {
+        const priorityOrder = {
+            CRITICAL: 0,
+            HIGH: 1,
+            MEDIUM: 2,
+            LOW: 3,
+        };
+        return recommendations.sort((a, b) => {
+            const priorityDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
+            if (priorityDiff !== 0)
+                return priorityDiff;
+            // Within same priority, sort by action
+            const actionOrder = {
+                REGENERATE: 0,
+                DELETE: 1,
+                UPDATE: 2,
+                ADD: 3,
+                VERIFY: 4,
+            };
+            return actionOrder[a.action] - actionOrder[b.action];
+        });
+    }
+}

package/build/tools/auth/loginTool.js

@@ -12,7 +12,7 @@ Logging into Skyramp provides access to additional platform features and service
                 .string()
                 .describe("The prompt user provided to login to Skyramp"),
         },
-        annotations: {
+        _meta: {
             keywords: ["login", "authenticate", "skyramp login"],
         },
     }, async (params) => {

package/build/tools/auth/logoutTool.js

@@ -11,7 +11,7 @@ Logout from Skyramp platform to end your authenticated session.`,
                 .string()
                 .describe("The prompt user provided to logout from Skyramp"),
         },
-        annotations: {
+        _meta: {
             keywords: ["logout", "sign out", "skyramp logout"],
         },
     }, async (params) => {

package/build/tools/code-refactor/codeReuseTool.js

@@ -37,16 +37,18 @@ export function registerCodeReuseTool(server) {
       3. **ONLY IF** reusable helpers are found in OTHER test files → move to utils and update imports
       4. **IF NO** existing utils or reusable helpers are found in test files then reuse is complete, without creating any files or changes to the test files
       5. Verify no duplicate code remains and tests work as expected
-      6. **MANDATORY**: After code reuse is complete, proceed to modularization by calling skyramp_modularization tool
+      6. **MANDATORY**: After code reuse is complete, proceed to modularization by calling skyramp_modularization tool ONLY for UI, E2E, INTEGRATION or LOAD tests generated from traces.
       
       **WHEN TO SKIP (DO NOTHING):**
       - Other test files only have repetitive patterns, no helper functions
       - Other test files are essentially identical to current file
       - No actual function definitions found in other test files
-      
+
+      **MANDATORY**: ONLY ALLOW CODE REUSE IF THE IS TRACE BASED FLAG IS SET TO TRUE ELSE DO NOT ALLOW CODE REUSE AND LEAVE THE TEST FILE AS IS.
+      **CRITICAL**: NON TRACE BASED TESTS ARE ALREADY MODULARIZED AND DO NOT NEED CODE REUSE.
       The tool will provide step-by-step instructions that MUST be followed exactly.`,
         inputSchema: codeReuseSchema.shape,
-        annotations: {
+        _meta: {
             keywords: [
                 "code reuse",
                 "duplicate code",