@skyramp/mcp 0.1.8 → 0.2.0-rc.2

package/build/services/TestHealthService.js

@@ -1,694 +0,0 @@
-import * as fs from "fs";
-import { logger } from "../utils/logger.js";
-export class TestHealthService {
-    /**
-     * Generate comprehensive health report for tests
-     */
-    async generateHealthReport(tests, driftResults) {
-        logger.info(`Generating health report for ${tests.length} tests`);
-        const healthAnalyses = [];
-        const recommendations = [];
-        // Analyze each test
-        for (const test of tests) {
-            const driftData = driftResults?.find((d) => d.testFile === test.testFile);
-            const analysis = await this.analyzeTestHealth(test.testFile, test.execution, driftData, test.apiSchema);
-            healthAnalyses.push(analysis);
-            recommendations.push(analysis.recommendation);
-        }
-        // Calculate summary statistics
-        const summary = {
-            totalTests: tests.length,
-            healthy: healthAnalyses.filter((a) => a.healthScore.status === "healthy")
-                .length,
-            atRisk: healthAnalyses.filter((a) => a.healthScore.status === "at_risk")
-                .length,
-            broken: healthAnalyses.filter((a) => a.healthScore.status === "broken")
-                .length,
-            unknown: healthAnalyses.filter((a) => a.healthScore.status === "unknown")
-                .length,
-            averageHealthScore: healthAnalyses.length > 0
-                ? Math.round(healthAnalyses.reduce((sum, a) => sum + a.healthScore.overall, 0) / healthAnalyses.length)
-                : 0,
-        };
-        // Analyze coverage if API schemas are available
-        let coverage;
-        const apiSchemas = tests
-            .map((t) => t.apiSchema)
-            .filter((s) => s);
-        if (apiSchemas.length > 0) {
-            try {
-                coverage = await this.analyzeCoverage(tests, apiSchemas[0]);
-            }
-            catch (error) {
-                logger.error(`Failed to analyze coverage: ${error.message}`);
-            }
-        }
-        return {
-            summary,
-            tests: healthAnalyses,
-            recommendations: this.prioritizeRecommendations(recommendations),
-            coverage,
-            generatedAt: new Date().toISOString(),
-        };
-    }
-    /**
-     * Analyze health of a single test
-     */
-    async analyzeTestHealth(testFile, execution, drift, apiSchema) {
-        // Calculate execution score
-        const executionScore = execution
-            ? this.calculateExecutionScore(execution)
-            : undefined;
-        // Calculate health score
-        const healthScore = this.calculateHealthScore(executionScore?.score, drift?.driftScore);
-        // Identify issues
-        const issues = this.identifyIssues(execution, drift);
-        // Extract API endpoint info from test
-        const apiEndpoint = apiSchema
-            ? await this.extractEndpointFromTest(testFile, apiSchema)
-            : undefined;
-        // Generate recommendation
-        const recommendation = this.generateRecommendation(testFile, healthScore, drift?.driftScore, execution, issues, apiEndpoint, drift?.apiSchemaChanges);
-        return {
-            testFile,
-            healthScore,
-            issues,
-            recommendation,
-            executionData: execution
-                ? {
-                    passed: execution.passed,
-                    duration: execution.duration,
-                    errors: execution.errors,
-                    warnings: execution.warnings,
-                }
-                : undefined,
-            driftData: drift
-                ? {
-                    driftScore: drift.driftScore,
-                    changes: drift.changes?.length || 0,
-                    affectedFiles: drift.affectedFiles?.files?.length || 0,
-                }
-                : undefined,
-            apiEndpoint,
-        };
-    }
-    /**
-     * Calculate execution score (0-100)
-     */
-    calculateExecutionScore(execution) {
-        let score;
-        let status;
-        if (execution.crashed) {
-            score = 0;
-            status = "crashed";
-        }
-        else if (execution.passed) {
-            if (execution.warnings.length > 0) {
-                // Passed with warnings: 50-100 based on warning count
-                score = Math.max(50, 100 - execution.warnings.length * 10);
-                status = "passed_with_warnings";
-            }
-            else {
-                score = 100;
-                status = "passed";
-            }
-        }
-        else if (execution.errors.length > 0) {
-            // Failed with errors: 30-50 based on error count
-            score = Math.max(30, 50 - execution.errors.length * 5);
-            status = "failed";
-        }
-        else if (execution.duration > 280000) {
-            // Near timeout (5 min = 300000ms)
-            score = 20;
-            status = "timeout";
-        }
-        else {
-            score = 30;
-            status = "failed";
-        }
-        return {
-            score,
-            status,
-            hasWarnings: execution.warnings.length > 0,
-            hasErrors: execution.errors.length > 0,
-            crashed: execution.crashed,
-        };
-    }
-    /**
-     * Calculate overall health score
-     *
-     * Health status is primarily drift-based with optional execution refinement:
-     * - Healthy: drift < 20 (and execution >= 80 if available)
-     * - At Risk: drift 20-40 (and execution >= 60 if available)
-     * - Broken: drift >= 40 (or execution < 60 if available)
-     */
-    calculateHealthScore(executionScore, driftScore) {
-        let overall;
-        let calculationMethod;
-        if (executionScore !== undefined && driftScore !== undefined) {
-            // Combined: 60% execution, 40% drift (inverted)
-            overall = Math.round(0.6 * executionScore + 0.4 * (100 - driftScore));
-            calculationMethod = "combined";
-        }
-        else if (executionScore !== undefined) {
-            overall = executionScore;
-            calculationMethod = "execution_only";
-        }
-        else if (driftScore !== undefined) {
-            overall = 100 - driftScore;
-            calculationMethod = "drift_only";
-        }
-        else {
-            overall = 0;
-            calculationMethod = "drift_only";
-        }
-        // Determine health status (primarily drift-based, execution is optional refinement)
-        // User requirements interpretation:
-        // - Healthy: drift < 20 AND (no execution OR execution >= 80)
-        // - At Risk: drift 20-40 AND (no execution OR execution >= 60)
-        // - Broken: drift >= 40 OR execution < 60
-        let status;
-        const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
-        const exec = executionScore !== undefined ? executionScore : -1; // -1 means no execution data
-        if (drift === -1 && exec === -1) {
-            // No data available
-            status = "unknown";
-        }
-        else if (drift !== -1) {
-            // Drift-based status (primary)
-            if (drift >= 40) {
-                // High drift (>= 40): Broken regardless of execution
-                status = "broken";
-            }
-            else if (drift >= 20 && drift < 40) {
-                // Medium drift (20-40): At risk, unless execution is very poor
-                status = exec !== -1 && exec < 60 ? "broken" : "at_risk";
-            }
-            else {
-                // Low drift (< 20): Healthy, but execution can downgrade to at_risk
-                // Since code hasn't changed much, even failing tests are "at_risk" not "broken"
-                if (exec !== -1 && exec < 80) {
-                    status = "at_risk"; // Any execution issues with low drift → at_risk
-                }
-                else {
-                    status = "healthy";
-                }
-            }
-        }
-        else if (exec !== -1) {
-            // Only execution data available (no drift)
-            if (exec >= 80) {
-                status = "healthy";
-            }
-            else if (exec >= 60) {
-                status = "at_risk";
-            }
-            else {
-                status = "broken";
-            }
-        }
-        else {
-            status = "unknown";
-        }
-        return {
-            overall,
-            executionScore,
-            driftScore,
-            status,
-            calculationMethod,
-        };
-    }
-    /**
-     * Identify specific issues with a test
-     */
-    identifyIssues(execution, drift) {
-        const issues = [];
-        // Check execution issues
-        if (execution) {
-            if (execution.crashed) {
-                issues.push({
-                    type: "crash",
-                    severity: "critical",
-                    description: "Test crashed during execution",
-                    details: execution.errors.join("; "),
-                });
-            }
-            else if (execution.errors.length > 0) {
-                issues.push({
-                    type: "test_failures",
-                    severity: "high",
-                    description: `Test failed with ${execution.errors.length} error(s)`,
-                    details: execution.errors.slice(0, 3).join("; "),
-                });
-            }
-            if (execution.duration > 280000) {
-                issues.push({
-                    type: "timeout",
-                    severity: "medium",
-                    description: "Test approaching timeout threshold",
-                    details: `Duration: ${(execution.duration / 1000).toFixed(1)}s`,
-                });
-            }
-        }
-        // Check drift issues
-        if (drift &&
-            drift.changes &&
-            Array.isArray(drift.changes) &&
-            drift.changes.length > 0) {
-            const hasCodeChanges = drift.changes.some((c) => ["code_change", "function_changed", "class_changed"].includes(c.type));
-            if (hasCodeChanges) {
-                issues.push({
-                    type: "code_changes",
-                    severity: "medium",
-                    description: "Code changes detected in dependencies",
-                    details: `${drift.affectedFiles?.files.length || 0} file(s) changed`,
-                });
-            }
-            const endpointsRenamed = drift.changes.filter((c) => c.type === "endpoint_renamed");
-            if (endpointsRenamed.length > 0) {
-                issues.push({
-                    type: "endpoints_renamed",
-                    severity: "high",
-                    description: `${endpointsRenamed.length} API endpoint(s) renamed`,
-                    details: endpointsRenamed.map((c) => c.description).join("; "),
-                });
-            }
-            const endpointsRemoved = drift.changes.filter((c) => c.type === "endpoint_removed");
-            if (endpointsRemoved.length > 0) {
-                issues.push({
-                    type: "endpoints_removed",
-                    severity: "high",
-                    description: `${endpointsRemoved.length} API endpoint(s) removed`,
-                    details: endpointsRemoved.map((c) => c.description).join("; "),
-                });
-            }
-            const schemaChanges = drift.changes.filter((c) => ["endpoint_modified", "authentication_changed"].includes(c.type));
-            if (schemaChanges.length > 0) {
-                issues.push({
-                    type: "schema_changes",
-                    severity: "high",
-                    description: "API schema changes detected",
-                    details: schemaChanges.map((c) => c.description).join("; "),
-                });
-            }
-            const authChanged = drift.changes.some((c) => c.type === "authentication_changed");
-            if (authChanged) {
-                issues.push({
-                    type: "authentication_changed",
-                    severity: "critical",
-                    description: "Authentication mechanism changed",
-                });
-            }
-        }
-        return issues;
-    }
-    /**
-     * Generate recommendation for a test
-     *
-     * Recommendation logic (primarily drift-based):
-     * 1. IF drift > 70: REGENERATE (HIGH priority)
-     * 2. ELSE IF endpoint missing: DELETE (HIGH priority)
-     * 3. ELSE IF 30 < drift <= 70: UPDATE (MEDIUM priority)
-     * 4. ELSE IF drift > 10: VERIFY (LOW priority)
-     * 5. ELSE: VERIFY (LOW priority, healthy test)
-     *
-     * Execution failures enhance rationale but don't change primary action
-     */
-    generateRecommendation(testFile, healthScore, driftScore, execution, issues, apiEndpoint, apiSchemaChanges) {
-        const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
-        let action;
-        let priority;
-        let rationale;
-        let estimatedWork = "SMALL";
-        // Handle missing drift data first
-        if (drift === -1) {
-            // No drift data available - base recommendation on health status and execution
-            if (healthScore.status === "unknown") {
-                action = "VERIFY";
-                priority = "MEDIUM";
-                rationale = "Unable to analyze drift - manual verification recommended";
-                estimatedWork = "SMALL";
-                // If execution data shows failure, escalate
-                if (execution && !execution.passed) {
-                    priority = "HIGH";
-                    rationale =
-                        "Drift analysis unavailable and test is failing - investigate immediately";
-                    estimatedWork = "MEDIUM";
-                }
-            }
-            else if (execution && !execution.passed) {
-                // No drift data but test is failing
-                action = "UPDATE";
-                priority = "HIGH";
-                rationale =
-                    "Test is failing but drift analysis unavailable - review test logic and dependencies";
-                estimatedWork = "MEDIUM";
-            }
-            else if (execution && execution.passed) {
-                // No drift data but test is passing
-                action = "VERIFY";
-                priority = "LOW";
-                rationale =
-                    "Drift analysis unavailable but test is passing - periodic verification recommended";
-                estimatedWork = "SMALL";
-            }
-            else {
-                // No drift data, no execution data
-                action = "VERIFY";
-                priority = "MEDIUM";
-                rationale = "No drift or execution data available - analysis needed";
-                estimatedWork = "SMALL";
-            }
-        }
-        else if (issues && issues.some((i) => i.type === "endpoints_renamed")) {
-            // Endpoint renamed -> UPDATE with path substitution (regardless of drift score)
-            action = "UPDATE";
-            priority = "HIGH";
-            rationale =
-                "Endpoint path renamed - test URLs must be updated to match new path";
-            estimatedWork = "SMALL";
-            const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
-            if (renameIssue?.details) {
-                rationale += `. ${renameIssue.details}`;
-            }
-            if (execution && !execution.passed) {
-                rationale += ". Test is currently failing due to the path change";
-            }
-        }
-        else if (drift > 70) {
-            // High drift -> REGENERATE
-            action = "REGENERATE";
-            priority = "HIGH";
-            rationale =
-                "High drift detected - significant code changes since test creation";
-            estimatedWork = "MEDIUM";
-            // Enhance rationale with test failures if present
-            if (execution && !execution.passed) {
-                rationale += ". Test is also failing";
-            }
-            // Add specific issues
-            if (issues && issues.length > 0) {
-                const criticalIssues = issues.filter((i) => ["critical", "high"].includes(i.severity));
-                if (criticalIssues.length > 0) {
-                    rationale += `. Critical issues: ${criticalIssues
-                        .map((i) => i.description)
-                        .join(", ")}`;
-                }
-            }
-        }
-        else if (apiEndpoint?.exists === false) {
-            // Endpoint removed from schema -> DELETE
-            action = "DELETE";
-            priority = "HIGH";
-            rationale = "Endpoint no longer exists in API schema";
-            estimatedWork = "SMALL";
-        }
-        else if (drift > 30 && drift <= 70) {
-            // Moderate drift -> UPDATE
-            action = "UPDATE";
-            priority = "MEDIUM";
-            rationale =
-                "Moderate drift detected - related code changes may affect test";
-            estimatedWork = "SMALL";
-            // Enhance with schema changes
-            const schemaChanges = issues?.filter((i) => [
-                "schema_changes",
-                "endpoints_removed",
-                "endpoints_renamed",
-                "authentication_changed",
-            ].includes(i.type));
-            if (schemaChanges && schemaChanges.length > 0) {
-                rationale += `. Schema changes: ${schemaChanges
-                    .map((i) => i.description)
-                    .join(", ")}`;
-                estimatedWork = "MEDIUM";
-            }
-            // Note execution failures
-            if (execution && !execution.passed) {
-                rationale += ". Test is also failing - requires immediate attention";
-                priority = "HIGH"; // Escalate priority for failing tests
-            }
-        }
-        else if (drift > 10) {
-            // Low drift -> VERIFY
-            action = "VERIFY";
-            priority = "LOW";
-            rationale = "Minor changes detected - test likely still valid";
-            estimatedWork = "SMALL";
-            // Escalate to UPDATE if test is actually failing
-            if (execution) {
-                if (!execution.passed) {
-                    action = "UPDATE";
-                    priority = "HIGH";
-                    rationale =
-                        "Minor drift but test is failing - update needed to match code changes";
-                    estimatedWork = "MEDIUM";
-                }
-                else if (execution.warnings.length > 0) {
-                    rationale += ". Test passed with warnings";
-                }
-            }
-        }
-        else {
-            // Minimal/no drift -> VERIFY
-            action = "VERIFY";
-            priority = "LOW";
-            rationale = "Test appears healthy - periodic verification recommended";
-            estimatedWork = "SMALL";
-            // Handle edge case: low drift but test is failing
-            if (execution && !execution.passed) {
-                action = "UPDATE";
-                priority = "HIGH";
-                rationale =
-                    "Test is failing despite low drift - may indicate environmental issues or test flakiness";
-                estimatedWork = "MEDIUM";
-            }
-        }
-        // Determine endpoint status
-        let endpointStatus;
-        const renameIssues = issues?.filter((i) => i.type === "endpoints_renamed");
-        if (renameIssues && renameIssues.length > 0) {
-            endpointStatus = "renamed";
-        }
-        else if (apiEndpoint === undefined) {
-            endpointStatus = undefined;
-        }
-        else if (apiEndpoint.exists) {
-            endpointStatus = "exists";
-        }
-        else {
-            endpointStatus = "missing";
-        }
-        // Extract rename mappings from apiSchemaChanges for downstream tools
-        const renamedEndpoints = apiSchemaChanges?.endpointsRenamed &&
-            apiSchemaChanges.endpointsRenamed.length > 0
-            ? apiSchemaChanges.endpointsRenamed
-            : undefined;
-        return {
-            testFile,
-            action,
-            priority,
-            rationale,
-            estimatedWork,
-            issues,
-            details: {
-                driftScore: drift,
-                executionPassed: execution?.passed,
-                endpointStatus,
-                renamedEndpoints,
-            },
-        };
-    }
-    /**
-     * Parse OpenAPI schema and extract endpoints
-     */
-    async parseApiSchema(schemaPath) {
-        let schema;
-        try {
-            if (schemaPath.startsWith("http://") ||
-                schemaPath.startsWith("https://")) {
-                // Fetch from URL
-                const response = await fetch(schemaPath);
-                if (!response.ok) {
-                    throw new Error(`HTTP error! status: ${response.status}`);
-                }
-                schema = await response.json();
-            }
-            else {
-                // Read from file
-                const content = fs.readFileSync(schemaPath, "utf-8");
-                schema = JSON.parse(content);
-            }
-        }
-        catch (error) {
-            logger.error(`Failed to parse API schema: ${error.message}`);
-            throw new Error(`Could not parse API schema at ${schemaPath}`);
-        }
-        const endpoints = [];
-        const paths = schema.paths || {};
-        for (const [pathStr, pathItem] of Object.entries(paths)) {
-            for (const [method, operation] of Object.entries(pathItem)) {
-                if (["get", "post", "put", "delete", "patch"].includes(method)) {
-                    const op = operation;
-                    const authRequired = this.checkAuthRequired(op, schema);
-                    endpoints.push({
-                        path: pathStr,
-                        method: method.toUpperCase(),
-                        operationId: op.operationId,
-                        authRequired,
-                        parameters: op.parameters || [],
-                        requestBody: op.requestBody,
-                        responses: op.responses,
-                    });
-                }
-            }
-        }
-        return endpoints;
-    }
-    /**
-     * Check if endpoint requires authentication
-     */
-    checkAuthRequired(operation, schema) {
-        // Check security at operation level
-        if (operation.security && operation.security.length > 0) {
-            return true;
-        }
-        // Check security at schema level
-        if (schema.security && schema.security.length > 0) {
-            return true;
-        }
-        return false;
-    }
-    /**
-     * Analyze test coverage
-     */
-    async analyzeCoverage(tests, apiSchema) {
-        const endpoints = await this.parseApiSchema(apiSchema);
-        const coverage = [];
-        for (const endpoint of endpoints) {
-            const endpointKey = `${endpoint.method} ${endpoint.path}`;
-            const coveredBy = tests.filter((t) => this.testCoversEndpoint(t.testFile, endpoint));
-            coverage.push({
-                endpoint: endpointKey,
-                method: endpoint.method,
-                covered: coveredBy.length > 0,
-                testFiles: coveredBy.map((t) => t.testFile),
-                endpointInfo: endpoint,
-            });
-        }
-        const coveredEndpoints = coverage.filter((c) => c.covered).length;
-        const uncoveredEndpoints = endpoints.filter((ep) => !coverage.find((c) => c.endpoint === `${ep.method} ${ep.path}`)
-            ?.covered);
-        return {
-            totalEndpoints: endpoints.length,
-            coveredEndpoints,
-            coveragePercentage: endpoints.length > 0
-                ? Math.round((coveredEndpoints / endpoints.length) * 100)
-                : 0,
-            uncoveredEndpoints,
-            coverage,
-        };
-    }
-    /**
-     * Check if test covers endpoint
-     */
-    testCoversEndpoint(testFile, endpoint) {
-        try {
-            const content = fs.readFileSync(testFile, "utf-8");
-            // Convert OpenAPI-style path to regex, e.g. /users/{id} -> /users/[^/]+
-            // Replace path parameters with regex pattern, escaping the path parts but not the regex itself
-            const pathRegexString = endpoint.path
-                .split(/(\{[^}]+\})/) // Split on path parameters, keeping them in the result
-                .map((part, index) => {
-                if (part.match(/^\{[^}]+\}$/)) {
-                    // This is a path parameter like {id}, replace with regex pattern
-                    return "[^/]+";
-                }
-                else {
-                    // This is a literal path part, escape special regex chars
-                    return part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-                }
-            })
-                .join("");
-            // pathRegexString is already escaped, so we can use it directly
-            const pathRegex = new RegExp(pathRegexString);
-            const method = endpoint.method.toUpperCase();
-            const methodLower = endpoint.method.toLowerCase();
-            // Check if path matches
-            if (!pathRegex.test(content)) {
-                return false;
-            }
-            // More precise method detection patterns
-            const methodPatterns = [
-                // Python: method="PUT" or method='PUT'
-                new RegExp(`method\\s*=\\s*["']${method}["']`, "i"),
-                // JavaScript/TypeScript: method: 'PUT' or .put( or .PUT(
-                new RegExp(`method\\s*:\\s*["']${method}["']`, "i"),
-                new RegExp(`\\.${methodLower}\\s*\\(`, "i"),
-                // HTTP request patterns: PUT /path
-                new RegExp(`${method}\\s+${pathRegexString}`, "i"),
-                // Axios/fetch: { method: 'PUT' }
-                new RegExp(`["']method["']\\s*:\\s*["']${method}["']`, "i"),
-                // RestAssured/Java: .put()
-                new RegExp(`\\.${methodLower}\\(`, "i"),
-                // Go: http.MethodPut or "PUT"
-                new RegExp(`http\\.Method${method.charAt(0) + methodLower.slice(1)}`, "i"),
-                new RegExp(`["']${method}["']`, "i"),
-            ];
-            // Check if any pattern matches
-            return methodPatterns.some((pattern) => pattern.test(content));
-        }
-        catch (error) {
-            return false;
-        }
-    }
-    /**
-     * Extract endpoint information from test file
-     */
-    async extractEndpointFromTest(testFile, apiSchema) {
-        try {
-            const content = fs.readFileSync(testFile, "utf-8");
-            const endpoints = await this.parseApiSchema(apiSchema);
-            // Find matching endpoint
-            for (const endpoint of endpoints) {
-                if (this.testCoversEndpoint(testFile, endpoint)) {
-                    return {
-                        path: endpoint.path,
-                        method: endpoint.method,
-                        exists: true,
-                    };
-                }
-            }
-            return { path: "unknown", method: "unknown", exists: false };
-        }
-        catch (error) {
-            return undefined;
-        }
-    }
-    /**
-     * Prioritize and sort recommendations
-     */
-    prioritizeRecommendations(recommendations) {
-        const priorityOrder = {
-            CRITICAL: 0,
-            HIGH: 1,
-            MEDIUM: 2,
-            LOW: 3,
-        };
-        return recommendations.sort((a, b) => {
-            const priorityDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
-            if (priorityDiff !== 0)
-                return priorityDiff;
-            // Within same priority, sort by action
-            const actionOrder = {
-                REGENERATE: 0,
-                DELETE: 1,
-                UPDATE: 2,
-                ADD: 3,
-                VERIFY: 4,
-            };
-            return actionOrder[a.action] - actionOrder[b.action];
-        });
-    }
-}