@skyramp/mcp 0.0.43 → 0.0.45

package/build/services/ModularizationService.js

@@ -9,8 +9,23 @@ export class ModularizationService {
             testType: params.testType,
             language: params.language,
         });
-        let prompt = "";
         const testType = params.testType;
+        // Check if the test type is one that should not be modularized
+        if (!params.isTraceBased) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `⚠️ **MODULARIZATION NOT APPLICABLE**
+
+Modularization is intentionally NOT applied to ${testType.toUpperCase()} tests if the test is not generated from traces.
+The test file \`${params.testFile}\` will remain unchanged. No further action is needed.`,
+                    },
+                ],
+                isError: false,
+            };
+        }
+        let prompt = "";
         switch (testType) {
             case TestType.UI:
                 prompt = getModularizationPromptForUI(params.testFile);

package/build/services/TestDiscoveryService.js

@@ -0,0 +1,237 @@
+import * as fs from "fs";
+import * as path from "path";
+import { simpleGit } from "simple-git";
+import { logger } from "../utils/logger.js";
+import fg from "fast-glob";
+export class TestDiscoveryService {
+    EXCLUDED_DIRS = [
+        "node_modules",
+        "venv",
+        ".venv",
+        "build",
+        "dist",
+        ".git",
+        "__pycache__",
+        ".pytest_cache",
+        "coverage",
+        ".next",
+        "out",
+        "target",
+    ];
+    SKYRAMP_MARKER = "Generated by Skyramp";
+    // Supported test file extensions
+    SUPPORTED_EXTENSIONS = [".py", ".js", ".ts", ".java"];
+    // Concurrency control for parallel operations
+    MAX_CONCURRENT_OPERATIONS = 10;
+    // Cache git client and repo status per repository
+    gitClientCache = new Map();
+    isGitRepoCache = new Map();
+    /**
+     * Discover all Skyramp tests in a repository
+     * Uses fast-glob for cross-platform file scanning
+     */
+    async discoverTests(repositoryPath) {
+        logger.info(`Starting test discovery in: ${repositoryPath}`);
+        if (!fs.existsSync(repositoryPath)) {
+            throw new Error(`Repository path does not exist: ${repositoryPath}`);
+        }
+        const stats = fs.statSync(repositoryPath);
+        if (!stats.isDirectory()) {
+            throw new Error(`Path is not a directory: ${repositoryPath}`);
+        }
+        // Initialize git client cache for this repository
+        await this.initializeGitClient(repositoryPath);
+        // Use cross-platform file search to find files containing Skyramp marker
+        const testFiles = this.findSkyrampTestsWithGrep(repositoryPath);
+        logger.info(`Found ${testFiles.length} Skyramp test files`);
+        // Process files in parallel with concurrency control
+        const skyrampTests = await this.processFilesInBatches(testFiles, repositoryPath);
+        logger.info(`Discovered ${skyrampTests.length} Skyramp tests`);
+        // Clean up cache to free memory
+        this.gitClientCache.clear();
+        this.isGitRepoCache.clear();
+        return {
+            tests: skyrampTests,
+        };
+    }
+    /**
+     * Initialize git client and check if repository is a git repo
+     */
+    async initializeGitClient(repositoryPath) {
+        try {
+            const git = simpleGit(repositoryPath);
+            this.gitClientCache.set(repositoryPath, git);
+            const isRepo = await git.checkIsRepo();
+            this.isGitRepoCache.set(repositoryPath, isRepo);
+            if (isRepo) {
+                logger.debug(`Git repository detected at: ${repositoryPath}`);
+            }
+            else {
+                logger.debug(`Not a git repository: ${repositoryPath}`);
+            }
+        }
+        catch (error) {
+            logger.debug(`Could not initialize git client: ${error.message}`);
+            this.isGitRepoCache.set(repositoryPath, false);
+        }
+    }
+    /**
+     * Process test files in parallel batches with concurrency control
+     */
+    async processFilesInBatches(testFiles, repositoryPath) {
+        const results = [];
+        // Process files in batches to control concurrency
+        for (let i = 0; i < testFiles.length; i += this.MAX_CONCURRENT_OPERATIONS) {
+            const batch = testFiles.slice(i, i + this.MAX_CONCURRENT_OPERATIONS);
+            const batchResults = await Promise.all(batch.map(async (testFile) => {
+                try {
+                    return await this.extractTestMetadata(testFile, repositoryPath);
+                }
+                catch (error) {
+                    logger.error(`Error processing test file ${testFile}: ${error}`);
+                    return null;
+                }
+            }));
+            // Filter out null results and add to final results
+            results.push(...batchResults.filter((test) => test !== null));
+        }
+        return results;
+    }
+    /**
+     * Find files containing Skyramp marker using cross-platform Node.js file search
+     */
+    findSkyrampTestsWithGrep(repositoryPath) {
+        try {
+            // Build glob patterns for supported extensions
+            const globPatterns = this.SUPPORTED_EXTENSIONS.map((ext) => `**/*${ext}`);
+            // Build ignore patterns for excluded directories
+            const ignorePatterns = this.EXCLUDED_DIRS.map((dir) => `**/${dir}/**`);
+            // Use fast-glob to find all matching files
+            const allFiles = fg.sync(globPatterns, {
+                cwd: repositoryPath,
+                ignore: ignorePatterns,
+                absolute: true,
+                caseSensitiveMatch: false,
+            });
+            logger.debug(`Found ${allFiles.length} candidate files to search`);
+            // Read files and check for Skyramp marker
+            const matchingFiles = [];
+            const marker = this.SKYRAMP_MARKER;
+            // Process files in batches to avoid memory issues
+            const batchSize = 100;
+            for (let i = 0; i < allFiles.length; i += batchSize) {
+                const batch = allFiles.slice(i, i + batchSize);
+                const batchResults = batch.filter((file) => {
+                    try {
+                        // Read file and check for marker
+                        const content = fs.readFileSync(file, "utf-8");
+                        return content.includes(marker);
+                    }
+                    catch (error) {
+                        // Skip files that can't be read (permissions, etc.)
+                        logger.debug(`Skipping file ${file}: ${error}`);
+                        return false;
+                    }
+                });
+                matchingFiles.push(...batchResults);
+            }
+            logger.debug(`Found ${matchingFiles.length} files with Skyramp marker`);
+            return matchingFiles;
+        }
+        catch (error) {
+            logger.error(`File search failed: ${error.message}`);
+            logger.info("Falling back to directory scanning method");
+            return [];
+        }
+    }
+    /**
+     * Extract metadata from a test file
+     * File is already confirmed to contain Skyramp marker by file search
+     */
+    async extractTestMetadata(testFile, repositoryPath) {
+        let content;
+        try {
+            // Read full file (file search already confirmed it has Skyramp marker)
+            content = fs.readFileSync(testFile, "utf-8");
+        }
+        catch (error) {
+            logger.debug(`Could not read file ${testFile}: ${error}`);
+            return null;
+        }
+        const language = this.detectLanguage(testFile);
+        const testType = this.detectTestType(content, testFile);
+        const apiSchema = this.extractApiSchema(content);
+        const framework = this.extractFramework(content);
+        return {
+            testFile,
+            testType,
+            language,
+            framework,
+            apiSchema,
+        };
+    }
+    /**
+     * Detect programming language from file extension
+     */
+    detectLanguage(testFile) {
+        const ext = path.extname(testFile);
+        const languageMap = {
+            ".py": "python",
+            ".js": "javascript",
+            ".ts": "typescript",
+            ".java": "java",
+        };
+        return languageMap[ext] || "unknown";
+    }
+    /**
+     * Detect test type from file content and name
+     * Checks Skyramp command line, filename, and content patterns
+     */
+    detectTestType(content, testFile) {
+        const testTypes = [
+            "smoke",
+            "integration",
+            "load",
+            "contract",
+            "fuzz",
+            "e2e",
+            "ui",
+        ];
+        // First, try to extract from Skyramp command line
+        // Pattern: skyramp generate smoke rest ...
+        const commandMatch = content.match(/skyramp generate (\w+)/i);
+        if (commandMatch && commandMatch[1]) {
+            const type = commandMatch[1].toLowerCase();
+            if (testTypes.includes(type)) {
+                return type;
+            }
+        }
+        return "";
+    }
+    /**
+     * Extract API schema path from test content
+     * Looks for Skyramp command line first, then fallback to other patterns
+     */
+    extractApiSchema(content) {
+        // First, try to extract from Skyramp command line
+        // Pattern: --api-schema http://localhost:8000/openapi.json
+        const apiSchemaMatch = content.match(/--api-schema\s+([^\s\\]+)/);
+        if (apiSchemaMatch && apiSchemaMatch[1]) {
+            return apiSchemaMatch[1];
+        }
+        return "";
+    }
+    /**
+     * Extract Framework from test content
+     * Looks for Skyramp command line first, then fallback to other patterns
+     */
+    extractFramework(content) {
+        // First, try to extract from Skyramp command line
+        // Pattern: --framework pytest
+        const frameworkMatch = content.match(/--framework\s+([^\s\\]+)/);
+        if (frameworkMatch && frameworkMatch[1]) {
+            return frameworkMatch[1];
+        }
+        return "";
+    }
+}

package/build/services/TestExecutionService.js

@@ -0,0 +1,311 @@
+import Docker from "dockerode";
+import path from "path";
+import fs from "fs";
+import { Writable } from "stream";
+import { stripVTControlCharacters } from "util";
+import { logger } from "../utils/logger.js";
+const DEFAULT_TIMEOUT = 300000; // 5 minutes
+const MAX_CONCURRENT_EXECUTIONS = 5;
+const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.2.38";
+const DOCKER_PLATFORM = "linux/amd64";
+// Files and directories to exclude when mounting workspace to Docker container
+export const EXCLUDED_MOUNT_ITEMS = [
+    "package-lock.json",
+    "package.json",
+    "node_modules",
+];
+export class TestExecutionService {
+    docker;
+    imageReady;
+    constructor() {
+        this.docker = new Docker();
+        this.imageReady = this.ensureDockerImage();
+    }
+    /**
+     * Execute multiple tests in parallel batches
+     */
+    async executeBatch(testOptions) {
+        logger.info(`Starting batch execution of ${testOptions.length} tests`);
+        const startTime = Date.now();
+        const results = [];
+        // Execute tests in batches to control concurrency
+        for (let i = 0; i < testOptions.length; i += MAX_CONCURRENT_EXECUTIONS) {
+            const batch = testOptions.slice(i, i + MAX_CONCURRENT_EXECUTIONS);
+            logger.debug(`Executing batch ${Math.floor(i / MAX_CONCURRENT_EXECUTIONS) + 1} (${batch.length} tests)`);
+            const batchPromises = batch.map((options) => this.executeTest(options).catch((error) => {
+                // If execution fails completely, return error result
+                return {
+                    testFile: options.testFile,
+                    passed: false,
+                    executedAt: new Date().toISOString(),
+                    duration: 0,
+                    errors: [error.message],
+                    warnings: [],
+                    crashed: true,
+                };
+            }));
+            const batchResults = await Promise.all(batchPromises);
+            results.push(...batchResults);
+        }
+        const totalDuration = Date.now() - startTime;
+        const passed = results.filter((r) => r.passed).length;
+        const failed = results.filter((r) => !r.passed && !r.crashed).length;
+        const crashed = results.filter((r) => r.crashed).length;
+        logger.info(`Batch execution complete: ${passed} passed, ${failed} failed, ${crashed} crashed`);
+        return {
+            totalTests: testOptions.length,
+            passed,
+            failed,
+            crashed,
+            totalDuration,
+            results,
+        };
+    }
+    /**
+     * Execute a single test
+     */
+    async executeTest(options) {
+        const startTime = Date.now();
+        const executedAt = new Date().toISOString();
+        logger.debug(`Executing test: ${options.testFile}`);
+        // Wait for Docker image to be ready
+        await this.imageReady;
+        // Validate workspace path - use accessSync for better validation
+        const workspacePath = path.resolve(options.workspacePath);
+        try {
+            fs.accessSync(workspacePath, fs.constants.R_OK);
+        }
+        catch (err) {
+            throw new Error(`Workspace path does not exist or is not readable: ${workspacePath}`);
+        }
+        // Validate test file - use basename for safer filename extraction
+        const filename = path.basename(options.testFile);
+        if (!filename) {
+            throw new Error("Invalid test file path: could not extract filename");
+        }
+        if (!fs.existsSync(options.testFile)) {
+            throw new Error(`Test file does not exist: ${options.testFile}`);
+        }
+        const containerMountPath = "/home/user";
+        const dockerSocketPath = "/var/run/docker.sock";
+        // Calculate relative test file path
+        let testFilePath = path.relative(workspacePath, options.testFile);
+        testFilePath = path.resolve(containerMountPath, testFilePath);
+        // Prepare Docker command
+        const command = [
+            "./root/runner.sh",
+            options.language,
+            testFilePath,
+            options.testType,
+        ];
+        // Prepare host config with mounts
+        const hostConfig = {
+            ExtraHosts: ["host.docker.internal:host-gateway"],
+            Mounts: [
+                {
+                    Type: "bind",
+                    Target: dockerSocketPath,
+                    Source: dockerSocketPath,
+                },
+            ],
+        };
+        // Mount workspace files (excluding unnecessary items)
+        const workspaceFiles = fs.readdirSync(workspacePath);
+        const filesToMount = workspaceFiles.filter((file) => !EXCLUDED_MOUNT_ITEMS.includes(file));
+        hostConfig.Mounts?.push(...filesToMount.map((file) => ({
+            Type: "bind",
+            Target: path.join(containerMountPath, file),
+            Source: path.join(workspacePath, file),
+        })));
+        // Prepare environment variables
+        const env = [
+            `SKYRAMP_TEST_TOKEN=${options.token || ""}`,
+            "SKYRAMP_IN_DOCKER=true",
+        ];
+        if (process.env.SKYRAMP_DEBUG) {
+            env.push(`SKYRAMP_DEBUG=${process.env.SKYRAMP_DEBUG}`);
+        }
+        if (process.env.API_KEY) {
+            env.push(`API_KEY=${process.env.API_KEY}`);
+        }
+        // Capture output
+        let output = "";
+        class DockerStream extends Writable {
+            _write(data, encode, cb) {
+                output += data.toString();
+                cb();
+            }
+        }
+        const stream = new DockerStream();
+        try {
+            let statusCode = 0;
+            let containerRef = null;
+            // Run container with timeout
+            const executionPromise = this.docker
+                .run(EXECUTOR_DOCKER_IMAGE, command, stream, {
+                Env: env,
+                HostConfig: hostConfig,
+            })
+                .then(function (data) {
+                const result = data[0];
+                const container = data[1];
+                containerRef = container; // Capture container reference for cleanup
+                stream.end();
+                statusCode = result.StatusCode;
+                logger.debug("Docker container execution completed");
+                return container.remove();
+            })
+                .then(function () {
+                logger.debug("Docker container removed successfully");
+                containerRef = null; // Container already removed
+                return statusCode;
+            })
+                .catch(function (err) {
+                logger.error("Docker container execution failed", {
+                    error: err.message,
+                });
+                throw err;
+            });
+            // Apply timeout
+            const timeout = options.timeout || DEFAULT_TIMEOUT;
+            let timeoutHandle;
+            const timeoutPromise = new Promise((_, reject) => {
+                timeoutHandle = setTimeout(() => reject(new Error(`Test execution timeout after ${timeout}ms`)), timeout);
+            });
+            try {
+                statusCode = await Promise.race([executionPromise, timeoutPromise]);
+                // Clear the timeout timer if execution completes successfully
+                if (timeoutHandle)
+                    clearTimeout(timeoutHandle);
+            }
+            catch (error) {
+                // Clear the timeout timer on any error
+                if (timeoutHandle)
+                    clearTimeout(timeoutHandle);
+                // Cleanup container on timeout or other errors
+                if (containerRef !== null) {
+                    const container = containerRef;
+                    try {
+                        logger.warning("Cleaning up orphaned Docker container after error");
+                        await container.stop({ t: 5 }); // Give 5 seconds to stop gracefully
+                        await container.remove({ force: true });
+                        logger.debug("Orphaned container cleaned up successfully");
+                    }
+                    catch (cleanupError) {
+                        logger.error("Failed to cleanup orphaned container", {
+                            error: cleanupError.message,
+                        });
+                    }
+                }
+                throw error; // Re-throw the original error
+            }
+            const duration = Date.now() - startTime;
+            const cleanOutput = stripVTControlCharacters(output);
+            logger.info("Test execution completed", {
+                output: cleanOutput.substring(0, 200) +
+                    (cleanOutput.length > 200 ? "..." : ""),
+                statusCode,
+            });
+            // Parse errors and warnings from output
+            const errors = this.parseErrors(cleanOutput);
+            const warnings = this.parseWarnings(cleanOutput);
+            // Check if test crashed
+            const crashed = statusCode !== 0 && statusCode !== 1;
+            const result = {
+                testFile: options.testFile,
+                passed: statusCode === 0,
+                executedAt,
+                duration,
+                errors,
+                warnings,
+                crashed,
+                output: cleanOutput,
+                exitCode: statusCode,
+            };
+            logger.debug(`Test ${result.passed ? "passed" : "failed"}: ${options.testFile}`);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            const cleanOutput = stripVTControlCharacters(output);
+            logger.error(`Test execution error: ${error.message}`);
+            return {
+                testFile: options.testFile,
+                passed: false,
+                executedAt,
+                duration,
+                errors: [error.message],
+                warnings: [],
+                crashed: true,
+                output: cleanOutput,
+            };
+        }
+    }
+    /**
+     * Ensure Docker image is available
+     */
+    async ensureDockerImage() {
+        try {
+            const images = await this.docker.listImages();
+            const imageExists = images.some((img) => (img.RepoTags && img.RepoTags.includes(EXECUTOR_DOCKER_IMAGE)) ||
+                (img.RepoDigests && img.RepoDigests.includes(EXECUTOR_DOCKER_IMAGE)));
+            if (imageExists) {
+                logger.debug(`Docker image ${EXECUTOR_DOCKER_IMAGE} already available`);
+                return;
+            }
+            logger.info(`Pulling Docker image ${EXECUTOR_DOCKER_IMAGE}...`);
+            await new Promise((resolve, reject) => {
+                this.docker.pull(EXECUTOR_DOCKER_IMAGE, { platform: DOCKER_PLATFORM }, (err, stream) => {
+                    if (err)
+                        return reject(err);
+                    if (!stream)
+                        return reject(new Error("No stream received from docker pull"));
+                    this.docker.modem.followProgress(stream, (err, res) => {
+                        if (err)
+                            return reject(err);
+                        logger.info(`Docker image ${EXECUTOR_DOCKER_IMAGE} pulled successfully`);
+                        resolve(res);
+                    });
+                });
+            });
+        }
+        catch (error) {
+            logger.error(`Failed to ensure Docker image: ${error.message}`);
+            throw new Error(`Docker image setup failed: ${error.message}`);
+        }
+    }
+    /**
+     * Parse errors from test output
+     */
+    parseErrors(output) {
+        const errors = [];
+        const lines = output.split("\n");
+        for (const line of lines) {
+            // Common error patterns
+            if (line.includes("Error:") ||
+                line.includes("ERROR") ||
+                line.includes("AssertionError") ||
+                line.includes("Exception") ||
+                line.includes("FAILED") ||
+                line.includes("Traceback")) {
+                errors.push(line.trim());
+            }
+        }
+        return errors.slice(0, 20); // Limit to first 20 errors
+    }
+    /**
+     * Parse warnings from test output
+     */
+    parseWarnings(output) {
+        const warnings = [];
+        const lines = output.split("\n");
+        for (const line of lines) {
+            if (line.includes("Warning:") ||
+                line.includes("WARN") ||
+                line.includes("Deprecated")) {
+                warnings.push(line.trim());
+            }
+        }
+        return warnings.slice(0, 10); // Limit to first 10 warnings
+    }
+}

package/build/services/TestGenerationService.js

@@ -44,12 +44,26 @@ export class TestGenerationService {
 `;
             }
             else if (params.modularizeCode) {
-                // Only modularization
-                postGenerationMessage += `
+                // Check if test type should be modularized
+                if (!params.trace && !params.scenarioFile && !params.playwrightInput) {
+                    postGenerationMessage += `
+ Test generated successfully!
+
+**NOTE**: Modularization is intentionally NOT applied to ${testType.toUpperCase()} tests.
+${testType.toUpperCase()} tests are designed to be simple, straightforward, and deterministic. 
+Modularization can introduce unpredictable results and unnecessary complexity for these test types.
+
+The generated test file remains unchanged and ready to use as-is.
+`;
+                }
+                else {
+                    // Only modularization for UI, E2E, Integration tests
+                    postGenerationMessage += `
  Test generated successfully!
 
 ⏭️**CRITICAL NEXT STEP**: Running modularization with skyramp_modularization tool.
 `;
+                }
             }
             return {
                 content: [