@skyramp/mcp 0.0.44 → 0.0.46

package/build/services/TestExecutionService.js

@@ -0,0 +1,504 @@
+import Docker from "dockerode";
+import path from "path";
+import fs from "fs";
+import { Writable } from "stream";
+import { stripVTControlCharacters } from "util";
+import { logger } from "../utils/logger.js";
+const DEFAULT_TIMEOUT = 300000; // 5 minutes
+const MAX_CONCURRENT_EXECUTIONS = 5;
+const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.2.40";
+const DOCKER_PLATFORM = "linux/amd64";
+// Files and directories to exclude when mounting workspace to Docker container
+export const EXCLUDED_MOUNT_ITEMS = [
+    "package-lock.json",
+    "package.json",
+    "node_modules",
+];
+/**
+ * Find the start index of a comment in a line, ignoring comment delimiters inside strings
+ * Returns -1 if no comment is found outside of strings
+ */
+function findCommentStart(line) {
+    let inSingleQuote = false;
+    let inDoubleQuote = false;
+    let escaped = false;
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+        const nextChar = i + 1 < line.length ? line[i + 1] : '';
+        // Handle escape sequences
+        if (escaped) {
+            escaped = false;
+            continue;
+        }
+        if (char === '\\') {
+            escaped = true;
+            continue;
+        }
+        // Track string boundaries
+        if (char === "'" && !inDoubleQuote) {
+            inSingleQuote = !inSingleQuote;
+            continue;
+        }
+        if (char === '"' && !inSingleQuote) {
+            inDoubleQuote = !inDoubleQuote;
+            continue;
+        }
+        // Check for comments only if not in a string
+        if (!inSingleQuote && !inDoubleQuote) {
+            if (char === '/' && nextChar === '/') {
+                return i;
+            }
+            if (char === '#') {
+                return i;
+            }
+        }
+    }
+    return -1; // No comment found
+}
+/**
+ * Filter out comments from code lines
+ * Returns array of non-comment lines with inline comments removed
+ */
+function filterComments(lines) {
+    const nonCommentLines = [];
+    let inMultiLineComment = false;
+    let inPythonMultiLineComment = false;
+    for (let line of lines) {
+        const trimmed = line.trim();
+        // Check for Python multi-line string comments (""" or ''')
+        // Count occurrences to handle single-line strings like '''comment'''
+        const tripleDoubleCount = (trimmed.match(/"""/g) || []).length;
+        const tripleSingleCount = (trimmed.match(/'''/g) || []).length;
+        if (tripleDoubleCount > 0) {
+            // Odd number: opening or closing a multi-line comment (toggle state)
+            // Even number: complete string on same line (e.g., """comment""" - don't toggle)
+            if (tripleDoubleCount % 2 === 1) {
+                inPythonMultiLineComment = !inPythonMultiLineComment;
+            }
+            continue; // Skip any line with triple quotes
+        }
+        if (tripleSingleCount > 0) {
+            if (tripleSingleCount % 2 === 1) {
+                inPythonMultiLineComment = !inPythonMultiLineComment;
+            }
+            continue;
+        }
+        if (inPythonMultiLineComment) {
+            continue;
+        }
+        // Check for multi-line comment start/end (/* */)
+        if (trimmed.includes('/*')) {
+            if (trimmed.includes('*/')) {
+                // Single-line multi-line comment (e.g., /* comment */)
+                // Don't change state, just skip the line
+                continue;
+            }
+            else {
+                // Opening a multi-line comment
+                inMultiLineComment = true;
+                continue;
+            }
+        }
+        if (inMultiLineComment) {
+            if (trimmed.includes('*/')) {
+                inMultiLineComment = false;
+            }
+            continue;
+        }
+        // Skip single-line comments
+        if (trimmed.startsWith('//') || trimmed.startsWith('#')) {
+            continue;
+        }
+        // Remove inline comments from the line before processing
+        // Use helper function to avoid removing comment delimiters inside strings
+        const commentIndex = findCommentStart(line);
+        if (commentIndex >= 0) {
+            line = line.substring(0, commentIndex);
+        }
+        nonCommentLines.push(line);
+    }
+    return nonCommentLines;
+}
+/**
+ * Detect session file paths referenced in test files
+ * Looks for storageState patterns in TypeScript/JavaScript/Python/Java/C# test files
+ * Excludes matches found in comments
+ */
+function detectSessionFiles(testFilePath) {
+    try {
+        const content = fs.readFileSync(testFilePath, "utf-8");
+        const lines = content.split('\n');
+        const sessionFiles = [];
+        // Pattern for TypeScript/JavaScript: storageState: '/path/to/file' or storageState: "/path/to/file"
+        const tsJsPattern = /storageState:\s*['"]([^'"]+)['"]/g;
+        // Pattern for Python: storage_state='/path/to/file' or storage_state="/path/to/file"
+        const pythonPattern = /storage_state\s*=\s*['"]([^'"]+)['"]/g;
+        // Pattern for Java: setStorageState(Paths.get("path")) or setStorageState("path")
+        // Enforces proper parenthesis matching: first ) is required, second ) is required when using Paths.get
+        const javaPattern = /setStorageState(?:Path)?\s*\(\s*(?:Paths\.get\s*\(\s*)?['"]([^'"]+)['"]\s*\)(?:\s*\))?/g;
+        // Pattern for C#: StorageStatePath = "path" or StorageState = "path"
+        const csharpPattern = /StorageState(?:Path)?\s*=\s*['"]([^'"]+)['"]/g;
+        // Filter out comments
+        const codeLines = filterComments(lines);
+        // Process each non-comment line
+        for (const line of codeLines) {
+            // Try all patterns on this line
+            let match;
+            tsJsPattern.lastIndex = 0;
+            while ((match = tsJsPattern.exec(line)) !== null) {
+                sessionFiles.push(match[1]);
+            }
+            pythonPattern.lastIndex = 0;
+            while ((match = pythonPattern.exec(line)) !== null) {
+                sessionFiles.push(match[1]);
+            }
+            javaPattern.lastIndex = 0;
+            while ((match = javaPattern.exec(line)) !== null) {
+                sessionFiles.push(match[1]);
+            }
+            csharpPattern.lastIndex = 0;
+            while ((match = csharpPattern.exec(line)) !== null) {
+                sessionFiles.push(match[1]);
+            }
+        }
+        return sessionFiles;
+    }
+    catch (error) {
+        logger.error(`Failed to detect session files in ${testFilePath}`, { error });
+        return [];
+    }
+}
+export class TestExecutionService {
+    docker;
+    imageReady;
+    constructor() {
+        this.docker = new Docker();
+        this.imageReady = this.ensureDockerImage();
+    }
+    /**
+     * Execute multiple tests in parallel batches
+     */
+    async executeBatch(testOptions) {
+        logger.info(`Starting batch execution of ${testOptions.length} tests`);
+        const startTime = Date.now();
+        const results = [];
+        // Execute tests in batches to control concurrency
+        for (let i = 0; i < testOptions.length; i += MAX_CONCURRENT_EXECUTIONS) {
+            const batch = testOptions.slice(i, i + MAX_CONCURRENT_EXECUTIONS);
+            logger.debug(`Executing batch ${Math.floor(i / MAX_CONCURRENT_EXECUTIONS) + 1} (${batch.length} tests)`);
+            const batchPromises = batch.map((options) => this.executeTest(options).catch((error) => {
+                // If execution fails completely, return error result
+                return {
+                    testFile: options.testFile,
+                    passed: false,
+                    executedAt: new Date().toISOString(),
+                    duration: 0,
+                    errors: [error.message],
+                    warnings: [],
+                    crashed: true,
+                };
+            }));
+            const batchResults = await Promise.all(batchPromises);
+            results.push(...batchResults);
+        }
+        const totalDuration = Date.now() - startTime;
+        const passed = results.filter((r) => r.passed).length;
+        const failed = results.filter((r) => !r.passed && !r.crashed).length;
+        const crashed = results.filter((r) => r.crashed).length;
+        logger.info(`Batch execution complete: ${passed} passed, ${failed} failed, ${crashed} crashed`);
+        return {
+            totalTests: testOptions.length,
+            passed,
+            failed,
+            crashed,
+            totalDuration,
+            results,
+        };
+    }
+    /**
+     * Execute a single test
+     */
+    async executeTest(options) {
+        const startTime = Date.now();
+        const executedAt = new Date().toISOString();
+        logger.debug(`Executing test: ${options.testFile}`);
+        // Wait for Docker image to be ready
+        await this.imageReady;
+        // Validate workspace path - use accessSync for better validation
+        const workspacePath = path.resolve(options.workspacePath);
+        try {
+            fs.accessSync(workspacePath, fs.constants.R_OK);
+        }
+        catch (err) {
+            throw new Error(`Workspace path does not exist or is not readable: ${workspacePath}`);
+        }
+        // Validate test file - use basename for safer filename extraction
+        const filename = path.basename(options.testFile);
+        if (!filename) {
+            throw new Error("Invalid test file path: could not extract filename");
+        }
+        if (!fs.existsSync(options.testFile)) {
+            throw new Error(`Test file does not exist: ${options.testFile}`);
+        }
+        const containerMountPath = "/home/user";
+        const dockerSocketPath = "/var/run/docker.sock";
+        // Calculate relative test file path
+        let testFilePath = path.relative(workspacePath, options.testFile);
+        testFilePath = path.resolve(containerMountPath, testFilePath);
+        // Prepare Docker command
+        const command = [
+            "/root/runner.sh",
+            options.language,
+            testFilePath,
+            options.testType,
+        ];
+        // Prepare host config with mounts
+        const hostConfig = {
+            ExtraHosts: ["host.docker.internal:host-gateway"],
+            Mounts: [
+                {
+                    Type: "bind",
+                    Target: dockerSocketPath,
+                    Source: dockerSocketPath,
+                },
+            ],
+        };
+        // Mount workspace files (excluding unnecessary items)
+        const workspaceFiles = fs.readdirSync(workspacePath);
+        const filesToMount = workspaceFiles.filter((file) => !EXCLUDED_MOUNT_ITEMS.includes(file));
+        hostConfig.Mounts?.push(...filesToMount.map((file) => ({
+            Type: "bind",
+            Target: path.join(containerMountPath, file),
+            Source: path.join(workspacePath, file),
+        })));
+        // Detect and mount session files
+        const sessionFiles = detectSessionFiles(options.testFile);
+        const mountedPaths = new Set(); // Track mounted file paths to prevent duplicates
+        for (const sessionFile of sessionFiles) {
+            let sessionFileSource;
+            let sessionFileTarget;
+            if (path.isAbsolute(sessionFile)) {
+                // Absolute path: mount at the same absolute path in container
+                sessionFileSource = sessionFile;
+                sessionFileTarget = sessionFile;
+            }
+            else {
+                // Relative path: resolve from test file directory on host
+                const testFileDir = path.dirname(options.testFile);
+                sessionFileSource = path.resolve(testFileDir, sessionFile);
+                // Mount at the corresponding relative path in the container
+                const testFileDirInContainer = path.dirname(testFilePath);
+                sessionFileTarget = path.resolve(testFileDirInContainer, sessionFile);
+            }
+            // Check if session file exists on host
+            if (fs.existsSync(sessionFileSource)) {
+                // Only mount if we haven't already mounted this path
+                if (!mountedPaths.has(sessionFileTarget)) {
+                    logger.info(` docker run -v ${sessionFileSource}:${sessionFileTarget}:ro ...`);
+                    hostConfig.Mounts?.push({
+                        Type: "bind",
+                        Target: sessionFileTarget,
+                        Source: sessionFileSource,
+                        ReadOnly: true,
+                    });
+                    mountedPaths.add(sessionFileTarget);
+                }
+            }
+            else {
+                logger.error(`✗ Session file not found: ${sessionFileSource}`);
+                logger.error(`  Referenced in test as: ${sessionFile}`);
+            }
+        }
+        // Prepare environment variables
+        const env = [
+            `SKYRAMP_TEST_TOKEN=${options.token || ""}`,
+            "SKYRAMP_IN_DOCKER=true",
+        ];
+        if (process.env.SKYRAMP_DEBUG) {
+            env.push(`SKYRAMP_DEBUG=${process.env.SKYRAMP_DEBUG}`);
+        }
+        if (process.env.API_KEY) {
+            env.push(`API_KEY=${process.env.API_KEY}`);
+        }
+        // Capture output
+        let output = "";
+        class DockerStream extends Writable {
+            _write(data, encode, cb) {
+                output += data.toString();
+                cb();
+            }
+        }
+        const stream = new DockerStream();
+        try {
+            let statusCode = 0;
+            let containerRef = null;
+            // Run container with timeout
+            const executionPromise = this.docker
+                .run(EXECUTOR_DOCKER_IMAGE, command, stream, {
+                Env: env,
+                HostConfig: hostConfig,
+                WorkingDir: containerMountPath, // Explicitly set working directory
+            })
+                .then(function (data) {
+                const result = data[0];
+                const container = data[1];
+                containerRef = container; // Capture container reference for cleanup
+                stream.end();
+                statusCode = result.StatusCode;
+                logger.debug("Docker container execution completed");
+                return container.remove();
+            })
+                .then(function () {
+                logger.debug("Docker container removed successfully");
+                containerRef = null; // Container already removed
+                return statusCode;
+            })
+                .catch(function (err) {
+                logger.error("Docker container execution failed", {
+                    error: err.message,
+                });
+                throw err;
+            });
+            // Apply timeout
+            const timeout = options.timeout || DEFAULT_TIMEOUT;
+            let timeoutHandle;
+            const timeoutPromise = new Promise((_, reject) => {
+                timeoutHandle = setTimeout(() => reject(new Error(`Test execution timeout after ${timeout}ms`)), timeout);
+            });
+            try {
+                statusCode = await Promise.race([executionPromise, timeoutPromise]);
+                // Clear the timeout timer if execution completes successfully
+                if (timeoutHandle)
+                    clearTimeout(timeoutHandle);
+            }
+            catch (error) {
+                // Clear the timeout timer on any error
+                if (timeoutHandle)
+                    clearTimeout(timeoutHandle);
+                // Cleanup container on timeout or other errors
+                if (containerRef !== null) {
+                    const container = containerRef;
+                    try {
+                        logger.warning("Cleaning up orphaned Docker container after error");
+                        await container.stop({ t: 5 }); // Give 5 seconds to stop gracefully
+                        await container.remove({ force: true });
+                        logger.debug("Orphaned container cleaned up successfully");
+                    }
+                    catch (cleanupError) {
+                        logger.error("Failed to cleanup orphaned container", {
+                            error: cleanupError.message,
+                        });
+                    }
+                }
+                throw error; // Re-throw the original error
+            }
+            const duration = Date.now() - startTime;
+            const cleanOutput = stripVTControlCharacters(output);
+            logger.info("Test execution completed", {
+                output: cleanOutput.substring(0, 200) +
+                    (cleanOutput.length > 200 ? "..." : ""),
+                statusCode,
+            });
+            // Parse errors and warnings from output
+            const errors = this.parseErrors(cleanOutput);
+            const warnings = this.parseWarnings(cleanOutput);
+            // Check if test crashed
+            const crashed = statusCode !== 0 && statusCode !== 1;
+            const result = {
+                testFile: options.testFile,
+                passed: statusCode === 0,
+                executedAt,
+                duration,
+                errors,
+                warnings,
+                crashed,
+                output: cleanOutput,
+                exitCode: statusCode,
+            };
+            logger.debug(`Test ${result.passed ? "passed" : "failed"}: ${options.testFile}`);
+            return result;
+        }
+        catch (error) {
+            const duration = Date.now() - startTime;
+            const cleanOutput = stripVTControlCharacters(output);
+            logger.error(`Test execution error: ${error.message}`);
+            return {
+                testFile: options.testFile,
+                passed: false,
+                executedAt,
+                duration,
+                errors: [error.message],
+                warnings: [],
+                crashed: true,
+                output: cleanOutput,
+            };
+        }
+    }
+    /**
+     * Ensure Docker image is available
+     */
+    async ensureDockerImage() {
+        try {
+            const images = await this.docker.listImages();
+            const imageExists = images.some((img) => (img.RepoTags && img.RepoTags.includes(EXECUTOR_DOCKER_IMAGE)) ||
+                (img.RepoDigests && img.RepoDigests.includes(EXECUTOR_DOCKER_IMAGE)));
+            if (imageExists) {
+                logger.debug(`Docker image ${EXECUTOR_DOCKER_IMAGE} already available`);
+                return;
+            }
+            logger.info(`Pulling Docker image ${EXECUTOR_DOCKER_IMAGE}...`);
+            await new Promise((resolve, reject) => {
+                this.docker.pull(EXECUTOR_DOCKER_IMAGE, { platform: DOCKER_PLATFORM }, (err, stream) => {
+                    if (err)
+                        return reject(err);
+                    if (!stream)
+                        return reject(new Error("No stream received from docker pull"));
+                    this.docker.modem.followProgress(stream, (err, res) => {
+                        if (err)
+                            return reject(err);
+                        logger.info(`Docker image ${EXECUTOR_DOCKER_IMAGE} pulled successfully`);
+                        resolve(res);
+                    });
+                });
+            });
+        }
+        catch (error) {
+            logger.error(`Failed to ensure Docker image: ${error.message}`);
+            throw new Error(`Docker image setup failed: ${error.message}`);
+        }
+    }
+    /**
+     * Parse errors from test output
+     */
+    parseErrors(output) {
+        const errors = [];
+        const lines = output.split("\n");
+        for (const line of lines) {
+            // Common error patterns
+            if (line.includes("Error:") ||
+                line.includes("ERROR") ||
+                line.includes("AssertionError") ||
+                line.includes("Exception") ||
+                line.includes("FAILED") ||
+                line.includes("Traceback")) {
+                errors.push(line.trim());
+            }
+        }
+        return errors.slice(0, 20); // Limit to first 20 errors
+    }
+    /**
+     * Parse warnings from test output
+     */
+    parseWarnings(output) {
+        const warnings = [];
+        const lines = output.split("\n");
+        for (const line of lines) {
+            if (line.includes("Warning:") ||
+                line.includes("WARN") ||
+                line.includes("Deprecated")) {
+                warnings.push(line.trim());
+            }
+        }
+        return warnings.slice(0, 10); // Limit to first 10 warnings
+    }
+}

package/build/services/TestGenerationService.js

@@ -44,12 +44,26 @@ export class TestGenerationService {
 `;
             }
             else if (params.modularizeCode) {
-                // Only modularization
-                postGenerationMessage += `
+                // Check if test type should be modularized
+                if (!params.trace && !params.scenarioFile && !params.playwrightInput) {
+                    postGenerationMessage += `
+ Test generated successfully!
+
+**NOTE**: Modularization is intentionally NOT applied to ${testType.toUpperCase()} tests.
+${testType.toUpperCase()} tests are designed to be simple, straightforward, and deterministic. 
+Modularization can introduce unpredictable results and unnecessary complexity for these test types.
+
+The generated test file remains unchanged and ready to use as-is.
+`;
+                }
+                else {
+                    // Only modularization for UI, E2E, Integration tests
+                    postGenerationMessage += `
  Test generated successfully!
 
 ⏭️**CRITICAL NEXT STEP**: Running modularization with skyramp_modularization tool.
 `;
+                }
             }
             return {
                 content: [