@skyramp/mcp 0.2.8 → 0.2.150-rc.local.de

package/build/prompts/testbot/testbot-prompts.d.ts

@@ -17,10 +17,5 @@ export interface RelatedRepository {
 export declare function parseRelatedRepositories(raw: string | undefined): RelatedRepository[] | undefined;
 export declare function getTestbotPrompt(prTitle: string, prDescription: string, summaryOutputFile: string, repositoryPath: string, baseBranch?: string, maxRecommendations?: number, maxGenerate?: number, _maxCritical?: number, // Reserved — accepted for API compat but not yet wired into prompt
 prNumber?: number, userPrompt?: string, services?: Service[], uiCredentials?: string, testsRepoDir?: string, relatedRepositories?: RelatedRepository[], primaryRepo?: string): string;
-/**
- * Read services from .skyramp/workspace.yml. Returns empty array if
- * the workspace file doesn't exist or can't be parsed.
- */
-export declare function readWorkspaceServices(repositoryPath: string): Promise<Service[]>;
 export declare function buildWorkspaceRecoveryPrefix(repositoryPath: string): string;
 export declare function registerTestbotPrompt(server: McpServer): void;

package/build/prompts/testbot/testbot-prompts.js

@@ -5,7 +5,7 @@ import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PA
 import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
 import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
 import { resolveServiceDetailsRef } from "../../utils/utils.js";
-import { readWorkspaceConfigRaw } from "../../utils/workspaceAuth.js";
+import { buildServiceContext, readWorkspaceServices, } from "../prompt-utils.js";
 // Cached at module-load — flags are process-wide and cannot change per call.
 const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
 const SERVICE_REFS = resolveServiceDetailsRef();
@@ -547,7 +547,9 @@ Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than o
 4. **Wait**: Do NOT proceed to test execution until steps 1–3 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
 Do not make any changes other than the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
 
-**Final execution (mandatory):** Do NOT call \`skyramp_execute_test\` until ALL maintenance edits AND ALL new test generation/enhancement are complete.
+**Execution timing:**
+- **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
+- **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection.
 - Only report test results for files you actually ran.
 **Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.
 
@@ -592,38 +594,6 @@ ${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modular
     // removing stateOutputFile from the prompt schema. Remove the RUNNER_TEMP branch in
     // AnalysisStateManager.ts when this is done.
 }
-function escapeXml(value) {
-    return value
-        .replaceAll('&', '&')
-        .replaceAll('<', '&lt;')
-        .replaceAll('>', '&gt;')
-        .replaceAll('"', '&quot;')
-        .replaceAll("'", '&apos;');
-}
-function buildServiceContext(services) {
-    const blocks = services.map(svc => {
-        const parts = [`<service name="${escapeXml(svc.serviceName)}">`];
-        if (svc.language)
-            parts.push(`  <language>${escapeXml(svc.language)}</language>`);
-        if (svc.framework)
-            parts.push(`  <framework>${escapeXml(svc.framework)}</framework>`);
-        if (svc.api?.baseUrl)
-            parts.push(`  <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
-        if (svc.testDirectory)
-            parts.push(`  <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
-        parts.push('</service>');
-        return parts.join('\n');
-    });
-    return `<services>\n${blocks.join('\n')}\n</services>`;
-}
-/**
- * Read services from .skyramp/workspace.yml. Returns empty array if
- * the workspace file doesn't exist or can't be parsed.
- */
-export async function readWorkspaceServices(repositoryPath) {
-    const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
-    return (rawConfig?.services ?? []);
-}
 export function buildWorkspaceRecoveryPrefix(repositoryPath) {
     return `IMPORTANT: The existing .skyramp/workspace.yml failed to parse or validate. Before proceeding with any tasks below, you MUST call skyramp_init_scan with workspacePath "${repositoryPath}" and force: true, then call skyramp_init_workspace with workspacePath "${repositoryPath}", the discovered services, scanToken, and force: true to regenerate the workspace file.\n\n`;
 }

package/build/resources/testbotResource.js

@@ -2,7 +2,8 @@ import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { logger } from "../utils/logger.js";
 import { AnalyticsService } from "../services/AnalyticsService.js";
 import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, } from "../prompts/test-recommendation/recommendationSections.js";
-import { getTestbotPrompt, readWorkspaceServices, parseRelatedRepositories, } from "../prompts/testbot/testbot-prompts.js";
+import { getTestbotPrompt, parseRelatedRepositories, } from "../prompts/testbot/testbot-prompts.js";
+import { readWorkspaceServices } from "../prompts/prompt-utils.js";
 export function registerTestbotResource(server) {
     logger.info("Registering testbot resource");
     // RFC 6570 {+rest} (reserved expansion) captures the entire query string

package/build/services/TestExecutionService.d.ts

@@ -1,5 +1,6 @@
 import { TestExecutionResult, BatchExecutionResult, TestExecutionOptions, ProgressCallback } from "../types/TestExecution.js";
-export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.29";
+import { EXECUTOR_DOCKER_IMAGE } from "../utils/versions.js";
+export { EXECUTOR_DOCKER_IMAGE };
 export declare const PLAYWRIGHT_CONFIG_FILES: string[];
 export declare const EXCLUDED_MOUNT_ITEMS: string[];
 export declare const MOUNT_NULL_ITEMS: string[];
@@ -25,19 +26,26 @@ export declare function getVideoSubdir(testFile: string): string;
  */
 export declare function detectSessionFiles(testFilePath: string): string[];
 export declare class TestExecutionService {
+    private static executionQueue;
     private docker;
     private imageReady;
     constructor();
+    private cleanupSkyrampExecutionContainers;
+    private prepareSutForExecution;
+    private stopSutAfterExecution;
     /**
      * Execute multiple tests in parallel batches
      */
     executeBatch(testOptions: TestExecutionOptions[]): Promise<BatchExecutionResult>;
     /**
-     * Execute a single test
+     * Execute a single test. Executions are serialized within one MCP server
+     * process because pre-run cleanup intentionally removes stale Skyramp
+     * worker/executor containers before each test.
      * @param options Test execution options
      * @param onProgress Optional callback for progress updates
      */
     executeTest(options: TestExecutionOptions, onProgress?: ProgressCallback): Promise<TestExecutionResult>;
+    private executeTestInternal;
     /**
      * Ensure Docker image is available
      * @param onProgress Optional callback for progress updates during pull

package/build/services/TestExecutionService.js

@@ -4,17 +4,24 @@ import fs from "fs";
 import os from "os";
 import crypto from "crypto";
 import { Writable } from "stream";
+import { exec } from "child_process";
+import { promisify } from "util";
 import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
 import { TestExecutionStatus, } from "../types/TestExecution.js";
 import { TestType } from "../types/TestTypes.js";
 import { buildContainerEnv } from "./containerEnv.js";
-import { SKYRAMP_IMAGE_VERSION } from "../utils/versions.js";
+import { readWorkspaceConfigRaw } from "../utils/workspaceAuth.js";
+import { EXECUTOR_DOCKER_IMAGE } from "../utils/versions.js";
+export { EXECUTOR_DOCKER_IMAGE };
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
-const MAX_CONCURRENT_EXECUTIONS = 5;
-export const EXECUTOR_DOCKER_IMAGE = `skyramp/executor:${SKYRAMP_IMAGE_VERSION}`;
+const OWNED_EXECUTION_CONTAINER_LABEL = "dev.skyramp.mcp.execution";
+const OWNED_EXECUTION_CONTAINER_LABEL_VALUE = "true";
 const DOCKER_PLATFORM = "linux/amd64";
 const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
+const execAsync = promisify(exec);
+const SUT_READY_TIMEOUT_MS = 15000;
+const SUT_READY_INTERVAL_MS = 1000;
 // Temp file with valid empty JSON — used instead of /dev/null for .json config files
 // so Node.js doesn't throw ERR_INVALID_PACKAGE_CONFIG when reading them.
 const EMPTY_JSON_PATH = path.join(os.tmpdir(), "skyramp-empty.json");
@@ -32,9 +39,7 @@ export const PLAYWRIGHT_CONFIG_FILES = [
     "playwright.config.mjs",
 ];
 // Directories to skip mounting entirely (cannot bind-mount /dev/null to a directory)
-export const EXCLUDED_MOUNT_ITEMS = [
-    "node_modules",
-];
+export const EXCLUDED_MOUNT_ITEMS = ["node_modules"];
 // Files to shadow with /dev/null recursively so the container ignores them
 export const MOUNT_NULL_ITEMS = [
     "package-lock.json",
@@ -65,6 +70,60 @@ export function getVideoSubdir(testFile) {
     const hash = crypto.createHash("sha256").update(testFile).digest("hex").slice(0, 8);
     return `${basename}-${hash}`;
 }
+function isOwnedStoppedExecutionContainer(container) {
+    return (container.Labels?.[OWNED_EXECUTION_CONTAINER_LABEL] ===
+        OWNED_EXECUTION_CONTAINER_LABEL_VALUE && container.State !== "running");
+}
+function isDockerRuntimeCommand(command) {
+    const trimmed = command.trim();
+    if (!trimmed)
+        return false;
+    if (/[;&|`<>\r\n]/.test(trimmed) || trimmed.includes("$("))
+        return false;
+    return /^(docker\s+(compose|run|start|stop|rm|pull|build|network|volume)\b|docker-compose\b)/.test(trimmed);
+}
+function assertDockerRuntimeCommand(command, fieldName) {
+    if (!isDockerRuntimeCommand(command)) {
+        throw new Error(`${fieldName} must be a Docker command for docker runtime services. Use docker compose/docker-compose or docker run/start/stop/rm/pull/build/network/volume.`);
+    }
+}
+async function getWorkspaceExecutionService(workspacePath, testFile, language) {
+    const rawConfig = await readWorkspaceConfigRaw(workspacePath);
+    const services = rawConfig?.services ?? [];
+    if (!services.length)
+        return undefined;
+    const absTestFile = path.isAbsolute(testFile)
+        ? testFile
+        : path.resolve(workspacePath, testFile);
+    let matches = services.filter((service) => {
+        if (!service.testDirectory)
+            return false;
+        const absTestDirectory = path.resolve(workspacePath, service.testDirectory);
+        return (absTestFile === absTestDirectory ||
+            absTestFile.startsWith(absTestDirectory + path.sep));
+    });
+    if (matches.length > 1 && language) {
+        const byLanguage = matches.filter((service) => service.language?.toLowerCase() === language.toLowerCase());
+        if (byLanguage.length > 0)
+            matches = byLanguage;
+    }
+    return matches.length === 1 ? matches[0] : undefined;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function probeHttpUrl(url, timeoutMs) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        // Any HTTP response means the SUT accepted and handled a request. We do not
+        // require 2xx because many APIs intentionally return 404/401 at the root.
+        await fetch(url, { method: "GET", signal: controller.signal });
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
 /**
  * For UI/E2E tests, create a per-execution video output directory under the
  * workspace and generate the language-specific runner config that enables
@@ -84,14 +143,14 @@ function setupVideoCapture(options, workspacePath, containerMountPath) {
     };
     if (!browserTest)
         return setup;
-    // Per-execution video output directory on host (cleared so a retry never
-    // reports a stale recording from a prior run).
-    const videoSubdir = getVideoSubdir(options.testFile);
+    // Each execution gets a unique video directory so there is nothing to clean up
+    // between runs. Avoids EACCES when the Docker executor (which runs as root) has
+    // left root-owned files in a previous run's directory. The timestamp keeps the
+    // name readable/ordered; the random suffix guarantees uniqueness even when two
+    // executions land in the same millisecond (retries, parallel runs).
+    const videoSubdir = `${getVideoSubdir(options.testFile)}-${Date.now()}-${crypto.randomBytes(3).toString("hex")}`;
     setup.videoHostDir = path.join(workspacePath, ".skyramp", "videos", videoSubdir);
     setup.videoContainerDir = path.join(containerMountPath, ".skyramp", "videos", videoSubdir);
-    if (fs.existsSync(setup.videoHostDir)) {
-        fs.rmSync(setup.videoHostDir, { recursive: true, force: true });
-    }
     fs.mkdirSync(setup.videoHostDir, { recursive: true });
     // TypeScript/JavaScript: generate a per-execution Playwright config with video.
     if (options.language === "typescript" || options.language === "javascript") {
@@ -353,11 +412,102 @@ function findExcludedPaths(dir, excludedItems) {
     return results;
 }
 export class TestExecutionService {
+    static executionQueue = Promise.resolve();
     docker;
     imageReady = null;
     constructor() {
         this.docker = new Docker();
     }
+    async cleanupSkyrampExecutionContainers() {
+        const containers = await this.docker.listContainers({ all: true });
+        const staleContainers = containers.filter(isOwnedStoppedExecutionContainer);
+        await Promise.all(staleContainers.map(async (containerInfo) => {
+            try {
+                await this.docker
+                    .getContainer(containerInfo.Id)
+                    .remove();
+            }
+            catch (error) {
+                logger.warning("Failed to remove stale Skyramp execution container", {
+                    containerId: containerInfo.Id,
+                    error: error.message,
+                });
+            }
+        }));
+        return staleContainers.length;
+    }
+    async prepareSutForExecution(options, workspacePath) {
+        if (options.testType === TestType.CONTRACT)
+            return undefined;
+        const service = await getWorkspaceExecutionService(workspacePath, options.testFile, options.language);
+        if (!service)
+            return undefined;
+        const startCommand = service.runtimeDetails?.serverStartCommand;
+        const stopCommand = service.runtimeDetails?.serverStopCommand;
+        if (service.runtimeDetails?.runtime === "docker") {
+            if (startCommand)
+                assertDockerRuntimeCommand(startCommand, "serverStartCommand");
+            if (stopCommand)
+                assertDockerRuntimeCommand(stopCommand, "serverStopCommand");
+        }
+        let shouldStop = false;
+        const teardown = async () => {
+            if (stopCommand) {
+                await this.stopSutAfterExecution(stopCommand, workspacePath);
+            }
+        };
+        try {
+            if (service.runtimeDetails?.runtime === "docker" && startCommand) {
+                try {
+                    await execAsync(startCommand, {
+                        cwd: workspacePath,
+                        timeout: 60000,
+                        env: process.env,
+                    });
+                    shouldStop = Boolean(stopCommand);
+                }
+                catch (error) {
+                    throw new Error(`SUT setup command failed before test execution (${startCommand}): ${error.message}`);
+                }
+            }
+            const baseUrl = service.api?.baseUrl;
+            if (!baseUrl)
+                return shouldStop ? teardown : undefined;
+            const deadline = Date.now() + SUT_READY_TIMEOUT_MS;
+            let lastError;
+            while (Date.now() < deadline) {
+                try {
+                    await probeHttpUrl(baseUrl, Math.min(5000, SUT_READY_TIMEOUT_MS));
+                    return shouldStop ? teardown : undefined;
+                }
+                catch (error) {
+                    lastError = error;
+                    await sleep(SUT_READY_INTERVAL_MS);
+                }
+            }
+            throw new Error(`SUT did not become reachable at ${baseUrl} before test execution: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
+        }
+        catch (error) {
+            if (shouldStop)
+                await teardown();
+            throw error;
+        }
+    }
+    async stopSutAfterExecution(stopCommand, workspacePath) {
+        try {
+            await execAsync(stopCommand, {
+                cwd: workspacePath,
+                timeout: 60000,
+                env: process.env,
+            });
+        }
+        catch (error) {
+            logger.warning("SUT teardown command failed after test execution", {
+                command: stopCommand,
+                error: error.message,
+            });
+        }
+    }
     /**
      * Execute multiple tests in parallel batches
      */
@@ -365,23 +515,21 @@ export class TestExecutionService {
         logger.info(`Starting batch execution of ${testOptions.length} tests`);
         const startTime = Date.now();
         const results = [];
-        // Execute tests in batches to control concurrency
-        for (let i = 0; i < testOptions.length; i += MAX_CONCURRENT_EXECUTIONS) {
-            const batch = testOptions.slice(i, i + MAX_CONCURRENT_EXECUTIONS);
-            logger.debug(`Executing batch ${Math.floor(i / MAX_CONCURRENT_EXECUTIONS) + 1} (${batch.length} tests)`);
-            const batchPromises = batch.map((options) => this.executeTest(options).catch((error) => {
-                // If execution fails completely, return error result
-                return {
+        // Execute serially because each run owns shared Docker worker/executor state.
+        for (const options of testOptions) {
+            try {
+                results.push(await this.executeTest(options));
+            }
+            catch (error) {
+                results.push({
                     testFile: options.testFile,
                     status: TestExecutionStatus.Error,
                     executedAt: new Date().toISOString(),
                     duration: 0,
                     errors: [error.message],
                     warnings: [],
-                };
-            }));
-            const batchResults = await Promise.all(batchPromises);
-            results.push(...batchResults);
+                });
+            }
         }
         const totalDuration = Date.now() - startTime;
         const passed = results.filter((r) => r.status === TestExecutionStatus.Pass).length;
@@ -398,11 +546,29 @@ export class TestExecutionService {
         };
     }
     /**
-     * Execute a single test
+     * Execute a single test. Executions are serialized within one MCP server
+     * process because pre-run cleanup intentionally removes stale Skyramp
+     * worker/executor containers before each test.
      * @param options Test execution options
      * @param onProgress Optional callback for progress updates
      */
     async executeTest(options, onProgress) {
+        const previousExecution = TestExecutionService.executionQueue;
+        let releaseCurrentExecution;
+        TestExecutionService.executionQueue = new Promise((resolve) => {
+            releaseCurrentExecution = resolve;
+        });
+        await previousExecution.catch(() => {
+            // Keep the queue moving even if an earlier execution failed.
+        });
+        try {
+            return await this.executeTestInternal(options, onProgress);
+        }
+        finally {
+            releaseCurrentExecution();
+        }
+    }
+    async executeTestInternal(options, onProgress) {
         const startTime = Date.now();
         const executedAt = new Date().toISOString();
         logger.debug(`Executing test: ${options.testFile}`);
@@ -440,7 +606,7 @@ export class TestExecutionService {
         try {
             fs.accessSync(workspacePath, fs.constants.R_OK);
         }
-        catch (err) {
+        catch {
             throw new Error(`Workspace path does not exist or is not readable: ${workspacePath}`);
         }
         // Validate test file - use basename for safer filename extraction
@@ -451,6 +617,19 @@ export class TestExecutionService {
         if (!fs.existsSync(options.testFile)) {
             throw new Error(`Test file does not exist: ${options.testFile}`);
         }
+        await onProgress?.({
+            phase: "preparing",
+            message: "Cleaning up stale Skyramp execution containers...",
+            percent: 28,
+        });
+        const cleanedContainers = await this.cleanupSkyrampExecutionContainers();
+        logger.info(`Removed ${cleanedContainers} stale Skyramp execution container(s) before test execution`);
+        await onProgress?.({
+            phase: "preparing",
+            message: "Preparing SUT for test execution...",
+            percent: 29,
+        });
+        const teardownSut = await this.prepareSutForExecution(options, workspacePath);
         await onProgress?.({
             phase: "preparing",
             message: "Preparing Docker container configuration...",
@@ -462,7 +641,7 @@ export class TestExecutionService {
         let testFilePath = path.relative(workspacePath, options.testFile);
         testFilePath = path.resolve(containerMountPath, testFilePath);
         // Video recording setup for UI/E2E browser tests (no-op for API tests).
-        const { browserTest, videoHostDir, videoContainerDir, playwrightConfigPath, pytestConfigPath } = setupVideoCapture(options, workspacePath, containerMountPath);
+        const { browserTest, videoHostDir, videoContainerDir, playwrightConfigPath, pytestConfigPath, } = setupVideoCapture(options, workspacePath, containerMountPath);
         // Prepare Docker command
         const command = [
             "/root/runner.sh",
@@ -562,9 +741,17 @@ export class TestExecutionService {
         const playwrightConfigPaths = findExcludedPaths(workspacePath, PLAYWRIGHT_CONFIG_FILES);
         for (const absolutePath of playwrightConfigPaths) {
             const rel = path.relative(workspacePath, absolutePath);
-            pushMount({ Type: "bind", Source: playwrightConfigPath, Target: path.join(containerMountPath, rel) });
+            pushMount({
+                Type: "bind",
+                Source: playwrightConfigPath,
+                Target: path.join(containerMountPath, rel),
+            });
             if (mirrorAtHostPath) {
-                pushMount({ Type: "bind", Source: playwrightConfigPath, Target: absolutePath });
+                pushMount({
+                    Type: "bind",
+                    Source: playwrightConfigPath,
+                    Target: absolutePath,
+                });
             }
         }
         // Mount video output directory for browser tests (writable)
@@ -694,6 +881,9 @@ export class TestExecutionService {
                 .run(EXECUTOR_DOCKER_IMAGE, command, stream, {
                 Env: env,
                 HostConfig: hostConfig,
+                Labels: {
+                    [OWNED_EXECUTION_CONTAINER_LABEL]: OWNED_EXECUTION_CONTAINER_LABEL_VALUE,
+                },
                 WorkingDir: containerMountPath, // Explicitly set working directory
             })
                 .then(function (data) {
@@ -791,9 +981,11 @@ export class TestExecutionService {
                 percent: 100,
             });
             logger.debug(`Test ${result.status}: ${options.testFile}`);
+            await teardownSut?.();
             return result;
         }
         catch (error) {
+            await teardownSut?.();
             const duration = Date.now() - startTime;
             const cleanOutput = stripVTControlCharacters(output);
             logger.error(`Test execution error: ${error.message}`);