@skyramp/mcp 0.2.7 → 0.2.150-rc.mntnc

package/build/services/TestExecutionService.js

@@ -2,9 +2,12 @@ import Docker from "dockerode";
 import path from "path";
 import fs from "fs";
 import os from "os";
+import crypto from "crypto";
 import { Writable } from "stream";
 import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
+import { TestExecutionStatus, } from "../types/TestExecution.js";
+import { TestType } from "../types/TestTypes.js";
 import { buildContainerEnv } from "./containerEnv.js";
 import { SKYRAMP_IMAGE_VERSION } from "../utils/versions.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
@@ -47,6 +50,109 @@ export const MOUNT_NULL_ITEMS = [
     "__init__.py",
     "conftest.py",
 ];
+/**
+ * Check if a test type requires browser execution (UI/E2E)
+ */
+export function isBrowserTest(testType) {
+    return testType === TestType.UI || testType === TestType.E2E;
+}
+/**
+ * Generate a deterministic unique subdirectory name for video output
+ * based on the test file path
+ */
+export function getVideoSubdir(testFile) {
+    const basename = path.basename(testFile, path.extname(testFile)).replace(/[^A-Za-z0-9._-]/g, "-");
+    const hash = crypto.createHash("sha256").update(testFile).digest("hex").slice(0, 8);
+    return `${basename}-${hash}`;
+}
+/**
+ * For UI/E2E tests, create a per-execution video output directory under the
+ * workspace and generate the language-specific runner config that enables
+ * recording (Playwright config for TS/JS, pytest.ini for Python). The returned
+ * paths are mounted into the executor container by the caller.
+ *
+ * No-op (returns disabled defaults) for non-browser test types.
+ */
+function setupVideoCapture(options, workspacePath, containerMountPath) {
+    const browserTest = isBrowserTest(options.testType);
+    const setup = {
+        browserTest,
+        videoHostDir: "",
+        videoContainerDir: "",
+        playwrightConfigPath: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
+        pytestConfigPath: "",
+    };
+    if (!browserTest)
+        return setup;
+    // Per-execution video output directory on host (cleared so a retry never
+    // reports a stale recording from a prior run).
+    const videoSubdir = getVideoSubdir(options.testFile);
+    setup.videoHostDir = path.join(workspacePath, ".skyramp", "videos", videoSubdir);
+    setup.videoContainerDir = path.join(containerMountPath, ".skyramp", "videos", videoSubdir);
+    if (fs.existsSync(setup.videoHostDir)) {
+        fs.rmSync(setup.videoHostDir, { recursive: true, force: true });
+    }
+    fs.mkdirSync(setup.videoHostDir, { recursive: true });
+    // TypeScript/JavaScript: generate a per-execution Playwright config with video.
+    if (options.language === "typescript" || options.language === "javascript") {
+        const configContent = `import { defineConfig } from "@playwright/test";
+export default defineConfig({
+  testDir: ".",
+  timeout: 30000,
+  retries: 0,
+  use: { video: "on" },
+  outputDir: "${setup.videoContainerDir}"
+});
+`;
+        setup.playwrightConfigPath = path.join(os.tmpdir(), `skyramp-playwright-${videoSubdir}.config.ts`);
+        fs.writeFileSync(setup.playwrightConfigPath, configContent);
+    }
+    // Python: generate a pytest.ini with the video flags.
+    if (options.language === "python") {
+        const pytestContent = `[pytest]
+addopts = --video on --output ${setup.videoContainerDir}
+`;
+        setup.pytestConfigPath = path.join(os.tmpdir(), `skyramp-pytest-${videoSubdir}.ini`);
+        fs.writeFileSync(setup.pytestConfigPath, pytestContent);
+    }
+    return setup;
+}
+/**
+ * Recursively scan a per-execution video output directory for the recorded
+ * `video.webm`. Returns its host path, or undefined if none was produced.
+ * Best-effort: scan errors are logged and treated as "no video".
+ */
+function collectVideoPath(videoHostDir) {
+    if (!videoHostDir)
+        return undefined;
+    const findVideo = (dir) => {
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isFile() && entry.name === "video.webm")
+                return fullPath;
+            if (entry.isDirectory()) {
+                const found = findVideo(fullPath);
+                if (found)
+                    return found;
+            }
+        }
+        return undefined;
+    };
+    try {
+        const videoPath = findVideo(videoHostDir);
+        if (videoPath) {
+            logger.info(`Video recording found: ${videoPath}`);
+        }
+        else {
+            logger.warning(`No video.webm found in ${videoHostDir}`);
+        }
+        return videoPath;
+    }
+    catch (err) {
+        logger.error(`Failed to scan for video: ${err.message}`);
+        return undefined;
+    }
+}
 /**
  * Find the start index of a comment in a line, ignoring comment delimiters inside strings
  * Returns -1 if no comment is found outside of strings
@@ -267,21 +373,20 @@ export class TestExecutionService {
                 // If execution fails completely, return error result
                 return {
                     testFile: options.testFile,
-                    passed: false,
+                    status: TestExecutionStatus.Error,
                     executedAt: new Date().toISOString(),
                     duration: 0,
                     errors: [error.message],
                     warnings: [],
-                    crashed: true,
                 };
             }));
             const batchResults = await Promise.all(batchPromises);
             results.push(...batchResults);
         }
         const totalDuration = Date.now() - startTime;
-        const passed = results.filter((r) => r.passed).length;
-        const failed = results.filter((r) => !r.passed && !r.crashed).length;
-        const crashed = results.filter((r) => r.crashed).length;
+        const passed = results.filter((r) => r.status === TestExecutionStatus.Pass).length;
+        const failed = results.filter((r) => r.status === TestExecutionStatus.Fail).length;
+        const crashed = results.filter((r) => r.status === TestExecutionStatus.Error).length;
         logger.info(`Batch execution complete: ${passed} passed, ${failed} failed, ${crashed} crashed`);
         return {
             totalTests: testOptions.length,
@@ -356,6 +461,8 @@ export class TestExecutionService {
         // Calculate relative test file path
         let testFilePath = path.relative(workspacePath, options.testFile);
         testFilePath = path.resolve(containerMountPath, testFilePath);
+        // Video recording setup for UI/E2E browser tests (no-op for API tests).
+        const { browserTest, videoHostDir, videoContainerDir, playwrightConfigPath, pytestConfigPath } = setupVideoCapture(options, workspacePath, containerMountPath);
         // Prepare Docker command
         const command = [
             "/root/runner.sh",
@@ -399,6 +506,28 @@ export class TestExecutionService {
             mountedPaths.add(mount.Target);
             hostConfig.Mounts.push(mount);
         };
+        // For browser tests, mount generated video-enabled configs at canonical container
+        // paths BEFORE the nullPaths loop so they WIN over any /dev/null shadows and work
+        // even if the repo has no config file. The pushMount dedup ensures later shadows
+        // for the same target are dropped.
+        if (browserTest) {
+            if (options.language === "typescript" || options.language === "javascript") {
+                // Mount generated Playwright config at canonical container path
+                pushMount({
+                    Type: "bind",
+                    Source: playwrightConfigPath,
+                    Target: path.join(containerMountPath, "playwright.config.ts"),
+                });
+            }
+            if (options.language === "python" && pytestConfigPath) {
+                // Mount generated pytest.ini at canonical container path
+                pushMount({
+                    Type: "bind",
+                    Source: pytestConfigPath,
+                    Target: path.join(containerMountPath, "pytest.ini"),
+                });
+            }
+        }
         const mirrorAtHostPath = workspacePath !== containerMountPath;
         for (const file of filesToMount) {
             const source = path.join(workspacePath, file);
@@ -425,23 +554,27 @@ export class TestExecutionService {
                 pushMount({ Type: "bind", Source: source, Target: absolutePath });
             }
         }
-        // Mount Playwright config files with minimal config (shadows repo configs that may
-        // import dotenv or other dependencies not available in the executor container)
+        // Mount Playwright config files with the per-execution config (minimal or generated).
+        // Shadows ANY repo playwright.config.{ts,js,mjs} recursively so repo configs that
+        // import dotenv/etc. don't crash the executor. Applies to ALL tests — including
+        // non-browser TypeScript tests (contract/integration/fuzz/smoke/load) which per
+        // runner.sh also run via `npx playwright test <file>` and read playwright.config.ts.
         const playwrightConfigPaths = findExcludedPaths(workspacePath, PLAYWRIGHT_CONFIG_FILES);
         for (const absolutePath of playwrightConfigPaths) {
             const rel = path.relative(workspacePath, absolutePath);
+            pushMount({ Type: "bind", Source: playwrightConfigPath, Target: path.join(containerMountPath, rel) });
+            if (mirrorAtHostPath) {
+                pushMount({ Type: "bind", Source: playwrightConfigPath, Target: absolutePath });
+            }
+        }
+        // Mount video output directory for browser tests (writable)
+        if (browserTest && videoHostDir) {
             pushMount({
                 Type: "bind",
-                Source: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
-                Target: path.join(containerMountPath, rel),
+                Source: videoHostDir,
+                Target: videoContainerDir,
+                ReadOnly: false,
             });
-            if (mirrorAtHostPath) {
-                pushMount({
-                    Type: "bind",
-                    Source: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
-                    Target: absolutePath,
-                });
-            }
         }
         // Detect and mount session files referenced outside the workspace
         // (anything inside the workspace is already covered by the dual mount
@@ -636,28 +769,28 @@ export class TestExecutionService {
             // Parse errors and warnings from output
             const errors = this.parseErrors(cleanOutput);
             const warnings = this.parseWarnings(cleanOutput);
-            // Check if test crashed
-            const crashed = statusCode !== 0 && statusCode !== 1;
+            // Collect the recorded video (browser tests only).
+            const videoPath = browserTest ? collectVideoPath(videoHostDir) : undefined;
             const result = {
                 testFile: options.testFile,
-                passed: statusCode === 0,
+                status: statusCode === 0 ? TestExecutionStatus.Pass
+                    : statusCode !== 1 ? TestExecutionStatus.Error
+                        : TestExecutionStatus.Fail,
                 executedAt,
                 duration,
                 errors,
                 warnings,
-                crashed,
                 output: cleanOutput,
                 exitCode: statusCode,
+                videoPath,
             };
             // Report completion
             await onProgress?.({
                 phase: "processing",
-                message: result.passed
-                    ? "Test passed successfully"
-                    : "Test execution completed with failures",
+                message: `Test execution completed: ${result.status}`,
                 percent: 100,
             });
-            logger.debug(`Test ${result.passed ? "passed" : "failed"}: ${options.testFile}`);
+            logger.debug(`Test ${result.status}: ${options.testFile}`);
             return result;
         }
         catch (error) {
@@ -666,12 +799,11 @@ export class TestExecutionService {
             logger.error(`Test execution error: ${error.message}`);
             return {
                 testFile: options.testFile,
-                passed: false,
+                status: TestExecutionStatus.Error,
                 executedAt,
                 duration,
                 errors: [error.message],
                 warnings: [],
-                crashed: true,
                 output: cleanOutput,
             };
         }

package/build/services/TestExecutionService.test.js

@@ -21,6 +21,9 @@ const mockReaddirSync = jest.fn().mockImplementation((_path, options) => {
     return ["test_file.py"];
 });
 const mockReadFileSync = jest.fn().mockReturnValue("");
+const mockWriteFileSync = jest.fn();
+const mockMkdirSync = jest.fn();
+const mockRmSync = jest.fn();
 jest.unstable_mockModule("fs", () => ({
     ...actualFs,
     default: {
@@ -29,12 +32,18 @@ jest.unstable_mockModule("fs", () => ({
         existsSync: mockExistsSync,
         readdirSync: mockReaddirSync,
         readFileSync: mockReadFileSync,
+        writeFileSync: mockWriteFileSync,
+        mkdirSync: mockMkdirSync,
+        rmSync: mockRmSync,
         constants: actualFs.default.constants,
     },
     accessSync: mockAccessSync,
     existsSync: mockExistsSync,
     readdirSync: mockReaddirSync,
     readFileSync: mockReadFileSync,
+    writeFileSync: mockWriteFileSync,
+    mkdirSync: mockMkdirSync,
+    rmSync: mockRmSync,
 }));
 // Mock logger
 jest.unstable_mockModule("../utils/logger.js", () => ({
@@ -197,6 +206,81 @@ test('x', () => {});
         expect(detectSessionFiles("/ws/spec.ts")).toEqual([]);
     });
 });
+describe("isBrowserTest", () => {
+    let isBrowserTest;
+    beforeAll(async () => {
+        const mod = await import("./TestExecutionService.js");
+        isBrowserTest = mod.isBrowserTest;
+    });
+    it("returns true for UI test type", () => {
+        expect(isBrowserTest("ui")).toBe(true);
+    });
+    it("returns true for E2E test type", () => {
+        expect(isBrowserTest("e2e")).toBe(true);
+    });
+    it("returns false for contract test type", () => {
+        expect(isBrowserTest("contract")).toBe(false);
+    });
+    it("returns false for integration test type", () => {
+        expect(isBrowserTest("integration")).toBe(false);
+    });
+    it("returns false for fuzz test type", () => {
+        expect(isBrowserTest("fuzz")).toBe(false);
+    });
+    it("returns false for smoke test type", () => {
+        expect(isBrowserTest("smoke")).toBe(false);
+    });
+    it("returns false for load test type", () => {
+        expect(isBrowserTest("load")).toBe(false);
+    });
+    it("returns false for mock test type", () => {
+        expect(isBrowserTest("mock")).toBe(false);
+    });
+});
+describe("getVideoSubdir", () => {
+    let getVideoSubdir;
+    beforeAll(async () => {
+        const mod = await import("./TestExecutionService.js");
+        getVideoSubdir = mod.getVideoSubdir;
+    });
+    it("generates deterministic subdirectory name with basename and hash", () => {
+        const testFile = "/workspace/tests/login.spec.ts";
+        const result = getVideoSubdir(testFile);
+        // Should be "basename-8hexchars" format
+        // Note: path.extname only strips the final extension (.ts), so .spec remains in basename
+        expect(result).toMatch(/^login\.spec-[0-9a-f]{8}$/);
+    });
+    it("strips file extension from basename", () => {
+        const result = getVideoSubdir("/workspace/checkout.spec.ts");
+        // path.extname('.spec.ts') returns '.ts', so 'checkout.spec' is the basename
+        expect(result).toMatch(/^checkout\.spec-[0-9a-f]{8}$/);
+        expect(result).toContain(".spec");
+    });
+    it("produces different hashes for different paths", () => {
+        const path1 = "/workspace/tests/login.spec.ts";
+        const path2 = "/workspace/tests/checkout.spec.ts";
+        const result1 = getVideoSubdir(path1);
+        const result2 = getVideoSubdir(path2);
+        expect(result1).not.toBe(result2);
+        // Different hashes
+        expect(result1.split("-")[1]).not.toBe(result2.split("-")[1]);
+    });
+    it("produces same hash for the same path (deterministic)", () => {
+        const testFile = "/workspace/tests/products.spec.ts";
+        const result1 = getVideoSubdir(testFile);
+        const result2 = getVideoSubdir(testFile);
+        expect(result1).toBe(result2);
+    });
+    it("handles paths with no extension", () => {
+        const result = getVideoSubdir("/workspace/test_file");
+        expect(result).toMatch(/^test_file-[0-9a-f]{8}$/);
+    });
+    it("handles paths with multiple dots", () => {
+        const result = getVideoSubdir("/workspace/test.spec.e2e.ts");
+        // Should strip only the last extension (.ts)
+        expect(result).toMatch(/^test\.spec\.e2e-[0-9a-f]{8}$/);
+    });
+});
 describe("TestExecutionService.executeTest - Docker env forwarding", () => {
     // Import after mocks are set up
     let TestExecutionService;
@@ -324,3 +408,225 @@ describe("TestExecutionService.executeTest - Docker env forwarding", () => {
         }
     });
 });
+describe("TestExecutionService.executeTest - Video capture for browser tests", () => {
+    let TestExecutionService;
+    let getVideoSubdir;
+    let EXECUTOR_DOCKER_IMAGE;
+    beforeAll(async () => {
+        const mod = await import("./TestExecutionService.js");
+        TestExecutionService = mod.TestExecutionService;
+        getVideoSubdir = mod.getVideoSubdir;
+        EXECUTOR_DOCKER_IMAGE = mod.EXECUTOR_DOCKER_IMAGE;
+    });
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockListImages.mockResolvedValue([{ RepoTags: [EXECUTOR_DOCKER_IMAGE] }]);
+        mockExistsSync.mockReturnValue(true);
+        mockReaddirSync.mockImplementation((_path, options) => {
+            if (options?.withFileTypes) {
+                return [
+                    { name: "test.spec.ts", isFile: () => true, isDirectory: () => false },
+                    { name: "test.py", isFile: () => true, isDirectory: () => false },
+                ];
+            }
+            return ["test.spec.ts", "test.py"];
+        });
+        // Reset writeFileSync mock to default behavior
+        mockWriteFileSync.mockImplementation(() => { });
+    });
+    it("mounts generated Playwright config at /home/user/playwright.config.ts for TypeScript UI test", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/login.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "ui",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        const playwrightConfigMount = dockerOptions.HostConfig.Mounts.find((m) => m.Target === "/home/user/playwright.config.ts");
+        expect(playwrightConfigMount).toBeDefined();
+        expect(playwrightConfigMount.Type).toBe("bind");
+        // Source should be a temp file containing the generated config
+        expect(playwrightConfigMount.Source).toContain("skyramp-playwright-");
+        expect(playwrightConfigMount.Source).toContain(".config.ts");
+    });
+    it("generated Playwright config for TypeScript contains video:on and outputDir", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        let capturedConfigContent = "";
+        mockWriteFileSync.mockImplementation((filePath, content) => {
+            if (filePath.includes("skyramp-playwright-") && filePath.endsWith(".config.ts")) {
+                capturedConfigContent = content;
+            }
+        });
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/checkout.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "e2e",
+        });
+        expect(capturedConfigContent).toContain('video: "on"');
+        expect(capturedConfigContent).toContain("outputDir:");
+        expect(capturedConfigContent).toContain(".skyramp/videos/");
+    });
+    it("mounts generated pytest.ini at /home/user/pytest.ini for Python UI test", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/test_login.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "ui",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        const pytestConfigMount = dockerOptions.HostConfig.Mounts.find((m) => m.Target === "/home/user/pytest.ini");
+        expect(pytestConfigMount).toBeDefined();
+        expect(pytestConfigMount.Type).toBe("bind");
+        expect(pytestConfigMount.Source).toContain("skyramp-pytest-");
+        expect(pytestConfigMount.Source).toContain(".ini");
+    });
+    it("generated pytest.ini for Python contains --video on and --output", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        let capturedPytestContent = "";
+        mockWriteFileSync.mockImplementation((filePath, content) => {
+            if (filePath.includes("skyramp-pytest-") && filePath.endsWith(".ini")) {
+                capturedPytestContent = content;
+            }
+        });
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/test_checkout.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "e2e",
+        });
+        expect(capturedPytestContent).toContain("--video on");
+        expect(capturedPytestContent).toContain("--output");
+        expect(capturedPytestContent).toContain(".skyramp/videos/");
+    });
+    it("mounts video output directory as writable for UI test", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/tests/login.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "ui",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        const videoSubdir = getVideoSubdir("/workspace/tests/login.spec.ts");
+        const videoMount = dockerOptions.HostConfig.Mounts.find((m) => m.Target === `/home/user/.skyramp/videos/${videoSubdir}`);
+        expect(videoMount).toBeDefined();
+        expect(videoMount.Type).toBe("bind");
+        expect(videoMount.ReadOnly).toBe(false);
+        expect(videoMount.Source).toContain(".skyramp/videos/");
+        expect(videoMount.Source).toContain(videoSubdir);
+    });
+    it("does not mount video directory for non-browser tests", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/contract_test.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "contract",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        const videoMounts = dockerOptions.HostConfig.Mounts.filter((m) => m.Target.includes(".skyramp/videos"));
+        expect(videoMounts).toHaveLength(0);
+    });
+    it("uses minimal static Playwright config for non-browser TypeScript tests", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        let usedGeneratedConfig = false;
+        mockWriteFileSync.mockImplementation((filePath, content) => {
+            if (filePath.includes("skyramp-playwright-") && content.includes('video: "on"')) {
+                usedGeneratedConfig = true;
+            }
+        });
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/contract_test.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "contract",
+        });
+        // Should NOT generate a video-enabled config for contract test
+        expect(usedGeneratedConfig).toBe(false);
+    });
+    it("populates videoPath when video.webm exists after execution", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const videoSubdir = getVideoSubdir("/workspace/test.spec.ts");
+        mockReaddirSync.mockImplementation((dir, options) => {
+            // Mock video directory scan
+            if (dir.includes(".skyramp/videos") && dir.includes(videoSubdir)) {
+                if (options?.withFileTypes) {
+                    return [{ name: "video.webm", isFile: () => true, isDirectory: () => false }];
+                }
+                return ["video.webm"];
+            }
+            // Default workspace files
+            if (options?.withFileTypes) {
+                return [{ name: "test.spec.ts", isFile: () => true, isDirectory: () => false }];
+            }
+            return ["test.spec.ts"];
+        });
+        const service = new TestExecutionService();
+        const result = await service.executeTest({
+            testFile: "/workspace/test.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "ui",
+        });
+        expect(result.videoPath).toBeDefined();
+        expect(result.videoPath).toContain(".skyramp/videos/");
+        expect(result.videoPath).toContain("video.webm");
+    });
+    it("leaves videoPath undefined when no video.webm found", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const videoSubdir = getVideoSubdir("/workspace/test.spec.ts");
+        mockReaddirSync.mockImplementation((dir, options) => {
+            // Mock empty video directory
+            if (dir.includes(".skyramp/videos") && dir.includes(videoSubdir)) {
+                if (options?.withFileTypes) {
+                    return [];
+                }
+                return [];
+            }
+            // Default workspace files
+            if (options?.withFileTypes) {
+                return [{ name: "test.spec.ts", isFile: () => true, isDirectory: () => false }];
+            }
+            return ["test.spec.ts"];
+        });
+        const service = new TestExecutionService();
+        const result = await service.executeTest({
+            testFile: "/workspace/test.spec.ts",
+            workspacePath: "/workspace",
+            language: "typescript",
+            testType: "ui",
+        });
+        expect(result.videoPath).toBeUndefined();
+    });
+    it("leaves videoPath undefined for non-browser tests", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        const result = await service.executeTest({
+            testFile: "/workspace/integration_test.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "integration",
+        });
+        expect(result.videoPath).toBeUndefined();
+    });
+});

package/build/tools/executeSkyrampTestTool.js

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { stripVTControlCharacters } from "util";
 import { TestExecutionService } from "../services/TestExecutionService.js";
 import { AnalyticsService } from "../services/AnalyticsService.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
 import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { StateManager } from "../utils/AnalysisStateManager.js";
@@ -66,7 +67,11 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             stateFile: z
                 .string()
                 .optional()
-                .describe("Optional path to state file from skyramp_analyze_changes. When provided, execution results (passed/failed, errors, duration) will be written back to enrich the test health analysis."),
+                .describe("Path to state file from skyramp_analyze_changes. Always pass when available — results are written back so skyramp_submit_report can override before/afterStatus with ground-truth pass/fail."),
+            phase: z
+                .enum(["before", "after"])
+                .optional()
+                .describe("Execution phase for maintained tests: 'before' captures pre-edit baseline; 'after' (default) records post-edit result."),
         },
         _meta: {
             keywords: ["run test", "execute test"],
@@ -147,7 +152,12 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                     if (stateData && stateData.existingTests) {
                         const testIndex = stateData.existingTests.findIndex((t) => t.testFile === params.testFile);
                         if (testIndex >= 0) {
-                            stateData.existingTests[testIndex].execution = result;
+                            if (params.phase === "before") {
+                                stateData.existingTests[testIndex].executionBefore = result;
+                            }
+                            else {
+                                stateData.existingTests[testIndex].executionAfter = result;
+                            }
                             await stateManager.writeData(stateData);
                             logger.info(`Updated stateFile with execution results for ${params.testFile}`);
                         }
@@ -163,7 +173,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             }
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
-            if (!result.passed) {
+            if (result.status !== TestExecutionStatus.Pass) {
                 errorResult = {
                     content: [
                         {
@@ -177,11 +187,14 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
+            const videoInfo = result.videoPath
+                ? `\n\nVideo recording: ${result.videoPath}`
+                : "";
             return {
                 content: [
                     {
                         type: "text",
-                        text: `Test execution result: ${stripVTControlCharacters(result.output || "")}`,
+                        text: `Test execution result: ${stripVTControlCharacters(result.output || "")}${videoInfo}`,
                     },
                 ],
             };