@skyramp/mcp 0.2.6 → 0.2.8-rc.1

package/build/services/TestDiscoveryService.d.ts

@@ -0,0 +1,128 @@
+import { TestAnalysisResult } from "../types/TestAnalysis.js";
+interface TestDiscoveryResult {
+    tests: TestAnalysisResult[];
+    /** Paths of external files that were deemed relevant to the PR (score > 0). */
+    relevantExternalTestPaths: string[];
+}
+export interface TestDiscoveryOptions {
+    /**
+     * Resource names derived from endpoints in changed files (e.g. ["orders", "products"]).
+     * - Non-empty array: external files partitioned by relevance; only relevant files
+     *   get full endpoint extraction. May be the sentinel `["unknown"]` when endpoints
+     *   exist but resource names are unresolvable — treated as "endpoints present" to
+     *   avoid skipping external coverage. Also used for "add field" PRs where the changed
+     *   files have no endpoint definitions but the scanner found related endpoints nearby.
+     * - Empty array `[]`: PR mode with no endpoints from diff or scanner — external
+     *   tests are skipped entirely to avoid flooding context.
+     * - `undefined`: full-repo mode — external tests capped at MAX_EXTERNAL_FULL_REPO.
+     */
+    changedResources?: string[];
+}
+export declare class TestDiscoveryService {
+    private readonly EXCLUDED_DIRS;
+    private readonly SKYRAMP_MARKER;
+    private readonly SUPPORTED_EXTENSIONS;
+    private readonly MAX_CONCURRENT_OPERATIONS;
+    private readonly MAX_EXTERNAL_FULL_REPO;
+    private readonly TEST_FILE_PATTERNS;
+    private readonly TEST_DIR_PATTERNS;
+    /**
+     * Discover all tests under testDir — both Skyramp-generated and external (user-written).
+     * Uses fast-glob for cross-platform file scanning, then classifies discovered files
+     * as Skyramp-generated tests, external tests, or not-a-test during processing.
+     *
+     * External test handling depends on `options.changedResources`:
+     * - `string[]` with entries (PR mode, endpoints detected): partition by relevance.
+     * - `[]` empty array (PR mode, scanner found no endpoints): skip external tests entirely
+     *   rather than flooding context with irrelevant files.
+     * - `undefined` (full-repo mode, no diff): cap at MAX_EXTERNAL_FULL_REPO.
+     */
+    discoverTests(testDir: string, options?: TestDiscoveryOptions): Promise<TestDiscoveryResult>;
+    /**
+     * Score an external test file's relevance to a set of changed resource names.
+     * Tokenises the last two segments of the file path (split by /, \, -, _, .)
+     * and counts overlapping tokens with the changed resources.
+     * Example: "test_orders_api.py" vs ["orders"] → score 1.
+     */
+    private scoreRelevance;
+    /**
+     * Partition external test files into relevant (score > 0) and low-relevance (score = 0).
+     * Signal: path/name token overlap with changedResources.
+     */
+    private partitionByRelevance;
+    /**
+     * Process test files in parallel batches with concurrency control
+     * @param isExternal When true, uses external test metadata extraction
+     * @param contentCache Optional pre-read file contents from classification pass
+     */
+    private processFilesInBatches;
+    /**
+     * Classify candidate files as skyramp (has marker), external (test file without
+     * marker), or not-a-test. Caches file contents so downstream metadata extraction
+     * doesn't need to re-read from disk.
+     */
+    private classifyTestFiles;
+    /**
+     * Check if a file is an external (non-Skyramp) test file based on naming convention
+     * or directory placement.
+     */
+    private isExternalTestFile;
+    /**
+     * Extract metadata from a test file
+     * File is already confirmed to contain Skyramp marker by file search
+     * @param cachedContent Optional pre-read content from classification pass
+     */
+    private extractTestMetadata;
+    /**
+     * Extract the HTTP methods and paths covered by this test file.
+     * Returns a comma-separated string like "GET /orders/{id}, DELETE /orders/{id}".
+     * Handles Python (send_request / check_schema) and TypeScript/JS (sendRequest).
+     */
+    private extractCoveredEndpoints;
+    /**
+     * Extract metadata from an external (non-Skyramp) test file.
+     * Uses heuristic patterns for endpoint extraction, test type, and framework detection.
+     * @param cachedContent Optional pre-read content from classification pass
+     */
+    private extractExternalTestMetadata;
+    /**
+     * Extract HTTP endpoints from external (non-Skyramp) test files.
+     * Matches common patterns: requests.get, axios.post, fetch, supertest, RestAssured.
+     */
+    private extractExternalEndpoints;
+    /**
+     * Normalize an endpoint path extracted from test code.
+     * Strips base URL prefixes and returns only the path portion.
+     * Returns null if the path doesn't look like an API endpoint.
+     */
+    private normalizeEndpointPath;
+    /**
+     * Detect test type from external test file path and content.
+     * Falls back to directory/filename heuristics since external tests lack Skyramp command lines.
+     */
+    private detectExternalTestType;
+    /**
+     * Detect test framework from external test file content by checking imports.
+     */
+    private detectExternalFramework;
+    /**
+     * Detect programming language from file extension
+     */
+    private detectLanguage;
+    /**
+     * Detect test type from file content and name
+     * Checks Skyramp command line, filename, and content patterns
+     */
+    private detectTestType;
+    /**
+     * Extract API schema path from test content
+     * Looks for Skyramp command line first, then fallback to other patterns
+     */
+    private extractApiSchema;
+    /**
+     * Extract Framework from test content
+     * Looks for Skyramp command line first, then fallback to other patterns
+     */
+    private extractFramework;
+}
+export {};

package/build/services/TestDiscoveryService.js

@@ -102,9 +102,7 @@ export class TestDiscoveryService {
                 this.partitionByRelevance(classified.external, changedResources));
         }
         else if (changedResources !== undefined) {
-            // PR mode with an explicit empty endpoint list from diff parsing — don't flood
-            // context with irrelevant external tests. The LLM will work from Skyramp tests
-            // and scanned endpoints only.
+            // PR mode with no path-resolvable endpoints — no path-based scoring available.
             relevantExternal = [];
             otherExternal = [];
         }
@@ -151,27 +149,39 @@ export class TestDiscoveryService {
         // Use the last two path segments to avoid false matches from deeply-nested dirs
         const segments = normalized.split("/").slice(-2).join("/");
         const tokens = new Set(segments.split(/[/\-_.]+/).filter(Boolean));
+        // Expand tokens with plural/singular variants so "deployment" matches
+        // "deployments" and vice versa. Only strip trailing "s" when the result
+        // is ≥4 chars (avoids "pas"→"pa") and doesn't end in "ss" (avoids "pass"→"pas").
+        const expanded = new Set(tokens);
+        for (const t of tokens) {
+            if (t.endsWith("s") && !t.endsWith("ss") && t.length > 4)
+                expanded.add(t.slice(0, -1));
+            else if (!t.endsWith("s"))
+                expanded.add(t + "s");
+        }
         return changedResources.filter(r => {
             const rLower = r.toLowerCase();
-            // Direct single-token match (e.g. "orders" in file path tokens)
-            if (tokens.has(rLower))
+            if (expanded.has(rLower))
                 return true;
-            // Compound resource names (e.g. "order-items" from /api/order-items) — check
-            // that ALL hyphen/underscore-separated parts appear as individual tokens.
-            // This handles test_order_items.py correctly matching resource "order-items".
+            // Compound resource names (e.g. "order-items") — all parts must appear as tokens.
             const parts = rLower.split(/[-_]/);
-            return parts.length > 1 && parts.every(p => p.length >= 3 && tokens.has(p));
+            return parts.length > 1 && parts.every(p => p.length >= 3 && expanded.has(p));
         }).length;
     }
     /**
-     * Partition external test files into relevant (score > 0) and low-relevance (score = 0)
-     * based on file path/name overlap with the changed resource names from the PR diff.
+     * Partition external test files into relevant (score > 0) and low-relevance (score = 0).
+     * Signal: path/name token overlap with changedResources.
      */
     partitionByRelevance(files, changedResources) {
         const relevant = [];
         const other = [];
         for (const f of files) {
-            (this.scoreRelevance(f, changedResources) > 0 ? relevant : other).push(f);
+            if (this.scoreRelevance(f, changedResources) > 0) {
+                relevant.push(f);
+            }
+            else {
+                other.push(f);
+            }
         }
         return { relevant, other };
     }

package/build/services/TestDiscoveryService.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/services/TestDiscoveryService.test.js

@@ -1,10 +1,12 @@
+// @ts-nocheck - Jest ESM type inference issues
+import { jest } from "@jest/globals";
 // Mock heavy dependencies
 jest.mock("simple-git", () => ({
     simpleGit: jest.fn(() => ({
         checkIsRepo: jest.fn().mockResolvedValue(false),
     })),
 }));
-jest.mock("../utils/logger.js", () => ({
+jest.unstable_mockModule("../utils/logger.js", () => ({
     logger: {
         info: jest.fn(),
         debug: jest.fn(),
@@ -14,12 +16,11 @@ jest.mock("../utils/logger.js", () => ({
 }));
 // fast-glob must return real file paths from the temp directory.
 // We use the real implementation by not mocking it.
-// Jest resolves .js → .ts via moduleNameMapper in jest.config.
 import * as fs from "fs";
 import * as path from "path";
 import * as os from "os";
-import { TestDiscoveryService } from "./TestDiscoveryService.js";
 import { TestSource } from "../types/TestAnalysis.js";
+const { TestDiscoveryService } = await import("./TestDiscoveryService.js");
 describe("TestDiscoveryService", () => {
     let service;
     let tmpDir;

package/build/services/TestExecutionService.d.ts

@@ -0,0 +1,55 @@
+import { TestExecutionResult, BatchExecutionResult, TestExecutionOptions, ProgressCallback } from "../types/TestExecution.js";
+export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.29";
+export declare const PLAYWRIGHT_CONFIG_FILES: string[];
+export declare const EXCLUDED_MOUNT_ITEMS: string[];
+export declare const MOUNT_NULL_ITEMS: string[];
+/**
+ * Check if a test type requires browser execution (UI/E2E)
+ */
+export declare function isBrowserTest(testType: string): boolean;
+/**
+ * Generate a deterministic unique subdirectory name for video output
+ * based on the test file path
+ */
+export declare function getVideoSubdir(testFile: string): string;
+/**
+ * Detect session file paths referenced in test files
+ * Looks for storageState patterns in TypeScript/JavaScript/Python/Java/C# test files
+ * Excludes matches found in comments
+ *
+ * Also handles the codegen pattern `path.join(__dirname, '<filename>')` (TS/JS) —
+ * the filename is resolved relative to the test file's directory on the host so
+ * the existing absolute-path mount branch makes it visible at the same path
+ * inside the container (Playwright's TS loader resolves __dirname to the host
+ * workspace path at runtime).
+ */
+export declare function detectSessionFiles(testFilePath: string): string[];
+export declare class TestExecutionService {
+    private docker;
+    private imageReady;
+    constructor();
+    /**
+     * Execute multiple tests in parallel batches
+     */
+    executeBatch(testOptions: TestExecutionOptions[]): Promise<BatchExecutionResult>;
+    /**
+     * Execute a single test
+     * @param options Test execution options
+     * @param onProgress Optional callback for progress updates
+     */
+    executeTest(options: TestExecutionOptions, onProgress?: ProgressCallback): Promise<TestExecutionResult>;
+    /**
+     * Ensure Docker image is available
+     * @param onProgress Optional callback for progress updates during pull
+     * @returns Object indicating whether image was cached or pulled
+     */
+    private ensureDockerImage;
+    /**
+     * Parse errors from test output
+     */
+    private parseErrors;
+    /**
+     * Parse warnings from test output
+     */
+    private parseWarnings;
+}

package/build/services/TestExecutionService.js

@@ -2,9 +2,12 @@ import Docker from "dockerode";
 import path from "path";
 import fs from "fs";
 import os from "os";
+import crypto from "crypto";
 import { Writable } from "stream";
 import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
+import { TestExecutionStatus, } from "../types/TestExecution.js";
+import { TestType } from "../types/TestTypes.js";
 import { buildContainerEnv } from "./containerEnv.js";
 import { SKYRAMP_IMAGE_VERSION } from "../utils/versions.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
@@ -47,6 +50,109 @@ export const MOUNT_NULL_ITEMS = [
     "__init__.py",
     "conftest.py",
 ];
+/**
+ * Check if a test type requires browser execution (UI/E2E)
+ */
+export function isBrowserTest(testType) {
+    return testType === TestType.UI || testType === TestType.E2E;
+}
+/**
+ * Generate a deterministic unique subdirectory name for video output
+ * based on the test file path
+ */
+export function getVideoSubdir(testFile) {
+    const basename = path.basename(testFile, path.extname(testFile)).replace(/[^A-Za-z0-9._-]/g, "-");
+    const hash = crypto.createHash("sha256").update(testFile).digest("hex").slice(0, 8);
+    return `${basename}-${hash}`;
+}
+/**
+ * For UI/E2E tests, create a per-execution video output directory under the
+ * workspace and generate the language-specific runner config that enables
+ * recording (Playwright config for TS/JS, pytest.ini for Python). The returned
+ * paths are mounted into the executor container by the caller.
+ *
+ * No-op (returns disabled defaults) for non-browser test types.
+ */
+function setupVideoCapture(options, workspacePath, containerMountPath) {
+    const browserTest = isBrowserTest(options.testType);
+    const setup = {
+        browserTest,
+        videoHostDir: "",
+        videoContainerDir: "",
+        playwrightConfigPath: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
+        pytestConfigPath: "",
+    };
+    if (!browserTest)
+        return setup;
+    // Per-execution video output directory on host (cleared so a retry never
+    // reports a stale recording from a prior run).
+    const videoSubdir = getVideoSubdir(options.testFile);
+    setup.videoHostDir = path.join(workspacePath, ".skyramp", "videos", videoSubdir);
+    setup.videoContainerDir = path.join(containerMountPath, ".skyramp", "videos", videoSubdir);
+    if (fs.existsSync(setup.videoHostDir)) {
+        fs.rmSync(setup.videoHostDir, { recursive: true, force: true });
+    }
+    fs.mkdirSync(setup.videoHostDir, { recursive: true });
+    // TypeScript/JavaScript: generate a per-execution Playwright config with video.
+    if (options.language === "typescript" || options.language === "javascript") {
+        const configContent = `import { defineConfig } from "@playwright/test";
+export default defineConfig({
+  testDir: ".",
+  timeout: 30000,
+  retries: 0,
+  use: { video: "on" },
+  outputDir: "${setup.videoContainerDir}"
+});
+`;
+        setup.playwrightConfigPath = path.join(os.tmpdir(), `skyramp-playwright-${videoSubdir}.config.ts`);
+        fs.writeFileSync(setup.playwrightConfigPath, configContent);
+    }
+    // Python: generate a pytest.ini with the video flags.
+    if (options.language === "python") {
+        const pytestContent = `[pytest]
+addopts = --video on --output ${setup.videoContainerDir}
+`;
+        setup.pytestConfigPath = path.join(os.tmpdir(), `skyramp-pytest-${videoSubdir}.ini`);
+        fs.writeFileSync(setup.pytestConfigPath, pytestContent);
+    }
+    return setup;
+}
+/**
+ * Recursively scan a per-execution video output directory for the recorded
+ * `video.webm`. Returns its host path, or undefined if none was produced.
+ * Best-effort: scan errors are logged and treated as "no video".
+ */
+function collectVideoPath(videoHostDir) {
+    if (!videoHostDir)
+        return undefined;
+    const findVideo = (dir) => {
+        for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isFile() && entry.name === "video.webm")
+                return fullPath;
+            if (entry.isDirectory()) {
+                const found = findVideo(fullPath);
+                if (found)
+                    return found;
+            }
+        }
+        return undefined;
+    };
+    try {
+        const videoPath = findVideo(videoHostDir);
+        if (videoPath) {
+            logger.info(`Video recording found: ${videoPath}`);
+        }
+        else {
+            logger.warning(`No video.webm found in ${videoHostDir}`);
+        }
+        return videoPath;
+    }
+    catch (err) {
+        logger.error(`Failed to scan for video: ${err.message}`);
+        return undefined;
+    }
+}
 /**
  * Find the start index of a comment in a line, ignoring comment delimiters inside strings
  * Returns -1 if no comment is found outside of strings
@@ -267,21 +373,20 @@ export class TestExecutionService {
                 // If execution fails completely, return error result
                 return {
                     testFile: options.testFile,
-                    passed: false,
+                    status: TestExecutionStatus.Error,
                     executedAt: new Date().toISOString(),
                     duration: 0,
                     errors: [error.message],
                     warnings: [],
-                    crashed: true,
                 };
             }));
             const batchResults = await Promise.all(batchPromises);
             results.push(...batchResults);
         }
         const totalDuration = Date.now() - startTime;
-        const passed = results.filter((r) => r.passed).length;
-        const failed = results.filter((r) => !r.passed && !r.crashed).length;
-        const crashed = results.filter((r) => r.crashed).length;
+        const passed = results.filter((r) => r.status === TestExecutionStatus.Pass).length;
+        const failed = results.filter((r) => r.status === TestExecutionStatus.Fail).length;
+        const crashed = results.filter((r) => r.status === TestExecutionStatus.Error).length;
         logger.info(`Batch execution complete: ${passed} passed, ${failed} failed, ${crashed} crashed`);
         return {
             totalTests: testOptions.length,
@@ -356,6 +461,8 @@ export class TestExecutionService {
         // Calculate relative test file path
         let testFilePath = path.relative(workspacePath, options.testFile);
         testFilePath = path.resolve(containerMountPath, testFilePath);
+        // Video recording setup for UI/E2E browser tests (no-op for API tests).
+        const { browserTest, videoHostDir, videoContainerDir, playwrightConfigPath, pytestConfigPath } = setupVideoCapture(options, workspacePath, containerMountPath);
         // Prepare Docker command
         const command = [
             "/root/runner.sh",
@@ -399,6 +506,28 @@ export class TestExecutionService {
             mountedPaths.add(mount.Target);
             hostConfig.Mounts.push(mount);
         };
+        // For browser tests, mount generated video-enabled configs at canonical container
+        // paths BEFORE the nullPaths loop so they WIN over any /dev/null shadows and work
+        // even if the repo has no config file. The pushMount dedup ensures later shadows
+        // for the same target are dropped.
+        if (browserTest) {
+            if (options.language === "typescript" || options.language === "javascript") {
+                // Mount generated Playwright config at canonical container path
+                pushMount({
+                    Type: "bind",
+                    Source: playwrightConfigPath,
+                    Target: path.join(containerMountPath, "playwright.config.ts"),
+                });
+            }
+            if (options.language === "python" && pytestConfigPath) {
+                // Mount generated pytest.ini at canonical container path
+                pushMount({
+                    Type: "bind",
+                    Source: pytestConfigPath,
+                    Target: path.join(containerMountPath, "pytest.ini"),
+                });
+            }
+        }
         const mirrorAtHostPath = workspacePath !== containerMountPath;
         for (const file of filesToMount) {
             const source = path.join(workspacePath, file);
@@ -425,23 +554,27 @@ export class TestExecutionService {
                 pushMount({ Type: "bind", Source: source, Target: absolutePath });
             }
         }
-        // Mount Playwright config files with minimal config (shadows repo configs that may
-        // import dotenv or other dependencies not available in the executor container)
+        // Mount Playwright config files with the per-execution config (minimal or generated).
+        // Shadows ANY repo playwright.config.{ts,js,mjs} recursively so repo configs that
+        // import dotenv/etc. don't crash the executor. Applies to ALL tests — including
+        // non-browser TypeScript tests (contract/integration/fuzz/smoke/load) which per
+        // runner.sh also run via `npx playwright test <file>` and read playwright.config.ts.
         const playwrightConfigPaths = findExcludedPaths(workspacePath, PLAYWRIGHT_CONFIG_FILES);
         for (const absolutePath of playwrightConfigPaths) {
             const rel = path.relative(workspacePath, absolutePath);
+            pushMount({ Type: "bind", Source: playwrightConfigPath, Target: path.join(containerMountPath, rel) });
+            if (mirrorAtHostPath) {
+                pushMount({ Type: "bind", Source: playwrightConfigPath, Target: absolutePath });
+            }
+        }
+        // Mount video output directory for browser tests (writable)
+        if (browserTest && videoHostDir) {
             pushMount({
                 Type: "bind",
-                Source: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
-                Target: path.join(containerMountPath, rel),
+                Source: videoHostDir,
+                Target: videoContainerDir,
+                ReadOnly: false,
             });
-            if (mirrorAtHostPath) {
-                pushMount({
-                    Type: "bind",
-                    Source: MINIMAL_PLAYWRIGHT_CONFIG_PATH,
-                    Target: absolutePath,
-                });
-            }
         }
         // Detect and mount session files referenced outside the workspace
         // (anything inside the workspace is already covered by the dual mount
@@ -636,28 +769,28 @@ export class TestExecutionService {
             // Parse errors and warnings from output
             const errors = this.parseErrors(cleanOutput);
             const warnings = this.parseWarnings(cleanOutput);
-            // Check if test crashed
-            const crashed = statusCode !== 0 && statusCode !== 1;
+            // Collect the recorded video (browser tests only).
+            const videoPath = browserTest ? collectVideoPath(videoHostDir) : undefined;
             const result = {
                 testFile: options.testFile,
-                passed: statusCode === 0,
+                status: statusCode === 0 ? TestExecutionStatus.Pass
+                    : statusCode !== 1 ? TestExecutionStatus.Error
+                        : TestExecutionStatus.Fail,
                 executedAt,
                 duration,
                 errors,
                 warnings,
-                crashed,
                 output: cleanOutput,
                 exitCode: statusCode,
+                videoPath,
             };
             // Report completion
             await onProgress?.({
                 phase: "processing",
-                message: result.passed
-                    ? "Test passed successfully"
-                    : "Test execution completed with failures",
+                message: `Test execution completed: ${result.status}`,
                 percent: 100,
             });
-            logger.debug(`Test ${result.passed ? "passed" : "failed"}: ${options.testFile}`);
+            logger.debug(`Test ${result.status}: ${options.testFile}`);
             return result;
         }
         catch (error) {
@@ -666,12 +799,11 @@ export class TestExecutionService {
             logger.error(`Test execution error: ${error.message}`);
             return {
                 testFile: options.testFile,
-                passed: false,
+                status: TestExecutionStatus.Error,
                 executedAt,
                 duration,
                 errors: [error.message],
                 warnings: [],
-                crashed: true,
                 output: cleanOutput,
             };
         }

package/build/services/TestExecutionService.test.d.ts

@@ -0,0 +1 @@
+export {};