@skyramp/mcp 0.0.55 → 0.0.56

package/build/index.js

@@ -32,7 +32,9 @@ import { registerActionsTool } from "./tools/test-maintenance/actionsTool.js";
 import { registerStateCleanupTool } from "./tools/test-maintenance/stateCleanupTool.js";
 import { registerTestbotPrompt } from "./prompts/testbot/testbot-prompts.js";
 import { registerInitTestbotTool } from "./tools/initTestbotTool.js";
+import { registerSubmitReportTool } from "./tools/submitReportTool.js";
 import { AnalyticsService } from "./services/AnalyticsService.js";
+import { initCheck } from "./utils/initAgent.js";
 const server = new McpServer({
     name: "Skyramp MCP Server",
     version: "1.0.0",
@@ -46,6 +48,24 @@ const server = new McpServer({
         },
     },
 });
+// Check for first-time invocation after version update (runs in background, doesn't block)
+let initCheckCalled = false;
+const originalRegisterTool = server.registerTool.bind(server);
+server.registerTool = function (name, definition, handler) {
+    const wrappedHandler = async (...args) => {
+        if (!initCheckCalled) {
+            initCheck()
+                .then(() => {
+                initCheckCalled = true; // Only set to true if initCheck succeeds, allowing retry on failure
+            })
+                .catch((err) => {
+                logger.error("Background initialization check failed", { error: err });
+            });
+        }
+        return handler(...args);
+    };
+    return originalRegisterTool(name, definition, wrappedHandler);
+};
 // Register prompts
 logger.info("Starting prompt registration process");
 const prompts = [
@@ -55,7 +75,7 @@ const prompts = [
 ];
 if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
     prompts.push(registerTestbotPrompt);
-    logger.info("TestBot prompt enabled via SKYRAMP_ENABLE_TESTBOT");
+    logger.info("TestBot prompt enabled via SKYRAMP_FEATURE_TESTBOT");
 }
 prompts.forEach((registerPrompt) => registerPrompt(server));
 logger.info("All prompts registered successfully");
@@ -99,7 +119,8 @@ const infrastructureTools = [
 ];
 if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
     infrastructureTools.push(registerInitTestbotTool);
-    logger.info("TestBot init tool enabled via SKYRAMP_FEATURE_TESTBOT");
+    infrastructureTools.push(registerSubmitReportTool);
+    logger.info("TestBot tools enabled via SKYRAMP_FEATURE_TESTBOT");
 }
 infrastructureTools.forEach((registerTool) => registerTool(server));
 // Global error handlers for crash telemetry

package/build/prompts/startTraceCollectionPrompts.js

@@ -22,13 +22,21 @@ export function registerStartTraceCollectionPrompt(mcpServer) {
 **Playwright Configuration Options:**
 When playwright is enabled for trace collection, you can optionally configure:
 
-1. **Playwright Storage Path** (playwrightStoragePath):
+1. **Browser** (browser):
+   - Choose which browser to use for trace collection
+   - Supported browsers:
+     * 'chromium' - Chrome/Chromium browser (default)
+     * 'firefox' - Mozilla Firefox browser
+     * 'webkit' - Safari/WebKit browser
+   - Use firefox or webkit when you need to test cross-browser compatibility or specific browser behaviors
+
+2. **Playwright Storage Path** (playwrightStoragePath):
    - Path to a playwright session storage file containing authentication data (cookies, localStorage, sessionStorage, etc.)
    - MUST be an absolute path like /path/to/storage.json
    - Use this when you have manually created a session from the login flow and want to reuse it for future trace collections to avoid manual login every time
    - The session file should be created beforehand using Playwright's storageState feature during the login flow
 
-2. **Playwright Viewport Size** (playwrightViewportSize):
+3. **Playwright Viewport Size** (playwrightViewportSize):
    - Defines the browser window size for trace collection
    - Supported formats:
      * 'hd' - 1280x720
@@ -47,6 +55,11 @@ When playwright is enabled for trace collection, you can optionally configure:
 **Example usage prompt for trace collection with playwright storage and viewport:**
 * Start playwright trace collection with storage path /Users/dev/session-storage.json and viewport size full-hd
 
+**Example usage prompt for trace collection with specific browser:**
+* Start trace collection with Firefox browser
+* Collect UI traces using webkit browser
+* Start playwright trace collection with chromium browser (default)
+
 **CRITICAL: NEVER SHOW THE CLI COMMANDS.**
 `,
                 },

package/build/prompts/testbot/testbot-prompts.js

@@ -1,102 +1,69 @@
 import { z } from "zod";
 import { logger } from "../../utils/logger.js";
-function getTestbotRecommendationPrompt(prTitle, prDescription, summaryOutputFile) {
+function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile) {
     return `<TITLE>${prTitle}</TITLE>
 <DESCRIPTION>${prDescription}</DESCRIPTION>
+<CODE CHANGES>${diffFile}</CODE CHANGES>
+<TEST DIRECTORY>${testDirectory}</TEST DIRECTORY>
 
-Use the MCP tools offered by Skyramp MCP server to analyze the pull request title and description to come up with a business justification
-case for this code change. Based on your analysis of the business case, perform
+For all the following work, use the tools offered by Skyramp MCP server.
 
-1. Get recommendations for new contract, e2e, smoke, load tests that can be generated by Skyramp MCP tools
-2. Use Skyramp MCP to execute the tests and validate the results.
-3. Generate a CONCISE summary to '${summaryOutputFile}' following this format:
+First analyze the pull request title, description, and code changes to determine a business case
+justification for this code change.
 
-## 🤖 Skyramp Test Bot Recommendations
+Then perform the following tasks based on your analysis:
 
-### 💡 Suggested Test Coverage
-- Brief list of new test types added and for which endpoint.
+## Task 1: New Test Recommendations
 
-### 🧪 Test Results
-- List each test executed with pass/fail status and brief details
+1. If, and only if, there are new endpoints in the code changes, get recommendations
+   for new contract, e2e, smoke, load tests specific to the code changes
+   and business case analysis. DO NOT use the whole repository for generating recommendations.
+2. If, and only if, there are recommendations for new tests, then use Skyramp MCP generate
+   tools to generate contract, e2e, smoke and load tests.
+3. Use Skyramp MCP to execute the generated tests and validate the results.
 
-### ⚠️ Issues Found
-- List any test failures or bugs discovered (if none, write 'None')
-
-Keep the summary brief and scannable. Avoid repeating code diff details that are already visible in the PR.
-
-Use the skyramp-mcp tools available to you for test analysis, generation, and execution.`;
-}
-function getTestbotMaintenancePrompt(diffFile, testDirectory, summaryOutputFile) {
-    return `Based on the git diff in ${diffFile} file, perform the following test maintenance tasks:
+## Task 2: Existing Test Maintenance
 
 1. Check if there are any relevant Skyramp tests in the '${testDirectory}' directory that correspond to the code changes.
 2. If there are relevant tests, use Skyramp MCP tools to analyze and modify them to ensure they pass with the new changes.
-3. If there are no tests that can validate the changes, use Skyramp MCP to generate appropriate Skyramp tests for the new or modified code.
-4. Use Skyramp MCP to execute the tests and validate the results.
-5. Generate a CONCISE summary to '${summaryOutputFile}' following this format:
-
-## 🤖 Skyramp Test Bot Summary
+3. Use Skyramp MCP to execute the updated tests and validate the results.
 
-**Tests Modified:** [number] | **Tests Created:** [number] | **Tests Executed:** [number] | **Passed:** [number] | **Failed:** [number]
+## Task 3: Submit Report
 
-### ✅ Maintenance Summary
-- Brief one-line summary of test changes (details are in the commit)
+After completing Tasks 1 and 2, you MUST call the Skyramp MCP tool "skyramp_submit_report" to submit your report.
+Pass '${summaryOutputFile}' as the summaryOutputFile parameter.
 
-### 🧪 Test Results
-- List each test executed with pass/fail status and brief details
+Do NOT write the report to a file yourself. Do NOT skip this step. The skyramp_submit_report tool is the ONLY way to submit the report.
 
-### ⚠️ Issues Found
-- List any test failures or bugs discovered (if none, write 'None')
-
-Keep the summary brief and scannable. Avoid repeating code diff details that are already visible in the PR.
-
-Use the skyramp-mcp tools available to you for test analysis, generation, and execution.`;
+Reminder: Use the Skyramp MCP tools available to you for test analysis, generation, and execution.`;
 }
 export function registerTestbotPrompt(server) {
     logger.info("Registering testbot prompt");
     server.registerPrompt("skyramp_testbot", {
         description: "Run Skyramp TestBot to generate test recommendations and perform test maintenance for a pull request.",
         argsSchema: {
-            prTitle: z
-                .string()
-                .describe("Pull request title"),
+            prTitle: z.string().describe("Pull request title"),
             prDescription: z
                 .string()
                 .describe("Pull request description/body"),
-            diffFile: z
-                .string()
-                .describe("Path to the git diff file"),
+            diffFile: z.string().describe("Path to the git diff file"),
             testDirectory: z
                 .string()
                 .default("tests")
                 .describe("Directory containing Skyramp tests"),
-            recommendationSummaryOutputFile: z
-                .string()
-                .describe("File path where the agent should write the recommendation summary"),
-            maintenanceSummaryOutputFile: z
+            summaryOutputFile: z
                 .string()
-                .describe("File path where the agent should write the maintenance summary"),
+                .describe("File path where the agent should write the testbot summary report"),
         },
     }, (args) => {
-        const recommendationPrompt = getTestbotRecommendationPrompt(args.prTitle, args.prDescription, args.recommendationSummaryOutputFile);
-        const maintenancePrompt = getTestbotMaintenancePrompt(args.diffFile, args.testDirectory, args.maintenanceSummaryOutputFile);
+        const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile);
         return {
             messages: [
                 {
                     role: "user",
                     content: {
                         type: "text",
-                        text: `# Skyramp TestBot Workflow
-
-Follow these two phases in order.
-
-## Phase 1: Test Recommendations
-
-${recommendationPrompt}
-
-## Phase 2: Test Maintenance
-
-${maintenancePrompt}`,
+                        text: prompt,
                     },
                 },
             ],

package/build/services/TestExecutionService.js

@@ -6,7 +6,7 @@ import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
 const MAX_CONCURRENT_EXECUTIONS = 5;
-const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.8";
+const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.9";
 const DOCKER_PLATFORM = "linux/amd64";
 const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
 // Files and directories to exclude when mounting workspace to Docker container

package/build/tools/submitReportTool.js

@@ -0,0 +1,97 @@
+import { z } from "zod";
+import { logger } from "../utils/logger.js";
+import * as fs from "fs/promises";
+import * as path from "path";
+import { AnalyticsService } from "../services/AnalyticsService.js";
+const TOOL_NAME = "skyramp_submit_report";
+const testResultSchema = z.object({
+    testType: z.string().describe("Type of test: Smoke, Contract, Integration, Fuzz, E2E, Load, etc."),
+    endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
+    status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
+    details: z.string().describe("Execution time and test file name, e.g. '10.8s, products_smoke_test.py'"),
+});
+const newTestSchema = z.object({
+    testType: z.string().describe("Type of test created: Smoke, Contract, Integration, etc."),
+    endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
+    fileName: z.string().describe("Name of the generated test file"),
+});
+const descriptionSchema = z.object({
+    description: z.string().describe("One-line description"),
+});
+export function registerSubmitReportTool(server) {
+    server.registerTool(TOOL_NAME, {
+        description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
+            "This is the ONLY way to submit the report — do NOT write the report to a file manually.",
+        inputSchema: {
+            summaryOutputFile: z
+                .string()
+                .describe("The file path where the report should be written (provided in the task instructions)"),
+            businessCaseAnalysis: z
+                .string()
+                .describe("2-3 sentence business justification for this PR"),
+            newTestsCreated: z
+                .array(newTestSchema)
+                .describe("List of new tests created. Use empty array [] if none."),
+            testMaintenance: z
+                .array(descriptionSchema)
+                .describe("List of existing test modifications. Use empty array [] if none."),
+            testResults: z
+                .array(testResultSchema)
+                .describe("List of ALL test execution results. One entry per test executed."),
+            issuesFound: z
+                .array(descriptionSchema)
+                .describe("List of issues, failures, or bugs found. Use empty array [] if none."),
+        },
+        _meta: {
+            keywords: ["report", "summary", "testbot", "submit"],
+        },
+    }, async (params) => {
+        const startTime = Date.now();
+        let errorResult;
+        const reportJson = JSON.stringify({
+            businessCaseAnalysis: params.businessCaseAnalysis,
+            newTestsCreated: params.newTestsCreated,
+            testMaintenance: params.testMaintenance,
+            testResults: params.testResults,
+            issuesFound: params.issuesFound,
+        }, null, 2);
+        logger.info("Submitting testbot report", {
+            outputFile: params.summaryOutputFile,
+            payloadBytes: reportJson.length,
+            testResultCount: params.testResults.length,
+        });
+        try {
+            await fs.mkdir(path.dirname(params.summaryOutputFile), { recursive: true });
+            await fs.writeFile(params.summaryOutputFile, reportJson, "utf-8");
+            const elapsed = Date.now() - startTime;
+            logger.info("Testbot report written successfully", {
+                outputFile: params.summaryOutputFile,
+                elapsedMs: elapsed,
+            });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Report submitted successfully to ${params.summaryOutputFile}`,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const elapsed = Date.now() - startTime;
+            const errorMessage = `Failed to write report: ${error.message}`;
+            logger.error(errorMessage, { error, elapsedMs: elapsed });
+            errorResult = {
+                content: [{ type: "text", text: errorMessage }],
+                isError: true,
+            };
+            return errorResult;
+        }
+        finally {
+            const recordParams = {
+                summary_output_file: params.summaryOutputFile,
+            };
+            AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, recordParams);
+        }
+    });
+}

package/build/tools/submitReportTool.test.js

@@ -0,0 +1,118 @@
+// @ts-ignore
+import { registerSubmitReportTool } from "./submitReportTool.js";
+import * as fs from "fs/promises";
+import * as path from "path";
+import * as os from "os";
+jest.mock("../utils/logger.js", () => ({
+    logger: { info: jest.fn(), error: jest.fn() },
+}));
+jest.mock("../services/AnalyticsService.js", () => ({
+    AnalyticsService: { pushMCPToolEvent: jest.fn() },
+}));
+function captureToolHandler() {
+    let handler;
+    const fakeServer = {
+        registerTool: (_name, _opts, fn) => {
+            handler = fn;
+        },
+    };
+    registerSubmitReportTool(fakeServer);
+    return handler;
+}
+function sampleReportParams(outputFile) {
+    return {
+        summaryOutputFile: outputFile,
+        businessCaseAnalysis: "This PR adds product search. Tests needed for filtering.",
+        newTestsCreated: [
+            { testType: "Smoke", endpoint: "GET /api/v1/products/search", fileName: "search_smoke_test.py" },
+        ],
+        testMaintenance: [{ description: "Updated auth flow in existing tests" }],
+        testResults: [
+            { testType: "Smoke", endpoint: "GET /api/v1/products", status: "Pass", details: "2.1s, products_smoke_test.py" },
+            { testType: "Smoke", endpoint: "GET /api/v1/products/search", status: "Fail", details: "3.4s, search_smoke_test.py" },
+        ],
+        issuesFound: [{ description: "Search endpoint returns 500 with category filter" }],
+    };
+}
+describe("registerSubmitReportTool", () => {
+    let handler;
+    let tmpDirs = [];
+    beforeAll(() => {
+        handler = captureToolHandler();
+    });
+    afterEach(async () => {
+        for (const dir of tmpDirs) {
+            await fs.rm(dir, { recursive: true, force: true });
+        }
+        tmpDirs = [];
+    });
+    it("should register the tool on the server", () => {
+        const registerToolMock = jest.fn();
+        const fakeServer = { registerTool: registerToolMock };
+        registerSubmitReportTool(fakeServer);
+        expect(registerToolMock).toHaveBeenCalledWith("skyramp_submit_report", expect.objectContaining({
+            description: expect.any(String),
+            inputSchema: expect.any(Object),
+        }), expect.any(Function));
+    });
+    it("should write JSON report to the specified file", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const result = await handler(sampleReportParams(outputFile));
+        expect(result.isError).toBeUndefined();
+        expect(result.content[0].text).toContain("Report submitted successfully");
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.businessCaseAnalysis).toBe("This PR adds product search. Tests needed for filtering.");
+        expect(written.newTestsCreated).toHaveLength(1);
+        expect(written.testResults).toHaveLength(2);
+        expect(written.issuesFound).toHaveLength(1);
+        expect(written.testMaintenance).toHaveLength(1);
+    });
+    it("should create parent directories if they don't exist", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "nested", "dirs", "report.json");
+        const result = await handler(sampleReportParams(outputFile));
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.businessCaseAnalysis).toBeDefined();
+    });
+    it("should handle empty arrays for optional sections", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const result = await handler({
+            summaryOutputFile: outputFile,
+            businessCaseAnalysis: "Simple config change.",
+            newTestsCreated: [],
+            testMaintenance: [],
+            testResults: [
+                { testType: "Smoke", endpoint: "GET /api/v1/products", status: "Pass", details: "1.0s, test.py" },
+            ],
+            issuesFound: [],
+        });
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.newTestsCreated).toEqual([]);
+        expect(written.testMaintenance).toEqual([]);
+        expect(written.issuesFound).toEqual([]);
+        expect(written.testResults).toHaveLength(1);
+    });
+    it("should return error for invalid file path", async () => {
+        const result = await handler(sampleReportParams("/nonexistent/readonly/path/report.json"));
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain("Failed to write report");
+    });
+    it("should produce valid JSON with pretty formatting", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        await handler(sampleReportParams(outputFile));
+        const raw = await fs.readFile(outputFile, "utf-8");
+        // Pretty-printed JSON should have indentation
+        expect(raw).toContain("  ");
+        // Should not contain summaryOutputFile in the output
+        expect(raw).not.toContain("summaryOutputFile");
+    });
+});

package/build/tools/trace/startTraceCollectionTool.js

@@ -30,6 +30,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advance
                 .boolean()
                 .describe("Whether to enable Playwright for trace collection. Set to true for UI interactions, false for API-only tracing")
                 .default(true),
+            browser: basePlaywrightSchema.shape.browser,
             playwrightStoragePath: basePlaywrightSchema.shape.playwrightStoragePath,
             playwrightSaveStoragePath: basePlaywrightSchema.shape.playwrightSaveStoragePath,
             playwrightViewportSize: basePlaywrightSchema.shape.playwrightViewportSize,
@@ -85,6 +86,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advance
         };
         logger.info("Starting trace collection", {
             playwright: params.playwright,
+            browser: params.browser,
             runtime: params.runtime,
             include: params.include,
             exclude: params.exclude,
@@ -116,6 +118,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advance
                 generateNoProxy: params.noProxy,
                 unblock: true,
                 playwright: params.playwright,
+                browser: params.browser,
                 playwrightStoragePath: params.playwrightStoragePath,
                 playwrightViewportSize: params.playwrightViewportSize,
                 entrypoint: TELEMETRY_entrypoint_FIELD_NAME,

package/build/types/TestTypes.js

@@ -96,6 +96,10 @@ export const basePlaywrightSchema = z.object({
         .string()
         .default("")
         .describe("Viewport size for playwright browser. THE VALUE MUST BE IN THE FORMAT ['', 'hd', 'full-hd', '2k', 'x,y' e.g. '1920,1080' for 1920x1080 resolution]. DEFAULT VALUE IS ''. If set to '', the browser will use its default viewport size."),
+    browser: z
+        .enum(["chromium", "firefox", "webkit"])
+        .default("chromium")
+        .describe("Browser to use for Playwright trace collection. Supported browsers: 'chromium' (default), 'firefox', 'webkit'. Choose based on your testing requirements and target browser compatibility."),
 });
 export const baseTraceSchema = z.object({
     trace: z

package/build/utils/initAgent.js

@@ -0,0 +1,34 @@
+import * as fs from "fs";
+import * as path from "path";
+import { fileURLToPath } from "url";
+import { SkyrampClient } from "@skyramp/skyramp";
+import { logger } from "./logger.js";
+/**
+ * Get the current MCP package version from package.json
+ */
+export function getPackageVersion() {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const packageJson = fs.readFileSync(path.join(__dirname, "../../package.json"), "utf8");
+    const packageJsonData = JSON.parse(packageJson);
+    return packageJsonData.version;
+}
+/**
+ * Checks if this is the first time after extension install and initializes the agent if needed.
+ */
+export async function initCheck() {
+    const currentVersion = getPackageVersion();
+    try {
+        const client = new SkyrampClient();
+        // @ts-ignore - Backend will be updated to add initAgent method that accepts version and entryPoint parameters
+        const initOutput = await client.initAgent({
+            version: currentVersion,
+            entryPoint: "mcp",
+        });
+        logger.info("Skyramp MCP agent initialized", { initOutput });
+    }
+    catch (error) {
+        logger.error("Error during first-time initialization", { error });
+        throw error;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.0.55",
+  "version": "0.0.56",
   "main": "build/index.js",
   "type": "module",
   "bin": {
@@ -46,7 +46,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.24.3",
     "@playwright/test": "^1.55.0",
-    "@skyramp/skyramp": "1.3.8",
+    "@skyramp/skyramp": "1.3.9",
     "dockerode": "^4.0.6",
     "fast-glob": "^3.3.3",
     "simple-git": "^3.30.0",