@skyramp/mcp 0.0.54 → 0.0.55

package/build/index.js

@@ -30,6 +30,8 @@ import { registerExecuteBatchTestsTool } from "./tools/test-maintenance/executeB
 import { registerCalculateHealthScoresTool } from "./tools/test-maintenance/calculateHealthScoresTool.js";
 import { registerActionsTool } from "./tools/test-maintenance/actionsTool.js";
 import { registerStateCleanupTool } from "./tools/test-maintenance/stateCleanupTool.js";
+import { registerTestbotPrompt } from "./prompts/testbot/testbot-prompts.js";
+import { registerInitTestbotTool } from "./tools/initTestbotTool.js";
 import { AnalyticsService } from "./services/AnalyticsService.js";
 const server = new McpServer({
     name: "Skyramp MCP Server",
@@ -51,6 +53,10 @@ const prompts = [
     registerStartTraceCollectionPrompt,
     registerTestHealthPrompt,
 ];
+if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
+    prompts.push(registerTestbotPrompt);
+    logger.info("TestBot prompt enabled via SKYRAMP_ENABLE_TESTBOT");
+}
 prompts.forEach((registerPrompt) => registerPrompt(server));
 logger.info("All prompts registered successfully");
 // Register test generation tools
@@ -91,6 +97,10 @@ const infrastructureTools = [
     registerTraceTool,
     registerTraceStopTool,
 ];
+if (process.env.SKYRAMP_FEATURE_TESTBOT === "1") {
+    infrastructureTools.push(registerInitTestbotTool);
+    logger.info("TestBot init tool enabled via SKYRAMP_FEATURE_TESTBOT");
+}
 infrastructureTools.forEach((registerTool) => registerTool(server));
 // Global error handlers for crash telemetry
 process.on("uncaughtException", async (error) => {

package/build/prompts/testbot/testbot-prompts.js

@@ -0,0 +1,105 @@
+import { z } from "zod";
+import { logger } from "../../utils/logger.js";
+function getTestbotRecommendationPrompt(prTitle, prDescription, summaryOutputFile) {
+    return `<TITLE>${prTitle}</TITLE>
+<DESCRIPTION>${prDescription}</DESCRIPTION>
+
+Use the MCP tools offered by Skyramp MCP server to analyze the pull request title and description to come up with a business justification
+case for this code change. Based on your analysis of the business case, perform
+
+1. Get recommendations for new contract, e2e, smoke, load tests that can be generated by Skyramp MCP tools
+2. Use Skyramp MCP to execute the tests and validate the results.
+3. Generate a CONCISE summary to '${summaryOutputFile}' following this format:
+
+## 🤖 Skyramp Test Bot Recommendations
+
+### 💡 Suggested Test Coverage
+- Brief list of new test types added and for which endpoint.
+
+### 🧪 Test Results
+- List each test executed with pass/fail status and brief details
+
+### ⚠️ Issues Found
+- List any test failures or bugs discovered (if none, write 'None')
+
+Keep the summary brief and scannable. Avoid repeating code diff details that are already visible in the PR.
+
+Use the skyramp-mcp tools available to you for test analysis, generation, and execution.`;
+}
+function getTestbotMaintenancePrompt(diffFile, testDirectory, summaryOutputFile) {
+    return `Based on the git diff in ${diffFile} file, perform the following test maintenance tasks:
+
+1. Check if there are any relevant Skyramp tests in the '${testDirectory}' directory that correspond to the code changes.
+2. If there are relevant tests, use Skyramp MCP tools to analyze and modify them to ensure they pass with the new changes.
+3. If there are no tests that can validate the changes, use Skyramp MCP to generate appropriate Skyramp tests for the new or modified code.
+4. Use Skyramp MCP to execute the tests and validate the results.
+5. Generate a CONCISE summary to '${summaryOutputFile}' following this format:
+
+## 🤖 Skyramp Test Bot Summary
+
+**Tests Modified:** [number] | **Tests Created:** [number] | **Tests Executed:** [number] | **Passed:** [number] | **Failed:** [number]
+
+### ✅ Maintenance Summary
+- Brief one-line summary of test changes (details are in the commit)
+
+### 🧪 Test Results
+- List each test executed with pass/fail status and brief details
+
+### ⚠️ Issues Found
+- List any test failures or bugs discovered (if none, write 'None')
+
+Keep the summary brief and scannable. Avoid repeating code diff details that are already visible in the PR.
+
+Use the skyramp-mcp tools available to you for test analysis, generation, and execution.`;
+}
+export function registerTestbotPrompt(server) {
+    logger.info("Registering testbot prompt");
+    server.registerPrompt("skyramp_testbot", {
+        description: "Run Skyramp TestBot to generate test recommendations and perform test maintenance for a pull request.",
+        argsSchema: {
+            prTitle: z
+                .string()
+                .describe("Pull request title"),
+            prDescription: z
+                .string()
+                .describe("Pull request description/body"),
+            diffFile: z
+                .string()
+                .describe("Path to the git diff file"),
+            testDirectory: z
+                .string()
+                .default("tests")
+                .describe("Directory containing Skyramp tests"),
+            recommendationSummaryOutputFile: z
+                .string()
+                .describe("File path where the agent should write the recommendation summary"),
+            maintenanceSummaryOutputFile: z
+                .string()
+                .describe("File path where the agent should write the maintenance summary"),
+        },
+    }, (args) => {
+        const recommendationPrompt = getTestbotRecommendationPrompt(args.prTitle, args.prDescription, args.recommendationSummaryOutputFile);
+        const maintenancePrompt = getTestbotMaintenancePrompt(args.diffFile, args.testDirectory, args.maintenanceSummaryOutputFile);
+        return {
+            messages: [
+                {
+                    role: "user",
+                    content: {
+                        type: "text",
+                        text: `# Skyramp TestBot Workflow
+
+Follow these two phases in order.
+
+## Phase 1: Test Recommendations
+
+${recommendationPrompt}
+
+## Phase 2: Test Maintenance
+
+${maintenancePrompt}`,
+                    },
+                },
+            ],
+        };
+    });
+}

package/build/services/TestExecutionService.js

@@ -6,7 +6,7 @@ import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
 const MAX_CONCURRENT_EXECUTIONS = 5;
-const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.6";
+const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.8";
 const DOCKER_PLATFORM = "linux/amd64";
 const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
 // Files and directories to exclude when mounting workspace to Docker container

package/build/tools/initTestbotTool.js

@@ -0,0 +1,187 @@
+import { z } from "zod";
+import { simpleGit } from "simple-git";
+import * as fs from "fs/promises";
+import * as path from "path";
+import { logger } from "../utils/logger.js";
+import { AnalyticsService } from "../services/AnalyticsService.js";
+const TOOL_NAME = "skyramp_init_testbot";
+const WORKFLOW_FILENAME = "skyramp-test-bot.yml";
+function generateWorkflow(targetBranches, agentType) {
+    const apiKeyLine = agentType === "copilot"
+        ? "          copilot_api_key: ${{ secrets.COPILOT_API_KEY }}"
+        : "          cursor_api_key: ${{ secrets.CURSOR_API_KEY }}";
+    const branchesBlock = targetBranches && targetBranches.length > 0
+        ? `\n    branches:\n${targetBranches.map((b) => `      - '${b}'`).join("\n")}`
+        : "";
+    return `name: Skyramp Test Automation
+on:
+  pull_request:${branchesBlock}
+
+jobs:
+  test-maintenance:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Skyramp Test Bot
+        uses: skyramp/test-bot@v1
+        with:
+          skyramp_license_file: \${{ secrets.SKYRAMP_LICENSE }}
+${apiKeyLine}
+`;
+}
+export function registerInitTestbotTool(server) {
+    server.registerTool(TOOL_NAME, {
+        description: `Initialize Skyramp Test Bot for a GitHub repository
+
+Sets up a GitHub Actions workflow that automatically generates and maintains tests on pull requests using Skyramp Test Bot. This tool validates the repository, creates the workflow file, and provides setup instructions for required secrets.`,
+        inputSchema: {
+            repository_path: z
+                .string()
+                .describe("Absolute path to the user's GitHub repository"),
+            agent_type: z
+                .enum(["cursor", "copilot"])
+                .default("cursor")
+                .describe("Which AI agent the user uses (cursor or copilot). Defaults to cursor."),
+            target_branches: z
+                .array(z.string())
+                .optional()
+                .describe("Branches to trigger the test bot on pull requests. If not specified, the workflow triggers on all pull requests."),
+        },
+        _meta: {
+            keywords: [
+                "init",
+                "testbot",
+                "test-bot",
+                "github actions",
+                "workflow",
+                "setup",
+                "ci",
+            ],
+        },
+    }, async (params) => {
+        let errorResult;
+        const repoPath = params.repository_path;
+        const agentType = params.agent_type;
+        const targetBranches = params.target_branches;
+        logger.info("Initializing Skyramp Test Bot", {
+            repository_path: repoPath,
+            agent_type: agentType,
+            target_branches: targetBranches,
+        });
+        try {
+            // Validate it's a git repo
+            const git = simpleGit(repoPath);
+            const isRepo = await git.checkIsRepo();
+            if (!isRepo) {
+                errorResult = {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Error: "${repoPath}" is not a Git repository. Please run this tool from a valid Git repository.`,
+                        },
+                    ],
+                    isError: true,
+                };
+                return errorResult;
+            }
+            // Validate it has a GitHub remote
+            const remotes = await git.getRemotes(true);
+            const hasGitHubRemote = remotes.some((remote) => remote.refs.fetch?.includes("github.com") ||
+                remote.refs.push?.includes("github.com"));
+            if (!hasGitHubRemote) {
+                errorResult = {
+                    content: [
+                        {
+                            type: "text",
+                            text: `Error: No GitHub remote found in "${repoPath}". Skyramp Test Bot requires a GitHub repository. Please add a GitHub remote and try again.`,
+                        },
+                    ],
+                    isError: true,
+                };
+                return errorResult;
+            }
+            // Create the workflow directory if needed and write the workflow file atomically
+            const workflowDir = path.join(repoPath, ".github", "workflows");
+            await fs.mkdir(workflowDir, { recursive: true });
+            const workflowPath = path.join(workflowDir, WORKFLOW_FILENAME);
+            const workflowContent = generateWorkflow(targetBranches, agentType);
+            try {
+                await fs.writeFile(workflowPath, workflowContent, {
+                    encoding: "utf-8",
+                    flag: "wx",
+                });
+            }
+            catch (err) {
+                if (err.code === "EEXIST") {
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: `Skyramp Test Bot workflow already exists at ".github/workflows/${WORKFLOW_FILENAME}". No changes were made. If you want to reconfigure it, delete the existing file and run this tool again.`,
+                            },
+                        ],
+                    };
+                }
+                throw err;
+            }
+            // Build success message with setup instructions
+            const secretName = agentType === "copilot" ? "COPILOT_API_KEY" : "CURSOR_API_KEY";
+            const agentLabel = agentType === "copilot" ? "Copilot" : "Cursor";
+            const successMessage = `Skyramp Test Bot workflow created successfully at ".github/workflows/${WORKFLOW_FILENAME}".
+
+To complete the setup, configure the following GitHub Secrets in your repository (Settings > Secrets and variables > Actions):
+
+1. **SKYRAMP_LICENSE** - The contents of your Skyramp license file (paste the full license file text)
+2. **${secretName}** - Your ${agentLabel} API key
+
+The workflow is configured to run on ${targetBranches && targetBranches.length > 0 ? `pull requests targeting: ${targetBranches.join(", ")}` : "all pull requests"}
+
+Required repository permissions (already configured in the workflow):
+- \`contents: write\` - To commit generated tests
+- \`pull-requests: write\` - To post comments on PRs
+
+Next steps:
+1. Add the required secrets to your GitHub repository
+2. Commit and push the new workflow file
+3. Open a pull request to see Skyramp Test Bot in action`;
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: successMessage,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const errorMessage = `Failed to initialize Skyramp Test Bot: ${error.message}`;
+            logger.error(errorMessage, { error });
+            errorResult = {
+                content: [
+                    {
+                        type: "text",
+                        text: errorMessage,
+                    },
+                ],
+                isError: true,
+            };
+            return errorResult;
+        }
+        finally {
+            const recordParams = {
+                repository_path: repoPath,
+                agent_type: agentType,
+                target_branches: targetBranches ? targetBranches.join(",") : "",
+            };
+            AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, recordParams);
+        }
+    });
+}

package/build/tools/initTestbotTool.test.js

@@ -0,0 +1,194 @@
+// @ts-ignore
+import { registerInitTestbotTool } from "./initTestbotTool.js";
+import { simpleGit } from "simple-git";
+import * as fs from "fs/promises";
+import * as fsSync from "fs";
+import * as path from "path";
+import * as os from "os";
+// Mock logger and AnalyticsService to avoid side effects
+jest.mock("../utils/logger.js", () => ({
+    logger: { info: jest.fn(), error: jest.fn() },
+}));
+jest.mock("../services/AnalyticsService.js", () => ({
+    AnalyticsService: { pushMCPToolEvent: jest.fn() },
+}));
+/**
+ * Captures the tool handler callback from registerInitTestbotTool
+ * by providing a fake McpServer with a mock registerTool method.
+ */
+function captureToolHandler() {
+    let handler;
+    const fakeServer = {
+        registerTool: (_name, _opts, fn) => {
+            handler = fn;
+        },
+    };
+    registerInitTestbotTool(fakeServer);
+    return handler;
+}
+/**
+ * Creates a temp directory, initializes a git repo, and optionally
+ * adds a GitHub remote.
+ */
+async function createTempRepo(options) {
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "init-testbot-test-"));
+    const git = simpleGit(tmpDir);
+    await git.init();
+    if (options?.addGithubRemote !== false) {
+        await git.addRemote("origin", "https://github.com/test-org/test-repo.git");
+    }
+    return tmpDir;
+}
+describe("registerInitTestbotTool", () => {
+    let handler;
+    let tmpDirs = [];
+    beforeAll(() => {
+        handler = captureToolHandler();
+    });
+    afterEach(async () => {
+        for (const dir of tmpDirs) {
+            await fs.rm(dir, { recursive: true, force: true });
+        }
+        tmpDirs = [];
+    });
+    it("should register the tool on the server", () => {
+        const registerToolMock = jest.fn();
+        const fakeServer = { registerTool: registerToolMock };
+        registerInitTestbotTool(fakeServer);
+        expect(registerToolMock).toHaveBeenCalledWith("skyramp_init_testbot", expect.objectContaining({
+            description: expect.any(String),
+            inputSchema: expect.any(Object),
+        }), expect.any(Function));
+    });
+    it("should return error for non-git directory", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "init-testbot-test-"));
+        tmpDirs.push(tmpDir);
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain("is not a Git repository");
+    });
+    it("should return error for git repo without GitHub remote", async () => {
+        const tmpDir = await createTempRepo({ addGithubRemote: false });
+        tmpDirs.push(tmpDir);
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain("No GitHub remote found");
+    });
+    it("should create workflow file on success", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        expect(result.isError).toBeUndefined();
+        expect(result.content[0].text).toContain("workflow created successfully");
+        const workflowPath = path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml");
+        expect(fsSync.existsSync(workflowPath)).toBe(true);
+    });
+    it("should return already-exists message when workflow file exists", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        // Create the workflow file first
+        const workflowDir = path.join(tmpDir, ".github", "workflows");
+        await fs.mkdir(workflowDir, { recursive: true });
+        await fs.writeFile(path.join(workflowDir, "skyramp-test-bot.yml"), "existing");
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        expect(result.isError).toBeUndefined();
+        expect(result.content[0].text).toContain("already exists");
+        // Verify original file was not overwritten
+        const content = await fs.readFile(path.join(workflowDir, "skyramp-test-bot.yml"), "utf-8");
+        expect(content).toBe("existing");
+    });
+    it("should generate workflow with cursor API key by default", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        const workflowContent = await fs.readFile(path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml"), "utf-8");
+        expect(workflowContent).toContain("cursor_api_key");
+        expect(workflowContent).not.toContain("copilot_api_key");
+    });
+    it("should generate workflow with copilot API key when agent_type is copilot", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        await handler({
+            repository_path: tmpDir,
+            agent_type: "copilot",
+        });
+        const workflowContent = await fs.readFile(path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml"), "utf-8");
+        expect(workflowContent).toContain("copilot_api_key");
+        expect(workflowContent).not.toContain("cursor_api_key");
+    });
+    it("should generate workflow without branches key when target_branches is not provided", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        const workflowContent = await fs.readFile(path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml"), "utf-8");
+        expect(workflowContent).toContain("pull_request:");
+        expect(workflowContent).not.toContain("branches:");
+    });
+    it("should generate workflow with branches key when target_branches is provided", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+            target_branches: ["main", "develop"],
+        });
+        const workflowContent = await fs.readFile(path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml"), "utf-8");
+        expect(workflowContent).toContain("branches:");
+        expect(workflowContent).toContain("- 'main'");
+        expect(workflowContent).toContain("- 'develop'");
+    });
+    it("should include setup instructions with correct secret names for cursor", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        expect(result.content[0].text).toContain("CURSOR_API_KEY");
+        expect(result.content[0].text).toContain("SKYRAMP_LICENSE");
+        expect(result.content[0].text).toContain("all pull requests");
+    });
+    it("should include setup instructions with correct secret names for copilot", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        const result = await handler({
+            repository_path: tmpDir,
+            agent_type: "copilot",
+            target_branches: ["main"],
+        });
+        expect(result.content[0].text).toContain("COPILOT_API_KEY");
+        expect(result.content[0].text).toContain("pull requests targeting: main");
+    });
+    it("should generate valid YAML workflow structure", async () => {
+        const tmpDir = await createTempRepo();
+        tmpDirs.push(tmpDir);
+        await handler({
+            repository_path: tmpDir,
+            agent_type: "cursor",
+        });
+        const workflowContent = await fs.readFile(path.join(tmpDir, ".github", "workflows", "skyramp-test-bot.yml"), "utf-8");
+        expect(workflowContent).toContain("name: Skyramp Test Automation");
+        expect(workflowContent).toContain("runs-on: ubuntu-latest");
+        expect(workflowContent).toContain("uses: actions/checkout@v4");
+        expect(workflowContent).toContain("uses: skyramp/test-bot@v1");
+        expect(workflowContent).toContain("skyramp_license_file:");
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.0.54",
+  "version": "0.0.55",
   "main": "build/index.js",
   "type": "module",
   "bin": {
@@ -45,8 +45,8 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.24.3",
-    "@skyramp/skyramp": "1.3.6",
     "@playwright/test": "^1.55.0",
+    "@skyramp/skyramp": "1.3.8",
     "dockerode": "^4.0.6",
     "fast-glob": "^3.3.3",
     "simple-git": "^3.30.0",