@skyramp/mcp 0.2.0-rc.3 → 0.2.0

package/build/prompts/testbot/testbot-prompts.js

@@ -331,7 +331,7 @@ Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}
 - **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
   - For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
   - For \`testType: "integration"\` or \`"e2e"\` entries: omit \`primaryEndpoint\` — use \`description\` to list the endpoints involved instead.
-- **testMaintenance**: Use \`[]\` **only** if no existing Skyramp tests were found in the repository. If existing tests were found (any score), include one entry per test. For UPDATE/REGENERATE/DELETE tests that were modified and executed, populate all fields from real before/after execution results. For IGNORE-scored tests (not modified or executed), derive \`beforeStatus\` from the \`skyramp_analyze_test_health\` health score (typically \`"Pass"\` if drift score is 0 and no health issues were flagged), set \`afterStatus\` to \`"Skipped"\`, and use \`afterDetails\` to explain why (e.g. "IGNORE: drift score 0 — endpoint not modified in this PR"). Do **not** add entries for tests that were not returned by the health analysis.
+- **testMaintenance**: Use \`[]\` **only** if no existing Skyramp tests were found in the repository. If existing tests were found (any score), include one entry per test. Set \`action\` to the exact drift action you chose from the Action Decision Matrix (\`UPDATE\`, \`REGENERATE\`, \`DELETE\`, \`VERIFY\`, or \`IGNORE\`). For UPDATE/REGENERATE/DELETE tests that were modified and executed, populate all fields from real before/after execution results. For VERIFY/IGNORE tests (not modified), derive \`beforeStatus\` from the \`skyramp_analyze_test_health\` health score (typically \`"Pass"\` if drift score is 0 and no health issues were flagged), set \`afterStatus\` to \`"Skipped"\`, and use \`afterDetails\` to explain why (e.g. "IGNORE: drift score 0 — endpoint not modified in this PR"). Do **not** add entries for tests that were not returned by the health analysis.
 
 ---
 

package/build/services/AnalyticsService.js

@@ -2,7 +2,7 @@ import { pushToolEvent } from "@skyramp/skyramp";
 import * as fs from "fs";
 import * as path from "path";
 import { fileURLToPath } from "url";
-import { getEntryPoint, getCIPlatform } from "../utils/telemetry.js";
+import { getEntryPoint, getCIPlatform, getRepositoryInfo, } from "../utils/telemetry.js";
 import { logger } from "../utils/logger.js";
 export class AnalyticsService {
     static async pushTestGenerationToolEvent(toolName, result, params) {
@@ -29,6 +29,10 @@ export class AnalyticsService {
             if (ciPlatform) {
                 params.ciPlatform = ciPlatform;
             }
+            // Prefer a repo path the tool already supplied — the MCP server's own
+            // process.cwd() is set by the IDE and may not be the user's repo.
+            const repoPath = params.repositoryPath || params.workspacePath;
+            Object.assign(params, await getRepositoryInfo(repoPath));
             await pushToolEvent(getEntryPoint(), toolName, errorMessage, params);
         }
         catch (error) {

package/build/tools/submitReportTool.js

@@ -5,6 +5,15 @@ import * as path from "path";
 import { AnalyticsService } from "../services/AnalyticsService.js";
 import { TEST_CATEGORIES, externalCategory } from "../types/TestRecommendation.js";
 import { TestType, HttpMethod } from "../types/TestTypes.js";
+import { DriftAction } from "../types/TestAnalysis.js";
+// Drift actions that actually modify a test file. VERIFY and IGNORE are
+// no-ops (the test was assessed but left unchanged), so they must not count
+// toward "tests maintained" telemetry.
+const MAINTENANCE_CHANGE_ACTIONS = new Set([
+    DriftAction.Update,
+    DriftAction.Regenerate,
+    DriftAction.Delete,
+]);
 const TOOL_NAME = "skyramp_submit_report";
 const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
 const testResultSchema = z.object({
@@ -73,12 +82,58 @@ const testMaintenanceSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test."),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
+    action: z.nativeEnum(DriftAction).optional().describe("The drift action taken for this test, exactly as decided by the Action Decision Matrix: UPDATE, REGENERATE, or DELETE modify the test; VERIFY or IGNORE leave it unchanged (no-op)."),
     description: z.string().describe("What was changed and why"),
     beforeStatus: z.enum(["Pass", "Fail", "Error"]).describe("Test result BEFORE modification"),
     beforeDetails: z.string().describe("Execution output/timing before modification, or 'baseline from CI workflow <name>' if a parallel workflow provided the baseline"),
     afterStatus: z.enum(["Pass", "Fail", "Error", "Skipped"]).describe("Test result AFTER modification"),
     afterDetails: z.string().describe("Execution output/timing after modification"),
 });
+/**
+ * Derive per-run analytics counts from a submitted report. These power the
+ * alpha-launch dashboards (tests generated/maintained, suite growth, bugs vs
+ * test failures, flakiness recovery). All values are stringified for Amplitude.
+ *
+ * Bug-vs-failure distinction:
+ *  - issuesFound* = bugs/issues the agent flagged (real defects + severity)
+ *  - testsFailed = test executions that returned "Fail" (test-level outcome)
+ * These are intentionally separate metrics — a failing test is not always a bug.
+ *
+ * maintenanceRecovered approximates flakiness/regression fixes: tests that were
+ * Fail/Error before maintenance and Pass afterward.
+ *
+ * testsMaintained counts only entries that actually changed a test file
+ * (action UPDATE/REGENERATE/DELETE). VERIFY/IGNORE entries are reported for
+ * transparency but are no-ops, so they are excluded. When `action` is absent
+ * (older reports), we fall back to the status heuristic: an IGNORE no-op sets
+ * afterStatus to "Skipped", so anything else is treated as a real change.
+ */
+function isMaintenanceChange(m) {
+    if (m.action) {
+        return MAINTENANCE_CHANGE_ACTIONS.has(m.action);
+    }
+    return m.afterStatus !== "Skipped";
+}
+function computeReportMetrics(params) {
+    const recommendations = params.additionalRecommendations ?? [];
+    const countBy = (items, pred) => items.filter(pred).length;
+    const changedMaintenance = params.testMaintenance.filter(isMaintenanceChange);
+    const maintenanceRecovered = countBy(changedMaintenance, (m) => m.beforeStatus !== "Pass" && m.afterStatus === "Pass");
+    return {
+        testsGenerated: String(params.newTestsCreated.length),
+        testsMaintained: String(changedMaintenance.length),
+        recommendationsCount: String(recommendations.length),
+        maintenanceRecovered: String(maintenanceRecovered),
+        testsPassed: String(countBy(params.testResults, (t) => t.status === "Pass")),
+        testsFailed: String(countBy(params.testResults, (t) => t.status === "Fail")),
+        testsSkipped: String(countBy(params.testResults, (t) => t.status === "Skipped")),
+        issuesFound: String(params.issuesFound.length),
+        issuesCritical: String(countBy(params.issuesFound, (i) => i.severity === "critical")),
+        issuesHigh: String(countBy(params.issuesFound, (i) => i.severity === "high")),
+        issuesMedium: String(countBy(params.issuesFound, (i) => i.severity === "medium")),
+        issuesLow: String(countBy(params.issuesFound, (i) => i.severity === "low")),
+    };
+}
 export function registerSubmitReportTool(server) {
     server.registerTool(TOOL_NAME, {
         description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
@@ -201,6 +256,7 @@ export function registerSubmitReportTool(server) {
                 summary_output_file: params.summaryOutputFile,
                 testResultCount: String(params.testResults.length),
                 payloadBytes: String(reportJson.length),
+                ...computeReportMetrics(params),
             }).catch(() => { });
         }
     });

package/build/tools/submitReportTool.test.js

@@ -1,6 +1,7 @@
 // @ts-ignore
 import { registerSubmitReportTool, additionalRecommendationSchema } from "./submitReportTool.js";
 import { TestType } from "../types/TestTypes.js";
+import { AnalyticsService } from "../services/AnalyticsService.js";
 import * as fs from "fs/promises";
 import * as path from "path";
 import * as os from "os";
@@ -10,6 +11,7 @@ jest.mock("../utils/logger.js", () => ({
 jest.mock("../services/AnalyticsService.js", () => ({
     AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) },
 }));
+const pushMCPToolEventMock = AnalyticsService.pushMCPToolEvent;
 function captureToolHandler() {
     let handler;
     const fakeServer = {
@@ -290,4 +292,108 @@ describe("registerSubmitReportTool", () => {
             expect(result.data.primaryEndpoint).toBeUndefined();
         }
     });
+    describe("analytics metrics", () => {
+        beforeEach(() => {
+            pushMCPToolEventMock.mockClear();
+        });
+        function lastAnalyticsParams() {
+            expect(pushMCPToolEventMock).toHaveBeenCalled();
+            const calls = pushMCPToolEventMock.mock.calls;
+            return calls[calls.length - 1][2];
+        }
+        it("emits per-run count metrics derived from the report", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const outputFile = path.join(tmpDir, "report.json");
+            // sampleReportParams: 1 new test, 1 maintenance (Fail→Pass), 2 results
+            // (1 Pass + 1 Fail), 1 issue (no severity).
+            await handler(sampleReportParams(outputFile));
+            const params = lastAnalyticsParams();
+            expect(params.testsGenerated).toBe("1");
+            expect(params.testsMaintained).toBe("1");
+            expect(params.recommendationsCount).toBe("0");
+            expect(params.maintenanceRecovered).toBe("1");
+            expect(params.testsPassed).toBe("1");
+            expect(params.testsFailed).toBe("1");
+            expect(params.testsSkipped).toBe("0");
+            expect(params.issuesFound).toBe("1");
+        });
+        it("breaks issues down by severity", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const outputFile = path.join(tmpDir, "report.json");
+            await handler({
+                ...sampleReportParams(outputFile),
+                issuesFound: [
+                    { description: "Data corruption on checkout", severity: "critical" },
+                    { description: "Wrong total returned", severity: "high" },
+                    { description: "Another wrong total", severity: "high" },
+                    { description: "Minor gap", severity: "medium" },
+                    { description: "Unlabeled issue" },
+                ],
+            });
+            const params = lastAnalyticsParams();
+            expect(params.issuesFound).toBe("5");
+            expect(params.issuesCritical).toBe("1");
+            expect(params.issuesHigh).toBe("2");
+            expect(params.issuesMedium).toBe("1");
+            expect(params.issuesLow).toBe("0");
+        });
+        it("reports zero metrics for an empty report", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const outputFile = path.join(tmpDir, "report.json");
+            await handler({
+                summaryOutputFile: outputFile,
+                businessCaseAnalysis: "Config-only change.",
+                newTestsCreated: [],
+                testMaintenance: [],
+                testResults: [],
+                issuesFound: [],
+            });
+            const params = lastAnalyticsParams();
+            expect(params.testsGenerated).toBe("0");
+            expect(params.testsMaintained).toBe("0");
+            expect(params.maintenanceRecovered).toBe("0");
+            expect(params.testsPassed).toBe("0");
+            expect(params.issuesFound).toBe("0");
+        });
+        it("testsMaintained counts only change actions, not VERIFY/IGNORE no-ops", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const outputFile = path.join(tmpDir, "report.json");
+            await handler({
+                ...sampleReportParams(outputFile),
+                testMaintenance: [
+                    // Real change
+                    { action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched auth", beforeStatus: "Fail", beforeDetails: "401 (1.5s)", afterStatus: "Pass", afterDetails: "passed (2.3s)" },
+                    { action: "REGENERATE", testType: TestType.SMOKE, endpoint: "GET /api/v1/orders", fileName: "orders_smoke_test.py", description: "Rewrote for new shape", beforeStatus: "Fail", beforeDetails: "shape mismatch", afterStatus: "Pass", afterDetails: "passed (1.0s)" },
+                    // No-ops — assessed but not modified
+                    { action: "IGNORE", testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeStatus: "Pass", beforeDetails: "drift score 0", afterStatus: "Skipped", afterDetails: "IGNORE: endpoint not in PR" },
+                    { action: "VERIFY", testType: TestType.SMOKE, endpoint: "GET /api/v1/coupons", fileName: "coupons_smoke_test.py", description: "Verified still valid", beforeStatus: "Pass", beforeDetails: "minor drift", afterStatus: "Skipped", afterDetails: "VERIFY: assertions still hold" },
+                ],
+            });
+            const params = lastAnalyticsParams();
+            // 2 change actions (UPDATE + REGENERATE), not 4
+            expect(params.testsMaintained).toBe("2");
+            // Both changes went Fail→Pass
+            expect(params.maintenanceRecovered).toBe("2");
+        });
+        it("falls back to afterStatus heuristic when action is absent", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const outputFile = path.join(tmpDir, "report.json");
+            await handler({
+                ...sampleReportParams(outputFile),
+                testMaintenance: [
+                    // No action field — afterStatus !== "Skipped" → counts as a change
+                    { testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched", beforeStatus: "Fail", beforeDetails: "401", afterStatus: "Pass", afterDetails: "passed" },
+                    // No action field — afterStatus === "Skipped" → treated as no-op
+                    { testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeStatus: "Pass", beforeDetails: "drift 0", afterStatus: "Skipped", afterDetails: "not in PR" },
+                ],
+            });
+            const params = lastAnalyticsParams();
+            expect(params.testsMaintained).toBe("1");
+        });
+    });
 });

package/build/utils/telemetry.js

@@ -3,7 +3,11 @@
  * "testbot" when running as part of the GitHub Action test bot,
  * "mcp" for regular IDE/MCP usage.
  */
+import { execFile } from "child_process";
+import { promisify } from "util";
 import { isTestbotEnabled } from "./featureFlags.js";
+import { logger } from "./logger.js";
+const execFileAsync = promisify(execFile);
 export function getEntryPoint() {
     return isTestbotEnabled() ? "testbot" : "mcp";
 }
@@ -22,3 +26,69 @@ export function getCIPlatform() {
         return "circleci";
     return undefined;
 }
+/**
+ * Parses "owner/repo" out of a git remote URL. Handles both HTTPS
+ * (https://github.com/owner/repo.git) and SSH (git@github.com:owner/repo.git)
+ * forms, with or without the trailing ".git". Returns undefined if no match.
+ */
+export function parseRepositoryFromRemoteUrl(remoteUrl) {
+    const match = remoteUrl.trim().match(/[:/]([^/]+)\/([^/.]+?)(?:\.git)?$/);
+    if (!match)
+        return undefined;
+    const [, owner, repo] = match;
+    return { repository: `${owner}/${repo}`, repositoryOwner: owner };
+}
+// Memoize the local git lookup per working directory so we shell out at most
+// once per cwd for the lifetime of the MCP server process. The value is the
+// resolved RepositoryInfo (possibly empty when not a git repo / no remote).
+const localRepoInfoCache = new Map();
+async function getLocalRepositoryInfo(cwd) {
+    const cached = localRepoInfoCache.get(cwd);
+    if (cached)
+        return cached;
+    const lookup = (async () => {
+        try {
+            const { stdout } = await execFileAsync("git", ["-C", cwd, "remote", "get-url", "origin"], { timeout: 5_000 });
+            return parseRepositoryFromRemoteUrl(stdout) ?? {};
+        }
+        catch (error) {
+            // Not a git repo, no "origin" remote, or git not installed — attribute
+            // nothing rather than failing the telemetry call.
+            logger.debug("Could not resolve local git repository for telemetry", {
+                cwd,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return {};
+        }
+    })();
+    localRepoInfoCache.set(cwd, lookup);
+    return lookup;
+}
+/**
+ * Resolves source-repository attribution for telemetry, segmenting usage per
+ * customer/repo.
+ *
+ * - In GitHub Actions (testbot), reads GITHUB_REPOSITORY / GITHUB_REPOSITORY_OWNER
+ *   which the MCP subprocess inherits — fast, no shelling out.
+ * - In local/IDE usage, derives "owner/repo" from `git remote get-url origin`.
+ *   The lookup directory is the caller-supplied `cwd` (a tool's repositoryPath
+ *   or workspacePath) when available, since the MCP server's own process.cwd()
+ *   is set by the IDE and is not necessarily the user's repo. Falls back to
+ *   process.cwd(). Cached per directory so git is invoked at most once each.
+ *
+ * Returns an empty object when attribution can't be determined (e.g. not a git
+ * repo, no origin remote).
+ */
+export async function getRepositoryInfo(cwd) {
+    if (process.env.GITHUB_ACTIONS === "true") {
+        const info = {};
+        if (process.env.GITHUB_REPOSITORY) {
+            info.repository = process.env.GITHUB_REPOSITORY;
+        }
+        if (process.env.GITHUB_REPOSITORY_OWNER) {
+            info.repositoryOwner = process.env.GITHUB_REPOSITORY_OWNER;
+        }
+        return info;
+    }
+    return getLocalRepositoryInfo(cwd || process.cwd());
+}

package/build/utils/telemetry.test.js

@@ -0,0 +1,70 @@
+import { getRepositoryInfo, parseRepositoryFromRemoteUrl } from "./telemetry.js";
+describe("parseRepositoryFromRemoteUrl", () => {
+    it("parses an HTTPS remote with .git suffix", () => {
+        expect(parseRepositoryFromRemoteUrl("https://github.com/acme-corp/their-api.git")).toEqual({ repository: "acme-corp/their-api", repositoryOwner: "acme-corp" });
+    });
+    it("parses an HTTPS remote without .git suffix", () => {
+        expect(parseRepositoryFromRemoteUrl("https://github.com/acme-corp/their-api")).toEqual({ repository: "acme-corp/their-api", repositoryOwner: "acme-corp" });
+    });
+    it("parses an SSH remote", () => {
+        expect(parseRepositoryFromRemoteUrl("git@github.com:acme-corp/their-api.git")).toEqual({ repository: "acme-corp/their-api", repositoryOwner: "acme-corp" });
+    });
+    it("trims trailing whitespace/newline from git output", () => {
+        expect(parseRepositoryFromRemoteUrl("git@github.com:acme-corp/their-api.git\n")).toEqual({ repository: "acme-corp/their-api", repositoryOwner: "acme-corp" });
+    });
+    it("returns undefined for an unparseable URL", () => {
+        expect(parseRepositoryFromRemoteUrl("not-a-remote")).toBeUndefined();
+    });
+});
+describe("getRepositoryInfo", () => {
+    const originalEnv = { ...process.env };
+    afterEach(() => {
+        process.env = { ...originalEnv };
+    });
+    describe("in GitHub Actions", () => {
+        beforeEach(() => {
+            process.env.GITHUB_ACTIONS = "true";
+        });
+        it("returns repository and owner from env vars", async () => {
+            process.env.GITHUB_REPOSITORY = "acme-corp/their-api";
+            process.env.GITHUB_REPOSITORY_OWNER = "acme-corp";
+            await expect(getRepositoryInfo()).resolves.toEqual({
+                repository: "acme-corp/their-api",
+                repositoryOwner: "acme-corp",
+            });
+        });
+        it("includes only the fields that are present", async () => {
+            process.env.GITHUB_REPOSITORY = "acme-corp/their-api";
+            delete process.env.GITHUB_REPOSITORY_OWNER;
+            await expect(getRepositoryInfo()).resolves.toEqual({
+                repository: "acme-corp/their-api",
+            });
+        });
+    });
+    describe("local / IDE", () => {
+        beforeEach(() => {
+            delete process.env.GITHUB_ACTIONS;
+        });
+        it("derives repository info from the local git remote", async () => {
+            // The test suite itself runs inside the mcp git repo (origin = letsramp/mcp),
+            // so a local lookup should resolve owner=letsramp, repo=mcp.
+            const info = await getRepositoryInfo();
+            expect(info.repositoryOwner).toBe("letsramp");
+            expect(info.repository).toBe("letsramp/mcp");
+        });
+        it("uses the caller-supplied cwd when provided", async () => {
+            // Passing the repo path explicitly (as tools do via repositoryPath)
+            // resolves the same remote even if process.cwd() differs.
+            const info = await getRepositoryInfo(process.cwd());
+            expect(info.repository).toBe("letsramp/mcp");
+        });
+        it("ignores GITHUB_REPOSITORY env when not in GitHub Actions", async () => {
+            // A stray env var must not be trusted outside Actions; the git remote is
+            // the source of truth locally.
+            process.env.GITHUB_REPOSITORY = "spoofed/repo";
+            process.env.GITHUB_REPOSITORY_OWNER = "spoofed";
+            const info = await getRepositoryInfo();
+            expect(info.repository).not.toBe("spoofed/repo");
+        });
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.2.0-rc.3",
+  "version": "0.2.0",
   "main": "build/index.js",
   "exports": {
     ".": "./build/index.js",