@skyramp/mcp 0.2.7 → 0.2.8-rc.1

package/build/tools/executeSkyrampTestTool.js

@@ -2,6 +2,7 @@ import { z } from "zod";
 import { stripVTControlCharacters } from "util";
 import { TestExecutionService } from "../services/TestExecutionService.js";
 import { AnalyticsService } from "../services/AnalyticsService.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
 import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { StateManager } from "../utils/AnalysisStateManager.js";
@@ -66,7 +67,15 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             stateFile: z
                 .string()
                 .optional()
-                .describe("Optional path to state file from skyramp_analyze_changes. When provided, execution results (passed/failed, errors, duration) will be written back to enrich the test health analysis."),
+                .describe("Path to state file from skyramp_analyze_changes. Always pass when available — results are written back so skyramp_submit_report can override before/afterStatus with ground-truth pass/fail."),
+            phase: z
+                .enum(["before", "after"])
+                .optional()
+                .describe("Execution phase for maintained tests: 'before' captures pre-edit baseline; 'after' (default) records post-edit result."),
+            repository: z
+                .string()
+                .optional()
+                .describe("The owner/repo whose analysis section to write execution results into (e.g. 'letsramp/api-insight'). In a multi-repo run, ALWAYS set it — the primary's owner/repo for a primary-repo test, or a related repo's owner/repo for that repo's section of the run-scoped stateFile. Omit only for a single-repo run."),
         },
         _meta: {
             keywords: ["run test", "execute test"],
@@ -139,16 +148,23 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                 playwrightSaveStoragePath: params.playwrightSaveStoragePath,
                 useHostNetwork,
             }, onExecutionProgress);
-            // Update stateFile with execution results if provided
+            // Update stateFile with execution results if provided. Multi-repo: write
+            // into the section for `repository` (primary when omitted/matching the
+            // root) of the run-scoped file.
             if (params.stateFile) {
                 try {
                     const stateManager = StateManager.fromStatePath(params.stateFile);
-                    const stateData = await stateManager.readData();
+                    const stateData = await stateManager.readRepoData(params.repository);
                     if (stateData && stateData.existingTests) {
                         const testIndex = stateData.existingTests.findIndex((t) => t.testFile === params.testFile);
                         if (testIndex >= 0) {
-                            stateData.existingTests[testIndex].execution = result;
-                            await stateManager.writeData(stateData);
+                            if (params.phase === "before") {
+                                stateData.existingTests[testIndex].executionBefore = result;
+                            }
+                            else {
+                                stateData.existingTests[testIndex].executionAfter = result;
+                            }
+                            await stateManager.writeRepoData(stateData, { repo: params.repository });
                             logger.info(`Updated stateFile with execution results for ${params.testFile}`);
                         }
                         else {
@@ -163,7 +179,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             }
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
-            if (!result.passed) {
+            if (result.status !== TestExecutionStatus.Pass) {
                 errorResult = {
                     content: [
                         {
@@ -177,11 +193,14 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
+            const videoInfo = result.videoPath
+                ? `\n\nVideo recording: ${result.videoPath}`
+                : "";
             return {
                 content: [
                     {
                         type: "text",
-                        text: `Test execution result: ${stripVTControlCharacters(result.output || "")}`,
+                        text: `Test execution result: ${stripVTControlCharacters(result.output || "")}${videoInfo}`,
                     },
                 ],
             };

package/build/tools/generate-tests/generateBatchScenarioRestTool.js

@@ -67,7 +67,7 @@ CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&li
     responseBody: z
         .string()
         .optional()
-        .describe("JSON string of the expected response body"),
+        .describe("JSON string of the expected response body. For steps that produce an id used by later steps, include that id field with a realistic UUID generated fresh for this scenario — the backend chains by value-matching any subsequent step path or requestBody that contains the same UUID."),
     statusCode: z
         .number()
         .optional()

package/build/tools/submitReportTool.d.ts

@@ -47,6 +47,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
     traceFile: z.ZodOptional<z.ZodString>;
     frontendTrace: z.ZodOptional<z.ZodString>;
     reasoning: z.ZodString;
+    repository: z.ZodOptional<z.ZodString>;
     targetElements: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
         role: z.ZodString;
         accessibleName: z.ZodString;
@@ -90,6 +91,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     description?: string | undefined;
+    repository?: string | undefined;
     traceFile?: string | undefined;
     scenarioFile?: string | undefined;
     frontendTrace?: string | undefined;
@@ -113,6 +115,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     description?: string | undefined;
+    repository?: string | undefined;
     category?: unknown;
     traceFile?: string | undefined;
     scenarioFile?: string | undefined;
@@ -138,6 +141,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     description?: string | undefined;
+    repository?: string | undefined;
     traceFile?: string | undefined;
     scenarioFile?: string | undefined;
     frontendTrace?: string | undefined;
@@ -161,6 +165,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     description?: string | undefined;
+    repository?: string | undefined;
     category?: unknown;
     traceFile?: string | undefined;
     scenarioFile?: string | undefined;
@@ -213,6 +218,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
     backendTrace: z.ZodOptional<z.ZodString>;
     frontendTrace: z.ZodOptional<z.ZodString>;
     reasoning: z.ZodString;
+    repository: z.ZodOptional<z.ZodString>;
     targetElements: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
         role: z.ZodString;
         accessibleName: z.ZodString;
@@ -263,6 +269,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
     }[];
     testId: string;
     reasoning: string;
+    repository?: string | undefined;
     scenarioName?: string | undefined;
     frontendTrace?: string | undefined;
     targetElements?: {
@@ -295,6 +302,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     priority?: unknown;
+    repository?: string | undefined;
     scenarioName?: string | undefined;
     category?: unknown;
     frontendTrace?: string | undefined;
@@ -329,6 +337,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
     }[];
     testId: string;
     reasoning: string;
+    repository?: string | undefined;
     scenarioName?: string | undefined;
     frontendTrace?: string | undefined;
     targetElements?: {
@@ -361,6 +370,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
     testId: string;
     reasoning: string;
     priority?: unknown;
+    repository?: string | undefined;
     scenarioName?: string | undefined;
     category?: unknown;
     frontendTrace?: string | undefined;

package/build/tools/submitReportTool.js

@@ -6,6 +6,9 @@ import { AnalyticsService } from "../services/AnalyticsService.js";
 import { TEST_CATEGORIES, externalCategory } from "../types/TestRecommendation.js";
 import { TestType, HttpMethod } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
+import { StateManager } from "../utils/AnalysisStateManager.js";
+import { toolError } from "../utils/utils.js";
 // Drift actions that actually modify a test file. VERIFY and IGNORE are
 // no-ops (the test was assessed but left unchanged), so they must not count
 // toward "tests maintained" telemetry.
@@ -16,11 +19,29 @@ const MAINTENANCE_CHANGE_ACTIONS = new Set([
 ]);
 const TOOL_NAME = "skyramp_submit_report";
 const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
+// Per-repo attribution. In a multi-repo run, every report item carries the
+// owner/repo it pertains to (the primary's owner/repo for primary findings, or a
+// related repo's owner/repo for findings from that repo's diff). Whitespace-only
+// values are normalized to undefined so a blank string is never treated as a
+// distinct repo. Single-repo runs may omit it.
+const repositoryField = z
+    .string()
+    .optional()
+    .describe("The 'owner/repo' this item pertains to (e.g. 'letsramp/api-insight'). In a multi-repo run, ALWAYS set it — the primary's owner/repo for primary-repo findings, or a related repo's owner/repo for findings derived from that repository's diff. May be omitted only in single-repo runs.");
+/** Normalize a `repository` attribution: trim, and map a blank/whitespace-only
+ *  value to undefined so it is never serialized as a distinct empty repo
+ *  (downstream consumers treat absence as "the primary repo"). */
+function normalizeRepository(item) {
+    const trimmed = item.repository?.trim();
+    return trimmed ? { ...item, repository: trimmed } : { ...item, repository: undefined };
+}
 const testResultSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test. Do not include priority or other metadata in this field."),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
     details: z.string().describe("One sentence — no embedded newlines, no markdown. e.g. '10.8s, products_contract_test.py' or 'failed: <one-line error summary>, products_contract_test.py'"),
+    videoPath: z.string().optional().describe("Path to video recording (UI/E2E tests only)"),
+    repository: repositoryField,
 });
 // Structured grounding for UI recommendations. Lifted verbatim from a
 // `browser_blueprint` element captured during this run. Source of truth for
@@ -61,6 +82,7 @@ export const newTestSchema = z.object({
     traceFile: z.string().optional().describe("Path to the backend trace file if used or created"),
     frontendTrace: z.string().optional().describe("Path to the Playwright/UI trace file if used or created"),
     reasoning: z.string().describe("Why this test was created: what production risk it mitigates, what code pattern it targets, or what coverage gap it fills"),
+    repository: repositoryField,
     targetElements: z.array(targetElementSchema).min(1).nullable().optional().describe("UI tests only: structured grounding for one or more elements the test targets. Most tests target a single element (array length 1); render-state and multi-step UI tests target several (array length 2+). Each entry must be lifted verbatim from a captured blueprint element. Set to null when blueprint capture failed (also requires '[no-blueprint-data]' marker in BOTH description and reasoning). See Blueprint Citation Invariant in testbot prompt."),
     pageContext: pageContextSchema.optional().describe("UI tests only: page metadata for the test. Lifted from the BlueprintCapture used during grounding."),
 }).superRefine((rec, ctx) => {
@@ -135,6 +157,7 @@ const descriptionSchema = z.object({
         .describe("Issue severity. critical = feature broken/unusable (e.g. page doesn't load, data corruption). " +
         "high = incorrect behavior (e.g. wrong calculation, stale data returned). " +
         "medium = minor functional gap. low = cosmetic or informational."),
+    repository: repositoryField,
 });
 const scenarioStepSchema = z.object({
     method: z.nativeEnum(HttpMethod).optional().describe("HTTP method. Required for API steps, omit for UI/E2E actions."),
@@ -158,6 +181,7 @@ export const additionalRecommendationSchema = z.object({
     backendTrace: z.string().optional().describe("Path to backend trace file if available, e.g. 'tests/skyramp-traces.json'. Used by integration and E2E tests."),
     frontendTrace: z.string().optional().describe("Path to Playwright/UI trace file if available, e.g. 'tests/skyramp-playwright.zip'. UI tests need this; E2E tests need both frontend and backend traces."),
     reasoning: z.string().describe("Why this test is recommended: the specific production risk, business rule, or security boundary it would validate"),
+    repository: repositoryField,
     targetElements: z.array(targetElementSchema).min(1).nullable().optional().describe("UI tests only: structured grounding for one or more elements the test targets. Most tests target a single element (array length 1); render-state and multi-step UI tests target several (array length 2+). Each entry must be lifted verbatim from a captured blueprint element. Set to null when blueprint capture failed (also requires '[no-blueprint-data]' marker in BOTH description and reasoning). See Blueprint Citation Invariant in testbot prompt."),
     pageContext: pageContextSchema.optional().describe("UI tests only: page metadata for the test. Lifted from the BlueprintCapture used during grounding."),
 }).superRefine((rec, ctx) => {
@@ -226,13 +250,25 @@ const testMaintenanceSchema = z.object({
     fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
     action: z.nativeEnum(DriftAction).describe("The drift action assigned to this test during maintenance triage."),
     description: z.string().describe("What was changed and why"),
-    beforeStatus: z.enum(["Pass", "Fail", "Error"]).describe("Test result BEFORE modification"),
-    beforeDetails: z.string().describe("Execution output/timing before modification, or 'baseline from CI workflow <name>' if a parallel workflow provided the baseline"),
-    afterStatus: z.enum(["Pass", "Fail", "Error", "Skipped"]).describe("Test result AFTER modification"),
-    afterDetails: z.string().describe("Execution output/timing after modification"),
-})
-    .refine(m => ![DriftAction.Verify, DriftAction.Ignore].includes(m.action) || m.afterStatus === "Skipped", { message: "VERIFY and IGNORE entries must have afterStatus: 'Skipped' — these actions do not modify the test file" })
-    .refine(m => ![DriftAction.Update, DriftAction.Regenerate, DriftAction.Delete].includes(m.action) || m.afterStatus !== "Skipped", { message: "UPDATE, REGENERATE, and DELETE entries must have a real afterStatus (not Skipped) — these actions modify the test file and must be executed" });
+    beforeDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '4 passed in 15.09s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — assert 403 got 200, auth middleware not enforced'. " +
+        "Empty string for VERIFY/IGNORE entries where no before-execution was run."),
+    afterDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '5 passed in 10.96s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — check_schema fails, order_id=1 has discount from prior PATCH test'. " +
+        "Empty string for VERIFY/IGNORE/DELETE entries where no after-execution was run."),
+    // Server-populated from stateFile execution records — never supplied by the LLM.
+    beforeStatus: z.nativeEnum(TestExecutionStatus),
+    afterStatus: z.nativeEnum(TestExecutionStatus),
+});
+// LLM input contract: status fields are server-populated from stateFile.
+const testMaintenanceLLMSchema = testMaintenanceSchema.omit({
+    beforeStatus: true,
+    afterStatus: true,
+});
 /**
  * Derive per-run analytics counts from a submitted report. These power the
  * alpha-launch dashboards (tests generated/maintained, suite growth, bugs vs
@@ -256,7 +292,7 @@ function computeReportMetrics(params) {
     const recommendations = params.additionalRecommendations ?? [];
     const countBy = (items, pred) => items.filter(pred).length;
     const changedMaintenance = (params.testMaintenance ?? []).filter(isMaintenanceChange);
-    const maintenanceRecovered = countBy(changedMaintenance, (m) => m.beforeStatus !== "Pass" && m.afterStatus === "Pass");
+    const maintenanceRecovered = countBy(changedMaintenance, (m) => (m.beforeStatus === TestExecutionStatus.Fail || m.beforeStatus === TestExecutionStatus.Error) && m.afterStatus === TestExecutionStatus.Pass);
     return {
         testsGenerated: String(params.newTestsCreated.length),
         testsMaintained: String(changedMaintenance.length),
@@ -272,6 +308,23 @@ function computeReportMetrics(params) {
         issuesLow: String(countBy(params.issuesFound, (i) => i.severity === "low")),
     };
 }
+function deduplicateById(items) {
+    const seen = new Set();
+    const result = [];
+    for (const item of items) {
+        let id = item.testId;
+        if (seen.has(id)) {
+            let suffix = 2;
+            while (seen.has(`${id}-${suffix}`))
+                suffix++;
+            id = `${id}-${suffix}`;
+            item.testId = id;
+        }
+        seen.add(id);
+        result.push(item);
+    }
+    return result;
+}
 export function registerSubmitReportTool(server) {
     server.registerTool(TOOL_NAME, {
         description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
@@ -292,9 +345,9 @@ export function registerSubmitReportTool(server) {
                 .default([])
                 .describe("Recommended tests that were not generated (lower priority). Only include recommendations that add distinct coverage beyond generated tests — do not pad with variants testing the same endpoint and flow."),
             testMaintenance: z
-                .array(testMaintenanceSchema)
+                .array(testMaintenanceLLMSchema)
                 .optional()
-                .describe("One entry per test assessed in the drift analysis step. Omit this field when no existing tests were found."),
+                .describe("One entry per test assessed in the drift analysis step, including IGNORE and VERIFY entries — these provide triage transparency even when no edits were needed. Omit this field entirely when no existing tests were found."),
             testResults: z
                 .array(testResultSchema)
                 .describe("List of ALL test execution results. One entry per test executed."),
@@ -317,6 +370,12 @@ export function registerSubmitReportTool(server) {
                 .describe("Succinct commit message (if possible, under 72 chars) summarizing what Testbot did, " +
                 "e.g. 'add contract tests for /products endpoint' or 'update smoke tests for order API changes'. " +
                 "Used as both the git commit subject and the side PR title — the consumer applies truncation as needed."),
+            stateFile: z
+                .string()
+                .describe("Path to the state file returned by skyramp_analyze_changes. Required — " +
+                "contains execution results for maintenance scoring (existingTests), " +
+                "discovered endpoints for coverage validation (newEndpoints), and " +
+                "UI page context for frontend test grounding (uiContext)."),
         },
         _meta: {
             keywords: ["report", "summary", "testbot", "submit"],
@@ -324,40 +383,51 @@ export function registerSubmitReportTool(server) {
     }, async (params) => {
         const startTime = Date.now();
         let errorResult;
-        const deduplicateById = (items) => {
-            const seen = new Set();
-            const result = [];
-            for (const item of items) {
-                let id = item.testId;
-                if (seen.has(id)) {
-                    let suffix = 2;
-                    while (seen.has(`${id}-${suffix}`))
-                        suffix++;
-                    id = `${id}-${suffix}`;
-                    item.testId = id;
-                }
-                seen.add(id);
-                result.push(item);
-            }
-            return result;
-        };
         const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
         const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
+        let testMaintenance;
+        if (params.testMaintenance) {
+            try {
+                const stateData = await StateManager.fromStatePath(params.stateFile).readData();
+                if (!stateData) {
+                    errorResult = toolError(`stateFile not found or empty: ${params.stateFile}`);
+                    return errorResult;
+                }
+                // Status fields are stateFile-authoritative. Details are LLM-supplied when present
+                // (single-line, with root-cause context) and fall back to server-side extraction.
+                testMaintenance = params.testMaintenance.map((m) => {
+                    const recorded = stateData.existingTests?.find((t) => t.testFile.endsWith(m.fileName) || path.basename(t.testFile) === path.basename(m.fileName));
+                    const defaultBeforeStatus = MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const defaultAfterStatus = m.action === DriftAction.Delete ? TestExecutionStatus.Skipped
+                        : MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
+                    const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
+                    logger.info(`${m.fileName}: before=${beforeStatus} after=${afterStatus}`);
+                    return { ...m, beforeStatus, afterStatus };
+                });
+            }
+            catch (err) {
+                errorResult = toolError(`Failed to load stateFile for maintenance enrichment: ${err.message}`);
+                return errorResult;
+            }
+        }
         // Strip generation-artifact fields from newTestsCreated before writing.
         // scenarioFile, traceFile, frontendTrace are internal paths used during
         // generation — downstream scoring scripts don't expect them and fail if
         // they encounter these string fields while traversing the object.
-        const sanitizedNewTests = dedupedNewTests.map(({ scenarioFile: _sf, traceFile: _tf, frontendTrace: _ft, ...rest }) => rest);
-        const reportJson = JSON.stringify({
+        // Also normalize each item's `repository` (blank → undefined).
+        const sanitizedNewTests = dedupedNewTests.map(({ scenarioFile: _sf, traceFile: _tf, frontendTrace: _ft, ...rest }) => normalizeRepository(rest));
+        const report = {
             businessCaseAnalysis: params.businessCaseAnalysis,
             newTestsCreated: sanitizedNewTests,
-            additionalRecommendations: dedupedRecommendations,
-            testMaintenance: params.testMaintenance,
-            testResults: params.testResults,
-            issuesFound: params.issuesFound,
+            additionalRecommendations: dedupedRecommendations.map(normalizeRepository),
+            testMaintenance,
+            testResults: params.testResults.map(normalizeRepository),
+            issuesFound: params.issuesFound.map(normalizeRepository),
             nextSteps: params.nextSteps ?? [],
             commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim() || DEFAULT_COMMIT_MESSAGE,
-        }, null, 2);
+        };
+        const reportJson = JSON.stringify(report, null, 2);
         logger.info("Submitting testbot report", {
             outputFile: params.summaryOutputFile,
             payloadBytes: reportJson.length,
@@ -384,10 +454,7 @@ export function registerSubmitReportTool(server) {
             const elapsed = Date.now() - startTime;
             const errorMessage = `Failed to write report: ${error.message}`;
             logger.error(errorMessage, { error, elapsedMs: elapsed });
-            errorResult = {
-                content: [{ type: "text", text: errorMessage }],
-                isError: true,
-            };
+            errorResult = toolError(errorMessage);
             return errorResult;
         }
         finally {
@@ -395,7 +462,7 @@ export function registerSubmitReportTool(server) {
                 summary_output_file: params.summaryOutputFile,
                 testResultCount: String(params.testResults.length),
                 payloadBytes: String(reportJson.length),
-                ...computeReportMetrics(params),
+                ...computeReportMetrics({ ...params, testMaintenance }),
             }).catch(() => { });
         }
     });