@skyramp/mcp 0.2.7 → 0.2.150-rc.mntnc

package/build/tools/generate-tests/generateBatchScenarioRestTool.js

@@ -67,7 +67,7 @@ CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&li
     responseBody: z
         .string()
         .optional()
-        .describe("JSON string of the expected response body"),
+        .describe("JSON string of the expected response body. For steps that produce an id used by later steps, include that id field with a realistic UUID generated fresh for this scenario — the backend chains by value-matching any subsequent step path or requestBody that contains the same UUID."),
     statusCode: z
         .number()
         .optional()

package/build/tools/submitReportTool.js

@@ -6,6 +6,9 @@ import { AnalyticsService } from "../services/AnalyticsService.js";
 import { TEST_CATEGORIES, externalCategory } from "../types/TestRecommendation.js";
 import { TestType, HttpMethod } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
+import { StateManager } from "../utils/AnalysisStateManager.js";
+import { toolError } from "../utils/utils.js";
 // Drift actions that actually modify a test file. VERIFY and IGNORE are
 // no-ops (the test was assessed but left unchanged), so they must not count
 // toward "tests maintained" telemetry.
@@ -21,6 +24,7 @@ const testResultSchema = z.object({
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
     details: z.string().describe("One sentence — no embedded newlines, no markdown. e.g. '10.8s, products_contract_test.py' or 'failed: <one-line error summary>, products_contract_test.py'"),
+    videoPath: z.string().optional().describe("Path to video recording (UI/E2E tests only)"),
 });
 // Structured grounding for UI recommendations. Lifted verbatim from a
 // `browser_blueprint` element captured during this run. Source of truth for
@@ -226,13 +230,25 @@ const testMaintenanceSchema = z.object({
     fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
     action: z.nativeEnum(DriftAction).describe("The drift action assigned to this test during maintenance triage."),
     description: z.string().describe("What was changed and why"),
-    beforeStatus: z.enum(["Pass", "Fail", "Error"]).describe("Test result BEFORE modification"),
-    beforeDetails: z.string().describe("Execution output/timing before modification, or 'baseline from CI workflow <name>' if a parallel workflow provided the baseline"),
-    afterStatus: z.enum(["Pass", "Fail", "Error", "Skipped"]).describe("Test result AFTER modification"),
-    afterDetails: z.string().describe("Execution output/timing after modification"),
-})
-    .refine(m => ![DriftAction.Verify, DriftAction.Ignore].includes(m.action) || m.afterStatus === "Skipped", { message: "VERIFY and IGNORE entries must have afterStatus: 'Skipped' — these actions do not modify the test file" })
-    .refine(m => ![DriftAction.Update, DriftAction.Regenerate, DriftAction.Delete].includes(m.action) || m.afterStatus !== "Skipped", { message: "UPDATE, REGENERATE, and DELETE entries must have a real afterStatus (not Skipped) — these actions modify the test file and must be executed" });
+    beforeDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '4 passed in 15.09s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — assert 403 got 200, auth middleware not enforced'. " +
+        "Empty string for VERIFY/IGNORE entries where no before-execution was run."),
+    afterDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '5 passed in 10.96s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — check_schema fails, order_id=1 has discount from prior PATCH test'. " +
+        "Empty string for VERIFY/IGNORE/DELETE entries where no after-execution was run."),
+    // Server-populated from stateFile execution records — never supplied by the LLM.
+    beforeStatus: z.nativeEnum(TestExecutionStatus),
+    afterStatus: z.nativeEnum(TestExecutionStatus),
+});
+// LLM input contract: status fields are server-populated from stateFile.
+const testMaintenanceLLMSchema = testMaintenanceSchema.omit({
+    beforeStatus: true,
+    afterStatus: true,
+});
 /**
  * Derive per-run analytics counts from a submitted report. These power the
  * alpha-launch dashboards (tests generated/maintained, suite growth, bugs vs
@@ -256,7 +272,7 @@ function computeReportMetrics(params) {
     const recommendations = params.additionalRecommendations ?? [];
     const countBy = (items, pred) => items.filter(pred).length;
     const changedMaintenance = (params.testMaintenance ?? []).filter(isMaintenanceChange);
-    const maintenanceRecovered = countBy(changedMaintenance, (m) => m.beforeStatus !== "Pass" && m.afterStatus === "Pass");
+    const maintenanceRecovered = countBy(changedMaintenance, (m) => (m.beforeStatus === TestExecutionStatus.Fail || m.beforeStatus === TestExecutionStatus.Error) && m.afterStatus === TestExecutionStatus.Pass);
     return {
         testsGenerated: String(params.newTestsCreated.length),
         testsMaintained: String(changedMaintenance.length),
@@ -272,6 +288,23 @@ function computeReportMetrics(params) {
         issuesLow: String(countBy(params.issuesFound, (i) => i.severity === "low")),
     };
 }
+function deduplicateById(items) {
+    const seen = new Set();
+    const result = [];
+    for (const item of items) {
+        let id = item.testId;
+        if (seen.has(id)) {
+            let suffix = 2;
+            while (seen.has(`${id}-${suffix}`))
+                suffix++;
+            id = `${id}-${suffix}`;
+            item.testId = id;
+        }
+        seen.add(id);
+        result.push(item);
+    }
+    return result;
+}
 export function registerSubmitReportTool(server) {
     server.registerTool(TOOL_NAME, {
         description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
@@ -292,9 +325,9 @@ export function registerSubmitReportTool(server) {
                 .default([])
                 .describe("Recommended tests that were not generated (lower priority). Only include recommendations that add distinct coverage beyond generated tests — do not pad with variants testing the same endpoint and flow."),
             testMaintenance: z
-                .array(testMaintenanceSchema)
+                .array(testMaintenanceLLMSchema)
                 .optional()
-                .describe("One entry per test assessed in the drift analysis step. Omit this field when no existing tests were found."),
+                .describe("One entry per test assessed in the drift analysis step, including IGNORE and VERIFY entries — these provide triage transparency even when no edits were needed. Omit this field entirely when no existing tests were found."),
             testResults: z
                 .array(testResultSchema)
                 .describe("List of ALL test execution results. One entry per test executed."),
@@ -317,6 +350,12 @@ export function registerSubmitReportTool(server) {
                 .describe("Succinct commit message (if possible, under 72 chars) summarizing what Testbot did, " +
                 "e.g. 'add contract tests for /products endpoint' or 'update smoke tests for order API changes'. " +
                 "Used as both the git commit subject and the side PR title — the consumer applies truncation as needed."),
+            stateFile: z
+                .string()
+                .describe("Path to the state file returned by skyramp_analyze_changes. Required — " +
+                "contains execution results for maintenance scoring (existingTests), " +
+                "discovered endpoints for coverage validation (newEndpoints), and " +
+                "UI page context for frontend test grounding (uiContext)."),
         },
         _meta: {
             keywords: ["report", "summary", "testbot", "submit"],
@@ -324,40 +363,50 @@ export function registerSubmitReportTool(server) {
     }, async (params) => {
         const startTime = Date.now();
         let errorResult;
-        const deduplicateById = (items) => {
-            const seen = new Set();
-            const result = [];
-            for (const item of items) {
-                let id = item.testId;
-                if (seen.has(id)) {
-                    let suffix = 2;
-                    while (seen.has(`${id}-${suffix}`))
-                        suffix++;
-                    id = `${id}-${suffix}`;
-                    item.testId = id;
-                }
-                seen.add(id);
-                result.push(item);
-            }
-            return result;
-        };
         const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
         const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
+        let testMaintenance;
+        if (params.testMaintenance) {
+            try {
+                const stateData = await StateManager.fromStatePath(params.stateFile).readData();
+                if (!stateData) {
+                    errorResult = toolError(`stateFile not found or empty: ${params.stateFile}`);
+                    return errorResult;
+                }
+                // Status fields are stateFile-authoritative. Details are LLM-supplied when present
+                // (single-line, with root-cause context) and fall back to server-side extraction.
+                testMaintenance = params.testMaintenance.map((m) => {
+                    const recorded = stateData.existingTests?.find((t) => t.testFile.endsWith(m.fileName) || path.basename(t.testFile) === path.basename(m.fileName));
+                    const defaultBeforeStatus = MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const defaultAfterStatus = m.action === DriftAction.Delete ? TestExecutionStatus.Skipped
+                        : MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
+                    const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
+                    logger.info(`${m.fileName}: before=${beforeStatus} after=${afterStatus}`);
+                    return { ...m, beforeStatus, afterStatus };
+                });
+            }
+            catch (err) {
+                errorResult = toolError(`Failed to load stateFile for maintenance enrichment: ${err.message}`);
+                return errorResult;
+            }
+        }
         // Strip generation-artifact fields from newTestsCreated before writing.
         // scenarioFile, traceFile, frontendTrace are internal paths used during
         // generation — downstream scoring scripts don't expect them and fail if
         // they encounter these string fields while traversing the object.
         const sanitizedNewTests = dedupedNewTests.map(({ scenarioFile: _sf, traceFile: _tf, frontendTrace: _ft, ...rest }) => rest);
-        const reportJson = JSON.stringify({
+        const report = {
             businessCaseAnalysis: params.businessCaseAnalysis,
             newTestsCreated: sanitizedNewTests,
             additionalRecommendations: dedupedRecommendations,
-            testMaintenance: params.testMaintenance,
+            testMaintenance,
             testResults: params.testResults,
             issuesFound: params.issuesFound,
             nextSteps: params.nextSteps ?? [],
             commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim() || DEFAULT_COMMIT_MESSAGE,
-        }, null, 2);
+        };
+        const reportJson = JSON.stringify(report, null, 2);
         logger.info("Submitting testbot report", {
             outputFile: params.summaryOutputFile,
             payloadBytes: reportJson.length,
@@ -384,10 +433,7 @@ export function registerSubmitReportTool(server) {
             const elapsed = Date.now() - startTime;
             const errorMessage = `Failed to write report: ${error.message}`;
             logger.error(errorMessage, { error, elapsedMs: elapsed });
-            errorResult = {
-                content: [{ type: "text", text: errorMessage }],
-                isError: true,
-            };
+            errorResult = toolError(errorMessage);
             return errorResult;
         }
         finally {
@@ -395,7 +441,7 @@ export function registerSubmitReportTool(server) {
                 summary_output_file: params.summaryOutputFile,
                 testResultCount: String(params.testResults.length),
                 payloadBytes: String(reportJson.length),
-                ...computeReportMetrics(params),
+                ...computeReportMetrics({ ...params, testMaintenance }),
             }).catch(() => { });
         }
     });