@skyramp/mcp 0.2.6 → 0.2.8-rc.1

package/build/tools/submitReportTool.test.js

@@ -1,17 +1,20 @@
-// @ts-ignore
-import { registerSubmitReportTool, additionalRecommendationSchema } from "./submitReportTool.js";
+// @ts-nocheck - Jest ESM type inference issues
+import { jest } from "@jest/globals";
 import { TestType } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
-import { AnalyticsService } from "../services/AnalyticsService.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
 import * as fs from "fs/promises";
 import * as path from "path";
 import * as os from "os";
-jest.mock("../utils/logger.js", () => ({
-    logger: { info: jest.fn(), error: jest.fn() },
+jest.unstable_mockModule("../utils/logger.js", () => ({
+    logger: { info: jest.fn(), error: jest.fn(), warning: jest.fn(), debug: jest.fn() },
 }));
-jest.mock("../services/AnalyticsService.js", () => ({
+jest.unstable_mockModule("../services/AnalyticsService.js", () => ({
     AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) },
 }));
+// @ts-ignore
+const { registerSubmitReportTool, additionalRecommendationSchema, newTestSchema, targetElementSchema, pageContextSchema } = await import("./submitReportTool.js");
+const { AnalyticsService } = await import("../services/AnalyticsService.js");
 const pushMCPToolEventMock = AnalyticsService.pushMCPToolEvent;
 function captureToolHandler() {
     let handler;
@@ -23,14 +26,20 @@ function captureToolHandler() {
     registerSubmitReportTool(fakeServer);
     return handler;
 }
-function sampleReportParams(outputFile) {
+async function writeStateFile(dir, existingTests) {
+    const stateFile = path.join(dir, "analyze-changes-state.json");
+    await fs.writeFile(stateFile, JSON.stringify({ existingTests }), "utf-8");
+    return stateFile;
+}
+function sampleReportParams(outputFile, stateFile) {
     return {
         summaryOutputFile: outputFile,
+        stateFile,
         businessCaseAnalysis: "This PR adds product search. Tests needed for filtering.",
         newTestsCreated: [
             { testId: "smoke-get-products-search", testType: TestType.SMOKE, category: "workflow", endpoint: "GET /api/v1/products/search", fileName: "search_smoke_test.py", reasoning: "Validates product search returns correct results for category filter" },
         ],
-        testMaintenance: [{ action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Updated auth flow in existing tests", beforeStatus: "Fail", beforeDetails: "401 Unauthorized (1.5s)", afterStatus: "Pass", afterDetails: "All assertions passed (2.3s)" }],
+        testMaintenance: [{ action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Updated auth flow in existing tests", beforeDetails: "", afterDetails: "" }],
         testResults: [
             { testType: TestType.SMOKE, endpoint: "GET /api/v1/products", status: "Pass", details: "2.1s, products_smoke_test.py" },
             { testType: TestType.SMOKE, endpoint: "GET /api/v1/products/search", status: "Fail", details: "3.4s, search_smoke_test.py" },
@@ -65,7 +74,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         expect(result.content[0].text).toContain("Report submitted successfully");
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
@@ -75,11 +85,44 @@ describe("registerSubmitReportTool", () => {
         expect(written.issuesFound).toHaveLength(1);
         expect(written.testMaintenance).toHaveLength(1);
     });
+    it("accepts and persists the per-item `repository` field", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const params = sampleReportParams(outputFile, stateFile);
+        // Attribute one item to a related repo, leave others on the primary.
+        params.newTestsCreated[0] = { ...params.newTestsCreated[0], repository: "acme/backend" };
+        params.testResults[0] = { ...params.testResults[0], repository: "acme/backend" };
+        params.issuesFound[0] = { ...params.issuesFound[0], repository: "acme/backend" };
+        const result = await handler(params);
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.newTestsCreated[0].repository).toBe("acme/backend");
+        expect(written.testResults[0].repository).toBe("acme/backend");
+        expect(written.issuesFound[0].repository).toBe("acme/backend");
+        // Items without a repository stay primary (undefined → omitted from JSON).
+        expect(written.testResults[1].repository).toBeUndefined();
+    });
+    it("normalizes a blank/whitespace-only `repository` to undefined", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const params = sampleReportParams(outputFile, stateFile);
+        params.testResults[0] = { ...params.testResults[0], repository: "   " };
+        const result = await handler(params);
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        // Blank string must not serialize as a distinct empty repo.
+        expect(written.testResults[0].repository).toBeUndefined();
+    });
     it("should create parent directories if they don't exist", async () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "nested", "dirs", "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.businessCaseAnalysis).toBeDefined();
@@ -88,11 +131,12 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
             summaryOutputFile: outputFile,
+            stateFile,
             businessCaseAnalysis: "Simple config change.",
             newTestsCreated: [],
-            testMaintenance: [],
             testResults: [
                 { testType: TestType.SMOKE, endpoint: "GET /api/v1/products", status: "Pass", details: "1.0s, test.py" },
             ],
@@ -101,12 +145,15 @@ describe("registerSubmitReportTool", () => {
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.newTestsCreated).toEqual([]);
-        expect(written.testMaintenance).toEqual([]);
+        expect(written.testMaintenance).toBeUndefined();
         expect(written.issuesFound).toEqual([]);
         expect(written.testResults).toHaveLength(1);
     });
     it("should return error for invalid file path", async () => {
-        const result = await handler(sampleReportParams("/nonexistent/readonly/path/report.json"));
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams("/nonexistent/readonly/path/report.json", stateFile));
         expect(result.isError).toBe(true);
         expect(result.content[0].text).toContain("Failed to write report");
     });
@@ -114,8 +161,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "add contract tests for /products endpoint",
         });
         expect(result.isError).toBeUndefined();
@@ -126,7 +174,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
@@ -135,8 +184,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "  line one\nline two\r\nline three  ",
         });
         expect(result.isError).toBeUndefined();
@@ -148,9 +198,10 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const longMessage = "add contract and integration tests for the new PATCH /orders/{order_id} endpoint including discount recalculation and line-item edits";
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: longMessage,
         });
         expect(result.isError).toBeUndefined();
@@ -158,24 +209,13 @@ describe("registerSubmitReportTool", () => {
         expect(written.commitMessage).toBe(longMessage);
         expect(written.commitMessage.length).toBeGreaterThan(72);
     });
-    it("should use default commitMessage when provided as empty string", async () => {
-        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
-        tmpDirs.push(tmpDir);
-        const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler({
-            ...sampleReportParams(outputFile),
-            commitMessage: "",
-        });
-        expect(result.isError).toBeUndefined();
-        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
-        expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
-    });
     it("should use default commitMessage when whitespace-only", async () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "   \n\r\n   ",
         });
         expect(result.isError).toBeUndefined();
@@ -186,8 +226,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             nextSteps: ["Check your targetSetupCommand — endpoints returned 404"],
         });
         expect(result.isError).toBeUndefined();
@@ -200,7 +241,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.nextSteps).toEqual([]);
@@ -209,7 +251,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        await handler(sampleReportParams(outputFile, stateFile));
         const raw = await fs.readFile(outputFile, "utf-8");
         // Pretty-printed JSON should have indentation
         expect(raw).toContain("  ");
@@ -220,8 +263,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const params = {
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             additionalRecommendations: [
                 {
                     testId: "integration-orders-patch-status",
@@ -293,6 +337,214 @@ describe("registerSubmitReportTool", () => {
             expect(result.data.primaryEndpoint).toBeUndefined();
         }
     });
+    // Structured grounding fields for UI recommendations.
+    // ----------------------------------------------------------------------
+    // Sub-schemas — targetElement and pageContext — populated by the agent
+    // from a captured browser_blueprint element. UI-only via superRefine on
+    // the parent newTestSchema / additionalRecommendationSchema.
+    describe("targetElementSchema", () => {
+        const validTarget = {
+            role: "button",
+            accessibleName: "Save changes",
+            testId: "save-changes-btn",
+            stableId: null,
+            contextText: null,
+            mutability: "mutable",
+            widgetType: "native",
+        };
+        it("accepts a fully-populated targetElement", () => {
+            expect(targetElementSchema.safeParse(validTarget).success).toBe(true);
+        });
+        it("accepts a minimal targetElement (only required fields)", () => {
+            const result = targetElementSchema.safeParse({
+                role: "heading",
+                accessibleName: "Notifications",
+                testId: null,
+                stableId: null,
+                contextText: null,
+            });
+            expect(result.success).toBe(true);
+        });
+        it("accepts contextText as a string array (repeating-row case)", () => {
+            const result = targetElementSchema.safeParse({
+                ...validTarget,
+                contextText: ["customer@example.com", "$129.99", "Pending"],
+            });
+            expect(result.success).toBe(true);
+        });
+        it("rejects targetElement missing accessibleName", () => {
+            const { accessibleName, ...withoutName } = validTarget;
+            const result = targetElementSchema.safeParse(withoutName);
+            expect(result.success).toBe(false);
+        });
+        it("rejects targetElement missing role", () => {
+            const { role, ...withoutRole } = validTarget;
+            const result = targetElementSchema.safeParse(withoutRole);
+            expect(result.success).toBe(false);
+        });
+        it("rejects an invalid mutability enum value", () => {
+            const result = targetElementSchema.safeParse({ ...validTarget, mutability: "sometimes" });
+            expect(result.success).toBe(false);
+        });
+        it("rejects an invalid widgetType enum value", () => {
+            const result = targetElementSchema.safeParse({ ...validTarget, widgetType: "fancy" });
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("pageContextSchema (UI recommendation grounding)", () => {
+        it("accepts pageContext with both url and pageHash", () => {
+            const result = pageContextSchema.safeParse({
+                url: "http://localhost:5173/orders/1",
+                pageHash: "10:434266447",
+            });
+            expect(result.success).toBe(true);
+        });
+        it("accepts pageContext without pageHash (optional)", () => {
+            const result = pageContextSchema.safeParse({ url: "http://localhost:5173/orders/1" });
+            expect(result.success).toBe(true);
+        });
+        it("rejects pageContext missing url", () => {
+            const result = pageContextSchema.safeParse({ pageHash: "10:abc" });
+            expect(result.success).toBe(false);
+        });
+    });
+    describe("newTestSchema with grounding fields (UI recommendation grounding)", () => {
+        const validTarget = {
+            role: "button",
+            accessibleName: "Save changes",
+            testId: "save-changes-btn",
+            stableId: null,
+            contextText: null,
+        };
+        const validPage = {
+            url: "http://localhost:5173/orders/1",
+            pageHash: "10:434266447",
+        };
+        const baseUITest = {
+            testId: "ui-edit-order-discount-validation",
+            testType: TestType.UI,
+            category: "business_rule",
+            endpoint: "PATCH /api/v1/orders/{id}",
+            fileName: "edit_order_discount.spec.ts",
+            description: "Navigates to the Edit Order form, enters a discount > 100, and asserts that the validation error appears and the Save button is disabled.",
+            reasoning: "Verifies the **Save changes** button (testid `save-changes-btn`) on the Edit Order form shows the validation error when discount > 100.",
+        };
+        it("accepts a UI test with single-element targetElements + pageContext", () => {
+            const result = newTestSchema.safeParse({
+                ...baseUITest,
+                targetElements: [validTarget],
+                pageContext: validPage,
+            });
+            expect(result.success).toBe(true);
+        });
+        it("accepts a UI test with multi-element targetElements (render-state test)", () => {
+            const result = newTestSchema.safeParse({
+                ...baseUITest,
+                targetElements: [
+                    {
+                        role: "heading",
+                        accessibleName: "Edit Order",
+                        testId: "edit-order-heading",
+                        stableId: null,
+                        contextText: null,
+                    },
+                    {
+                        role: "spinbutton",
+                        accessibleName: "Discount (%)",
+                        testId: "edit-order-input-discount",
+                        stableId: null,
+                        contextText: null,
+                    },
+                ],
+                pageContext: validPage,
+            });
+            expect(result.success).toBe(true);
+        });
+        it("rejects a UI test without targetElements (required for UI grounding)", () => {
+            const result = newTestSchema.safeParse(baseUITest);
+            expect(result.success).toBe(false);
+            expect(JSON.stringify(result.error?.issues)).toContain("targetElements is required for testType: 'ui'");
+        });
+        it("accepts a UI test with targetElements: null (fallback case)", () => {
+            const result = newTestSchema.safeParse({
+                ...baseUITest,
+                targetElements: null,
+                description: "[no-blueprint-data] Navigates to Edit Order form and checks discount validation error.",
+                reasoning: "[no-blueprint-data] Verifies the discount validation error appears on the Edit Order form when discount > 100.",
+            });
+            expect(result.success).toBe(true);
+        });
+        it("rejects a contract test that includes targetElements (UI-only)", () => {
+            const result = newTestSchema.safeParse({
+                testId: "contract-orders-patch",
+                testType: TestType.CONTRACT,
+                category: "business_rule",
+                endpoint: "PATCH /api/v1/orders/{id}",
+                fileName: "orders_patch_contract.py",
+                reasoning: "Validates request shape and 200/422 status codes",
+                targetElements: [validTarget],
+            });
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.error.issues.some((i) => i.path.includes("targetElements"))).toBe(true);
+            }
+        });
+        it("rejects an integration test that includes pageContext (UI-only)", () => {
+            const result = newTestSchema.safeParse({
+                testId: "integration-orders",
+                testType: TestType.INTEGRATION,
+                category: "business_rule",
+                endpoint: "POST /api/v1/orders",
+                fileName: "orders_integration.py",
+                reasoning: "Multi-step order lifecycle",
+                pageContext: validPage,
+            });
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.error.issues.some((i) => i.path.includes("pageContext"))).toBe(true);
+            }
+        });
+    });
+    describe("additionalRecommendationSchema with grounding fields (UI recommendation grounding)", () => {
+        const validTarget = {
+            role: "button",
+            accessibleName: "Save changes",
+            testId: "save-changes-btn",
+            stableId: null,
+            contextText: null,
+        };
+        const validPage = { url: "http://localhost:5173/orders/1" };
+        it("accepts a UI rec with targetElements + pageContext", () => {
+            const result = additionalRecommendationSchema.safeParse({
+                testId: "ui-edit-order-discount-validation",
+                testType: TestType.UI,
+                category: "business_rule",
+                steps: [{ description: "Open Edit Order, type 150 in discount, click Save" }],
+                description: "Verify discount > 100 shows validation error",
+                priority: "medium",
+                reasoning: "Verify that the **Save changes** button (testid `save-changes-btn`) on the Edit Order form shows the validation error when discount > 100.",
+                targetElements: [validTarget],
+                pageContext: validPage,
+            });
+            expect(result.success).toBe(true);
+        });
+        it("rejects a non-UI rec that includes targetElements", () => {
+            const result = additionalRecommendationSchema.safeParse({
+                testId: "integration-orders-patch",
+                testType: TestType.INTEGRATION,
+                category: "business_rule",
+                steps: [{ method: "PATCH", path: "/api/v1/orders/1", description: "Patch order" }],
+                description: "Verifies PATCH workflow",
+                priority: "high",
+                reasoning: "Catches mutation bugs",
+                targetElements: [validTarget],
+            });
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.error.issues.some((i) => i.path.includes("targetElements"))).toBe(true);
+            }
+        });
+    });
     describe("analytics metrics", () => {
         beforeEach(() => {
             pushMCPToolEventMock.mockClear();
@@ -306,14 +558,15 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
-            // sampleReportParams: 1 new test, 1 maintenance (Fail→Pass), 2 results
+            // sampleReportParams: 1 new test, 1 maintenance (no matching stateFile entry), 2 results
             // (1 Pass + 1 Fail), 1 issue (no severity).
-            await handler(sampleReportParams(outputFile));
+            const stateFile = await writeStateFile(tmpDir, []);
+            await handler(sampleReportParams(outputFile, stateFile));
             const params = lastAnalyticsParams();
             expect(params.testsGenerated).toBe("1");
             expect(params.testsMaintained).toBe("1");
             expect(params.recommendationsCount).toBe("0");
-            expect(params.maintenanceRecovered).toBe("1");
+            expect(params.maintenanceRecovered).toBe("0"); // no matching stateFile entry → no execution data
             expect(params.testsPassed).toBe("1");
             expect(params.testsFailed).toBe("1");
             expect(params.testsSkipped).toBe("0");
@@ -323,8 +576,9 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
-                ...sampleReportParams(outputFile),
+                ...sampleReportParams(outputFile, stateFile),
                 issuesFound: [
                     { description: "Data corruption on checkout", severity: "critical" },
                     { description: "Wrong total returned", severity: "high" },
@@ -344,8 +598,10 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
                 summaryOutputFile: outputFile,
+                stateFile,
                 businessCaseAnalysis: "Config-only change.",
                 newTestsCreated: [],
                 testMaintenance: [],
@@ -363,36 +619,236 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
-                ...sampleReportParams(outputFile),
+                ...sampleReportParams(outputFile, stateFile),
                 testMaintenance: [
                     // Real change
-                    { action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched auth", beforeStatus: "Fail", beforeDetails: "401 (1.5s)", afterStatus: "Pass", afterDetails: "passed (2.3s)" },
-                    { action: "REGENERATE", testType: TestType.SMOKE, endpoint: "GET /api/v1/orders", fileName: "orders_smoke_test.py", description: "Rewrote for new shape", beforeStatus: "Fail", beforeDetails: "shape mismatch", afterStatus: "Pass", afterDetails: "passed (1.0s)" },
+                    { action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched auth", beforeDetails: "", afterDetails: "" },
+                    { action: "REGENERATE", testType: TestType.SMOKE, endpoint: "GET /api/v1/orders", fileName: "orders_smoke_test.py", description: "Rewrote for new shape", beforeDetails: "", afterDetails: "" },
                     // No-ops — assessed but not modified
-                    { action: "IGNORE", testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeStatus: "Pass", beforeDetails: "drift score 0", afterStatus: "Skipped", afterDetails: "IGNORE: endpoint not in PR" },
-                    { action: "VERIFY", testType: TestType.SMOKE, endpoint: "GET /api/v1/coupons", fileName: "coupons_smoke_test.py", description: "Verified still valid", beforeStatus: "Pass", beforeDetails: "minor drift", afterStatus: "Skipped", afterDetails: "VERIFY: assertions still hold" },
+                    { action: "IGNORE", testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeDetails: "", afterDetails: "" },
+                    { action: "VERIFY", testType: TestType.SMOKE, endpoint: "GET /api/v1/coupons", fileName: "coupons_smoke_test.py", description: "Verified still valid", beforeDetails: "", afterDetails: "" },
                 ],
             });
             const params = lastAnalyticsParams();
             // 2 change actions (UPDATE + REGENERATE), not 4
             expect(params.testsMaintained).toBe("2");
-            // Both changes went Fail→Pass
-            expect(params.maintenanceRecovered).toBe("2");
+            // No matching stateFile entries — execution data unavailable
+            expect(params.maintenanceRecovered).toBe("0");
         });
-        it("counts UPDATE/REGENERATE/DELETE as maintenance changes, not VERIFY/IGNORE", async () => {
-            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
-            tmpDirs.push(tmpDir);
+    });
+    // ─────────────────────────────────────────────────────────────────────────────
+    // testMaintenance output shapes
+    // Each case documents exactly what the MCP writes to JSON, which is what the
+    // testbot renderer reads. Keeping these in sync prevents silent regressions
+    // when toOutput or the stateFile enrichment path changes.
+    // ─────────────────────────────────────────────────────────────────────────────
+    describe("testMaintenance output shapes", () => {
+        const baseMaintenance = {
+            testType: TestType.CONTRACT,
+            endpoint: "GET /api/v1/orders",
+            fileName: "orders_contract_test.py",
+            description: "Updated discount fields in assertions",
+            beforeDetails: "",
+            afterDetails: "",
+        };
+        async function writeAndRead(tmpDir, maintenance, extra) {
             const outputFile = path.join(tmpDir, "report.json");
             await handler({
-                ...sampleReportParams(outputFile),
-                testMaintenance: [
-                    { testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", action: DriftAction.Update, description: "Patched", beforeStatus: "Fail", beforeDetails: "401", afterStatus: "Pass", afterDetails: "passed" },
-                    { testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", action: DriftAction.Ignore, description: "No action required", beforeStatus: "Pass", beforeDetails: "drift 0", afterStatus: "Skipped", afterDetails: "not in PR" },
-                ],
+                summaryOutputFile: outputFile,
+                businessCaseAnalysis: "Test",
+                newTestsCreated: [],
+                testMaintenance: maintenance,
+                testResults: [],
+                issuesFound: [],
+                ...extra,
             });
-            const params = lastAnalyticsParams();
-            expect(params.testsMaintained).toBe("1");
+            const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+            return written.testMaintenance;
+        }
+        // ── No matching stateFile entry ───────────────────────────────────────────
+        it("UPDATE with no matching stateFile entry: beforeStatus=Unknown, afterStatus=Unknown", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Unknown");
+            expect(result[0].beforeDetails).toBe("");
+            expect(result[0].afterDetails).toBe("");
+        });
+        it("IGNORE with no matching stateFile entry: both Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Ignore }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Skipped");
+            expect(result[0].afterStatus).toBe("Skipped");
+            expect(result[0].beforeDetails).toBe("");
+            expect(result[0].afterDetails).toBe("");
         });
+        it("DELETE with no matching stateFile entry: beforeStatus=Unknown, afterStatus=Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Delete }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Skipped");
+        });
+        // ── With matching stateFile entry ─────────────────────────────────────────
+        it("UPDATE with before=Pass after=Pass: status from stateFile, details from LLM", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15090 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 12880 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance, action: DriftAction.Update,
+                    beforeDetails: "4 passed in 15.09s",
+                    afterDetails: "5 passed in 12.88s",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].beforeDetails).toBe("4 passed in 15.09s");
+            expect(result[0].afterStatus).toBe("Pass");
+            expect(result[0].afterDetails).toBe("5 passed in 12.88s");
+        });
+        it("UPDATE with before=Pass after=Fail: status from stateFile, details from LLM", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15090 },
+                    executionAfter: { status: TestExecutionStatus.Fail, errors: [], duration: 10940 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance, action: DriftAction.Update,
+                    beforeDetails: "4 passed in 15.09s",
+                    afterDetails: "FAILED test_orders_get — check_schema fails, test isolation bug",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Fail");
+            expect(result[0].afterDetails).toBe("FAILED test_orders_get — check_schema fails, test isolation bug");
+        });
+        it("UPDATE with crashed execution: status is Error", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Error, errors: ["Connection refused"], duration: 500 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Error");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+        it("UPDATE Fail→Pass: counted as maintenanceRecovered in analytics", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Fail, errors: ["FAILED test_foo - assert False"], duration: 5000 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 5000 },
+                }]);
+            await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(pushMCPToolEventMock.mock.calls.at(-1)?.[2].maintenanceRecovered).toBe("1");
+        });
+        it("UPDATE with no before-execution recorded: beforeStatus=Unknown, afterStatus=Pass", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    // executionBefore absent — agent skipped before-phase
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+        it("DELETE with before-execution, no after (file deleted): beforeStatus=Pass, afterStatus=Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                    // executionAfter absent — file was deleted, nothing to run
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Delete }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Skipped");
+        });
+        it("fileName matched by suffix: absolute stateFile path vs relative fileName", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/python/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15000 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 12000 },
+                }]);
+            // fileName is relative — should still match via endsWith
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance,
+                    action: DriftAction.Update,
+                    fileName: "orders_contract_test.py",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+    });
+    it("testResult accepts videoPath field (optional)", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler({
+            summaryOutputFile: outputFile,
+            stateFile,
+            businessCaseAnalysis: "UI changes for login flow",
+            newTestsCreated: [],
+            testMaintenance: [],
+            testResults: [
+                {
+                    testType: TestType.UI,
+                    endpoint: "GET /login",
+                    status: "Pass",
+                    details: "5.2s, login_ui_test.spec.ts",
+                    videoPath: "/workspace/.skyramp/videos/login_ui_test-abc12345/video.webm",
+                },
+            ],
+            issuesFound: [],
+        });
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.testResults).toHaveLength(1);
+        expect(written.testResults[0].videoPath).toBe("/workspace/.skyramp/videos/login_ui_test-abc12345/video.webm");
+    });
+    it("testResult accepts entry without videoPath (optional field)", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler({
+            summaryOutputFile: outputFile,
+            stateFile,
+            businessCaseAnalysis: "API endpoint changes",
+            newTestsCreated: [],
+            testMaintenance: [],
+            testResults: [
+                {
+                    testType: TestType.CONTRACT,
+                    endpoint: "POST /api/v1/products",
+                    status: "Pass",
+                    details: "2.1s, products_contract_test.py",
+                    // videoPath intentionally omitted
+                },
+            ],
+            issuesFound: [],
+        });
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.testResults).toHaveLength(1);
+        expect(written.testResults[0].videoPath).toBeUndefined();
     });
 });