@skyramp/mcp 0.2.7 → 0.2.150-rc.mntnc

package/build/tools/submitReportTool.test.js

@@ -2,11 +2,12 @@
 import { jest } from "@jest/globals";
 import { TestType } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
 import * as fs from "fs/promises";
 import * as path from "path";
 import * as os from "os";
 jest.unstable_mockModule("../utils/logger.js", () => ({
-    logger: { info: jest.fn(), error: jest.fn() },
+    logger: { info: jest.fn(), error: jest.fn(), warning: jest.fn(), debug: jest.fn() },
 }));
 jest.unstable_mockModule("../services/AnalyticsService.js", () => ({
     AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) },
@@ -25,14 +26,20 @@ function captureToolHandler() {
     registerSubmitReportTool(fakeServer);
     return handler;
 }
-function sampleReportParams(outputFile) {
+async function writeStateFile(dir, existingTests) {
+    const stateFile = path.join(dir, "analyze-changes-state.json");
+    await fs.writeFile(stateFile, JSON.stringify({ existingTests }), "utf-8");
+    return stateFile;
+}
+function sampleReportParams(outputFile, stateFile) {
     return {
         summaryOutputFile: outputFile,
+        stateFile,
         businessCaseAnalysis: "This PR adds product search. Tests needed for filtering.",
         newTestsCreated: [
             { testId: "smoke-get-products-search", testType: TestType.SMOKE, category: "workflow", endpoint: "GET /api/v1/products/search", fileName: "search_smoke_test.py", reasoning: "Validates product search returns correct results for category filter" },
         ],
-        testMaintenance: [{ action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Updated auth flow in existing tests", beforeStatus: "Fail", beforeDetails: "401 Unauthorized (1.5s)", afterStatus: "Pass", afterDetails: "All assertions passed (2.3s)" }],
+        testMaintenance: [{ action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Updated auth flow in existing tests", beforeDetails: "", afterDetails: "" }],
         testResults: [
             { testType: TestType.SMOKE, endpoint: "GET /api/v1/products", status: "Pass", details: "2.1s, products_smoke_test.py" },
             { testType: TestType.SMOKE, endpoint: "GET /api/v1/products/search", status: "Fail", details: "3.4s, search_smoke_test.py" },
@@ -67,7 +74,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         expect(result.content[0].text).toContain("Report submitted successfully");
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
@@ -81,7 +89,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "nested", "dirs", "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.businessCaseAnalysis).toBeDefined();
@@ -90,11 +99,12 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
             summaryOutputFile: outputFile,
+            stateFile,
             businessCaseAnalysis: "Simple config change.",
             newTestsCreated: [],
-            testMaintenance: [],
             testResults: [
                 { testType: TestType.SMOKE, endpoint: "GET /api/v1/products", status: "Pass", details: "1.0s, test.py" },
             ],
@@ -103,12 +113,15 @@ describe("registerSubmitReportTool", () => {
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.newTestsCreated).toEqual([]);
-        expect(written.testMaintenance).toEqual([]);
+        expect(written.testMaintenance).toBeUndefined();
         expect(written.issuesFound).toEqual([]);
         expect(written.testResults).toHaveLength(1);
     });
     it("should return error for invalid file path", async () => {
-        const result = await handler(sampleReportParams("/nonexistent/readonly/path/report.json"));
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams("/nonexistent/readonly/path/report.json", stateFile));
         expect(result.isError).toBe(true);
         expect(result.content[0].text).toContain("Failed to write report");
     });
@@ -116,8 +129,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "add contract tests for /products endpoint",
         });
         expect(result.isError).toBeUndefined();
@@ -128,7 +142,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
@@ -137,8 +152,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "  line one\nline two\r\nline three  ",
         });
         expect(result.isError).toBeUndefined();
@@ -150,9 +166,10 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const longMessage = "add contract and integration tests for the new PATCH /orders/{order_id} endpoint including discount recalculation and line-item edits";
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: longMessage,
         });
         expect(result.isError).toBeUndefined();
@@ -160,24 +177,13 @@ describe("registerSubmitReportTool", () => {
         expect(written.commitMessage).toBe(longMessage);
         expect(written.commitMessage.length).toBeGreaterThan(72);
     });
-    it("should use default commitMessage when provided as empty string", async () => {
-        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
-        tmpDirs.push(tmpDir);
-        const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler({
-            ...sampleReportParams(outputFile),
-            commitMessage: "",
-        });
-        expect(result.isError).toBeUndefined();
-        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
-        expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
-    });
     it("should use default commitMessage when whitespace-only", async () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             commitMessage: "   \n\r\n   ",
         });
         expect(result.isError).toBeUndefined();
@@ -188,8 +194,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const result = await handler({
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             nextSteps: ["Check your targetSetupCommand — endpoints returned 404"],
         });
         expect(result.isError).toBeUndefined();
@@ -202,7 +209,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        const result = await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler(sampleReportParams(outputFile, stateFile));
         expect(result.isError).toBeUndefined();
         const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
         expect(written.nextSteps).toEqual([]);
@@ -211,7 +219,8 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
-        await handler(sampleReportParams(outputFile));
+        const stateFile = await writeStateFile(tmpDir, []);
+        await handler(sampleReportParams(outputFile, stateFile));
         const raw = await fs.readFile(outputFile, "utf-8");
         // Pretty-printed JSON should have indentation
         expect(raw).toContain("  ");
@@ -222,8 +231,9 @@ describe("registerSubmitReportTool", () => {
         const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
         tmpDirs.push(tmpDir);
         const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
         const params = {
-            ...sampleReportParams(outputFile),
+            ...sampleReportParams(outputFile, stateFile),
             additionalRecommendations: [
                 {
                     testId: "integration-orders-patch-status",
@@ -516,14 +526,15 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
-            // sampleReportParams: 1 new test, 1 maintenance (Fail→Pass), 2 results
+            // sampleReportParams: 1 new test, 1 maintenance (no matching stateFile entry), 2 results
             // (1 Pass + 1 Fail), 1 issue (no severity).
-            await handler(sampleReportParams(outputFile));
+            const stateFile = await writeStateFile(tmpDir, []);
+            await handler(sampleReportParams(outputFile, stateFile));
             const params = lastAnalyticsParams();
             expect(params.testsGenerated).toBe("1");
             expect(params.testsMaintained).toBe("1");
             expect(params.recommendationsCount).toBe("0");
-            expect(params.maintenanceRecovered).toBe("1");
+            expect(params.maintenanceRecovered).toBe("0"); // no matching stateFile entry → no execution data
             expect(params.testsPassed).toBe("1");
             expect(params.testsFailed).toBe("1");
             expect(params.testsSkipped).toBe("0");
@@ -533,8 +544,9 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
-                ...sampleReportParams(outputFile),
+                ...sampleReportParams(outputFile, stateFile),
                 issuesFound: [
                     { description: "Data corruption on checkout", severity: "critical" },
                     { description: "Wrong total returned", severity: "high" },
@@ -554,8 +566,10 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
                 summaryOutputFile: outputFile,
+                stateFile,
                 businessCaseAnalysis: "Config-only change.",
                 newTestsCreated: [],
                 testMaintenance: [],
@@ -573,36 +587,236 @@ describe("registerSubmitReportTool", () => {
             const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
             tmpDirs.push(tmpDir);
             const outputFile = path.join(tmpDir, "report.json");
+            const stateFile = await writeStateFile(tmpDir, []);
             await handler({
-                ...sampleReportParams(outputFile),
+                ...sampleReportParams(outputFile, stateFile),
                 testMaintenance: [
                     // Real change
-                    { action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched auth", beforeStatus: "Fail", beforeDetails: "401 (1.5s)", afterStatus: "Pass", afterDetails: "passed (2.3s)" },
-                    { action: "REGENERATE", testType: TestType.SMOKE, endpoint: "GET /api/v1/orders", fileName: "orders_smoke_test.py", description: "Rewrote for new shape", beforeStatus: "Fail", beforeDetails: "shape mismatch", afterStatus: "Pass", afterDetails: "passed (1.0s)" },
+                    { action: "UPDATE", testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", description: "Patched auth", beforeDetails: "", afterDetails: "" },
+                    { action: "REGENERATE", testType: TestType.SMOKE, endpoint: "GET /api/v1/orders", fileName: "orders_smoke_test.py", description: "Rewrote for new shape", beforeDetails: "", afterDetails: "" },
                     // No-ops — assessed but not modified
-                    { action: "IGNORE", testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeStatus: "Pass", beforeDetails: "drift score 0", afterStatus: "Skipped", afterDetails: "IGNORE: endpoint not in PR" },
-                    { action: "VERIFY", testType: TestType.SMOKE, endpoint: "GET /api/v1/coupons", fileName: "coupons_smoke_test.py", description: "Verified still valid", beforeStatus: "Pass", beforeDetails: "minor drift", afterStatus: "Skipped", afterDetails: "VERIFY: assertions still hold" },
+                    { action: "IGNORE", testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", description: "No action required", beforeDetails: "", afterDetails: "" },
+                    { action: "VERIFY", testType: TestType.SMOKE, endpoint: "GET /api/v1/coupons", fileName: "coupons_smoke_test.py", description: "Verified still valid", beforeDetails: "", afterDetails: "" },
                 ],
             });
             const params = lastAnalyticsParams();
             // 2 change actions (UPDATE + REGENERATE), not 4
             expect(params.testsMaintained).toBe("2");
-            // Both changes went Fail→Pass
-            expect(params.maintenanceRecovered).toBe("2");
+            // No matching stateFile entries — execution data unavailable
+            expect(params.maintenanceRecovered).toBe("0");
         });
-        it("counts UPDATE/REGENERATE/DELETE as maintenance changes, not VERIFY/IGNORE", async () => {
-            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
-            tmpDirs.push(tmpDir);
+    });
+    // ─────────────────────────────────────────────────────────────────────────────
+    // testMaintenance output shapes
+    // Each case documents exactly what the MCP writes to JSON, which is what the
+    // testbot renderer reads. Keeping these in sync prevents silent regressions
+    // when toOutput or the stateFile enrichment path changes.
+    // ─────────────────────────────────────────────────────────────────────────────
+    describe("testMaintenance output shapes", () => {
+        const baseMaintenance = {
+            testType: TestType.CONTRACT,
+            endpoint: "GET /api/v1/orders",
+            fileName: "orders_contract_test.py",
+            description: "Updated discount fields in assertions",
+            beforeDetails: "",
+            afterDetails: "",
+        };
+        async function writeAndRead(tmpDir, maintenance, extra) {
             const outputFile = path.join(tmpDir, "report.json");
             await handler({
-                ...sampleReportParams(outputFile),
-                testMaintenance: [
-                    { testType: TestType.CONTRACT, endpoint: "GET /api/v1/products", fileName: "products_contract_test.py", action: DriftAction.Update, description: "Patched", beforeStatus: "Fail", beforeDetails: "401", afterStatus: "Pass", afterDetails: "passed" },
-                    { testType: TestType.SMOKE, endpoint: "GET /api/v1/reviews", fileName: "reviews_smoke_test.py", action: DriftAction.Ignore, description: "No action required", beforeStatus: "Pass", beforeDetails: "drift 0", afterStatus: "Skipped", afterDetails: "not in PR" },
-                ],
+                summaryOutputFile: outputFile,
+                businessCaseAnalysis: "Test",
+                newTestsCreated: [],
+                testMaintenance: maintenance,
+                testResults: [],
+                issuesFound: [],
+                ...extra,
             });
-            const params = lastAnalyticsParams();
-            expect(params.testsMaintained).toBe("1");
+            const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+            return written.testMaintenance;
+        }
+        // ── No matching stateFile entry ───────────────────────────────────────────
+        it("UPDATE with no matching stateFile entry: beforeStatus=Unknown, afterStatus=Unknown", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Unknown");
+            expect(result[0].beforeDetails).toBe("");
+            expect(result[0].afterDetails).toBe("");
+        });
+        it("IGNORE with no matching stateFile entry: both Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Ignore }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Skipped");
+            expect(result[0].afterStatus).toBe("Skipped");
+            expect(result[0].beforeDetails).toBe("");
+            expect(result[0].afterDetails).toBe("");
+        });
+        it("DELETE with no matching stateFile entry: beforeStatus=Unknown, afterStatus=Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, []);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Delete }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Skipped");
         });
+        // ── With matching stateFile entry ─────────────────────────────────────────
+        it("UPDATE with before=Pass after=Pass: status from stateFile, details from LLM", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15090 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 12880 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance, action: DriftAction.Update,
+                    beforeDetails: "4 passed in 15.09s",
+                    afterDetails: "5 passed in 12.88s",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].beforeDetails).toBe("4 passed in 15.09s");
+            expect(result[0].afterStatus).toBe("Pass");
+            expect(result[0].afterDetails).toBe("5 passed in 12.88s");
+        });
+        it("UPDATE with before=Pass after=Fail: status from stateFile, details from LLM", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15090 },
+                    executionAfter: { status: TestExecutionStatus.Fail, errors: [], duration: 10940 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance, action: DriftAction.Update,
+                    beforeDetails: "4 passed in 15.09s",
+                    afterDetails: "FAILED test_orders_get — check_schema fails, test isolation bug",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Fail");
+            expect(result[0].afterDetails).toBe("FAILED test_orders_get — check_schema fails, test isolation bug");
+        });
+        it("UPDATE with crashed execution: status is Error", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Error, errors: ["Connection refused"], duration: 500 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Error");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+        it("UPDATE Fail→Pass: counted as maintenanceRecovered in analytics", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Fail, errors: ["FAILED test_foo - assert False"], duration: 5000 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 5000 },
+                }]);
+            await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(pushMCPToolEventMock.mock.calls.at(-1)?.[2].maintenanceRecovered).toBe("1");
+        });
+        it("UPDATE with no before-execution recorded: beforeStatus=Unknown, afterStatus=Pass", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    // executionBefore absent — agent skipped before-phase
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Update }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Unknown");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+        it("DELETE with before-execution, no after (file deleted): beforeStatus=Pass, afterStatus=Skipped", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 2000 },
+                    // executionAfter absent — file was deleted, nothing to run
+                }]);
+            const result = await writeAndRead(tmpDir, [{ ...baseMaintenance, action: DriftAction.Delete }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Skipped");
+        });
+        it("fileName matched by suffix: absolute stateFile path vs relative fileName", async () => {
+            const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+            tmpDirs.push(tmpDir);
+            const stateFile = await writeStateFile(tmpDir, [{
+                    testFile: "/repo/backend/tests/python/orders_contract_test.py",
+                    executionBefore: { status: TestExecutionStatus.Pass, errors: [], duration: 15000 },
+                    executionAfter: { status: TestExecutionStatus.Pass, errors: [], duration: 12000 },
+                }]);
+            // fileName is relative — should still match via endsWith
+            const result = await writeAndRead(tmpDir, [{
+                    ...baseMaintenance,
+                    action: DriftAction.Update,
+                    fileName: "orders_contract_test.py",
+                }], { stateFile });
+            expect(result[0].beforeStatus).toBe("Pass");
+            expect(result[0].afterStatus).toBe("Pass");
+        });
+    });
+    it("testResult accepts videoPath field (optional)", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler({
+            summaryOutputFile: outputFile,
+            stateFile,
+            businessCaseAnalysis: "UI changes for login flow",
+            newTestsCreated: [],
+            testMaintenance: [],
+            testResults: [
+                {
+                    testType: TestType.UI,
+                    endpoint: "GET /login",
+                    status: "Pass",
+                    details: "5.2s, login_ui_test.spec.ts",
+                    videoPath: "/workspace/.skyramp/videos/login_ui_test-abc12345/video.webm",
+                },
+            ],
+            issuesFound: [],
+        });
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.testResults).toHaveLength(1);
+        expect(written.testResults[0].videoPath).toBe("/workspace/.skyramp/videos/login_ui_test-abc12345/video.webm");
+    });
+    it("testResult accepts entry without videoPath (optional field)", async () => {
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
+        tmpDirs.push(tmpDir);
+        const outputFile = path.join(tmpDir, "report.json");
+        const stateFile = await writeStateFile(tmpDir, []);
+        const result = await handler({
+            summaryOutputFile: outputFile,
+            stateFile,
+            businessCaseAnalysis: "API endpoint changes",
+            newTestsCreated: [],
+            testMaintenance: [],
+            testResults: [
+                {
+                    testType: TestType.CONTRACT,
+                    endpoint: "POST /api/v1/products",
+                    status: "Pass",
+                    details: "2.1s, products_contract_test.py",
+                    // videoPath intentionally omitted
+                },
+            ],
+            issuesFound: [],
+        });
+        expect(result.isError).toBeUndefined();
+        const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
+        expect(written.testResults).toHaveLength(1);
+        expect(written.testResults[0].videoPath).toBeUndefined();
     });
 });

package/build/tools/test-management/actionsTool.js

@@ -6,6 +6,7 @@ import * as fs from "fs";
 import * as path from "path";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { toolError } from "../../utils/utils.js";
+import { stageGeneratedPaths } from "../../utils/gitStaging.js";
 import { buildRenameStrategy, buildFileRenameStrategy, buildUpdateStrategy, buildRegenerateStrategy, buildUpdateFileInstruction, buildRegenerateFileInstruction, } from "../../prompts/test-maintenance/actionsInstructions.js";
 /**
  * Compute a suggested new filename when an endpoint is renamed.
@@ -365,6 +366,15 @@ export function registerActionsTool(server) {
                 });
                 llmInstructionsObj.regenerate_strategy = buildRegenerateStrategy();
             }
+            // Stage UPDATE and REGENERATE target paths so testbot's re-stage step
+            // commits the agent's edits even when files are outside testDirectory.
+            const stagedPaths = [
+                ...updateRecommendations.map((r) => r.testFile),
+                ...regenerateRecommendations.map((r) => r.testFile),
+            ];
+            for (const p of stagedPaths) {
+                await stageGeneratedPaths(p).catch((e) => logger.warning(`Failed to stage maintained file ${p}: ${e.message}`));
+            }
             const llmInstructions = JSON.stringify(llmInstructionsObj, null, 2);
             const contentBlocks = [
                 {

package/build/tools/test-management/analyzeChangesTool.d.ts

@@ -18,7 +18,6 @@ export declare const analyzeChangesInputSchema: {
     topN: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
     maxGenerate: z.ZodOptional<z.ZodNumber>;
     prNumber: z.ZodOptional<z.ZodNumber>;
-    stateOutputFile: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
     testsRepoDir: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
 };
 export declare const NO_UI_INSTRUCTIONS = "No UI changes detected \u2014 no blueprint capture needed.";