@skyramp/mcp 0.2.9 → 0.2.10-rc.1

package/build/tools/submitReportTool.js

@@ -8,7 +8,7 @@ import { TestType, HttpMethod } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
 import { TestExecutionStatus } from "../types/TestExecution.js";
 import { StateManager } from "../utils/AnalysisStateManager.js";
-import { toolError } from "../utils/utils.js";
+import { toolError, testFileMatches } from "../utils/utils.js";
 // Drift actions that actually modify a test file. VERIFY and IGNORE are
 // no-ops (the test was assessed but left unchanged), so they must not count
 // toward "tests maintained" telemetry.
@@ -244,10 +244,14 @@ export const additionalRecommendationSchema = z.object({
         }
     }
 });
+// TODO(multi-repo maintenance): no `repository` field yet — see readData() TODO below.
 const testMaintenanceSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test."),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
-    fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
+    testFilePath: z
+        .string()
+        .refine((p) => path.isAbsolute(p), { message: "testFilePath must be an absolute path" })
+        .describe("Absolute path of the test file that was maintained, e.g. '/repo/tests/products_smoke_test.py' — the same path you passed to skyramp_execute_test's testFile param. Consumers should basename this for display."),
     action: z.nativeEnum(DriftAction).describe("The drift action assigned to this test during maintenance triage."),
     description: z.string().describe("What was changed and why"),
     beforeDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
@@ -269,6 +273,14 @@ const testMaintenanceLLMSchema = testMaintenanceSchema.omit({
     beforeStatus: true,
     afterStatus: true,
 });
+// LLM only drafts beforeDetails/afterDetails per file — action/testType/endpoint/description
+// come from stateData.maintenanceVerdicts. Picked from testMaintenanceLLMSchema to avoid
+// redeclaring those fields.
+const testMaintenanceDetailSchema = testMaintenanceLLMSchema.pick({
+    testFilePath: true,
+    beforeDetails: true,
+    afterDetails: true,
+});
 /**
  * Derive per-run analytics counts from a submitted report. These power the
  * alpha-launch dashboards (tests generated/maintained, suite growth, bugs vs
@@ -341,7 +353,8 @@ export function registerSubmitReportTool(server) {
                 .describe("The file path where the report should be written (provided in the task instructions)"),
             businessCaseAnalysis: z
                 .string()
-                .describe("1-2 sentences describing what user-facing interactions this PR enables or changes (e.g. 'customers can now leave and view product reviews'). Focus on the user journey, not technical implementation. Flag backend-only or frontend-only gaps."),
+                .min(1, "businessCaseAnalysis is required — describe what the PR does, even for no-test runs (config-only, non-application, Testbot onboarding). It is never optional and must not be empty.")
+                .describe("1-2 sentences describing what user-facing interactions this PR enables or changes (e.g. 'customers can now leave and view product reviews'). Focus on the user journey, not technical implementation. Flag backend-only or frontend-only gaps. Always required — even when no tests apply (config-only diffs, Testbot onboarding PRs); describe the PR's purpose and why no tests apply."),
             newTestsCreated: z
                 .array(newTestSchema)
                 .describe("List of new tests created. Use empty array [] if none."),
@@ -350,10 +363,10 @@ export function registerSubmitReportTool(server) {
                 .optional()
                 .default([])
                 .describe("Recommended tests that were not generated (lower priority). Only include recommendations that add distinct coverage beyond generated tests — do not pad with variants testing the same endpoint and flow."),
-            testMaintenance: z
-                .array(testMaintenanceLLMSchema)
+            testMaintenanceDetails: z
+                .array(testMaintenanceDetailSchema)
                 .optional()
-                .describe("One entry per test that required action in the drift analysis step: UPDATE, REGENERATE, DELETE, and VERIFY. IGNORE entries are omitted — a clean test is not a maintenance action. Omit this field entirely when no existing tests were found."),
+                .describe("One entry per maintained test that had a before/after execution recorded, with beforeDetails/afterDetails drafted from what you actually saw when you ran it. Omit an entry entirely when no execution was recorded for that test."),
             testResults: z
                 .array(testResultSchema)
                 .describe("List of ALL test execution results. One entry per test executed."),
@@ -392,31 +405,93 @@ export function registerSubmitReportTool(server) {
         let errorResult;
         const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
         const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
+        const stateManager = StateManager.fromStatePath(params.stateFile);
+        let stateData;
+        try {
+            // Only reads the primary repo — see the related-repo check below for why related
+            // repos' maintenanceVerdicts are never folded in here.
+            stateData = await stateManager.readData();
+        }
+        catch (err) {
+            errorResult = toolError(`Failed to load stateFile: ${err.message}`);
+            return errorResult;
+        }
+        if (!stateData) {
+            errorResult = toolError(`stateFile not found or empty: ${params.stateFile}`);
+            return errorResult;
+        }
+        // undefined + existing tests present means skyramp_actions was never called this
+        // run — not "no maintenance needed" (that case writes an empty array). Reject rather
+        // than silently reporting an empty testMaintenance section indistinguishable from a
+        // real "nothing to do". With zero existing tests there was nothing to lose, so
+        // undefined is safe there — no round-trip through skyramp_actions required.
+        if (stateData.maintenanceVerdicts === undefined && (stateData.existingTests?.length ?? 0) > 0) {
+            errorResult = toolError("stateFile has existingTests but no maintenanceVerdicts — skyramp_actions was not called for this run. " +
+                "Call skyramp_actions (with recommendations: [] if no existing tests needed action) before skyramp_submit_report.");
+            return errorResult;
+        }
+        // TestbotReport.testMaintenance has no repository field, so a related repo's
+        // reconciled verdicts have nowhere to go in this report. Fail loud instead of
+        // silently omitting real maintenance work skyramp_actions already did and persisted.
+        // Multi-repo maintenance reporting is a separate, unscoped feature (would also need
+        // testbot prompt / drift-analysis / test-bot.git renderer changes) — not done here.
+        const relatedReposWithVerdicts = [];
+        for (const repo of await stateManager.listRelatedRepos()) {
+            const relatedData = await stateManager.readRepoData(repo);
+            if ((relatedData?.maintenanceVerdicts?.length ?? 0) > 0)
+                relatedReposWithVerdicts.push(repo);
+        }
+        if (relatedReposWithVerdicts.length > 0) {
+            errorResult = toolError(`Related repo(s) have reconciled maintenance verdicts that cannot be included in this report yet ` +
+                `(multi-repo maintenance reporting is not supported — testMaintenance has no repository field): ${relatedReposWithVerdicts.join(", ")}.`);
+            return errorResult;
+        }
+        // action/testType/endpoint/description come entirely from stateData.maintenanceVerdicts
+        // (written by skyramp_actions right after drift analysis) — the LLM never supplies
+        // testMaintenance directly, so there's nothing to re-transcribe here.
+        const rawMaintenance = (stateData.maintenanceVerdicts ?? []).map((v) => ({
+            testFilePath: v.testFilePath,
+            testType: v.testType,
+            endpoint: v.endpoint,
+            action: v.action,
+            description: v.rationale,
+            beforeDetails: "",
+            afterDetails: "",
+        }));
         let testMaintenance;
-        if (params.testMaintenance) {
-            try {
-                const stateData = await StateManager.fromStatePath(params.stateFile).readData();
-                if (!stateData) {
-                    errorResult = toolError(`stateFile not found or empty: ${params.stateFile}`);
-                    return errorResult;
-                }
-                // Status fields are stateFile-authoritative. Details are LLM-supplied when present
-                // (single-line, with root-cause context) and fall back to server-side extraction.
-                testMaintenance = params.testMaintenance.map((m) => {
-                    const recorded = stateData.existingTests?.find((t) => t.testFile.endsWith(m.fileName) || path.basename(t.testFile) === path.basename(m.fileName));
-                    const defaultBeforeStatus = MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
-                    const defaultAfterStatus = m.action === DriftAction.Delete ? TestExecutionStatus.Skipped
-                        : MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
-                    const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
-                    const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
-                    logger.info(`${m.fileName}: before=${beforeStatus} after=${afterStatus}`);
-                    return { ...m, beforeStatus, afterStatus };
-                });
-            }
-            catch (err) {
-                errorResult = toolError(`Failed to load stateFile for maintenance enrichment: ${err.message}`);
-                return errorResult;
-            }
+        // Rows missing a required detail (a real execution happened but the LLM didn't
+        // draft a summary for it) — collected here and rejected below, rather than
+        // silently shipping a blank field.
+        const missingDetails = [];
+        if (rawMaintenance.length > 0) {
+            // beforeStatus/afterStatus are always stateFile-authoritative. beforeDetails/
+            // afterDetails come from the LLM's drafted summary (testMaintenanceDetails) —
+            // required whenever the stateFile shows a matching execution actually ran.
+            testMaintenance = rawMaintenance.map((m) => {
+                const recorded = stateData.existingTests?.find((t) => testFileMatches(t.testFile, m.testFilePath));
+                const detail = params.testMaintenanceDetails?.find((d) => d.testFilePath === m.testFilePath);
+                const displayName = path.basename(m.testFilePath);
+                const defaultBeforeStatus = MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                const defaultAfterStatus = m.action === DriftAction.Delete ? TestExecutionStatus.Skipped
+                    : MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
+                const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
+                // Trim before checking — a whitespace-only string is semantically blank and
+                // must not satisfy the "drafted a summary" requirement.
+                const beforeDetails = detail?.beforeDetails?.trim() ?? "";
+                const afterDetails = detail?.afterDetails?.trim() ?? "";
+                if (recorded?.executionBefore && !beforeDetails)
+                    missingDetails.push(`${displayName} (beforeDetails)`);
+                if (recorded?.executionAfter && !afterDetails)
+                    missingDetails.push(`${displayName} (afterDetails)`);
+                logger.info(`${displayName}: before=${beforeStatus} after=${afterStatus}`);
+                return { ...m, beforeDetails, afterDetails, beforeStatus, afterStatus };
+            });
+        }
+        if (missingDetails.length > 0) {
+            errorResult = toolError(`${missingDetails.length} maintenance row(s) have a recorded execution but no drafted summary: ${missingDetails.join(", ")}. ` +
+                "Add a testMaintenanceDetails entry with the missing field(s), drafted from the execution output you already saw.");
+            return errorResult;
         }
         // Strip generation-artifact fields from newTestsCreated before writing.
         // scenarioFile, traceFile, frontendTrace are internal paths used during
@@ -428,7 +503,14 @@ export function registerSubmitReportTool(server) {
             businessCaseAnalysis: params.businessCaseAnalysis,
             newTestsCreated: sanitizedNewTests,
             additionalRecommendations: dedupedRecommendations.map(normalizeRepository),
-            testMaintenance,
+            // Report wire format keeps main's original `fileName` (basename) — testFilePath is
+            // an internal-only field, needed for matching but never meant to reach the report.
+            // TODO(multi-repo maintenance): map(normalizeRepository) once testMaintenanceSchema
+            // has a repository field (see TODO above).
+            testMaintenance: testMaintenance?.map(({ testFilePath, ...row }) => ({
+                ...row,
+                fileName: path.basename(testFilePath),
+            })),
             testResults: params.testResults.map(normalizeRepository),
             issuesFound: params.issuesFound.map(normalizeRepository),
             nextSteps: params.nextSteps ?? [],