@skyramp/mcp 0.2.6 → 0.2.150-rc.mntnc

package/build/tools/submitReportTool.js

@@ -6,6 +6,9 @@ import { AnalyticsService } from "../services/AnalyticsService.js";
 import { TEST_CATEGORIES, externalCategory } from "../types/TestRecommendation.js";
 import { TestType, HttpMethod } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
+import { TestExecutionStatus } from "../types/TestExecution.js";
+import { StateManager } from "../utils/AnalysisStateManager.js";
+import { toolError } from "../utils/utils.js";
 // Drift actions that actually modify a test file. VERIFY and IGNORE are
 // no-ops (the test was assessed but left unchanged), so they must not count
 // toward "tests maintained" telemetry.
@@ -21,12 +24,37 @@ const testResultSchema = z.object({
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
     details: z.string().describe("One sentence — no embedded newlines, no markdown. e.g. '10.8s, products_contract_test.py' or 'failed: <one-line error summary>, products_contract_test.py'"),
+    videoPath: z.string().optional().describe("Path to video recording (UI/E2E tests only)"),
+});
+// Structured grounding for UI recommendations. Lifted verbatim from a
+// `browser_blueprint` element captured during this run. Source of truth for
+// codegen (prefers testId > stableId > role+name for locator selection),
+// the verifier (cross-checks each field against captured blueprints), and
+// any future tooling that wants reliable grounding data without prose
+// parsing. `reasoning` is free-form prose constrained by the Blueprint
+// Citation Invariant — every element cited must appear in `targetElements`.
+// See testbot prompt step 4 for the full populate-and-render rules.
+export const targetElementSchema = z.object({
+    role: z.string().describe("ARIA role (e.g. 'button', 'heading', 'textbox', 'link'). Lifted verbatim from the captured blueprint element's `role` field."),
+    accessibleName: z.string().describe("Computed accessible name (e.g. 'Save changes', 'Order Details'). Lifted verbatim from the captured blueprint element's `accessibleName` field. Must match the bolded name in `reasoning` character-for-character."),
+    testId: z.string().nullable().describe("data-testid attribute value (preferred locator handle), or null when the element has no data-testid. Lifted from the blueprint element's `testId`."),
+    stableId: z.string().nullable().describe("Unique HTML id attribute value (fallback locator when no testId), or null. Lifted from the blueprint element's `stableId`."),
+    contextText: z.array(z.string()).nullable().describe("Disambiguating row text for elements inside repeating sections (table rows, list items): the row's surrounding non-interactive text. Lifted from the blueprint repeatingElement's items[].contextText. null for non-repeating elements."),
+    mutability: z.enum(["mutable", "immutable", "unknown"]).optional().describe("Whether the element's content/state is expected to change between captures. 'mutable' elements are behavioral-test targets; 'immutable' are smoke-test targets. Copied from the blueprint element's `mutability`."),
+    widgetType: z.enum(["native", "custom", "unknown"]).optional().describe("'native' = HTML built-ins (button, input). 'custom' or 'unknown' = non-standard composites; fall back to snapshot-driven trial clicks for interaction. Copied from the blueprint element's `widgetType`."),
+});
+// Page metadata for a UI recommendation. Lifted from the BlueprintCapture
+// the agent used to populate `targetElements`. Codegen reads `url` to drive
+// page.goto(); the verifier uses `pageHash` to detect stale captures.
+export const pageContextSchema = z.object({
+    url: z.string().describe("URL of the page where the test runs. Lifted from BlueprintCapture.url."),
+    pageHash: z.string().optional().describe("Opaque hash of the captured page state (BlueprintCapture.pageHash). Lets the verifier confirm the recommendation was grounded in a still-current capture."),
 });
 // TODO: Unify newTestSchema and additionalRecommendationSchema into a single
 // interface that adds an `implemented: boolean` field. Both describe the same
 // concept (a test recommendation) — the only difference is whether it was
 // generated in this run or left for later. Tracked per Archit's review comment.
-const newTestSchema = z.object({
+export const newTestSchema = z.object({
     testId: z.string().describe("Human-readable kebab-case identifier, e.g. 'contract-get-products' or 'integration-users-orders-workflow'. Format: '<testType>-<method>-<resource>' for single-endpoint tests or '<testType>-<scenario-slug>' for multi-step tests. Must be unique within the report."),
     testType: z.nativeEnum(TestType).describe("Type of test created. Do not include priority or other metadata in this field."),
     category: z.preprocess((val) => externalCategory(val), z.enum(TEST_CATEGORIES)).describe("Test category — critical categories (security_boundary, business_rule, data_integrity, breaking_change) get generation priority over workflow"),
@@ -37,6 +65,71 @@ const newTestSchema = z.object({
     traceFile: z.string().optional().describe("Path to the backend trace file if used or created"),
     frontendTrace: z.string().optional().describe("Path to the Playwright/UI trace file if used or created"),
     reasoning: z.string().describe("Why this test was created: what production risk it mitigates, what code pattern it targets, or what coverage gap it fills"),
+    targetElements: z.array(targetElementSchema).min(1).nullable().optional().describe("UI tests only: structured grounding for one or more elements the test targets. Most tests target a single element (array length 1); render-state and multi-step UI tests target several (array length 2+). Each entry must be lifted verbatim from a captured blueprint element. Set to null when blueprint capture failed (also requires '[no-blueprint-data]' marker in BOTH description and reasoning). See Blueprint Citation Invariant in testbot prompt."),
+    pageContext: pageContextSchema.optional().describe("UI tests only: page metadata for the test. Lifted from the BlueprintCapture used during grounding."),
+}).superRefine((rec, ctx) => {
+    // targetElements / pageContext are UI-only.
+    if (rec.testType !== TestType.UI) {
+        for (const field of ["targetElements", "pageContext"]) {
+            if (rec[field] !== undefined) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: [field],
+                    message: `${field} is only valid for testType: 'ui'`,
+                });
+            }
+        }
+    }
+    else {
+        // description is required for UI tests (walkthrough of steps/assertions).
+        if (!rec.description) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["description"],
+                message: "description is required for testType: 'ui'.",
+            });
+        }
+        // targetElements must be present (array or null — not omitted entirely).
+        if (rec.targetElements === undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["targetElements"],
+                message: "targetElements is required for testType: 'ui'. Use null when blueprint capture failed (and add '[no-blueprint-data]' to both description and reasoning).",
+            });
+        }
+        // pageContext is required when targetElements is an array (grounded), forbidden when null.
+        if (Array.isArray(rec.targetElements) && rec.pageContext === undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["pageContext"],
+                message: "pageContext is required for testType: 'ui' when targetElements is an array.",
+            });
+        }
+        if (rec.targetElements === null && rec.pageContext !== undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["pageContext"],
+                message: "pageContext must be omitted when targetElements is null (no blueprint data).",
+            });
+        }
+        // When targetElements is null, both description and reasoning must carry the '[no-blueprint-data]' marker.
+        if (rec.targetElements === null) {
+            if (!rec.reasoning.includes("[no-blueprint-data]")) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ["reasoning"],
+                    message: "reasoning must contain '[no-blueprint-data]' when targetElements is null.",
+                });
+            }
+            if (rec.description && !rec.description.includes("[no-blueprint-data]")) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ["description"],
+                    message: "description must contain '[no-blueprint-data]' when targetElements is null.",
+                });
+            }
+        }
+    }
 });
 const descriptionSchema = z.object({
     description: z.string().describe("One-line description. Do NOT prefix with the severity level — severity is a separate field. Include code logic bugs from the diff, test generation/execution failures, and environment misconfiguration."),
@@ -63,12 +156,14 @@ export const additionalRecommendationSchema = z.object({
     scenarioName: z.string().optional().describe("Proposed scenario name for future generation, e.g. 'products-orders-workflow'. No file exists yet — this is a suggestion only. Omit if not applicable."),
     // TODO: replace text with max(3) and check for regression
     steps: z.array(scenarioStepSchema).describe("Ordered sequence of API/UI steps in this test scenario (at most 3). Each step must include method and path so the endpoints are explicit. Omit requestBody and responseBody from steps."),
-    description: z.string().describe("What this test does and why it is valuable. For multi-step scenarios, list the endpoints involved (e.g. 'POST /api/v1/products → POST /api/v1/orders → PATCH /api/v1/orders/{id}') so the recommendation is actionable without primaryEndpoint."),
+    description: z.string().describe("Walkthrough of what the test does — the steps and assertions. For multi-step scenarios, list the endpoints involved. The 'why it is valuable' belongs in reasoning."),
     priority: z.preprocess((val) => (typeof val === "string" ? val.toLowerCase() : val), z.enum(["high", "medium", "low"])).describe("Priority level: high, medium, or low. First check diff relevance — does the test target an endpoint changed in this PR? HIGH: diff-relevant security/auth/error tests, cross-resource isolation for diff endpoints, CRUD lifecycle for NEW endpoints in the diff. MEDIUM: diff-relevant business-rule happy paths, multi-resource workflows involving diff endpoints, security/error tests for NON-diff endpoints. LOW: tests targeting only unchanged endpoints, trivially discoverable happy paths duplicating generated tests."),
     openApiSpec: z.string().optional().describe("Path to OpenAPI/Swagger spec file if available, e.g. 'openapi.yaml'"),
     backendTrace: z.string().optional().describe("Path to backend trace file if available, e.g. 'tests/skyramp-traces.json'. Used by integration and E2E tests."),
     frontendTrace: z.string().optional().describe("Path to Playwright/UI trace file if available, e.g. 'tests/skyramp-playwright.zip'. UI tests need this; E2E tests need both frontend and backend traces."),
     reasoning: z.string().describe("Why this test is recommended: the specific production risk, business rule, or security boundary it would validate"),
+    targetElements: z.array(targetElementSchema).min(1).nullable().optional().describe("UI tests only: structured grounding for one or more elements the test targets. Most tests target a single element (array length 1); render-state and multi-step UI tests target several (array length 2+). Each entry must be lifted verbatim from a captured blueprint element. Set to null when blueprint capture failed (also requires '[no-blueprint-data]' marker in BOTH description and reasoning). See Blueprint Citation Invariant in testbot prompt."),
+    pageContext: pageContextSchema.optional().describe("UI tests only: page metadata for the test. Lifted from the BlueprintCapture used during grounding."),
 }).superRefine((rec, ctx) => {
     if (rec.testType === TestType.CONTRACT && !rec.primaryEndpoint) {
         ctx.addIssue({
@@ -77,6 +172,57 @@ export const additionalRecommendationSchema = z.object({
             message: "primaryEndpoint is required for contract tests — provide the HTTP verb and path, e.g. 'PATCH /api/v1/orders/{order_id}'.",
         });
     }
+    // targetElements / pageContext are UI-only.
+    if (rec.testType !== TestType.UI) {
+        for (const field of ["targetElements", "pageContext"]) {
+            if (rec[field] !== undefined) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: [field],
+                    message: `${field} is only valid for testType: 'ui'`,
+                });
+            }
+        }
+    }
+    else {
+        if (rec.targetElements === undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["targetElements"],
+                message: "targetElements is required for testType: 'ui'. Use null when blueprint capture failed (and add '[no-blueprint-data]' to both description and reasoning).",
+            });
+        }
+        if (Array.isArray(rec.targetElements) && rec.pageContext === undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["pageContext"],
+                message: "pageContext is required for testType: 'ui' when targetElements is an array.",
+            });
+        }
+        if (rec.targetElements === null && rec.pageContext !== undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ["pageContext"],
+                message: "pageContext must be omitted when targetElements is null (no blueprint data).",
+            });
+        }
+        if (rec.targetElements === null) {
+            if (!rec.reasoning.includes("[no-blueprint-data]")) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ["reasoning"],
+                    message: "reasoning must contain '[no-blueprint-data]' when targetElements is null.",
+                });
+            }
+            if (!rec.description.includes("[no-blueprint-data]")) {
+                ctx.addIssue({
+                    code: z.ZodIssueCode.custom,
+                    path: ["description"],
+                    message: "description must contain '[no-blueprint-data]' when targetElements is null.",
+                });
+            }
+        }
+    }
 });
 const testMaintenanceSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test."),
@@ -84,13 +230,25 @@ const testMaintenanceSchema = z.object({
     fileName: z.string().describe("Test file that was maintained, e.g. 'products_smoke_test.py'"),
     action: z.nativeEnum(DriftAction).describe("The drift action assigned to this test during maintenance triage."),
     description: z.string().describe("What was changed and why"),
-    beforeStatus: z.enum(["Pass", "Fail", "Error"]).describe("Test result BEFORE modification"),
-    beforeDetails: z.string().describe("Execution output/timing before modification, or 'baseline from CI workflow <name>' if a parallel workflow provided the baseline"),
-    afterStatus: z.enum(["Pass", "Fail", "Error", "Skipped"]).describe("Test result AFTER modification"),
-    afterDetails: z.string().describe("Execution output/timing after modification"),
-})
-    .refine(m => ![DriftAction.Verify, DriftAction.Ignore].includes(m.action) || m.afterStatus === "Skipped", { message: "VERIFY and IGNORE entries must have afterStatus: 'Skipped' — these actions do not modify the test file" })
-    .refine(m => ![DriftAction.Update, DriftAction.Regenerate, DriftAction.Delete].includes(m.action) || m.afterStatus !== "Skipped", { message: "UPDATE, REGENERATE, and DELETE entries must have a real afterStatus (not Skipped) — these actions modify the test file and must be executed" });
+    beforeDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '4 passed in 15.09s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — assert 403 got 200, auth middleware not enforced'. " +
+        "Empty string for VERIFY/IGNORE entries where no before-execution was run."),
+    afterDetails: z.string().describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
+        "For passing runs: count and timing, e.g. '5 passed in 10.96s'. " +
+        "For failing runs: failure name and one-line root cause, e.g. " +
+        "'FAILED test_foo — check_schema fails, order_id=1 has discount from prior PATCH test'. " +
+        "Empty string for VERIFY/IGNORE/DELETE entries where no after-execution was run."),
+    // Server-populated from stateFile execution records — never supplied by the LLM.
+    beforeStatus: z.nativeEnum(TestExecutionStatus),
+    afterStatus: z.nativeEnum(TestExecutionStatus),
+});
+// LLM input contract: status fields are server-populated from stateFile.
+const testMaintenanceLLMSchema = testMaintenanceSchema.omit({
+    beforeStatus: true,
+    afterStatus: true,
+});
 /**
  * Derive per-run analytics counts from a submitted report. These power the
  * alpha-launch dashboards (tests generated/maintained, suite growth, bugs vs
@@ -114,7 +272,7 @@ function computeReportMetrics(params) {
     const recommendations = params.additionalRecommendations ?? [];
     const countBy = (items, pred) => items.filter(pred).length;
     const changedMaintenance = (params.testMaintenance ?? []).filter(isMaintenanceChange);
-    const maintenanceRecovered = countBy(changedMaintenance, (m) => m.beforeStatus !== "Pass" && m.afterStatus === "Pass");
+    const maintenanceRecovered = countBy(changedMaintenance, (m) => (m.beforeStatus === TestExecutionStatus.Fail || m.beforeStatus === TestExecutionStatus.Error) && m.afterStatus === TestExecutionStatus.Pass);
     return {
         testsGenerated: String(params.newTestsCreated.length),
         testsMaintained: String(changedMaintenance.length),
@@ -130,6 +288,23 @@ function computeReportMetrics(params) {
         issuesLow: String(countBy(params.issuesFound, (i) => i.severity === "low")),
     };
 }
+function deduplicateById(items) {
+    const seen = new Set();
+    const result = [];
+    for (const item of items) {
+        let id = item.testId;
+        if (seen.has(id)) {
+            let suffix = 2;
+            while (seen.has(`${id}-${suffix}`))
+                suffix++;
+            id = `${id}-${suffix}`;
+            item.testId = id;
+        }
+        seen.add(id);
+        result.push(item);
+    }
+    return result;
+}
 export function registerSubmitReportTool(server) {
     server.registerTool(TOOL_NAME, {
         description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
@@ -150,9 +325,9 @@ export function registerSubmitReportTool(server) {
                 .default([])
                 .describe("Recommended tests that were not generated (lower priority). Only include recommendations that add distinct coverage beyond generated tests — do not pad with variants testing the same endpoint and flow."),
             testMaintenance: z
-                .array(testMaintenanceSchema)
+                .array(testMaintenanceLLMSchema)
                 .optional()
-                .describe("One entry per test assessed in the drift analysis step. Omit this field when no existing tests were found."),
+                .describe("One entry per test assessed in the drift analysis step, including IGNORE and VERIFY entries — these provide triage transparency even when no edits were needed. Omit this field entirely when no existing tests were found."),
             testResults: z
                 .array(testResultSchema)
                 .describe("List of ALL test execution results. One entry per test executed."),
@@ -175,6 +350,12 @@ export function registerSubmitReportTool(server) {
                 .describe("Succinct commit message (if possible, under 72 chars) summarizing what Testbot did, " +
                 "e.g. 'add contract tests for /products endpoint' or 'update smoke tests for order API changes'. " +
                 "Used as both the git commit subject and the side PR title — the consumer applies truncation as needed."),
+            stateFile: z
+                .string()
+                .describe("Path to the state file returned by skyramp_analyze_changes. Required — " +
+                "contains execution results for maintenance scoring (existingTests), " +
+                "discovered endpoints for coverage validation (newEndpoints), and " +
+                "UI page context for frontend test grounding (uiContext)."),
         },
         _meta: {
             keywords: ["report", "summary", "testbot", "submit"],
@@ -182,40 +363,50 @@ export function registerSubmitReportTool(server) {
     }, async (params) => {
         const startTime = Date.now();
         let errorResult;
-        const deduplicateById = (items) => {
-            const seen = new Set();
-            const result = [];
-            for (const item of items) {
-                let id = item.testId;
-                if (seen.has(id)) {
-                    let suffix = 2;
-                    while (seen.has(`${id}-${suffix}`))
-                        suffix++;
-                    id = `${id}-${suffix}`;
-                    item.testId = id;
-                }
-                seen.add(id);
-                result.push(item);
-            }
-            return result;
-        };
         const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
         const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
+        let testMaintenance;
+        if (params.testMaintenance) {
+            try {
+                const stateData = await StateManager.fromStatePath(params.stateFile).readData();
+                if (!stateData) {
+                    errorResult = toolError(`stateFile not found or empty: ${params.stateFile}`);
+                    return errorResult;
+                }
+                // Status fields are stateFile-authoritative. Details are LLM-supplied when present
+                // (single-line, with root-cause context) and fall back to server-side extraction.
+                testMaintenance = params.testMaintenance.map((m) => {
+                    const recorded = stateData.existingTests?.find((t) => t.testFile.endsWith(m.fileName) || path.basename(t.testFile) === path.basename(m.fileName));
+                    const defaultBeforeStatus = MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const defaultAfterStatus = m.action === DriftAction.Delete ? TestExecutionStatus.Skipped
+                        : MAINTENANCE_CHANGE_ACTIONS.has(m.action) ? TestExecutionStatus.Unknown : TestExecutionStatus.Skipped;
+                    const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
+                    const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
+                    logger.info(`${m.fileName}: before=${beforeStatus} after=${afterStatus}`);
+                    return { ...m, beforeStatus, afterStatus };
+                });
+            }
+            catch (err) {
+                errorResult = toolError(`Failed to load stateFile for maintenance enrichment: ${err.message}`);
+                return errorResult;
+            }
+        }
         // Strip generation-artifact fields from newTestsCreated before writing.
         // scenarioFile, traceFile, frontendTrace are internal paths used during
         // generation — downstream scoring scripts don't expect them and fail if
         // they encounter these string fields while traversing the object.
         const sanitizedNewTests = dedupedNewTests.map(({ scenarioFile: _sf, traceFile: _tf, frontendTrace: _ft, ...rest }) => rest);
-        const reportJson = JSON.stringify({
+        const report = {
             businessCaseAnalysis: params.businessCaseAnalysis,
             newTestsCreated: sanitizedNewTests,
             additionalRecommendations: dedupedRecommendations,
-            testMaintenance: params.testMaintenance,
+            testMaintenance,
             testResults: params.testResults,
             issuesFound: params.issuesFound,
             nextSteps: params.nextSteps ?? [],
             commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim() || DEFAULT_COMMIT_MESSAGE,
-        }, null, 2);
+        };
+        const reportJson = JSON.stringify(report, null, 2);
         logger.info("Submitting testbot report", {
             outputFile: params.summaryOutputFile,
             payloadBytes: reportJson.length,
@@ -242,10 +433,7 @@ export function registerSubmitReportTool(server) {
             const elapsed = Date.now() - startTime;
             const errorMessage = `Failed to write report: ${error.message}`;
             logger.error(errorMessage, { error, elapsedMs: elapsed });
-            errorResult = {
-                content: [{ type: "text", text: errorMessage }],
-                isError: true,
-            };
+            errorResult = toolError(errorMessage);
             return errorResult;
         }
         finally {
@@ -253,7 +441,7 @@ export function registerSubmitReportTool(server) {
                 summary_output_file: params.summaryOutputFile,
                 testResultCount: String(params.testResults.length),
                 payloadBytes: String(reportJson.length),
-                ...computeReportMetrics(params),
+                ...computeReportMetrics({ ...params, testMaintenance }),
             }).catch(() => { });
         }
     });

package/build/tools/submitReportTool.test.d.ts

@@ -0,0 +1 @@
+export {};