@skyramp/mcp 0.3.3 → 0.3.5

package/build/tools/submitReportTool.js

@@ -7,11 +7,13 @@ import { TEST_CATEGORIES, externalCategory } from "../types/TestRecommendation.j
 import { TestType, HttpMethod } from "../types/TestTypes.js";
 import { DriftAction } from "../types/TestAnalysis.js";
 import { TestExecutionStatus } from "../types/TestExecution.js";
-import { StateManager } from "../utils/AnalysisStateManager.js";
+import { IssueFoundCategory } from "../types/TestbotReport.js";
+import { StateManager, runArtifactDir, getTestsRepoDir } from "../utils/AnalysisStateManager.js";
 import { toolError, testFileMatches } from "../utils/utils.js";
 import { matchesApprovedPlan } from "../utils/planMatchKeys.js";
 import { isTestbotEnabled } from "../utils/featureFlags.js";
-import { findUnbackedClaims, listChangedFiles } from "../utils/reportVerification.js";
+import { findUnbackedClaims, listChangedFiles, listChangedFilesAcross } from "../utils/reportVerification.js";
+import { getReportLanguage, isEnforcedReportLanguage, findLanguageViolations, findLanguageNearMisses, reportLanguageDisplayName, } from "../utils/reportLanguage.js";
 import { rederiveReuseOutcome } from "./code-refactor/reuse-state.js";
 // SKYR-3879 Path B: which testTypes the register-plan checkpoint gates. Mirrors
 // the generation tools actually wired to planGuard (batch-scenario/integration,
@@ -19,13 +21,49 @@ import { rederiveReuseOutcome } from "./code-refactor/reuse-state.js";
 // NOT gated at generation time (see planGuard.ts wiring), so they are excluded
 // here too rather than surprising the agent with a report-time-only gate.
 const PLAN_GATED_TEST_TYPES = new Set([TestType.CONTRACT, TestType.INTEGRATION]);
-/** Split an `endpoint` field ("GET /api/v1/products") into method + path. */
+/**
+ * Filename of the report, written beside the state file. SKYR-4147: the report path is
+ * derived here rather than accepted as a parameter. The caller builds the state file and
+ * the report path from a single temp dir (testbot's `skyrampTempDir()`), so the state
+ * file's directory IS the directory the caller reads the report back from. Taking the
+ * path from the agent instead meant the agent had to retype it out of a 600-character
+ * encoded prompt URI; one wrong character wrote the report somewhere nobody reads, and
+ * the tool still reported success, throwing away the whole run without a word.
+ *
+ * Must stay in sync with testbot, which hardcodes the same string in main.ts, post.ts and
+ * eval-entry.ts — the same duplication across the two repos that already exists for
+ * `RUN_STATE_FILE_NAME` / testbot's `STATE_FILE_NAME`.
+ */
+export const REPORT_FILE_NAME = "testbot-result.txt";
+/**
+ * The names to match a report entry against the approved plan's `name:` keys.
+ * `testId` is documented as `<testType>-<scenario-slug>`, so the plan's bare
+ * scenarioName only matches once that prefix is dropped — but agents also submit
+ * the bare name (both forms appear in SKYR-4123's runs), so try each.
+ */
+function planNameCandidates(testId, testType) {
+    const id = (testId ?? "").trim();
+    if (!id)
+        return [];
+    const prefix = `${testType}-`;
+    return id.toLowerCase().startsWith(prefix) ? [id, id.slice(prefix.length)] : [id];
+}
+/**
+ * Split an `endpoint` field into one {method, path} per endpoint it names.
+ * A multi-step test writes them comma-separated ("POST /orders, DELETE
+ * /orders/{id}"); read as a single endpoint the whole string becomes one
+ * unmatchable path, so such an entry could match nothing (SKYR-4123).
+ */
 function parseEndpointField(endpoint) {
-    const trimmed = (endpoint ?? "").trim();
-    const spaceIdx = trimmed.indexOf(" ");
-    if (spaceIdx <= 0)
-        return { path: trimmed || undefined };
-    return { method: trimmed.slice(0, spaceIdx), path: trimmed.slice(spaceIdx + 1).trim() };
+    const parts = (endpoint ?? "").split(",").map((p) => p.trim()).filter(Boolean);
+    if (parts.length === 0)
+        return [{}];
+    return parts.map((part) => {
+        const spaceIdx = part.indexOf(" ");
+        if (spaceIdx <= 0)
+            return { path: part };
+        return { method: part.slice(0, spaceIdx), path: part.slice(spaceIdx + 1).trim() };
+    });
 }
 // Drift actions that actually modify a test file. VERIFY and IGNORE are
 // no-ops (the test was assessed but left unchanged), so they must not count
@@ -53,12 +91,22 @@ function normalizeRepository(item) {
     const trimmed = item.repository?.trim();
     return trimmed ? { ...item, repository: trimmed } : { ...item, repository: undefined };
 }
+// videoPath is deliberately absent from this input contract: it is attached server-side
+// from the run's execution records (see attachVideoPath), and zod strips any the model
+// supplies anyway. SKYR-4156 is what happens when the agent owns that field instead.
 const testResultSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test. Do not include priority or other metadata in this field."),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
     details: z.string().describe("One sentence — no embedded newlines, no markdown. e.g. '10.8s, products_contract_test.py' or 'failed: <one-line error summary>, products_contract_test.py'"),
-    videoPath: z.string().optional().describe("Path to video recording (UI/E2E tests only)"),
+    // Required for every row: each one reports a specific test file the agent ran, so it
+    // can always name it. It is what identifies the row server-side — `endpoint` cannot,
+    // since several tests routinely exercise one endpoint — and for ui/e2e it is what
+    // attaches the recorded video (SKYR-4156). Not included in the report itself.
+    testFilePath: z
+        .string()
+        .refine((p) => path.isAbsolute(p), { message: "testFilePath must be an absolute path" })
+        .describe("Absolute path of the test file this result is for — the same path you passed to skyramp_execute_test's testFile param. Consumers basename it for display."),
     repository: repositoryField,
 });
 // Structured grounding for UI recommendations. Lifted verbatim from a
@@ -85,11 +133,39 @@ export const pageContextSchema = z.object({
     url: z.string().describe("URL of the page where the test runs. Lifted from BlueprintCapture.url."),
     pageHash: z.string().optional().describe("Opaque hash of the captured page state (BlueprintCapture.pageHash). Lets the verifier confirm the recommendation was grounded in a still-current capture."),
 });
+/**
+ * SKYR-4193: LLMs habitually emit every key a schema declares, using a
+ * literal null for "not applicable". A null is stripped (normalized to
+ * omitted) before validation wherever null carries no meaning, instead of
+ * costing the agent a rejection/rewrite round-trip (run 32182635458 lost
+ * ~2m40s to exactly this):
+ * - `pageContext: null` is always stripped — the field is optional but never
+ *   nullable, so null is never a valid value, UI or not.
+ * - `targetElements: null` is stripped only on non-UI entries — on UI tests
+ *   null is load-bearing ("blueprint capture failed", requires the
+ *   '[no-blueprint-data]' markers) and must reach the refinements intact.
+ * Non-null values on non-UI entries still reach the refinements and are
+ * still rejected.
+ */
+function stripNullGroundingFields(val) {
+    if (!val || typeof val !== "object" || Array.isArray(val))
+        return val;
+    const entry = val;
+    const stripTargetElements = entry.targetElements === null && entry.testType !== TestType.UI;
+    if (!stripTargetElements && entry.pageContext !== null)
+        return val;
+    const copy = { ...entry };
+    if (stripTargetElements)
+        delete copy.targetElements;
+    if (copy.pageContext === null)
+        delete copy.pageContext;
+    return copy;
+}
 // TODO: Unify newTestSchema and additionalRecommendationSchema into a single
 // interface that adds an `implemented: boolean` field. Both describe the same
 // concept (a test recommendation) — the only difference is whether it was
 // generated in this run or left for later. Tracked per Archit's review comment.
-export const newTestSchema = z.object({
+export const newTestSchema = z.preprocess(stripNullGroundingFields, z.object({
     testId: z.string().describe("Human-readable kebab-case identifier, e.g. 'contract-get-products' or 'integration-users-orders-workflow'. Format: '<testType>-<method>-<resource>' for single-endpoint tests or '<testType>-<scenario-slug>' for multi-step tests. Must be unique within the report."),
     testType: z.nativeEnum(TestType).describe("Type of test created. Do not include priority or other metadata in this field."),
     category: z.preprocess((val) => externalCategory(val), z.enum(TEST_CATEGORIES)).describe("Test category — critical categories (security_boundary, business_rule, data_integrity, breaking_change) get generation priority over workflow"),
@@ -166,7 +242,7 @@ export const newTestSchema = z.object({
             }
         }
     }
-});
+}));
 const issueFoundSchema = z.object({
     description: z.string().describe("One-line description. Do NOT prefix with the severity level — severity is a separate field. Include code logic bugs from the diff, test generation/execution failures, and environment misconfiguration."),
     severity: z
@@ -175,6 +251,13 @@ const issueFoundSchema = z.object({
         .describe("Issue severity. critical = feature broken/unusable (e.g. page doesn't load, data corruption). " +
         "high = incorrect behavior (e.g. wrong calculation, stale data returned). " +
         "medium = minor functional gap. low = cosmetic or informational."),
+    category: z
+        .nativeEnum(IssueFoundCategory)
+        .describe("Issue classification. bug = a product/code defect, e.g. found by a test or in the diff. " +
+        "lint = a linter or formatter finding (eslint, flake8, prettier). " +
+        "type = a type-check failure (tsc, mypy). " +
+        "config = environment or tooling misconfiguration (wrong workspace auth type, missing env var, setup command failure). " +
+        "The report renders lint/type/config entries in a separate 'Configuration Errors' section so product bugs stay prominent under 'Issues Found'."),
     repository: repositoryField,
 });
 const scenarioStepSchema = z.object({
@@ -185,7 +268,7 @@ const scenarioStepSchema = z.object({
     requestBody: z.record(z.any()).optional().describe("Example request body with realistic field values"),
     responseBody: z.record(z.any()).optional().describe("Key response fields to verify, e.g. { id: 'number', name: 'string', in_stock: 'boolean?' }"),
 });
-export const additionalRecommendationSchema = z.object({
+export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFields, z.object({
     testId: z.string().describe("Human-readable kebab-case identifier, e.g. 'integration-products-orders-workflow' or 'e2e-checkout-flow'. Format: '<testType>-<scenario-slug>'. Must be unique within the report."),
     testType: z.nativeEnum(TestType).describe("Type of test. Do not include priority or other metadata in this field."),
     category: z.preprocess((val) => externalCategory(val), z.enum(TEST_CATEGORIES)).describe("Test category — critical categories get generation priority over workflow"),
@@ -261,7 +344,7 @@ export const additionalRecommendationSchema = z.object({
             }
         }
     }
-});
+}));
 // TODO(multi-repo maintenance): no `repository` field yet — see readData() TODO below.
 const testMaintenanceSchema = z.object({
     testType: z.nativeEnum(TestType).describe("Type of test."),
@@ -366,6 +449,20 @@ async function attachReuseOutcome(test, outcomes) {
     const reuse = await rederiveReuseOutcome(found);
     return reuse ? { ...test, reuse } : test;
 }
+/**
+ * Attach the video recorded for this execution, matched by the row's testFilePath
+ * basename — the key skyramp_execute_test records under.
+ *
+ * Server-derived, never taken from the LLM, and the recorded value always wins: any
+ * videoPath the model supplied is discarded. That is the whole point of SKYR-4156 —
+ * the field used to be the agent's job, so it decayed to nothing on every row and
+ * testbot (which uploads only the video directories the report references) shipped
+ * no recordings at all. A row with no recorded execution gets no video, which
+ * consumers already treat as "no recording for this test".
+ */
+function attachVideoPath(row, videos) {
+    return { ...row, videoPath: videos?.[path.basename(row.testFilePath)]?.videoPath };
+}
 function deduplicateById(items) {
     const seen = new Set();
     const result = [];
@@ -387,16 +484,13 @@ export function registerSubmitReportTool(server) {
     server.registerTool(TOOL_NAME, {
         annotations: {
             readOnlyHint: false,
-            destructiveHint: true, // overwrites summaryOutputFile on disk
+            destructiveHint: true, // overwrites the report file on disk
             idempotentHint: false,
             openWorldHint: false,
         },
         description: "Submit the final testbot report. Call this tool once after completing all test analysis, generation, and execution. " +
             "This is the ONLY way to submit the report — do NOT write the report to a file manually.",
         inputSchema: {
-            summaryOutputFile: z
-                .string()
-                .describe("The file path where the report should be written (provided in the task instructions)"),
             businessCaseAnalysis: z
                 .string()
                 .min(1, "businessCaseAnalysis is required — describe what the PR does, even for no-test runs (config-only, non-application, Testbot onboarding). It is never optional and must not be empty.")
@@ -440,8 +534,9 @@ export function registerSubmitReportTool(server) {
                 .string()
                 .describe("Path to the state file returned by skyramp_analyze_changes. Required — " +
                 "contains execution results for maintenance scoring (existingTests), " +
-                "discovered endpoints for coverage validation (newEndpoints), and " +
-                "UI page context for frontend test grounding (uiContext)."),
+                "discovered endpoints for coverage validation (newEndpoints), " +
+                "UI page context for frontend test grounding (uiContext), and the " +
+                "videos recorded for each executed browser test (executionVideos)."),
         },
         _meta: {
             keywords: ["report", "summary", "testbot", "submit"],
@@ -449,6 +544,98 @@ export function registerSubmitReportTool(server) {
     }, async (params) => {
         const startTime = Date.now();
         let errorResult;
+        // Residual-leak telemetry (SKYR-4185): filled by the language guardrail
+        // below when a report language is enforced, spread into the analytics
+        // event in the finally. Empty (no properties) for non-enforced runs.
+        const languageTelemetry = {};
+        // The report goes next to the state file, so in a Testbot run the state file has to
+        // be in the run directory — that is where the action reads the report back from.
+        // Two ways that breaks, both ending in a report nobody reads while this tool says
+        // it succeeded, which is the failure SKYR-4147 fixed: no run directory at all (the
+        // state file then falls back to a temp path of its own), or a readable but stale
+        // state file from somewhere else. Reject both rather than write. Checked only in a
+        // Testbot run; local and IDE use have no run directory and no consumer waiting.
+        if (isTestbotEnabled()) {
+            const runDir = runArtifactDir();
+            if (!runDir) {
+                return toolError("Cannot write the report: this is a Testbot run but RUNNER_TEMP is not set, so " +
+                    "there is no run directory to write it to. The Testbot action reads the report " +
+                    "from $RUNNER_TEMP/skyramp. Set RUNNER_TEMP for the run. Retrying or changing " +
+                    "the report will not help — this is an environment problem.");
+            }
+            const stateDir = path.dirname(params.stateFile);
+            if (path.resolve(stateDir) !== path.resolve(runDir)) {
+                return toolError(`Cannot write the report: stateFile is in ${stateDir}, but this run's directory ` +
+                    `is ${runDir}. The report is written next to the state file and the Testbot ` +
+                    `action reads it from the run directory, so writing it would lose it. Pass the ` +
+                    `stateFile from this run's skyramp_analyze_changes output — not a path from an ` +
+                    `earlier run or one typed by hand.`);
+            }
+        }
+        // SKYR-4185: report-language guardrail. The language was captured when the
+        // testbot prompt was rendered (see reportLanguage.ts) — the instruction to
+        // write free-text fields in it is prose the agent sometimes ignores
+        // (letsramp/api-insight#215 shipped English testResults[].details in an
+        // otherwise-Japanese report). Reject rather than ship a half-translated
+        // report; the listed paths tell the agent exactly what to rewrite. Fields
+        // the agent does not author here (testMaintenance description from
+        // skyramp_actions verdicts, the server default commitMessage) are exempt —
+        // rejecting those would loop, since resubmitting cannot change them.
+        const reportLanguage = getReportLanguage();
+        if (reportLanguage && !isEnforcedReportLanguage(reportLanguage)) {
+            logger.warning(`Report language '${reportLanguage}' has no enforcement rules — skipping the language guardrail`);
+        }
+        if (isEnforcedReportLanguage(reportLanguage)) {
+            const textFields = [
+                { path: "businessCaseAnalysis", text: params.businessCaseAnalysis },
+            ];
+            params.newTestsCreated.forEach((t, i) => {
+                textFields.push({ path: `newTestsCreated[${i}].description`, text: t.description });
+                textFields.push({ path: `newTestsCreated[${i}].reasoning`, text: t.reasoning });
+            });
+            (params.additionalRecommendations ?? []).forEach((r, i) => {
+                textFields.push({ path: `additionalRecommendations[${i}].description`, text: r.description });
+                textFields.push({ path: `additionalRecommendations[${i}].reasoning`, text: r.reasoning });
+                r.steps.forEach((s, j) => {
+                    textFields.push({ path: `additionalRecommendations[${i}].steps[${j}].description`, text: s.description });
+                });
+            });
+            params.testResults.forEach((t, i) => {
+                textFields.push({ path: `testResults[${i}].details`, text: t.details });
+            });
+            params.issuesFound.forEach((f, i) => {
+                textFields.push({ path: `issuesFound[${i}].description`, text: f.description });
+            });
+            (params.nextSteps ?? []).forEach((s, i) => {
+                textFields.push({ path: `nextSteps[${i}]`, text: s });
+            });
+            (params.testMaintenanceDetails ?? []).forEach((d, i) => {
+                textFields.push({ path: `testMaintenanceDetails[${i}].beforeDetails`, text: d.beforeDetails });
+                textFields.push({ path: `testMaintenanceDetails[${i}].afterDetails`, text: d.afterDetails });
+            });
+            if (params.commitMessage && params.commitMessage !== DEFAULT_COMMIT_MESSAGE) {
+                textFields.push({ path: "commitMessage", text: params.commitMessage });
+            }
+            const languageViolations = findLanguageViolations(textFields, reportLanguage);
+            if (languageViolations.length > 0) {
+                const languageName = reportLanguageDisplayName(reportLanguage);
+                errorResult = toolError(`This run's report language is ${languageName}, but ${languageViolations.length} free-text field(s) are written in English:\n` +
+                    languageViolations.map((v) => `  - ${v}`).join("\n") +
+                    `\nRewrite ONLY the listed fields in ${languageName} and resubmit the report. ` +
+                    `Keep code identifiers, endpoint paths, file names, test IDs, enum values ` +
+                    `(Pass/Fail/Skipped, severity values, testType), and anything inside backticks untranslated.`);
+                return errorResult;
+            }
+            // The sub-threshold band (a single stray English word) is accepted by
+            // design; count what ships so the leniency's real-world leak rate is
+            // visible in Amplitude before anyone tightens the threshold.
+            const nearMisses = findLanguageNearMisses(textFields, reportLanguage);
+            languageTelemetry.reportLanguage = reportLanguage;
+            languageTelemetry.languageNearMissCount = String(nearMisses.length);
+            if (nearMisses.length > 0) {
+                logger.info(`Report language ${reportLanguage}: accepting ${nearMisses.length} field(s) with a single stray English word`, { fields: nearMisses });
+            }
+        }
         const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
         const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
         const stateManager = StateManager.fromStatePath(params.stateFile);
@@ -483,17 +670,31 @@ export function registerSubmitReportTool(server) {
         // is non-empty), so a generation the gate permitted is never rejected at
         // report time. Absent approvedPlan (register tool never ran / never
         // pre-seeded anything) → no check, back-compat.
+        //
+        // SKYR-4123: the name must be part of the query for that parity to hold.
+        // Every generation tool passes scenarioName; this side did not, so the plan
+        // item's `name:` key was dead here and a test the gate had already let
+        // through was rejected while being listed as approved in the same message.
         if (stateData.approvedPlan) {
             const approvedPlan = stateData.approvedPlan;
             const unapproved = dedupedNewTests.filter((t) => {
                 if (!PLAN_GATED_TEST_TYPES.has(t.testType))
                     return false;
-                const { method, path } = parseEndpointField(t.endpoint);
-                return !matchesApprovedPlan(approvedPlan, { testType: t.testType, method, path });
+                const endpoints = parseEndpointField(t.endpoint);
+                const names = planNameCandidates(t.testId, t.testType);
+                return !(names.length > 0 ? names : [undefined]).some((scenarioName) => endpoints.some(({ method, path }) => matchesApprovedPlan(approvedPlan, { scenarioName, testType: t.testType, method, path })));
             });
             if (unapproved.length > 0) {
                 const approvedList = approvedPlan.generate.length > 0
-                    ? approvedPlan.generate.map((item) => `[${item.testType}] ${item.scenarioName}`).join(", ")
+                    // Show each item's endpoint keys: without them a rejection that turns on
+                    // the endpoint reads as self-contradicting, because the entry's own name
+                    // is printed in this same list (SKYR-4123).
+                    ? approvedPlan.generate
+                        .map((item) => {
+                        const eps = item.matchKeys.filter((k) => k.startsWith("ep:"));
+                        return `[${item.testType}] ${item.scenarioName}${eps.length > 0 ? ` covering ${eps.join(", ")}` : ""}`;
+                    })
+                        .join("; ")
                     : "(none)";
                 errorResult = toolError(`${unapproved.length} newTestsCreated entr${unapproved.length === 1 ? "y" : "ies"} not in the approved plan from ` +
                     `skyramp_register_test_plan (plan ${approvedPlan.planId}) — neither its GENERATE list nor its ADDITIONAL backfill pool: ` +
@@ -578,7 +779,23 @@ export function registerSubmitReportTool(server) {
             const repoRoot = fullState?.metadata?.repositoryPath;
             if (repoRoot && repoRoot !== "unknown") {
                 try {
-                    const changedFiles = await listChangedFiles(repoRoot);
+                    // A run writes tests into up to three kinds of trees: the primary
+                    // checkout, the tests-repo checkout (testRepoPath), and related
+                    // repos. Scanning only the primary rejected a real, executed UI
+                    // test in the tests-repo checkout as "unbacked", forcing the agent
+                    // to demote it to additionalRecommendations (SKYR-4204 reopen,
+                    // run 32510028428).
+                    // The primary scan keeps its throwing form: an unscannable primary
+                    // degrades the whole check to a pass (SKYR-3883 semantics) via the
+                    // catch below. The extra trees are best-effort — a missing or
+                    // non-git one contributes nothing.
+                    const changedFiles = [
+                        ...(await listChangedFiles(repoRoot)),
+                        ...(await listChangedFilesAcross([
+                            getTestsRepoDir(),
+                            ...Object.values(fullState?.relatedRepos ?? {}).map((section) => section.repositoryPath),
+                        ])),
+                    ];
                     const unbacked = findUnbackedClaims({
                         repoRoot,
                         changedFiles,
@@ -633,30 +850,39 @@ export function registerSubmitReportTool(server) {
                 ...row,
                 fileName: path.basename(testFilePath),
             })),
-            testResults: params.testResults.map(normalizeRepository),
+            // videoPath is filled from the run's execution records; testFilePath is the
+            // match-only key and is stripped from the wire format, the same line drawn for
+            // testMaintenance's own testFilePath above (downstream scoring scripts traverse
+            // these objects and don't expect it).
+            testResults: params.testResults.map((row) => {
+                const { testFilePath: _tfp, ...wire } = attachVideoPath(normalizeRepository(row), stateData.executionVideos);
+                return wire;
+            }),
             issuesFound: params.issuesFound.map(normalizeRepository),
             nextSteps: params.nextSteps ?? [],
             commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim() || DEFAULT_COMMIT_MESSAGE,
         };
         const reportJson = JSON.stringify(report, null, 2);
+        // Beside the state file, which was read successfully above — so this directory is
+        // known to exist and no mkdir is needed. See REPORT_FILE_NAME.
+        const reportPath = path.join(path.dirname(params.stateFile), REPORT_FILE_NAME);
         logger.info("Submitting testbot report", {
-            outputFile: params.summaryOutputFile,
+            outputFile: reportPath,
             payloadBytes: reportJson.length,
             testResultCount: params.testResults.length,
         });
         try {
-            await fs.mkdir(path.dirname(params.summaryOutputFile), { recursive: true });
-            await fs.writeFile(params.summaryOutputFile, reportJson, "utf-8");
+            await fs.writeFile(reportPath, reportJson, "utf-8");
             const elapsed = Date.now() - startTime;
             logger.info("Testbot report written successfully", {
-                outputFile: params.summaryOutputFile,
+                outputFile: reportPath,
                 elapsedMs: elapsed,
             });
             return {
                 content: [
                     {
                         type: "text",
-                        text: `Report submitted successfully to ${params.summaryOutputFile}`,
+                        text: `Report submitted successfully to ${reportPath}`,
                     },
                 ],
             };
@@ -670,10 +896,11 @@ export function registerSubmitReportTool(server) {
         }
         finally {
             AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
-                summary_output_file: params.summaryOutputFile,
+                summary_output_file: reportPath,
                 testResultCount: String(params.testResults.length),
                 payloadBytes: String(reportJson.length),
                 ...computeReportMetrics({ ...params, testMaintenance }),
+                ...languageTelemetry,
             }).catch(() => { });
         }
     });

package/build/tools/test-management/actionsTool.js

@@ -6,7 +6,7 @@ import { TestType } from "../../types/TestTypes.js";
 import * as fs from "fs";
 import * as path from "path";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
-import { toolError } from "../../utils/utils.js";
+import { dualChannelResult, toolError } from "../../utils/utils.js";
 import { isTestFile } from "../../prompts/test-recommendation/scopeAssessment.js";
 import { buildRenameStrategy, buildFileRenameStrategy, buildUpdateStrategy, buildRegenerateStrategy, buildDeleteStrategy, buildUpdateFileInstruction, buildRegenerateFileInstruction, } from "../../prompts/test-maintenance/actionsInstructions.js";
 /**
@@ -136,6 +136,16 @@ export function registerActionsTool(server) {
             openWorldHint: true,
         },
         description: `Execute test maintenance actions — final step of the unified Test Health Analysis Flow. Call this once per run even when no tests need action (pass \`recommendations: []\`) — it is where the maintenance verdicts are persisted before the run moves on. Supply only the tests that need an action; any discovered test you omit from \`recommendations\` is treated as needing no action and recorded automatically, so you never enumerate the no-action tests yourself.`,
+        // SKYR-4194: the payload is returned via structuredContent, which requires a
+        // declared outputSchema — see dualChannelResult.
+        outputSchema: {
+            summary: z
+                .string()
+                .describe("Human-readable report of the maintenance actions to take."),
+            instructions: z
+                .string()
+                .describe("JSON instructions for carrying out those actions — file lists, per-file edit context, and the update/regenerate/delete strategies. Read and follow this field."),
+        },
         inputSchema: actionsSchema,
     }, async (args) => {
         let errorResult;
@@ -166,6 +176,7 @@ export function registerActionsTool(server) {
             const externalTestFiles = new Set(testAnalysisResults
                 .filter((t) => t.source === TestSource.External)
                 .map((t) => t.testFile));
+            const catalogByFile = new Map(testAnalysisResults.map((t) => [t.testFile, t]));
             // ── Build recommendations from LLM-supplied drift assessment ──
             // The LLM performs the drift assessment in context after skyramp_analyze_test_health
             // and passes results here directly — analyzeTestHealthTool never writes assessment
@@ -403,8 +414,7 @@ export function registerActionsTool(server) {
             const testFilesToDelete = deleteRecommendations.map((rec) => rec.testFilePath);
             const fileInstructions = [];
             const testFilesToUpdate = [];
-            const testFileContentMap = new Map();
-            // ── UPDATE: read file, emit targeted edit instructions ──
+            // ── UPDATE: emit targeted edit instructions ──
             for (const rec of updateRecommendations) {
                 if (!rec.testFilePath) {
                     logger.warning("Recommendation missing testFilePath", rec);
@@ -414,11 +424,12 @@ export function registerActionsTool(server) {
                 // rather than the spec. testFilePath (the spec) remains in the report.
                 const editTarget = rec.pomFile ?? rec.testFilePath;
                 testFilesToUpdate.push(editTarget);
-                try {
-                    testFileContentMap.set(editTarget, fs.readFileSync(editTarget, "utf-8"));
-                }
-                catch (error) {
-                    logger.error(`Failed to read ${rec.pomFile ? "page object" : "test"} file ${editTarget}: ${error.message}`);
+                // Existence check only — the agent reads the file itself before editing.
+                // Skips this file's instruction block exactly as the readFileSync failure did
+                // before. The path still reaches files_to_update and update_context; that was
+                // true before this change too.
+                if (!fs.existsSync(editTarget)) {
+                    logger.error(`Missing ${rec.pomFile ? "page object" : "test"} file: ${editTarget}`);
                     continue;
                 }
                 const renames = rec.renamedEndpoints || [];
@@ -440,22 +451,15 @@ export function registerActionsTool(server) {
                     rationale: rec.rationale,
                 }));
             }
-            // ── REGENERATE: read file for context, emit overwrite instructions ──
+            // ── REGENERATE: emit overwrite instructions ──
             const regenerateInstructions = [];
             const testFilesToRegenerate = [];
-            const regenerateContentMap = new Map();
             for (const rec of regenerateRecommendations) {
                 if (!rec.testFilePath) {
                     logger.warning("Recommendation missing testFilePath", rec);
                     continue;
                 }
                 testFilesToRegenerate.push(rec.testFilePath);
-                try {
-                    regenerateContentMap.set(rec.testFilePath, fs.readFileSync(rec.testFilePath, "utf-8"));
-                }
-                catch (error) {
-                    logger.warning(`Could not read file for REGENERATE context ${rec.testFilePath}: ${error.message}`);
-                }
                 regenerateInstructions.push(buildRegenerateFileInstruction({
                     testFile: rec.testFilePath,
                     updateInstructions: rec.updateInstructions,
@@ -533,12 +537,10 @@ export function registerActionsTool(server) {
                 .filter((rec) => !!rec.updateInstructions)
                 .map((rec) => {
                 const editTarget = rec.pomFile ?? rec.testFilePath;
-                const current_content = testFileContentMap.get(editTarget);
                 return {
                     file: editTarget,
                     context: rec.updateInstructions,
                     ...(rec.editSites?.length ? { edit_sites: rec.editSites } : {}),
-                    ...(current_content !== undefined && { current_content }),
                 };
             });
             if (updateContext.length > 0) {
@@ -549,8 +551,22 @@ export function registerActionsTool(server) {
             // endpoint URL, auth pattern, test type, and language to replicate.
             if (regenerateRecommendations.length > 0) {
                 llmInstructionsObj.regenerate_context = regenerateRecommendations.map((rec) => {
-                    const existing_content = regenerateContentMap.get(rec.testFilePath);
-                    return { file: rec.testFilePath, rationale: rec.updateInstructions || rec.rationale, ...(existing_content !== undefined && { existing_content }) };
+                    // The generation tool needs test type, language, framework and endpoint to
+                    // replicate the file. Those come from the discovery catalog, so the file
+                    // text is not needed to carry them.
+                    const cataloged = catalogByFile.get(rec.testFilePath);
+                    return {
+                        file: rec.testFilePath,
+                        rationale: rec.updateInstructions || rec.rationale,
+                        // test_type and endpoint are always emitted — the strategy tells the agent to
+                        // use them instead of reading the file, so an absent one leaves the generation
+                        // call without a target. The catalog's apiEndpoint is optional and can be an
+                        // empty string; both fields are required on the recommendation itself.
+                        test_type: cataloged?.testType ?? rec.testType,
+                        endpoint: cataloged?.apiEndpoint || rec.endpoint,
+                        ...(cataloged?.language && { language: cataloged.language }),
+                        ...(cataloged?.framework && { framework: cataloged.framework }),
+                    };
                 });
                 llmInstructionsObj.regenerate_strategy = buildRegenerateStrategy();
             }
@@ -558,25 +574,10 @@ export function registerActionsTool(server) {
                 llmInstructionsObj.delete_strategy = buildDeleteStrategy();
             }
             const llmInstructions = JSON.stringify(llmInstructionsObj, null, 2);
-            const contentBlocks = [
-                {
-                    type: "text",
-                    text: responseText,
-                    annotations: {
-                        audience: ["user"],
-                    },
-                },
-                {
-                    type: "text",
-                    text: llmInstructions,
-                    annotations: {
-                        audience: ["assistant"],
-                    },
-                },
-            ];
-            return {
-                content: contentBlocks,
-            };
+            // The two payloads were previously separate content[] blocks annotated
+            // audience: ["user"] and ["assistant"]. Nothing in this repo read those
+            // annotations, and the field names carry the same distinction.
+            return dualChannelResult({ summary: responseText, instructions: llmInstructions });
         }
         catch (error) {
             logger.error(`Actions tool failed: ${error.message}`, error);

package/build/tools/test-management/analyzeChangesTool.d.ts

@@ -1,5 +1,6 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
+import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
 import { CandidateUiPage } from "../../utils/uiPageEnumerator.js";
 import type { FrontendFileIntegration } from "../../types/FrontendIntegration.js";
 import { TraceFile } from "../../types/RepositoryAnalysis.js";
@@ -34,4 +35,14 @@ export declare function buildCaptureInstructions(pages: CandidateUiPage[]): stri
  * (`buildCaptureInstructions` still runs for the rest in that case).
  */
 export declare function buildUnintegratedInstructions(results: FrontendFileIntegration[]): string;
+/**
+ * Assemble the tool result. See `dualChannelResult` for why the payload is
+ * carried on both result channels.
+ */
+export declare function buildAnalyzeChangesResult(parts: {
+    structuredSummary: string;
+    uiInstructions: string;
+    outputText: string;
+    recommendationPrompt: string;
+}): CallToolResult;
 export declare function registerAnalyzeChangesTool(server: McpServer): void;