@skyramp/mcp 0.3.7 → 0.3.9-rc.1

package/build/recommendation/planRanker.d.ts

@@ -4,8 +4,10 @@ import { ScenarioCategory } from "../types/TestRecommendation.js";
 export interface RankOptions {
     /**
      * Categories that take the top carve-out tier ahead of everything else.
-     * Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught only —
-     * new_endpoint is MEDIUM, not carved out). Exposed so phase 2 can tune the carve-out
+     * Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught and
+     * requirement_conflict — new_endpoint is MEDIUM, not carved out). The two CRITICAL
+     * categories are carved out independently, so a requirement conflict never competes
+     * with a code-review bug for one slot. Exposed so phase 2 can tune the carve-out
      * WITHOUT reintroducing the agent's priority tag as a ranking input.
      */
     carveOutCategories?: ScenarioCategory[];
@@ -32,7 +34,7 @@ export interface SelectPlanContext extends BudgetContext {
  * independent of input order (the final tiebreak is the stable `candidateId`).
  *
  * Ordering (highest first):
- *   1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
+ *   1. Carve-out — CRITICAL-category scenarios (bug_caught, requirement_conflict), preserving the
  *      protected-first convention of `roundRobinByType` /
  *      `prioritizeAttackSurfaceBundles`.
  *   2. Verified discriminators — candidates whose declared discriminator survived

package/build/recommendation/planRanker.js

@@ -14,7 +14,7 @@ const DEFAULT_CARVE_OUT_CATEGORIES = Object.keys(CATEGORY_PRIORITY).filter((cate
  * independent of input order (the final tiebreak is the stable `candidateId`).
  *
  * Ordering (highest first):
- *   1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
+ *   1. Carve-out — CRITICAL-category scenarios (bug_caught, requirement_conflict), preserving the
  *      protected-first convention of `roundRobinByType` /
  *      `prioritizeAttackSurfaceBundles`.
  *   2. Verified discriminators — candidates whose declared discriminator survived

package/build/services/containerEnv.d.ts

@@ -22,4 +22,4 @@ export declare function isValidEnvVarName(name: string): boolean;
 /**
  * Build the environment variable array for the Docker executor container.
  */
-export declare function buildContainerEnv(options: Pick<TestExecutionOptions, "token" | "language" | "useHostNetwork">, saveStoragePath?: string, hostEnv?: Record<string, string | undefined>, passthroughNames?: string[]): string[];
+export declare function buildContainerEnv(options: Pick<TestExecutionOptions, "token" | "language" | "useHostNetwork" | "rebaselineSnapshots">, saveStoragePath?: string, hostEnv?: Record<string, string | undefined>, passthroughNames?: string[]): string[];

package/build/services/containerEnv.js

@@ -77,6 +77,18 @@ export function buildContainerEnv(options, saveStoragePath, hostEnv = process.en
         ...(options.token ? [`SKYRAMP_TEST_TOKEN=${options.token}`] : []),
         "SKYRAMP_IN_DOCKER=true",
     ];
+    // Visual-snapshot baselines this run replaces (SKYR-4298). SmartPlaywright
+    // reads SKYRAMP_UPDATE_SNAPSHOTS and re-captures each named baseline through
+    // its first-run path instead of comparing. Set here, first-class, because the
+    // SKYRAMP_ prefix is reserved from workspace passthrough below — nothing else
+    // can put it in the container. Omitted entirely when nothing is to refresh, so
+    // the default stays "compare".
+    const rebaseline = (options.rebaselineSnapshots ?? [])
+        .map((n) => n.trim())
+        .filter(Boolean);
+    if (rebaseline.length > 0) {
+        env.push(`SKYRAMP_UPDATE_SNAPSHOTS=${rebaseline.join(",")}`);
+    }
     // Skyramp-generated tests are standalone HTTP tests that never need host repo
     // conftest.py files or pytest configuration. --noconftest prevents loading any
     // conftest in the test directory tree (avoids missing deps like boto3, django).

package/build/tools/executeSkyrampTestTool.d.ts

@@ -1,6 +1,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { TestExecutionResult } from "../types/TestExecution.js";
 import { TestType } from "../types/TestTypes.js";
+import { MaintenanceActionCore, TestAnalysisResult } from "../types/TestAnalysis.js";
 export declare const CONTRACT_EXECUTION_MODES: readonly ["provider", "consumer"];
 export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
 /**
@@ -20,6 +21,16 @@ export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractM
  * its own attachment line.
  */
 export declare function withVideoInfo(output: string, videoPath?: string): string;
+/**
+ * Normalize the `rebaselineSnapshots` request (SKYR-4298): dedupe and drop blanks,
+ * and refuse it on the pre-edit run. A refresh there would overwrite the very
+ * evidence that the baseline is stale, and `beforeStatus` would read Pass for a
+ * test that failed against the committed image.
+ */
+export declare function resolveRebaselineSnapshots(requested: string[] | undefined, phase: "before" | "after" | undefined): {
+    snapshots: string[];
+    error?: string;
+};
 /**
  * The failure text the agent receives. Everything it needs has to be in here:
  * only a tool's return value reaches the transcript, and Claude Code does not
@@ -37,5 +48,74 @@ export declare function withVideoInfo(output: string, videoPath?: string): strin
  * agent at authentication on a run whose output was empty, and then it keyed off
  * the bare number, which pointed it there on a test count or a line number.
  */
+/** The on-disk files a requested baseline name resolves to; empty when none matches. */
+export type BaselineFileState = Array<{
+    file: string;
+    size: number;
+    mtimeMs: number;
+}>;
+/**
+ * Snapshot of the requested baselines under `<spec>-snapshots/` (SKYR-4298): for
+ * each requested name, every PNG whose name matches the stem (a spec may hold both
+ * `<stem>-linux.png` and `<stem>-chromium-linux.png`; latching onto one of them would
+ * misreport the other) with its size and mtime. Taken before and after the run so
+ * the tool can tell the agent which baselines were actually rewritten — SmartPlaywright
+ * is the only party that knows the exact filename, and an executor image that lacks
+ * SKYRAMP_UPDATE_SNAPSHOTS (or a name matching no toHaveScreenshot call) leaves
+ * every file untouched.
+ */
+export declare function readBaselineState(specFile: string, requested: string[]): Record<string, BaselineFileState>;
+/**
+ * Which requested baselines changed on disk between two readBaselineState calls, and
+ * which files carried the change (the ones to stage).
+ */
+export declare function diffBaselineState(before: Record<string, BaselineFileState>, after: Record<string, BaselineFileState>): {
+    refreshed: string[];
+    notRefreshed: string[];
+    refreshedFiles: Record<string, string[]>;
+};
+/**
+ * Authorize a refresh against the persisted maintenance state (SKYR-4298). The
+ * parameter description tells the agent to pass exactly what skyramp_actions
+ * returned, but prose cannot tell an authorized list from an invented one, so the
+ * tool checks: an UPDATE verdict for this spec must list every requested name, and
+ * the spec's phase:"before" run must already be recorded — a refresh before that
+ * run would overwrite the very evidence that the baseline is stale, whatever
+ * `phase` says or omits. A newly generated test has no verdict and no committed
+ * baseline, so it can never be refreshed here.
+ */
+export declare function authorizeRebaseline(stateData: {
+    existingTests?: TestAnalysisResult[];
+    maintenanceVerdicts?: MaintenanceActionCore[];
+} | null | undefined, testFile: string, requested: string[]): {
+    error?: string;
+};
+/**
+ * Reconcile the persisted verdict with what the executor actually did (SKYR-4298).
+ * An executor image whose @skyramp/skyramp predates SKYRAMP_UPDATE_SNAPSHOTS rewrites
+ * nothing; left alone, the verdict would still promise a refresh, the report gate
+ * would refuse the report, and nothing in the prompt makes the agent's way out
+ * deterministic. So: names that were not refreshed are dropped from the verdict; a
+ * rebaseline-only UPDATE with nothing left becomes VERIFY (the test stays red, the
+ * rationale says why), and an UPDATE that also carried edits keeps UPDATE and is
+ * held to its edit. The report then reflects what happened, not what was asked.
+ */
+export declare function applyRefreshOutcomeToVerdicts(verdicts: MaintenanceActionCore[], testFile: string, outcome: {
+    refreshed: string[];
+    notRefreshed: string[];
+}, executorImage: string): {
+    verdicts: MaintenanceActionCore[];
+    note?: string;
+};
+/**
+ * The refresh outcome line appended to the execution result, on pass and on fail
+ * alike. Without it a stale-baseline run that could not refresh reads as a bare
+ * "Screenshot comparison failed", and the report gate then tells the agent to run
+ * with rebaselineSnapshots set — which it already did.
+ */
+export declare function describeRefreshOutcome(outcome: {
+    refreshed: string[];
+    notRefreshed: string[];
+}): string;
 export declare function buildExecutionFailureText(result: TestExecutionResult): string;
 export declare function registerExecuteSkyrampTestTool(server: McpServer): void;

package/build/tools/executeSkyrampTestTool.js

@@ -11,10 +11,14 @@ import { TestExecutionStatus, } from "../types/TestExecution.js";
 import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { StateManager, getTestsRepoDir, } from "../utils/AnalysisStateManager.js";
-import { TestSource } from "../types/TestAnalysis.js";
+import { DriftAction, TestSource } from "../types/TestAnalysis.js";
 import { logger } from "../utils/logger.js";
 import { toolError } from "../utils/utils.js";
 import { recordExecutionVideo } from "./execution-video-state.js";
+import { stageGeneratedPaths } from "../utils/gitStaging.js";
+import * as fs from "fs";
+import { baselineFileMatchesStem, baselineStem, rebaselineSnapshotNameSchema, snapshotDirFor, } from "../utils/rebaselineSnapshots.js";
+import { EXECUTOR_DOCKER_IMAGE } from "../utils/versions.js";
 const TOOL_NAME = "skyramp_execute_test";
 export const CONTRACT_EXECUTION_MODES = ["provider", "consumer"];
 /**
@@ -97,22 +101,172 @@ const HTTP_401_SHAPES = [
     /\bwhere\s+401\s*=/i,
 ];
 /**
- * The failure text the agent receives. Everything it needs has to be in here:
- * only a tool's return value reaches the transcript, and Claude Code does not
- * capture an MCP server's stderr, so anything this omits is unrecoverable once
- * the run ends.
- *
- * `output` alone was not enough. When the executor produced none, the message
- * read "Test execution failed:" and stopped — so the agent inferred a cause and
- * reported it as fact. Measured in eval run 32220788735: it blamed an
- * unreachable backend while the app was answering 200 on both the host and the
- * docker bridge, and that invented cause reached `issuesFound`.
- *
- * The 401 hint is emitted only when the output carries a 401 in a status-shaped
- * position — see HTTP_401_SHAPES. It used to be unconditional, which pointed the
- * agent at authentication on a run whose output was empty, and then it keyed off
- * the bare number, which pointed it there on a test count or a line number.
+ * Normalize the `rebaselineSnapshots` request (SKYR-4298): dedupe and drop blanks,
+ * and refuse it on the pre-edit run. A refresh there would overwrite the very
+ * evidence that the baseline is stale, and `beforeStatus` would read Pass for a
+ * test that failed against the committed image.
+ */
+export function resolveRebaselineSnapshots(requested, phase) {
+    const snapshots = [
+        ...new Set((requested ?? []).map((s) => s.trim()).filter(Boolean)),
+    ];
+    if (snapshots.length > 0 && phase === "before") {
+        return {
+            snapshots: [],
+            error: `rebaselineSnapshots cannot be combined with phase: "before". Run the pre-edit baseline without it to record the mismatch, then pass rebaselineSnapshots on the final run only.`,
+        };
+    }
+    return { snapshots };
+}
+/**
+ * Snapshot of the requested baselines under `<spec>-snapshots/` (SKYR-4298): for
+ * each requested name, every PNG whose name matches the stem (a spec may hold both
+ * `<stem>-linux.png` and `<stem>-chromium-linux.png`; latching onto one of them would
+ * misreport the other) with its size and mtime. Taken before and after the run so
+ * the tool can tell the agent which baselines were actually rewritten — SmartPlaywright
+ * is the only party that knows the exact filename, and an executor image that lacks
+ * SKYRAMP_UPDATE_SNAPSHOTS (or a name matching no toHaveScreenshot call) leaves
+ * every file untouched.
  */
+export function readBaselineState(specFile, requested) {
+    const dir = snapshotDirFor(specFile);
+    let entries = [];
+    try {
+        entries = fs.readdirSync(dir).sort();
+    }
+    catch {
+        entries = [];
+    }
+    const state = {};
+    for (const name of requested) {
+        const stem = baselineStem(name);
+        const files = [];
+        for (const file of entries.filter((e) => baselineFileMatchesStem(e, stem))) {
+            try {
+                const st = fs.statSync(path.join(dir, file));
+                files.push({ file, size: st.size, mtimeMs: st.mtimeMs });
+            }
+            catch {
+                // vanished between readdir and stat — treat as absent
+            }
+        }
+        state[name] = files;
+    }
+    return state;
+}
+/**
+ * Which requested baselines changed on disk between two readBaselineState calls, and
+ * which files carried the change (the ones to stage).
+ */
+export function diffBaselineState(before, after) {
+    const refreshed = [];
+    const notRefreshed = [];
+    const refreshedFiles = {};
+    for (const name of Object.keys(after)) {
+        const prev = new Map((before[name] ?? []).map((f) => [f.file, f]));
+        const changed = after[name]
+            .filter((a) => {
+            const b = prev.get(a.file);
+            return !b || a.size !== b.size || a.mtimeMs !== b.mtimeMs;
+        })
+            .map((a) => a.file);
+        if (changed.length > 0) {
+            refreshed.push(name);
+            refreshedFiles[name] = changed;
+        }
+        else {
+            notRefreshed.push(name);
+        }
+    }
+    return { refreshed, notRefreshed, refreshedFiles };
+}
+/**
+ * Authorize a refresh against the persisted maintenance state (SKYR-4298). The
+ * parameter description tells the agent to pass exactly what skyramp_actions
+ * returned, but prose cannot tell an authorized list from an invented one, so the
+ * tool checks: an UPDATE verdict for this spec must list every requested name, and
+ * the spec's phase:"before" run must already be recorded — a refresh before that
+ * run would overwrite the very evidence that the baseline is stale, whatever
+ * `phase` says or omits. A newly generated test has no verdict and no committed
+ * baseline, so it can never be refreshed here.
+ */
+export function authorizeRebaseline(stateData, testFile, requested) {
+    if (requested.length === 0)
+        return {};
+    const verdict = (stateData?.maintenanceVerdicts ?? []).find((v) => v.testFilePath === testFile && v.action === DriftAction.Update);
+    if (!verdict) {
+        return {
+            error: `rebaselineSnapshots refused: no UPDATE verdict for ${testFile} in the stateFile. Only a maintained test whose skyramp_analyze_test_health verdict listed the baselines can be refreshed — a newly generated test writes its own baseline on its first run and has nothing to refresh.`,
+        };
+    }
+    const authorized = new Set(verdict.rebaselineSnapshots ?? []);
+    const unauthorized = requested.filter((n) => !authorized.has(n));
+    if (unauthorized.length > 0) {
+        return {
+            error: `rebaselineSnapshots refused: ${unauthorized.join(", ")} not in the UPDATE verdict for ${path.basename(testFile)} (authorized: ${authorized.size ? [...authorized].join(", ") : "none"}). Pass exactly the rebaseline_snapshots skyramp_actions returned, using the name the test passes to toHaveScreenshot (page-001.png), not the on-disk file (page-001-chromium-linux.png).`,
+        };
+    }
+    const entry = (stateData?.existingTests ?? []).find((t) => t.testFile === testFile);
+    if (!entry?.executionBefore) {
+        return {
+            error: `rebaselineSnapshots refused: no phase: "before" execution is recorded for ${path.basename(testFile)}. Run the pre-edit baseline first (it is the evidence the baseline is stale), then pass rebaselineSnapshots on the final run.`,
+        };
+    }
+    return {};
+}
+/**
+ * Reconcile the persisted verdict with what the executor actually did (SKYR-4298).
+ * An executor image whose @skyramp/skyramp predates SKYRAMP_UPDATE_SNAPSHOTS rewrites
+ * nothing; left alone, the verdict would still promise a refresh, the report gate
+ * would refuse the report, and nothing in the prompt makes the agent's way out
+ * deterministic. So: names that were not refreshed are dropped from the verdict; a
+ * rebaseline-only UPDATE with nothing left becomes VERIFY (the test stays red, the
+ * rationale says why), and an UPDATE that also carried edits keeps UPDATE and is
+ * held to its edit. The report then reflects what happened, not what was asked.
+ */
+export function applyRefreshOutcomeToVerdicts(verdicts, testFile, outcome, executorImage) {
+    if (outcome.notRefreshed.length === 0)
+        return { verdicts };
+    let note;
+    const next = verdicts.map((v) => {
+        if (v.testFilePath !== testFile || v.action !== DriftAction.Update)
+            return v;
+        const remaining = (v.rebaselineSnapshots ?? []).filter((n) => !outcome.notRefreshed.includes(n));
+        const reason = `baseline refresh of ${outcome.notRefreshed.join(", ")} was not applied by ${executorImage} (its @skyramp/skyramp lacks SKYRAMP_UPDATE_SNAPSHOTS, or the name matches no toHaveScreenshot call)`;
+        if (remaining.length === 0 && v.rebaselineOnly) {
+            note = `Verdict for ${path.basename(testFile)} downgraded UPDATE → VERIFY: ${reason}. The test stays as it is; report it honestly.`;
+            const { rebaselineSnapshots: _dropped, rebaselineOnly: _only, ...rest } = v;
+            void _dropped;
+            void _only;
+            return { ...rest, action: DriftAction.Verify, rationale: `${v.rationale} [${reason}]` };
+        }
+        note = `Verdict for ${path.basename(testFile)}: ${reason}; the UPDATE is held to its spec/POM edit only.`;
+        const { rebaselineSnapshots: _dropped, ...rest } = v;
+        void _dropped;
+        return {
+            ...rest,
+            ...(remaining.length > 0 ? { rebaselineSnapshots: remaining } : {}),
+            rationale: `${v.rationale} [${reason}]`,
+        };
+    });
+    return { verdicts: next, note };
+}
+/**
+ * The refresh outcome line appended to the execution result, on pass and on fail
+ * alike. Without it a stale-baseline run that could not refresh reads as a bare
+ * "Screenshot comparison failed", and the report gate then tells the agent to run
+ * with rebaselineSnapshots set — which it already did.
+ */
+export function describeRefreshOutcome(outcome) {
+    const parts = [];
+    if (outcome.refreshed.length > 0) {
+        parts.push(`Visual baselines refreshed: ${outcome.refreshed.join(", ")}.`);
+    }
+    if (outcome.notRefreshed.length > 0) {
+        parts.push(`Visual baselines NOT refreshed: ${outcome.notRefreshed.join(", ")} — the executor image may lack SKYRAMP_UPDATE_SNAPSHOTS support (needs @skyramp/skyramp with SKYR-4298), or the name matches no toHaveScreenshot() call in this spec. Do not report these as refreshed.`);
+    }
+    return parts.join(" ");
+}
 export function buildExecutionFailureText(result) {
     const output = stripVTControlCharacters(result.output || "").trim();
     const facts = [`status=${result.status}`];
@@ -199,8 +353,14 @@ export function registerExecuteSkyrampTestTool(server) {
                 .describe("Path to state file from skyramp_analyze_changes. Always pass when available — results are written back so skyramp_submit_report can override before/afterStatus with ground-truth pass/fail."),
             phase: z
                 .enum(["before", "after"])
-                .optional()
+                .default("after")
                 .describe("Execution phase for maintained tests: 'before' captures pre-edit baseline; 'after' (default) records post-edit result."),
+            rebaselineSnapshots: z
+                .array(rebaselineSnapshotNameSchema)
+                .optional()
+                .describe("UI tests only. toHaveScreenshot() baseline filenames (e.g. [\"page-001.png\"]) this run REPLACES instead of comparing against, because the PR intentionally changed how the captured page/element/region looks (SKYR-4298). " +
+                "Pass exactly the list skyramp_actions returned as rebaseline_snapshots for this spec — the tool checks it against the persisted UPDATE verdict in stateFile (so stateFile is required) and refuses names the verdict did not authorize, a spec with no such verdict, or a spec whose phase: 'before' run has not been recorded yet. Use the name as the test passes it (page-001.png), not the on-disk file (page-001-chromium-linux.png). " +
+                "The refreshed PNGs land beside the spec and are delivered with the Testbot PR as image diffs; the result names which baselines were refreshed and which were not. Never use this to silence a screenshot mismatch the diff does not explain."),
             repository: z
                 .string()
                 .optional()
@@ -227,6 +387,24 @@ export function registerExecuteSkyrampTestTool(server) {
                 errorResult = toolError(`testFile must be an absolute path, got: ${params.testFile}`);
                 return errorResult;
             }
+            const rebaseline = resolveRebaselineSnapshots(params.rebaselineSnapshots, params.phase);
+            if (rebaseline.error) {
+                errorResult = toolError(rebaseline.error);
+                return errorResult;
+            }
+            const rebaselineSnapshots = rebaseline.snapshots;
+            if (rebaselineSnapshots.length > 0) {
+                if (!params.stateFile) {
+                    errorResult = toolError("rebaselineSnapshots requires stateFile: the refresh is authorized against the UPDATE verdict skyramp_actions persisted there.");
+                    return errorResult;
+                }
+                const authState = await StateManager.fromStatePath(params.stateFile).readRepoData(params.repository);
+                const auth = authorizeRebaseline(authState, params.testFile, rebaselineSnapshots);
+                if (auth.error) {
+                    errorResult = toolError(auth.error);
+                    return errorResult;
+                }
+            }
             // Deterministic external-test guard (SKYR-3924): this tool runs Skyramp-generated
             // tests in the executor and cannot run a repo's native (user-written) suite, so a
             // run on an external test only errors (e.g. pytest import/collection failure).
@@ -343,7 +521,13 @@ export function registerExecuteSkyrampTestTool(server) {
                 playwrightSaveStoragePath: params.playwrightSaveStoragePath,
                 dockerNetwork,
                 useHostNetwork: false,
+                ...(rebaselineSnapshots.length > 0 ? { rebaselineSnapshots } : {}),
             };
+            // Identity of the requested baselines before the run, to report afterwards
+            // which ones SmartPlaywright actually rewrote (SKYR-4298).
+            const baselinesBefore = rebaselineSnapshots.length > 0
+                ? readBaselineState(params.testFile, rebaselineSnapshots)
+                : {};
             let result;
             try {
                 result = await executionService.executeTest(execOptions, onExecutionProgress);
@@ -396,10 +580,53 @@ export function registerExecuteSkyrampTestTool(server) {
             // populate testResults[].videoPath — testbot uploads only the video
             // directories the report references, so an unrecorded video is never seen.
             await recordExecutionVideo(result, params.stateFile);
+            // Which requested baselines were actually rewritten (SKYR-4298). Reported on
+            // pass and fail alike; a refreshed PNG is a deliverable like a generated
+            // spec, so stage the spec's snapshot directory whenever one was rewritten —
+            // even on a failing run, since the report gate checks the PNG, not the
+            // status — so the eval harness commit and the artifact collector see it
+            // (production delivery adds the whole test directory anyway). No-op outside
+            // a testbot run, like every other stageGeneratedPaths call; never fails the
+            // execution.
+            let refreshOutcomeText = "";
+            if (rebaselineSnapshots.length > 0) {
+                const after = readBaselineState(params.testFile, rebaselineSnapshots);
+                const outcome = diffBaselineState(baselinesBefore, after);
+                refreshOutcomeText = describeRefreshOutcome(outcome);
+                // Stage the rewritten files themselves, never the directory: `git add` on
+                // the directory would ship anything else sitting there under one
+                // authorized baseline's authority.
+                for (const name of outcome.refreshed) {
+                    for (const file of outcome.refreshedFiles[name] ?? []) {
+                        try {
+                            await stageGeneratedPaths(path.join(snapshotDirFor(params.testFile), file));
+                        }
+                        catch (err) {
+                            logger.warning(`Could not stage refreshed visual baseline ${file} for ${params.testFile}: ${err.message}`);
+                        }
+                    }
+                }
+                if (outcome.notRefreshed.length > 0 && params.stateFile) {
+                    try {
+                        const stateManager = StateManager.fromStatePath(params.stateFile);
+                        const stateData = await stateManager.readRepoData(params.repository);
+                        if (stateData?.maintenanceVerdicts) {
+                            const reconciled = applyRefreshOutcomeToVerdicts(stateData.maintenanceVerdicts, params.testFile, outcome, EXECUTOR_DOCKER_IMAGE);
+                            await stateManager.writeRepoData({ ...stateData, maintenanceVerdicts: reconciled.verdicts }, { repo: params.repository });
+                            if (reconciled.note)
+                                refreshOutcomeText += ` ${reconciled.note}`;
+                        }
+                    }
+                    catch (err) {
+                        logger.warning(`Could not reconcile the maintenance verdict with the refresh outcome: ${err.message}`);
+                    }
+                }
+            }
+            const withRefreshOutcome = (text) => refreshOutcomeText ? `${text}\n\n${refreshOutcomeText}` : text;
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
             if (result.status !== TestExecutionStatus.Pass) {
-                errorResult = toolError(withVideoInfo(buildExecutionFailureText(result), result.videoPath));
+                errorResult = toolError(withRefreshOutcome(withVideoInfo(buildExecutionFailureText(result), result.videoPath)));
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
@@ -407,7 +634,7 @@ export function registerExecuteSkyrampTestTool(server) {
                 content: [
                     {
                         type: "text",
-                        text: withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath),
+                        text: withRefreshOutcome(withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath)),
                     },
                 ],
             };

package/build/tools/submitReportTool.d.ts

@@ -101,8 +101,8 @@ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     description: string;
     testType: TestType;
     category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
-    fileName: string;
     endpoint: string;
+    fileName: string;
     testId: string;
     reasoning: string;
     repository?: string | undefined;
@@ -125,8 +125,8 @@ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
 }, {
     description: string;
     testType: TestType;
-    fileName: string;
     endpoint: string;
+    fileName: string;
     testId: string;
     reasoning: string;
     repository?: string | undefined;
@@ -151,8 +151,8 @@ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     description: string;
     testType: TestType;
     category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
-    fileName: string;
     endpoint: string;
+    fileName: string;
     testId: string;
     reasoning: string;
     repository?: string | undefined;
@@ -175,8 +175,8 @@ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
 }, {
     description: string;
     testType: TestType;
-    fileName: string;
     endpoint: string;
+    fileName: string;
     testId: string;
     reasoning: string;
     repository?: string | undefined;
@@ -201,8 +201,8 @@ export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     description: string;
     testType: TestType;
     category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
-    fileName: string;
     endpoint: string;
+    fileName: string;
     testId: string;
     reasoning: string;
     repository?: string | undefined;

package/build/tools/submitReportTool.js

@@ -13,6 +13,7 @@ import { toolError, testFileMatches } from "../utils/utils.js";
 import { matchesApprovedPlan } from "../utils/planMatchKeys.js";
 import { isTestbotEnabled } from "../utils/featureFlags.js";
 import { findInvalidSourceCitations, findUnchangedFileClaims, listChangedFiles, listChangedFilesAcross, listChangedFilesAbs, } from "../utils/reportVerification.js";
+import { isPlanOnlyMode } from "../utils/planOnlyMode.js";
 import { getReportLanguage, isEnforcedReportLanguage, findLanguageViolations, findLanguageNearMisses, reportLanguageDisplayName, } from "../utils/reportLanguage.js";
 import { canonicalTestPath, findAssertionRecordByFileName, rederiveAssertionOutcome, } from "./code-refactor/assertion-state.js";
 import { rederiveReuseOutcome, reuseChainSkipped, samePath, } from "./code-refactor/reuse-state.js";
@@ -343,7 +344,8 @@ const issueFoundSchema = z
         .optional()
         .describe("Issue severity. critical = feature broken/unusable (e.g. page doesn't load, data corruption). " +
         "high = incorrect behavior (e.g. wrong calculation, stale data returned). " +
-        "medium = minor functional gap. low = cosmetic or informational."),
+        "medium = minor functional gap. low = cosmetic or informational. " +
+        "Floor: a requirement conflict — the PR title/description (or a requirements file it references) states a requirement the implemented behavior contradicts — is 'high' or 'critical', never 'medium' or below."),
     category: z
         .nativeEnum(IssueFoundCategory)
         .describe("Issue classification. bug = a product/code defect, e.g. found by a test or in the diff. " +
@@ -1019,7 +1021,11 @@ export function registerSubmitReportTool(server) {
             testType: v.testType,
             endpoint: v.endpoint,
             action: v.action,
-            description: v.rationale,
+            // A baseline refresh (SKYR-4298) leaves the spec untouched, so name the
+            // replaced PNGs here — the reviewer's cue to open the image diff in the PR.
+            description: v.rebaselineSnapshots?.length
+                ? `${v.rationale} (visual baseline refreshed: ${v.rebaselineSnapshots.join(", ")})`
+                : v.rationale,
             beforeDetails: "",
             afterDetails: "",
         }));
@@ -1109,6 +1115,9 @@ export function registerSubmitReportTool(server) {
                         newTests: dedupedNewTests,
                         verdicts: stateData.maintenanceVerdicts ?? [],
                         primaryRepository: fullState?.metadata?.repository,
+                        // SKYR-4250: captured at prompt-render time, since planOnly is a
+                        // prompt-only argument (mirrors the SKYR-4185 language capture).
+                        planOnly: isPlanOnlyMode(),
                     });
                     if (unbacked.length > 0) {
                         // Offer the attribution-correction path explicitly and show what DID change: