@skyramp/mcp 0.3.8 → 0.4.0-rc.1

package/build/tools/executeSkyrampTestTool.d.ts

@@ -1,6 +1,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { TestExecutionResult } from "../types/TestExecution.js";
 import { TestType } from "../types/TestTypes.js";
+import { MaintenanceActionCore, TestAnalysisResult } from "../types/TestAnalysis.js";
 export declare const CONTRACT_EXECUTION_MODES: readonly ["provider", "consumer"];
 export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
 /**
@@ -20,6 +21,16 @@ export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractM
  * its own attachment line.
  */
 export declare function withVideoInfo(output: string, videoPath?: string): string;
+/**
+ * Normalize the `rebaselineSnapshots` request (SKYR-4298): dedupe and drop blanks,
+ * and refuse it on the pre-edit run. A refresh there would overwrite the very
+ * evidence that the baseline is stale, and `beforeStatus` would read Pass for a
+ * test that failed against the committed image.
+ */
+export declare function resolveRebaselineSnapshots(requested: string[] | undefined, phase: "before" | "after" | undefined): {
+    snapshots: string[];
+    error?: string;
+};
 /**
  * The failure text the agent receives. Everything it needs has to be in here:
  * only a tool's return value reaches the transcript, and Claude Code does not
@@ -37,5 +48,74 @@ export declare function withVideoInfo(output: string, videoPath?: string): strin
  * agent at authentication on a run whose output was empty, and then it keyed off
  * the bare number, which pointed it there on a test count or a line number.
  */
+/** The on-disk files a requested baseline name resolves to; empty when none matches. */
+export type BaselineFileState = Array<{
+    file: string;
+    size: number;
+    mtimeMs: number;
+}>;
+/**
+ * Snapshot of the requested baselines under `<spec>-snapshots/` (SKYR-4298): for
+ * each requested name, every PNG whose name matches the stem (a spec may hold both
+ * `<stem>-linux.png` and `<stem>-chromium-linux.png`; latching onto one of them would
+ * misreport the other) with its size and mtime. Taken before and after the run so
+ * the tool can tell the agent which baselines were actually rewritten — SmartPlaywright
+ * is the only party that knows the exact filename, and an executor image that lacks
+ * SKYRAMP_UPDATE_SNAPSHOTS (or a name matching no toHaveScreenshot call) leaves
+ * every file untouched.
+ */
+export declare function readBaselineState(specFile: string, requested: string[]): Record<string, BaselineFileState>;
+/**
+ * Which requested baselines changed on disk between two readBaselineState calls, and
+ * which files carried the change (the ones to stage).
+ */
+export declare function diffBaselineState(before: Record<string, BaselineFileState>, after: Record<string, BaselineFileState>): {
+    refreshed: string[];
+    notRefreshed: string[];
+    refreshedFiles: Record<string, string[]>;
+};
+/**
+ * Authorize a refresh against the persisted maintenance state (SKYR-4298). The
+ * parameter description tells the agent to pass exactly what skyramp_actions
+ * returned, but prose cannot tell an authorized list from an invented one, so the
+ * tool checks: an UPDATE verdict for this spec must list every requested name, and
+ * the spec's phase:"before" run must already be recorded — a refresh before that
+ * run would overwrite the very evidence that the baseline is stale, whatever
+ * `phase` says or omits. A newly generated test has no verdict and no committed
+ * baseline, so it can never be refreshed here.
+ */
+export declare function authorizeRebaseline(stateData: {
+    existingTests?: TestAnalysisResult[];
+    maintenanceVerdicts?: MaintenanceActionCore[];
+} | null | undefined, testFile: string, requested: string[]): {
+    error?: string;
+};
+/**
+ * Reconcile the persisted verdict with what the executor actually did (SKYR-4298).
+ * An executor image whose @skyramp/skyramp predates SKYRAMP_UPDATE_SNAPSHOTS rewrites
+ * nothing; left alone, the verdict would still promise a refresh, the report gate
+ * would refuse the report, and nothing in the prompt makes the agent's way out
+ * deterministic. So: names that were not refreshed are dropped from the verdict; a
+ * rebaseline-only UPDATE with nothing left becomes VERIFY (the test stays red, the
+ * rationale says why), and an UPDATE that also carried edits keeps UPDATE and is
+ * held to its edit. The report then reflects what happened, not what was asked.
+ */
+export declare function applyRefreshOutcomeToVerdicts(verdicts: MaintenanceActionCore[], testFile: string, outcome: {
+    refreshed: string[];
+    notRefreshed: string[];
+}, executorImage: string): {
+    verdicts: MaintenanceActionCore[];
+    note?: string;
+};
+/**
+ * The refresh outcome line appended to the execution result, on pass and on fail
+ * alike. Without it a stale-baseline run that could not refresh reads as a bare
+ * "Screenshot comparison failed", and the report gate then tells the agent to run
+ * with rebaselineSnapshots set — which it already did.
+ */
+export declare function describeRefreshOutcome(outcome: {
+    refreshed: string[];
+    notRefreshed: string[];
+}): string;
 export declare function buildExecutionFailureText(result: TestExecutionResult): string;
 export declare function registerExecuteSkyrampTestTool(server: McpServer): void;

package/build/tools/executeSkyrampTestTool.js

@@ -11,10 +11,14 @@ import { TestExecutionStatus, } from "../types/TestExecution.js";
 import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { StateManager, getTestsRepoDir, } from "../utils/AnalysisStateManager.js";
-import { TestSource } from "../types/TestAnalysis.js";
+import { DriftAction, TestSource } from "../types/TestAnalysis.js";
 import { logger } from "../utils/logger.js";
 import { toolError } from "../utils/utils.js";
 import { recordExecutionVideo } from "./execution-video-state.js";
+import { stageGeneratedPaths } from "../utils/gitStaging.js";
+import * as fs from "fs";
+import { baselineFileMatchesStem, baselineStem, rebaselineSnapshotNameSchema, snapshotDirFor, } from "../utils/rebaselineSnapshots.js";
+import { EXECUTOR_DOCKER_IMAGE } from "../utils/versions.js";
 const TOOL_NAME = "skyramp_execute_test";
 export const CONTRACT_EXECUTION_MODES = ["provider", "consumer"];
 /**
@@ -97,22 +101,172 @@ const HTTP_401_SHAPES = [
     /\bwhere\s+401\s*=/i,
 ];
 /**
- * The failure text the agent receives. Everything it needs has to be in here:
- * only a tool's return value reaches the transcript, and Claude Code does not
- * capture an MCP server's stderr, so anything this omits is unrecoverable once
- * the run ends.
- *
- * `output` alone was not enough. When the executor produced none, the message
- * read "Test execution failed:" and stopped — so the agent inferred a cause and
- * reported it as fact. Measured in eval run 32220788735: it blamed an
- * unreachable backend while the app was answering 200 on both the host and the
- * docker bridge, and that invented cause reached `issuesFound`.
- *
- * The 401 hint is emitted only when the output carries a 401 in a status-shaped
- * position — see HTTP_401_SHAPES. It used to be unconditional, which pointed the
- * agent at authentication on a run whose output was empty, and then it keyed off
- * the bare number, which pointed it there on a test count or a line number.
+ * Normalize the `rebaselineSnapshots` request (SKYR-4298): dedupe and drop blanks,
+ * and refuse it on the pre-edit run. A refresh there would overwrite the very
+ * evidence that the baseline is stale, and `beforeStatus` would read Pass for a
+ * test that failed against the committed image.
+ */
+export function resolveRebaselineSnapshots(requested, phase) {
+    const snapshots = [
+        ...new Set((requested ?? []).map((s) => s.trim()).filter(Boolean)),
+    ];
+    if (snapshots.length > 0 && phase === "before") {
+        return {
+            snapshots: [],
+            error: `rebaselineSnapshots cannot be combined with phase: "before". Run the pre-edit baseline without it to record the mismatch, then pass rebaselineSnapshots on the final run only.`,
+        };
+    }
+    return { snapshots };
+}
+/**
+ * Snapshot of the requested baselines under `<spec>-snapshots/` (SKYR-4298): for
+ * each requested name, every PNG whose name matches the stem (a spec may hold both
+ * `<stem>-linux.png` and `<stem>-chromium-linux.png`; latching onto one of them would
+ * misreport the other) with its size and mtime. Taken before and after the run so
+ * the tool can tell the agent which baselines were actually rewritten — SmartPlaywright
+ * is the only party that knows the exact filename, and an executor image that lacks
+ * SKYRAMP_UPDATE_SNAPSHOTS (or a name matching no toHaveScreenshot call) leaves
+ * every file untouched.
  */
+export function readBaselineState(specFile, requested) {
+    const dir = snapshotDirFor(specFile);
+    let entries = [];
+    try {
+        entries = fs.readdirSync(dir).sort();
+    }
+    catch {
+        entries = [];
+    }
+    const state = {};
+    for (const name of requested) {
+        const stem = baselineStem(name);
+        const files = [];
+        for (const file of entries.filter((e) => baselineFileMatchesStem(e, stem))) {
+            try {
+                const st = fs.statSync(path.join(dir, file));
+                files.push({ file, size: st.size, mtimeMs: st.mtimeMs });
+            }
+            catch {
+                // vanished between readdir and stat — treat as absent
+            }
+        }
+        state[name] = files;
+    }
+    return state;
+}
+/**
+ * Which requested baselines changed on disk between two readBaselineState calls, and
+ * which files carried the change (the ones to stage).
+ */
+export function diffBaselineState(before, after) {
+    const refreshed = [];
+    const notRefreshed = [];
+    const refreshedFiles = {};
+    for (const name of Object.keys(after)) {
+        const prev = new Map((before[name] ?? []).map((f) => [f.file, f]));
+        const changed = after[name]
+            .filter((a) => {
+            const b = prev.get(a.file);
+            return !b || a.size !== b.size || a.mtimeMs !== b.mtimeMs;
+        })
+            .map((a) => a.file);
+        if (changed.length > 0) {
+            refreshed.push(name);
+            refreshedFiles[name] = changed;
+        }
+        else {
+            notRefreshed.push(name);
+        }
+    }
+    return { refreshed, notRefreshed, refreshedFiles };
+}
+/**
+ * Authorize a refresh against the persisted maintenance state (SKYR-4298). The
+ * parameter description tells the agent to pass exactly what skyramp_actions
+ * returned, but prose cannot tell an authorized list from an invented one, so the
+ * tool checks: an UPDATE verdict for this spec must list every requested name, and
+ * the spec's phase:"before" run must already be recorded — a refresh before that
+ * run would overwrite the very evidence that the baseline is stale, whatever
+ * `phase` says or omits. A newly generated test has no verdict and no committed
+ * baseline, so it can never be refreshed here.
+ */
+export function authorizeRebaseline(stateData, testFile, requested) {
+    if (requested.length === 0)
+        return {};
+    const verdict = (stateData?.maintenanceVerdicts ?? []).find((v) => v.testFilePath === testFile && v.action === DriftAction.Update);
+    if (!verdict) {
+        return {
+            error: `rebaselineSnapshots refused: no UPDATE verdict for ${testFile} in the stateFile. Only a maintained test whose skyramp_analyze_test_health verdict listed the baselines can be refreshed — a newly generated test writes its own baseline on its first run and has nothing to refresh.`,
+        };
+    }
+    const authorized = new Set(verdict.rebaselineSnapshots ?? []);
+    const unauthorized = requested.filter((n) => !authorized.has(n));
+    if (unauthorized.length > 0) {
+        return {
+            error: `rebaselineSnapshots refused: ${unauthorized.join(", ")} not in the UPDATE verdict for ${path.basename(testFile)} (authorized: ${authorized.size ? [...authorized].join(", ") : "none"}). Pass exactly the rebaseline_snapshots skyramp_actions returned, using the name the test passes to toHaveScreenshot (page-001.png), not the on-disk file (page-001-chromium-linux.png).`,
+        };
+    }
+    const entry = (stateData?.existingTests ?? []).find((t) => t.testFile === testFile);
+    if (!entry?.executionBefore) {
+        return {
+            error: `rebaselineSnapshots refused: no phase: "before" execution is recorded for ${path.basename(testFile)}. Run the pre-edit baseline first (it is the evidence the baseline is stale), then pass rebaselineSnapshots on the final run.`,
+        };
+    }
+    return {};
+}
+/**
+ * Reconcile the persisted verdict with what the executor actually did (SKYR-4298).
+ * An executor image whose @skyramp/skyramp predates SKYRAMP_UPDATE_SNAPSHOTS rewrites
+ * nothing; left alone, the verdict would still promise a refresh, the report gate
+ * would refuse the report, and nothing in the prompt makes the agent's way out
+ * deterministic. So: names that were not refreshed are dropped from the verdict; a
+ * rebaseline-only UPDATE with nothing left becomes VERIFY (the test stays red, the
+ * rationale says why), and an UPDATE that also carried edits keeps UPDATE and is
+ * held to its edit. The report then reflects what happened, not what was asked.
+ */
+export function applyRefreshOutcomeToVerdicts(verdicts, testFile, outcome, executorImage) {
+    if (outcome.notRefreshed.length === 0)
+        return { verdicts };
+    let note;
+    const next = verdicts.map((v) => {
+        if (v.testFilePath !== testFile || v.action !== DriftAction.Update)
+            return v;
+        const remaining = (v.rebaselineSnapshots ?? []).filter((n) => !outcome.notRefreshed.includes(n));
+        const reason = `baseline refresh of ${outcome.notRefreshed.join(", ")} was not applied by ${executorImage} (its @skyramp/skyramp lacks SKYRAMP_UPDATE_SNAPSHOTS, or the name matches no toHaveScreenshot call)`;
+        if (remaining.length === 0 && v.rebaselineOnly) {
+            note = `Verdict for ${path.basename(testFile)} downgraded UPDATE → VERIFY: ${reason}. The test stays as it is; report it honestly.`;
+            const { rebaselineSnapshots: _dropped, rebaselineOnly: _only, ...rest } = v;
+            void _dropped;
+            void _only;
+            return { ...rest, action: DriftAction.Verify, rationale: `${v.rationale} [${reason}]` };
+        }
+        note = `Verdict for ${path.basename(testFile)}: ${reason}; the UPDATE is held to its spec/POM edit only.`;
+        const { rebaselineSnapshots: _dropped, ...rest } = v;
+        void _dropped;
+        return {
+            ...rest,
+            ...(remaining.length > 0 ? { rebaselineSnapshots: remaining } : {}),
+            rationale: `${v.rationale} [${reason}]`,
+        };
+    });
+    return { verdicts: next, note };
+}
+/**
+ * The refresh outcome line appended to the execution result, on pass and on fail
+ * alike. Without it a stale-baseline run that could not refresh reads as a bare
+ * "Screenshot comparison failed", and the report gate then tells the agent to run
+ * with rebaselineSnapshots set — which it already did.
+ */
+export function describeRefreshOutcome(outcome) {
+    const parts = [];
+    if (outcome.refreshed.length > 0) {
+        parts.push(`Visual baselines refreshed: ${outcome.refreshed.join(", ")}.`);
+    }
+    if (outcome.notRefreshed.length > 0) {
+        parts.push(`Visual baselines NOT refreshed: ${outcome.notRefreshed.join(", ")} — the executor image may lack SKYRAMP_UPDATE_SNAPSHOTS support (needs @skyramp/skyramp with SKYR-4298), or the name matches no toHaveScreenshot() call in this spec. Do not report these as refreshed.`);
+    }
+    return parts.join(" ");
+}
 export function buildExecutionFailureText(result) {
     const output = stripVTControlCharacters(result.output || "").trim();
     const facts = [`status=${result.status}`];
@@ -199,8 +353,14 @@ export function registerExecuteSkyrampTestTool(server) {
                 .describe("Path to state file from skyramp_analyze_changes. Always pass when available — results are written back so skyramp_submit_report can override before/afterStatus with ground-truth pass/fail."),
             phase: z
                 .enum(["before", "after"])
-                .optional()
+                .default("after")
                 .describe("Execution phase for maintained tests: 'before' captures pre-edit baseline; 'after' (default) records post-edit result."),
+            rebaselineSnapshots: z
+                .array(rebaselineSnapshotNameSchema)
+                .optional()
+                .describe("UI tests only. toHaveScreenshot() baseline filenames (e.g. [\"page-001.png\"]) this run REPLACES instead of comparing against, because the PR intentionally changed how the captured page/element/region looks (SKYR-4298). " +
+                "Pass exactly the list skyramp_actions returned as rebaseline_snapshots for this spec — the tool checks it against the persisted UPDATE verdict in stateFile (so stateFile is required) and refuses names the verdict did not authorize, a spec with no such verdict, or a spec whose phase: 'before' run has not been recorded yet. Use the name as the test passes it (page-001.png), not the on-disk file (page-001-chromium-linux.png). " +
+                "The refreshed PNGs land beside the spec and are delivered with the Testbot PR as image diffs; the result names which baselines were refreshed and which were not. Never use this to silence a screenshot mismatch the diff does not explain."),
             repository: z
                 .string()
                 .optional()
@@ -227,6 +387,24 @@ export function registerExecuteSkyrampTestTool(server) {
                 errorResult = toolError(`testFile must be an absolute path, got: ${params.testFile}`);
                 return errorResult;
             }
+            const rebaseline = resolveRebaselineSnapshots(params.rebaselineSnapshots, params.phase);
+            if (rebaseline.error) {
+                errorResult = toolError(rebaseline.error);
+                return errorResult;
+            }
+            const rebaselineSnapshots = rebaseline.snapshots;
+            if (rebaselineSnapshots.length > 0) {
+                if (!params.stateFile) {
+                    errorResult = toolError("rebaselineSnapshots requires stateFile: the refresh is authorized against the UPDATE verdict skyramp_actions persisted there.");
+                    return errorResult;
+                }
+                const authState = await StateManager.fromStatePath(params.stateFile).readRepoData(params.repository);
+                const auth = authorizeRebaseline(authState, params.testFile, rebaselineSnapshots);
+                if (auth.error) {
+                    errorResult = toolError(auth.error);
+                    return errorResult;
+                }
+            }
             // Deterministic external-test guard (SKYR-3924): this tool runs Skyramp-generated
             // tests in the executor and cannot run a repo's native (user-written) suite, so a
             // run on an external test only errors (e.g. pytest import/collection failure).
@@ -343,7 +521,13 @@ export function registerExecuteSkyrampTestTool(server) {
                 playwrightSaveStoragePath: params.playwrightSaveStoragePath,
                 dockerNetwork,
                 useHostNetwork: false,
+                ...(rebaselineSnapshots.length > 0 ? { rebaselineSnapshots } : {}),
             };
+            // Identity of the requested baselines before the run, to report afterwards
+            // which ones SmartPlaywright actually rewrote (SKYR-4298).
+            const baselinesBefore = rebaselineSnapshots.length > 0
+                ? readBaselineState(params.testFile, rebaselineSnapshots)
+                : {};
             let result;
             try {
                 result = await executionService.executeTest(execOptions, onExecutionProgress);
@@ -396,10 +580,53 @@ export function registerExecuteSkyrampTestTool(server) {
             // populate testResults[].videoPath — testbot uploads only the video
             // directories the report references, so an unrecorded video is never seen.
             await recordExecutionVideo(result, params.stateFile);
+            // Which requested baselines were actually rewritten (SKYR-4298). Reported on
+            // pass and fail alike; a refreshed PNG is a deliverable like a generated
+            // spec, so stage the spec's snapshot directory whenever one was rewritten —
+            // even on a failing run, since the report gate checks the PNG, not the
+            // status — so the eval harness commit and the artifact collector see it
+            // (production delivery adds the whole test directory anyway). No-op outside
+            // a testbot run, like every other stageGeneratedPaths call; never fails the
+            // execution.
+            let refreshOutcomeText = "";
+            if (rebaselineSnapshots.length > 0) {
+                const after = readBaselineState(params.testFile, rebaselineSnapshots);
+                const outcome = diffBaselineState(baselinesBefore, after);
+                refreshOutcomeText = describeRefreshOutcome(outcome);
+                // Stage the rewritten files themselves, never the directory: `git add` on
+                // the directory would ship anything else sitting there under one
+                // authorized baseline's authority.
+                for (const name of outcome.refreshed) {
+                    for (const file of outcome.refreshedFiles[name] ?? []) {
+                        try {
+                            await stageGeneratedPaths(path.join(snapshotDirFor(params.testFile), file));
+                        }
+                        catch (err) {
+                            logger.warning(`Could not stage refreshed visual baseline ${file} for ${params.testFile}: ${err.message}`);
+                        }
+                    }
+                }
+                if (outcome.notRefreshed.length > 0 && params.stateFile) {
+                    try {
+                        const stateManager = StateManager.fromStatePath(params.stateFile);
+                        const stateData = await stateManager.readRepoData(params.repository);
+                        if (stateData?.maintenanceVerdicts) {
+                            const reconciled = applyRefreshOutcomeToVerdicts(stateData.maintenanceVerdicts, params.testFile, outcome, EXECUTOR_DOCKER_IMAGE);
+                            await stateManager.writeRepoData({ ...stateData, maintenanceVerdicts: reconciled.verdicts }, { repo: params.repository });
+                            if (reconciled.note)
+                                refreshOutcomeText += ` ${reconciled.note}`;
+                        }
+                    }
+                    catch (err) {
+                        logger.warning(`Could not reconcile the maintenance verdict with the refresh outcome: ${err.message}`);
+                    }
+                }
+            }
+            const withRefreshOutcome = (text) => refreshOutcomeText ? `${text}\n\n${refreshOutcomeText}` : text;
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
             if (result.status !== TestExecutionStatus.Pass) {
-                errorResult = toolError(withVideoInfo(buildExecutionFailureText(result), result.videoPath));
+                errorResult = toolError(withRefreshOutcome(withVideoInfo(buildExecutionFailureText(result), result.videoPath)));
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
@@ -407,7 +634,7 @@ export function registerExecuteSkyrampTestTool(server) {
                 content: [
                     {
                         type: "text",
-                        text: withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath),
+                        text: withRefreshOutcome(withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath)),
                     },
                 ],
             };

package/build/tools/generate-tests/generateBatchScenarioRestTool.js

@@ -490,6 +490,12 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
                 + chainLint.warnings.map((w) => `  - ${w}`).join("\n")
             : "";
         try {
+            // The contract and integration generators create the test directory through
+            // the SDK, so a batch scenario that is the FIRST call into a new directory
+            // wrote into one that was not there and returned ENOENT. Same shape as
+            // batchMockGenerationTool.ts, and inside the try so a mkdir failure reads
+            // like a write failure.
+            fs.mkdirSync(path.dirname(filePath), { recursive: true });
             fs.writeFileSync(filePath, JSON.stringify(traceRequests, null, 2), "utf8");
         }
         catch (error) {

package/build/tools/generate-tests/generateContractRestTool.js

@@ -34,8 +34,8 @@ const baseContractTestSchema = {
     scenarioName: z
         .string()
         .optional()
-        .describe("The EXACT scenarioName of this test from the approved plan returned by skyramp_register_test_plan. " +
-        "In a testbot run, ALWAYS pass it verbatim from the plan's GENERATE list — it is the primary key the " +
+        .describe("The EXACT scenarioName of this test as you registered it with skyramp_register_test_plan. " +
+        "In a testbot run, ALWAYS pass it verbatim from your registered plan — it is the primary key the " +
         "plan gate matches on, so generation is admitted even when the endpoint path drifts from the plan's. " +
         "Omit in standalone/IDE usage (no registered plan)."),
     ...baseTestSchema,
@@ -223,7 +223,7 @@ ${this.buildSampleDataSection(params)}
 ${step5}
 
 **What NOT to do — any of these is a violation:**
-- Do NOT change function signatures, method names, class names, imports, or variable names.
+- Do NOT change function signatures, method names, class names, or variable names. Add an import only when the code you add needs it.
 - Do NOT add, remove, or reorder any functions, classes, or test cases.
 - Do NOT change assertion logic, HTTP methods, URLs, headers, or status code checks.
 - Do NOT reformat, reorder, or rewrite any code outside the identified JSON/dict literals.

package/build/tools/generate-tests/planGuard.d.ts

@@ -7,7 +7,7 @@ export type PlanGuardQuery = PlanMatchQuery;
 export { pathFromEndpointURL } from "../../utils/urlPath.js";
 /**
  * Returns null when the generation call is allowed, or a human-readable error
- * message (for `toolError`) when it must be rejected because it doesn't match
- * any item in the approved plan's GENERATE list.
+ * message (for `toolError`) when it must be rejected because it matches no
+ * planned test in the registered plan.
  */
 export declare function guardScenarioApproved(query: PlanGuardQuery): Promise<string | null>;

package/build/tools/generate-tests/planGuard.js

@@ -1,11 +1,11 @@
 /**
- * SKYR-3879 Path B — hard gate wired into the generation tools' handlers.
- * Conditional on an ACTIVE testbot run carrying an `approvedPlan`: standalone
- * / IDE MCP usage (no active run) and runs that never pre-seeded or
- * registered a plan are both unaffected (return null = allowed).
+ * Hard gate wired into the generation tools' handlers.
+ *
+ * Conditional on an ACTIVE run whose agent has registered a plan: standalone use
+ * and runs with no plan are unaffected. The report gate refuses a missing plan.
  */
 import { resolveRunStatePath, StateManager } from "../../utils/AnalysisStateManager.js";
-import { matchesApprovedPlan } from "../../utils/planMatchKeys.js";
+import { matchesPlanCandidates, nameMatchedEndpointMismatches, nameMatchedTypeMismatches, } from "../../utils/planMatchKeys.js";
 import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
 /** Re-exported for the generation tools that already import it from here.
  *  The implementation moved to utils/urlPath.ts (SKYR-4127) once workspace
@@ -14,8 +14,8 @@ import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
 export { pathFromEndpointURL } from "../../utils/urlPath.js";
 /**
  * Returns null when the generation call is allowed, or a human-readable error
- * message (for `toolError`) when it must be rejected because it doesn't match
- * any item in the approved plan's GENERATE list.
+ * message (for `toolError`) when it must be rejected because it matches no
+ * planned test in the registered plan.
  */
 export async function guardScenarioApproved(query) {
     const statePath = resolveRunStatePath();
@@ -29,18 +29,78 @@ export async function guardScenarioApproved(query) {
         // Unreadable/corrupt state must never block generation on an infra hiccup.
         return null;
     }
-    const approvedPlan = stateData?.approvedPlan;
-    if (!approvedPlan)
+    const plan = stateData?.plan;
+    if (!plan)
         return null;
-    // GENERATE items, plus ADDITIONAL as the documented backfill pool — but only
-    // when there's a non-empty GENERATE list to backfill (see matchesApprovedPlan).
-    if (matchesApprovedPlan(approvedPlan, query))
+    // The plan is read off disk, so its declared array type holds only on the
+    // validated path. An unreadable `plannedTests` is no evidence a scenario went
+    // unplanned, so it ALLOWS, as the report gate does; becoming `[]` refused every
+    // call. An EMPTY array is not that state — it says no test was warranted.
+    if (!Array.isArray(plan.plannedTests))
         return null;
-    const outstanding = approvedPlan.generate.length > 0
-        ? approvedPlan.generate.map((item) => `  - [${item.testType}] ${item.scenarioName}`).join("\n")
+    const plannedTests = plan.plannedTests;
+    if (matchesPlanCandidates(plannedTests, query))
+        return null;
+    // A planned test whose NAME and testType match is not an unplanned scenario — it is
+    // the planned one, called against an endpoint its steps do not name. That case
+    // gets its own message naming the step that disagreed.
+    const nearMisses = nameMatchedEndpointMismatches(plannedTests, query);
+    if (nearMisses.length > 0) {
+        // The plannedTestId is printed with the name because two planned tests can share
+        // a scenarioName, and a missing method or path is printed as such: the
+        // disagreement is then a field the plan left empty, not an endpoint the
+        // agent invented, and the message must not say otherwise.
+        const plannedSteps = nearMisses
+            .flatMap((item) => {
+            const steps = Array.isArray(item?.scenario?.steps) ? item.scenario.steps : [];
+            return steps
+                .filter((step) => step?.method || step?.path)
+                .map((step) => `  - [${item?.scenario?.testType ?? "?"}] ${item?.scenario?.scenarioName ?? "(unnamed)"} ` +
+                `(${item?.plannedTestId}): ${step?.method ?? "(no method)"} ${step?.path ?? "(no path)"}`);
+        })
+            .join("\n");
+        const head = `Scenario "${query.scenarioName}" IS in the plan you registered with ${TOOL_REGISTER_TEST_PLAN} ` +
+            `(registration ${plan.registrationNumber}), but `;
+        const called = `You called: ${query.method || "any method"} ${query.path}\n`;
+        // No step at all is a different refusal: there is nothing to retry
+        // generation against, so asking for that would send the agent in a circle.
+        if (!plannedSteps) {
+            return (head +
+                `it holds no step, so there is nothing to check this endpoint against.\n\n` +
+                called +
+                `Call ${TOOL_REGISTER_TEST_PLAN} again with the step this scenario tests, then retry generation.`);
+        }
+        return (head +
+            `no step of it agrees with the endpoint of this call.\n\n` +
+            called +
+            `The plan's steps for it:\n${plannedSteps}\n\n` +
+            `The two have to agree on the method and the path. Retry generation against the step the plan holds, or call ` +
+            `${TOOL_REGISTER_TEST_PLAN} again with the step corrected to the endpoint you mean to test.`);
+    }
+    // The plan holds this scenario under a DIFFERENT test type, so "not in the plan"
+    // is the same wrong message the endpoint near miss fixed. Neither the step nor
+    // the planned test is wrong: the two sides disagree about the KIND of test.
+    const typeMisses = nameMatchedTypeMismatches(plannedTests, query);
+    if (typeMisses.length > 0) {
+        // The plannedTestId is printed with the type because two planned tests can share
+        // a scenarioName, so the agent can see which entry to change.
+        const plannedTypes = typeMisses
+            .map((item) => `  - [${item?.scenario?.testType ?? "?"}] ${item?.plannedTestId}`)
+            .join("\n");
+        return (`Scenario "${query.scenarioName}" IS in the plan you registered with ${TOOL_REGISTER_TEST_PLAN} ` +
+            `(registration ${plan.registrationNumber}), under a different test type.\n\n` +
+            `The plan holds it as:\n${plannedTypes}\n\n` +
+            `You called the ${query.testType ?? "(untyped)"} generator.\n\n` +
+            `Either call the generator for the type the plan holds, or call ${TOOL_REGISTER_TEST_PLAN} again with the type you mean and retry. ` +
+            `Changing the plan is only the right move when the plan has the type wrong.`);
+    }
+    const outstanding = plannedTests.length > 0
+        ? plannedTests
+            .map((item) => `  - [${item?.scenario?.testType ?? "?"}] ${item?.scenario?.scenarioName ?? item?.plannedTestId}`)
+            .join("\n")
         : "  (none)";
-    return (`This scenario is not in the approved plan from ${TOOL_REGISTER_TEST_PLAN} (plan ${approvedPlan.planId}) — neither its GENERATE nor its ADDITIONAL list.\n\n` +
-        `Outstanding approved GENERATE items:\n${outstanding}\n\n` +
-        `Call ${TOOL_REGISTER_TEST_PLAN} first with this candidate included in \`candidates\`, then retry generation using the ` +
-        `exact scenarioName/testType/endpoint from the returned GENERATE list.`);
+    return (`This scenario is not in the plan you registered with ${TOOL_REGISTER_TEST_PLAN} (registration ${plan.registrationNumber}).\n\n` +
+        `Planned planned tests:\n${outstanding}\n\n` +
+        `Call ${TOOL_REGISTER_TEST_PLAN} again with this planned test included in \`planned tests\`, then retry generation using the ` +
+        `exact scenarioName/testType/endpoint you registered.`);
 }

package/build/tools/one-click/oneClickTool.d.ts

@@ -9,7 +9,6 @@ export type OneClickParams = {
     repositoryPath: string;
     endpointURL?: string;
     apiSchema?: string;
-    topN?: number;
     token?: string;
     baseBranch?: string;
     servicesToMock?: string;

package/build/tools/one-click/oneClickTool.js

@@ -160,10 +160,6 @@ const baseOneClickSchema = z.object({
         .string()
         .optional()
         .describe("Path or URL to OpenAPI/Swagger schema. If omitted, use schema from workspace or analysis."),
-    topN: z
-        .number()
-        .optional()
-        .describe("Number of top recommended test types to generate (e.g. 3–5). Default is 3. Used by full_repo_scan_recommend_generate_and_execute_top_n_tests."),
     token: z
         .string()
         .optional()
@@ -314,7 +310,6 @@ export function registerOneClickTool(server) {
                 repositoryPath: params.repositoryPath,
                 endpointURL: params.endpointURL,
                 apiSchema: params.apiSchema,
-                topN: params.topN ?? 3,
                 token: params.token ?? "",
                 baseBranch: params.baseBranch,
                 servicesToMock: params.servicesToMock,