@skyramp/mcp 0.3.4 → 0.3.6-rc.1

package/build/tools/code-refactor/utils-verify-gates.d.ts

@@ -0,0 +1,26 @@
+import { type UtilsVerifyResult } from "../../utils/utils-verify/index.js";
+/** How many verify rounds the agent may spend before it must document what remains.
+ *  SKYR-4059: an unbounded repair loop made six execution attempts against a cap of
+ *  two and destroyed verified reuse in between. The bound is stated in the text the
+ *  agent reads, not only here. */
+export declare const UTILS_VERIFY_MAX_ROUNDS = 3;
+/**
+ * The text the agent receives from a utils-path `verify: true` call. Precedence:
+ * violations (with the exact allow line per violation) → malformed markers → PASSED.
+ *
+ * Every remediation the agent could need is in the text: the file and line of each
+ * violation, what to change, the exact comment to paste when the violation is
+ * deliberate, and the round bound. A `<placeholder>` the agent must interpolate is
+ * how a gate never clears (mcp#716, run 30965653402), so the paste-ready lines carry
+ * the real kind and helper name and leave only the reason to fill in.
+ */
+export interface UtilsVerifyContext {
+    /** Files the verify pass actually staged (empty outside a testbot run). */
+    staged?: string[];
+    /** Utils files `git add` refused in a testbot run — a failed verification on its own. */
+    unstaged?: string[];
+    /** Whether the verdict reached run state. When it did not, the checkpoints fail
+     *  open on the unwritable state file — the agent is told rather than left to loop. */
+    recorded?: boolean;
+}
+export declare function formatUtilsVerifyText(r: UtilsVerifyResult, ctx?: UtilsVerifyContext): string;

package/build/tools/code-refactor/utils-verify-gates.js

@@ -0,0 +1,100 @@
+import * as path from "path";
+import { allowMarkerLine, } from "../../utils/utils-verify/index.js";
+/** How many verify rounds the agent may spend before it must document what remains.
+ *  SKYR-4059: an unbounded repair loop made six execution attempts against a cap of
+ *  two and destroyed verified reuse in between. The bound is stated in the text the
+ *  agent reads, not only here. */
+export const UTILS_VERIFY_MAX_ROUNDS = 3;
+/** Paths print as basenames only when every located utils file shares one directory
+ *  with the file shown; otherwise in full. Two utils files both called
+ *  `SkyrampUtils.py` under one repo is a designed-for state. */
+function show(file, r) {
+    const dirs = new Set(r.utilsFiles.map((f) => path.dirname(f)));
+    return dirs.size <= 1 && dirs.has(path.dirname(file))
+        ? path.basename(file)
+        : file;
+}
+export function formatUtilsVerifyText(r, ctx = {}) {
+    const tail = verdictTail(ctx) + unstagedTail(r, ctx) + namingAdvisory(r);
+    const spec = r.spec;
+    if (r.utilsFiles.length === 0) {
+        return ("VERIFICATION PASSED — no shared utils file is written or imported for this test; nothing to verify. " +
+            "(If you did create one, it carries no `Generated by Skyramp on …` header line, so it cannot be recognised or staged — add the header and re-verify.)" +
+            tail);
+    }
+    const files = r.utilsFiles.map((f) => show(f, r)).join(", ");
+    if (r.ok) {
+        const allowed = r.allowed.length > 0
+            ? ` ${r.allowed.length} documented decline${r.allowed.length === 1 ? "" : "s"}.`
+            : "";
+        return (`VERIFICATION PASSED — ${files}: ${r.helpers} helper${r.helpers === 1 ? "" : "s"}, ${r.imported} imported by this test.${allowed} ` +
+            `${ctx.staged && ctx.staged.length > 0 ? "The utils file is staged for the output commit. " : ""}Continue.` +
+            tail +
+            inlineCallSitesAdvisory(r));
+    }
+    const lines = [];
+    if (r.violations.length > 0) {
+        lines.push(`VERIFICATION FAILED — the shared utils file violates ${r.violations.length} invariant${r.violations.length === 1 ? "" : "s"}:`);
+        for (const v of r.violations) {
+            lines.push(`- ${v.kind} · ${show(v.file, r)}:${v.line} · ${v.helper}: ${v.detail}`);
+        }
+        lines.push("", `Fix each violation in the utils file (${r.family.family === "browser" ? "a shared browser helper holds actions and structural waits only; names are the user intent" : "a shared API helper asserts only the status code; one helper per method+path, extended with defaulted parameters; names are method + resource only"}), update any call site whose signature changed, then call skyramp_reuse_code again with the SAME arguments and \`verify: true\`. Re-verify after every fix pass; finish only on PASSED.`, `A violation that is deliberate may be documented instead of fixed. Copy the matching line below into the utils file as a comment, replacing only the <reason…> part — nothing else satisfies the gate:`);
+        if (spec) {
+            for (const v of r.violations)
+                lines.push(`    ${allowMarkerLine(spec, v.kind, v.helper)}`);
+        }
+        lines.push(`Do not spend more than ${UTILS_VERIFY_MAX_ROUNDS} verify rounds on this file: if violations remain after round ${UTILS_VERIFY_MAX_ROUNDS}, document each remaining one with its line above and re-verify once more.`);
+    }
+    if (r.malformedAllows.length > 0) {
+        lines.push(r.violations.length > 0
+            ? ""
+            : "VERIFICATION FAILED — every violation is documented, but not every marker can be read back.", "Fix these `reuse-verify: allow` lines — the report cannot read them, so the declines they document would be dropped silently:", ...r.malformedAllows.map((m) => `    ${show(m.file, r)}: ${m.line}`), `The form is \`${spec?.commentPrefix ?? "#"} reuse-verify: allow <kind> <helper> — <reason>\` with the reason separated by an em dash or a hyphen with a space on BOTH sides, the kind one of duplicate-helper / body-assertion / scenario-name, and the reason written out (not the <reason…> placeholder).`);
+    }
+    return lines.join("\n") + tail + inlineCallSitesAdvisory(r);
+}
+/** SKYR-4219: the sibling call-site pass the prompt asks for and the agent skips (0/2
+ *  seeded eval runs). Stated here because the verify call is the one point every run
+ *  reaches. Advisory — the sibling is a pre-existing test the maintenance flow owns, so
+ *  this informs and names the exact edit; it never blocks. */
+function inlineCallSitesAdvisory(r) {
+    if (r.inlineCallSites.length === 0)
+        return "";
+    const utils = r.utilsFiles.length === 1
+        ? path.basename(r.utilsFiles[0])
+        : "the utils file";
+    const n = r.inlineCallSites.length;
+    return [
+        "",
+        "",
+        `ADVISORY — ${n} inline request call${n === 1 ? "" : "s"} in other Skyramp-generated tests that a shared helper already wraps:`,
+        ...r.inlineCallSites.map((c) => `- ${show(c.file, r)}:${c.line} · ${c.method} ${c.path} · ${utils}.${c.helper}`),
+        `Each is a candidate: the request's method, path and call shape match the helper's. Replace the block with a call to the helper only if the remaining differences are literals (they become the arguments; keep the block's response variable; import the helper into that file; change nothing else there), then re-verify. This does not block execution.`,
+    ].join("\n");
+}
+function unstagedTail(r, ctx) {
+    if (!ctx.unstaged || ctx.unstaged.length === 0)
+        return "";
+    return (`\n\nVERIFICATION FAILED — ${ctx.unstaged.length === 1 ? "this utils file" : "these utils files"} could not be staged for the output commit: ` +
+        ctx.unstaged.map((f) => show(f, r)).join(", ") +
+        `. The tests importing it will fail for everyone else. Usually the file is gitignored (check \`git check-ignore -v <path>\`) or lies outside the repository; fix that (move the file, or un-ignore it) and re-verify.`);
+}
+/** Naming is advised, never enforced: a heuristic that blocked execution on
+ *  `create_test_case` or `list_expired_subscriptions` would be the worse outcome. */
+function namingAdvisory(r) {
+    if (r.advisories.length === 0)
+        return "";
+    const n = r.advisories.length;
+    return [
+        "",
+        "",
+        `ADVISORY — ${n} helper name${n === 1 ? "" : "s"} look${n === 1 ? "s" : ""} scenario-flavoured (does not block):`,
+        ...r.advisories.map((v) => `- ${show(v.file, r)}:${v.line} · ${v.helper}: ${v.detail}`),
+        "Shared helpers are named by method + resource (API) or user intent (browser); scenario values and adjectives belong in arguments. Rename if the name would stop another test from reusing the helper; otherwise continue.",
+    ].join("\n");
+}
+function verdictTail(ctx) {
+    if (ctx.recorded !== false)
+        return "";
+    return ("\n\nNOTE — this verdict could not be recorded in the run state (the state file is missing or not writable). " +
+        "The checkpoints fail open when the state file cannot be written, so this does not block you; if skyramp_execute_test still refuses, report that rather than re-running verification.");
+}

package/build/tools/code-refactor/verify-gates.d.ts

@@ -13,6 +13,7 @@ import { languageSchema } from "../../types/TestTypes.js";
 /** The spec a verify call targets — the subset of the tool params the gates need. */
 export type VerifyTarget = Pick<z.infer<typeof languageSchema>, "language" | "framework"> & {
     testFile: string;
+    testType?: string;
 };
 /** Renders a `VerifyResult` (from `verifyReuse`) as the text report returned to the agent. */
 export declare function formatVerifyReport(r: VerifyResult): string;
@@ -22,7 +23,7 @@ export declare function formatVerifyReport(r: VerifyResult): string;
 export declare function formatInfraGateFailure(): string;
 /** Renders the zero-reuse compliance-gate failure: a spec with 0 verified POM calls whose raw
  * selectors nonetheless overlap tier-1 POM candidates — i.e. reuse was skipped, not unavailable. */
-export declare function formatZeroReuseGateFailure(tier1: ScopedPom[]): string;
+export declare function formatZeroReuseGateFailure(tier1: ScopedPom[], unparsable?: string[]): string;
 /** Infrastructure gate: fires regardless of reuse/binding counts — a POM-aware spec whose Skyramp
  * runtime imports (or the `newSkyrampPlaywrightPage` wrapper) were stripped — e.g. replaced by the
  * repo's own fixtures under rule 3d's convention-matching — is a hard failure even when its POM

package/build/tools/code-refactor/verify-gates.js

@@ -1,16 +1,18 @@
 import { readFile } from "fs/promises";
 import * as path from "path";
 import { logger } from "../../utils/logger.js";
-import { isPomAwareTarget } from "../../prompts/code-reuse.js";
-import { selectScopedPoms } from "../../utils/pom-scope/index.js";
+import { isPomAwareTarget } from "../../utils/reuseRouting.js";
+import { selectScopedPoms, } from "../../utils/pom-scope/index.js";
 import { stripComments, stripStrings } from "../../utils/pom-scope/strip.js";
 import { escapeRegExp } from "../../utils/regex.js";
+import { findUnparsableKeptInline } from "./reuse-outcome.js";
 /** Renders a `VerifyResult` (from `verifyReuse`) as the text report returned to the agent. */
 export function formatVerifyReport(r) {
     if (r.ok) {
         const lines = [`VERIFICATION PASSED — ${r.checkedCalls} calls checked.`];
-        if (r.unverifiable.length > 0) {
-            lines.push(`Unverifiable (informational): ${r.unverifiable.join(", ")}`);
+        const informational = [...r.unverifiableCalls, ...r.unrecognizedBindings];
+        if (informational.length > 0) {
+            lines.push(`Unverifiable (informational): ${informational.join(", ")}`);
         }
         return lines.join("\n");
     }
@@ -36,35 +38,85 @@ export function formatInfraGateFailure() {
 }
 /** Renders the zero-reuse compliance-gate failure: a spec with 0 verified POM calls whose raw
  * selectors nonetheless overlap tier-1 POM candidates — i.e. reuse was skipped, not unavailable. */
-export function formatZeroReuseGateFailure(tier1) {
+export function formatZeroReuseGateFailure(tier1, unparsable = []) {
+    if (tier1.length === 0)
+        return formatMalformedMarkerFailure(unparsable);
     const lines = [
         "VERIFICATION FAILED — this test contains no page-object reuse, but detection found reusable POM candidates for its selectors:",
         ...tier1.map((p) => `- ${p.file}${p.matchedTokens.length > 0 ? ` (matched: ${p.matchedTokens.join(", ")})` : " (imported by a matched page-object file)"}`),
-        "Map the matching actions to these files' methods/properties (re-run skyramp_reuse_code for the full workflow if needed). A raw locator that matches a POM property verbatim is evidence TO substitute, never a reason to stay inline. Only finish with raw locators if you add a comment for each file above — `// kept inline: <file basename> — <reason it cannot cover this test>` — then re-run this verification, which will accept the documented files.",
+        "Map the matching actions to these files' methods/properties (re-run skyramp_reuse_code for the full workflow if needed). A raw locator that matches a POM property verbatim is evidence TO substitute, never a reason to stay inline.",
+        // The exact marker per candidate rather than a `<file basename>` placeholder: the
+        // gate matches on the name it prints here, and an agent asked to interpolate
+        // "basename" writes the stem (`channels`) about as often as `channels.ts`, which
+        // read as undocumented and left the gate unclearable (run 30965653402).
+        "Only finish with raw locators if you document EVERY file above. Copy these lines verbatim into the spec, replacing each <reason...> — nothing else satisfies the gate:",
+        ...tier1.map((p) => `    // kept inline: ${path.basename(p.file)} — <reason it cannot cover this test>`),
+        "Then re-run this verification, which will accept the documented files.",
     ];
+    if (unparsable.length > 0)
+        lines.push(malformedMarkerLines(unparsable).join("\n"));
     return lines.join("\n");
 }
+/** The remediation block for markers the gate accepts but `parseKeptInline` cannot read.
+ * Shared by both entry points so the corrected form is spelled the same way in each. */
+function malformedMarkerLines(unparsable) {
+    return [
+        "Also fix these `kept inline:` lines — they satisfy this gate but the report cannot read them, so their declines would be dropped silently:",
+        ...unparsable.map((line) => `    ${line}`),
+        "The reason must be separated by an em dash, or by a hyphen with a space on BOTH sides: `// kept inline: <file basename> — <reason>`.",
+    ];
+}
+/** Renders the malformed-marker failure on its own — every candidate is documented, so the
+ * zero-reuse gate itself would clear, and the only remaining defect is a line the report
+ * cannot parse. Reported to the agent, which can still fix it, rather than to a log line at
+ * report time, which nobody reads in time. */
+function formatMalformedMarkerFailure(unparsable) {
+    return [
+        "VERIFICATION FAILED — every POM candidate is documented, but not every decline can be read back.",
+        ...malformedMarkerLines(unparsable),
+    ].join("\n");
+}
 /** True when `specContent` already carries a `// kept inline: <basename> — <reason>` comment line
  * naming this candidate's file. This is the one canonical marker format: both instruction sites —
  * `formatZeroReuseGateFailure` above and `formatVerifyReport`'s demote flow — spell out exactly
- * this shape, so a demoted call's comment also satisfies the gate on the re-verify pass. */
+ * this shape, so a demoted call's comment also satisfies the gate on the re-verify pass.
+ *
+ * The extension is optional. `parseKeptInline` — which builds the report's `skipped` list from
+ * these same markers — accepts the stem, so requiring it here made the two disagree: a spec
+ * documenting `channels` reported all six declines correctly while the gate read every one as
+ * undocumented and blocked test execution outright (run 30965653402). A stem is unambiguous
+ * anyway; two POM files differing only by extension are not a case worth failing closed on. */
 function isKeptInlineDocumented(specContent, file) {
-    const basename = escapeRegExp(path.basename(file));
+    const base = path.basename(file);
+    const stem = base.slice(0, base.length - path.extname(base).length);
+    const name = escapeRegExp(base) +
+        (stem && stem !== base ? `|${escapeRegExp(stem)}` : "");
     // Anchored full-format match: `// kept inline: <basename> — <non-empty reason>`.
-    // The basename must start right after the marker (no substring collisions with
-    // longer filenames) and a dash-separated reason is required (a bare basename is
-    // not documentation — the reason is what makes the escape PR-reviewable).
-    const re = new RegExp(`//\\s*kept inline:\\s*${basename}\\s*[—–-]+\\s*\\S`);
+    // The name must start right after the marker and be followed by the separator, so a
+    // stem cannot match a longer filename sharing its prefix. A dash-separated reason is
+    // required (a bare basename is not documentation — the reason is what makes the
+    // escape PR-reviewable).
+    // `(?!<reason)` rejects the placeholder from formatZeroReuseGateFailure's paste-ready
+    // line when it was pasted unedited. Accepting it would publish "<reason it cannot cover
+    // this test>" to the PR as the stated decline, which is worse than no marker at all.
+    const re = new RegExp(`//\\s*kept inline:\\s*(?:${name})\\s*[—–-]+\\s*(?!<reason)\\S`);
     // String literals are blanked first so a marker embedded in quoted text can't satisfy the
     // gate — only a real comment counts (standalone or trailing after code; both are legitimate).
-    return stripStrings(specContent).split("\n").some((line) => re.test(line));
+    return stripStrings(specContent)
+        .split("\n")
+        .some((line) => re.test(line));
 }
-/** Verification ran clean but checked nothing: no verified POM calls, no violations, no deep-chain
- * (unverifiable) entries, no parse error. Such a spec still carries all of its raw selectors
- * verbatim, so running POM detection against it is meaningful — any other outcome (real bindings,
- * deep-chain-only reuse, broken syntax) needs its own remediation, not the mapping-instructions gate. */
+/** Verification ran clean but checked nothing: no verified POM calls, no violations, no unverifiable
+ * calls and no unrecognized bindings, no parse error. Such a spec still carries all of its raw
+ * selectors verbatim, so running POM detection against it is meaningful — any other outcome (real
+ * bindings, deep-chain-only reuse, broken syntax) needs its own remediation, not the
+ * mapping-instructions gate. */
 function isCleanButEmpty(r) {
-    return r.checkedCalls === 0 && r.violations.length === 0 && r.unverifiable.length === 0 && !r.parseError;
+    return (r.checkedCalls === 0 &&
+        r.violations.length === 0 &&
+        r.unverifiableCalls.length === 0 &&
+        r.unrecognizedBindings.length === 0 &&
+        !r.parseError);
 }
 /** Infrastructure gate: fires regardless of reuse/binding counts — a POM-aware spec whose Skyramp
  * runtime imports (or the `newSkyrampPlaywrightPage` wrapper) were stripped — e.g. replaced by the
@@ -73,14 +125,17 @@ function isCleanButEmpty(r) {
  * fire. Broken syntax (parseError) never reaches it: that needs its own remediation
  * (formatVerifyReport surfaces it), not an infra compliance failure. */
 export async function infraGateFailure(params, r) {
-    if (!isPomAwareTarget(params.language, params.framework) || r.parseError)
+    if (!isPomAwareTarget(params.language, params.framework, params.testType) ||
+        r.parseError)
         return undefined;
     let specContent;
     try {
         specContent = await readFile(params.testFile, "utf8");
     }
     catch (err) {
-        logger.warning("Verify-mode infra gate skipped — spec unreadable", { error: String(err) });
+        logger.warning("Verify-mode infra gate skipped — spec unreadable", {
+            error: String(err),
+        });
         return undefined;
     }
     // A commented-out import or wrapper call is not live infrastructure, and quoted text mentioning
@@ -88,7 +143,8 @@ export async function infraGateFailure(params, r) {
     // `'@skyramp/skyramp'` is itself a string literal (must survive), while the wrapper check wants
     // strings blanked too so `"newSkyrampPlaywrightPage("` in prose can't satisfy it.
     const commentless = stripComments(specContent);
-    if (!commentless.includes("@skyramp/skyramp") || !stripStrings(commentless).includes("newSkyrampPlaywrightPage(")) {
+    if (!commentless.includes("@skyramp/skyramp") ||
+        !stripStrings(commentless).includes("newSkyrampPlaywrightPage(")) {
         return formatInfraGateFailure();
     }
     return undefined;
@@ -99,19 +155,28 @@ export async function infraGateFailure(params, r) {
  * scoping errors (or a legitimately POM-less repo, tier1 === []) also return undefined: this gate
  * must never turn an infra error into a false compliance failure. */
 export async function zeroReuseGateFailure(params, r) {
-    if (!isCleanButEmpty(r) || !isPomAwareTarget(params.language, params.framework))
+    if (!isCleanButEmpty(r) ||
+        !isPomAwareTarget(params.language, params.framework, params.testType))
         return undefined;
     try {
         const specContent = await readFile(params.testFile, "utf8");
-        const { tier1, diagnostics } = await selectScopedPoms({ testFile: params.testFile, specContent });
+        const { tier1, diagnostics } = await selectScopedPoms({
+            testFile: params.testFile,
+            specContent,
+        });
         // Only fire on candidates found via conventional POM dirs (globs) — the selector-grep
         // discovery fallback also matches ordinary in-repo frontend source (app components
         // legitimately contain the spec's test-ids by construction), which is not evidence of
         // skipped reuse in a POM-less repo.
         if (diagnostics.discovery === "globs" && tier1.length > 0) {
             const undocumented = tier1.filter((p) => !isKeptInlineDocumented(specContent, p.file));
-            if (undocumented.length > 0) {
-                return formatZeroReuseGateFailure(undocumented);
+            // A marker this gate accepts but `parseKeptInline` cannot read clears the escape
+            // hatch while its decline never reaches the report — the silent loss this feature
+            // exists to close. Caught here rather than only logged at report time: the agent is
+            // still running and can fix the line; the log reader is not.
+            const unparsable = findUnparsableKeptInline(specContent);
+            if (undocumented.length > 0 || unparsable.length > 0) {
+                return formatZeroReuseGateFailure(undocumented, unparsable);
             }
         }
     }

package/build/tools/executeSkyrampTestTool.d.ts

@@ -1,4 +1,5 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { TestExecutionResult } from "../types/TestExecution.js";
 import { TestType } from "../types/TestTypes.js";
 export declare const CONTRACT_EXECUTION_MODES: readonly ["provider", "consumer"];
 export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
@@ -19,4 +20,22 @@ export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractM
  * its own attachment line.
  */
 export declare function withVideoInfo(output: string, videoPath?: string): string;
+/**
+ * The failure text the agent receives. Everything it needs has to be in here:
+ * only a tool's return value reaches the transcript, and Claude Code does not
+ * capture an MCP server's stderr, so anything this omits is unrecoverable once
+ * the run ends.
+ *
+ * `output` alone was not enough. When the executor produced none, the message
+ * read "Test execution failed:" and stopped — so the agent inferred a cause and
+ * reported it as fact. Measured in eval run 32220788735: it blamed an
+ * unreachable backend while the app was answering 200 on both the host and the
+ * docker bridge, and that invented cause reached `issuesFound`.
+ *
+ * The 401 hint is emitted only when the output carries a 401 in a status-shaped
+ * position — see HTTP_401_SHAPES. It used to be unconditional, which pointed the
+ * agent at authentication on a run whose output was empty, and then it keyed off
+ * the bare number, which pointed it there on a test count or a line number.
+ */
+export declare function buildExecutionFailureText(result: TestExecutionResult): string;
 export declare function registerExecuteSkyrampTestTool(server: McpServer): void;

package/build/tools/executeSkyrampTestTool.js

@@ -1,13 +1,16 @@
 import { z } from "zod";
+import { pendingReuseDebt } from "./code-refactor/reuse-state.js";
+import { pendingAssertionEnhancement } from "./code-refactor/enhance-state.js";
+import { stageUtilsArtifacts } from "../utils/utils-verify/index.js";
 import path from "path";
 import { stripVTControlCharacters } from "util";
 import { TestExecutionService } from "../services/TestExecutionService.js";
 import { AnalyticsService } from "../services/AnalyticsService.js";
 import { makeProgressReporter } from "../utils/progress.js";
-import { TestExecutionStatus } from "../types/TestExecution.js";
+import { TestExecutionStatus, } from "../types/TestExecution.js";
 import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
-import { StateManager, getTestsRepoDir } from "../utils/AnalysisStateManager.js";
+import { StateManager, getTestsRepoDir, } from "../utils/AnalysisStateManager.js";
 import { TestSource } from "../types/TestAnalysis.js";
 import { logger } from "../utils/logger.js";
 import { toolError } from "../utils/utils.js";
@@ -39,6 +42,119 @@ export function shouldInjectSkyrampBaseUrl(testType, contractMode) {
 export function withVideoInfo(output, videoPath) {
     return videoPath ? `${output}\n\nVideo recording: ${videoPath}` : output;
 }
+/**
+ * Where a real HTTP 401 shows up in runner output. Every entry is a shape taken from
+ * actual skyramp_execute_test output in the eval logs, not from guesswork:
+ *
+ *   "status_code": 401,                    the Skyramp client's per-request JSON block
+ *   statusCode":401 (backslash-escaped)     the same, inside a "response_body" string
+ *   Received: 401                          Playwright's expect diff
+ *   assert 401 == 201 / assert 404 == 401  pytest assertion rewriting
+ *   where 401 = ResponseV2(...)            pytest's explanation of the value above
+ *   .status_code == 401 / .toBe(401)       the test source echoed in a code frame
+ *   HTTP/1.1 401 / 401 Unauthorized        the status line and the reason phrase
+ *
+ * The reason phrase counts only next to the number. On its own, "Unauthorized" is
+ * as often the name of a test as the answer to one — a Playwright case titled
+ * "should return Unauthorized for an expired token" prints that word on a run
+ * whose received status was 500.
+ *
+ * Matching the bare number instead was the defect: `collected 401 items`,
+ * `Running 401 tests using 4 workers`, `foo_test.py:401: AssertionError`,
+ * `duration: 401 ms` and a path holding `pr-401` all satisfied it, and each one then
+ * told the agent the run had failed on auth and to go and ask the user for a token.
+ */
+const HTTP_401_SHAPES = [
+    // Reason phrase, bound to the number it explains. A serialized error body
+    // naming the phrase without a status is covered by the status-field pattern
+    // below, which reads the code the response actually carried.
+    // Anchored to the start of a line: a status line or a bare reason phrase sits
+    // there, while a test TITLE ("should return 401 Unauthorized for an expired
+    // token") always has words in front of it and is not evidence of a response.
+    /^\s*401\s+unauthori[sz]ed\b/im,
+    // Status line, as curl -i and Go's httputil print it.
+    /\bHTTP\/[\d.]+\s+401\b/i,
+    // A status FIELD set to 401 — the value the response carried. Only `:` is
+    // accepted: `== 401` and `= 401` are an assertion or echoed test source, which
+    // state what the test WANTED, and a failing test means it did not get it.
+    // A line naming `expected` is what the test WANTED, so an expected object
+    // (`Expected: {"status": 401}`) alongside `Received: {"status": 500}` is not a
+    // 401 the app sent. Reject the line rather than the value.
+    /^(?!.*\bexpected\b).*\b(?:status|status[_-]?code|statuscode|code|errorcode)\\?"?\s*:\s*401\b/im,
+    // Playwright prints both compared values. `Received` is what the app sent;
+    // `Expected` is what the test wanted, so it is not evidence of a 401.
+    /^\s*Received:\s*401\b/im,
+    // pytest assertion rewriting. A Skyramp-generated test reads
+    // `assert response.status_code == N`, so the OBSERVED value is on the left and
+    // `assert 404 == 401` means the test wanted 401 and the app sent 404.
+    //
+    // `skyramp_execute_test` also runs user-authored files, and a suite written
+    // the other way round (`assert 401 == resp.status_code`) fails as
+    // `assert 401 == 500` — a 500 the app sent, matching these two shapes. The
+    // two conventions are textually identical, so nothing here can separate them;
+    // the hint's wording carries the uncertainty instead of asserting a 401.
+    /\bassert\s+401\s*==/i,
+    /\bwhere\s+401\s*=/i,
+];
+/**
+ * The failure text the agent receives. Everything it needs has to be in here:
+ * only a tool's return value reaches the transcript, and Claude Code does not
+ * capture an MCP server's stderr, so anything this omits is unrecoverable once
+ * the run ends.
+ *
+ * `output` alone was not enough. When the executor produced none, the message
+ * read "Test execution failed:" and stopped — so the agent inferred a cause and
+ * reported it as fact. Measured in eval run 32220788735: it blamed an
+ * unreachable backend while the app was answering 200 on both the host and the
+ * docker bridge, and that invented cause reached `issuesFound`.
+ *
+ * The 401 hint is emitted only when the output carries a 401 in a status-shaped
+ * position — see HTTP_401_SHAPES. It used to be unconditional, which pointed the
+ * agent at authentication on a run whose output was empty, and then it keyed off
+ * the bare number, which pointed it there on a test count or a line number.
+ */
+export function buildExecutionFailureText(result) {
+    const output = stripVTControlCharacters(result.output || "").trim();
+    const facts = [`status=${result.status}`];
+    if (typeof result.exitCode === "number")
+        facts.push(`exitCode=${result.exitCode}`);
+    if (typeof result.duration === "number")
+        facts.push(`duration=${result.duration}ms`);
+    const sections = [`Test execution failed (${facts.join(", ")}).`];
+    const errors = (result.errors ?? []).filter((e) => e && e.trim());
+    if (output) {
+        sections.push(output);
+    }
+    else if (errors.length > 0) {
+        // The executor reported a cause of its own (e.g. "Docker image setup
+        // failed"). Saying the cause cannot be determined would contradict the
+        // Errors line below and throw away the only thing known about the failure.
+        sections.push("The executor captured no test output, so there are no per-test diagnostics. " +
+            "It did report the error below, which is the cause on record — use it. Do " +
+            "NOT infer anything the Errors line does not say, and leave the generated " +
+            "test unchanged.");
+    }
+    else {
+        sections.push("The executor captured no output, so this failure carries no diagnostics of " +
+            "its own and the cause cannot be determined from it. The test runner, the " +
+            "test process, or the application could each have died silently. Do NOT " +
+            "report the app as unreachable or misconfigured on this basis — nothing " +
+            "here shows that. Check any Errors line below and that the test runner " +
+            "started, then retry once; if it fails the same way, report the execution " +
+            "failure with its cause unresolved and leave the generated test unchanged.");
+    }
+    if (errors.length > 0)
+        sections.push(`Errors: ${errors.join("; ")}`);
+    if (HTTP_401_SHAPES.some((shape) => shape.test(output))) {
+        sections.push("The output carries a 401 in a status position. If the app returned it, ask " +
+            "the user for the authentication token (or where to read it from) and " +
+            "retry. Check which side of the comparison the 401 sits on first: a suite " +
+            "that writes the expected value on the left fails as `assert 401 == 500`, " +
+            "and there the 401 is what the test wanted, not what it got. Leave the " +
+            "generated test file unchanged either way.");
+    }
+    return sections.join("\n\n");
+}
 export function registerExecuteSkyrampTestTool(server) {
     server.registerTool(TOOL_NAME, {
         description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing. Call this once a test file exists on disk (from a skyramp_*_test_generation tool). First-time execution may take longer while Docker images download — this is expected, not a failure.`,
@@ -111,6 +227,17 @@ export function registerExecuteSkyrampTestTool(server) {
                 errorResult = toolError(`testFile must be an absolute path, got: ${params.testFile}`);
                 return errorResult;
             }
+            // SKYR-4262: block execution when skyramp_enhance_assertions handed out
+            // instructions for this spec and the file has not changed since — the
+            // agent acknowledged the enhancement step without acting on it. The debt
+            // clears through the file itself (any edit, including a documented
+            // `// assertions reviewed:` no-change decision), so the agent can never
+            // be stuck here. Same enforcement shape as the reuse checkpoint above.
+            const owedEnhancement = await pendingAssertionEnhancement(params.testFile, params.stateFile);
+            if (owedEnhancement) {
+                errorResult = toolError(owedEnhancement);
+                return errorResult;
+            }
             // Deterministic external-test guard (SKYR-3924): this tool runs Skyramp-generated
             // tests in the executor and cannot run a repo's native (user-written) suite, so a
             // run on an external test only errors (e.g. pytest import/collection failure).
@@ -139,6 +266,25 @@ export function registerExecuteSkyrampTestTool(server) {
                     logger.warning(`External-test guard could not read stateFile (${err.message}); proceeding with execution`);
                 }
             }
+            // SKYR-4220: stage the shared utils file (see stageUtilsArtifacts) — execution is
+            // the last step before delivery, and a utils file left unstaged here ships a
+            // test importing a module the PR lacks. After the external-test skip: a native
+            // suite this tool will not run gets no scan and no staging.
+            await stageUtilsArtifacts(params.testFile);
+            // SKYR-4115 backstop: enhance_assertions carries the same check, but nothing
+            // guarantees the agent calls it, and execution is the last step that still
+            // precedes reporting. See pendingReuseVerification for why enforcing the verify
+            // CALL needs no retry budget.
+            //
+            // Pass params.stateFile through: without it the state path resolves from the
+            // CI/Testbot anchor alone and the check returns early — failing OPEN — for a
+            // caller that supplies a valid stateFile outside CI. The external-test guard
+            // below already reads params.stateFile, so this one honours the same input.
+            const owedReuseVerification = await pendingReuseDebt(params.testFile, params.stateFile, params.testType);
+            if (owedReuseVerification) {
+                errorResult = toolError(owedReuseVerification);
+                return errorResult;
+            }
             // Send initial progress
             await sendProgress(5, 100, "Starting test execution...");
             // Cross-repo run (SKYR-3819): tests delivered under the run's testsRepoDir
@@ -172,7 +318,9 @@ export function registerExecuteSkyrampTestTool(server) {
             }
             const executionService = new TestExecutionService();
             const effectiveToken = resolveEffectiveToken(params.unauthenticated, params.token, process.env.SKYRAMP_TEST_TOKEN);
-            if (!effectiveToken && !params.unauthenticated && params.token === undefined) {
+            if (!effectiveToken &&
+                !params.unauthenticated &&
+                params.token === undefined) {
                 logger.warning("No auth token available — authenticated endpoints will likely return 401. Set SKYRAMP_TEST_TOKEN or pass token/unauthenticated.");
             }
             // Execute test with progress callback - reports Docker cache/pull status.
@@ -223,7 +371,9 @@ export function registerExecuteSkyrampTestTool(server) {
                             else {
                                 stateData.existingTests[testIndex].executionAfter = result;
                             }
-                            await stateManager.writeRepoData(stateData, { repo: params.repository });
+                            await stateManager.writeRepoData(stateData, {
+                                repo: params.repository,
+                            });
                             logger.info(`Updated stateFile with execution results for ${params.testFile}`);
                         }
                         else {
@@ -244,9 +394,7 @@ export function registerExecuteSkyrampTestTool(server) {
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
             if (result.status !== TestExecutionStatus.Pass) {
-                errorResult = toolError(withVideoInfo(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
-
-If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged.`, result.videoPath));
+                errorResult = toolError(withVideoInfo(buildExecutionFailureText(result), result.videoPath));
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
@@ -277,7 +425,9 @@ If this failed with a 401 status code, ask the user for the authentication token
                 workspacePath: params.workspacePath,
                 language: params.language,
                 testType: params.testType,
-            }).catch((err) => { logger.warning("Analytics event failed", { error: String(err) }); });
+            }).catch((err) => {
+                logger.warning("Analytics event failed", { error: String(err) });
+            });
         }
     });
 }

package/build/tools/generate-tests/generateBatchScenarioRestTool.js

@@ -481,9 +481,9 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
             responseBody: t.ResponseBody,
         })));
         if (chainLint.errors.length > 0) {
-            return toolError(`**Scenario step-ordering errors**\n\n`
+            return toolError(`**Scenario chaining errors**\n\n`
                 + chainLint.errors.map((e) => `- ${e}`).join("\n")
-                + `\n\nReorder or fix the steps and resubmit the full batch.`);
+                + `\n\nApply the remedy each error names and resubmit the full batch.`);
         }
         const chainLintWarning = chainLint.warnings.length > 0
             ? `\n\n⚠️ **Chaining warnings** — the generated test may hardcode these instead of chaining:\n`

package/build/tools/generate-tests/generateE2ERestTool.js

@@ -2,11 +2,20 @@ import { z } from "zod";
 import { baseSchema, baseTraceSchema, TestType, codeRefactoringSchema, } from "../../types/TestTypes.js";
 import { TestGenerationService, } from "../../services/TestGenerationService.js";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
+import { toolError } from "../../utils/utils.js";
+import { checkTraceReuse, recordTraceOutput } from "./trace-reuse-guard.js";
 const e2eTestSchema = {
     ...baseTraceSchema.shape,
     playwrightInput: z
         .string()
         .describe("MUST be absolute path to the playwright input file like /path/to/playwright-ui-test.zip and MUST be a zip file captured using start_trace_collection tool"),
+    allowTraceReuse: z
+        .boolean()
+        .default(false)
+        .describe("Explicit override for the trace-reuse guard. By default, generating a second differently-named test " +
+        "from a Playwright zip that already produced one is rejected — one recording covers one scenario, and reusing " +
+        "it manufactures identical tests under names the recording never covered. Set true only when the same " +
+        "recording genuinely covers the new scenario end-to-end."),
     ...codeRefactoringSchema.shape,
     ...baseSchema.shape,
 };
@@ -42,8 +51,15 @@ If \`trace\` parameter is provided (path to a trace file), DO NOT pass \`apiSche
             keywords: ["e2e test", "end-to-end test"],
         },
     }, async (params) => {
+        // SKYR-4262: same trace-reuse guard as skyramp_ui_test_generation — this
+        // tool consumes the same Playwright zips, so the same fan-out applies.
+        const traceReuse = await checkTraceReuse(params.playwrightInput, params.output, params.outputDir, !!params.allowTraceReuse, "e2e");
+        if (traceReuse.rejection)
+            return toolError(traceReuse.rejection);
         const service = new E2ETestService();
         const result = await service.generateTest(params);
+        if (!result.isError && traceReuse.key && traceReuse.identity)
+            recordTraceOutput(traceReuse.key, traceReuse.identity);
         AnalyticsService.pushTestGenerationToolEvent(TOOL_NAME, result, params).catch(() => {
             // Silently ignore analytics errors
         });