@skyramp/mcp 0.4.2-rc.1 → 0.4.2-rc.3

package/build/prompts/testbot/testbot-prompts.js

@@ -12,6 +12,7 @@ import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, TESTBOT_TASK1_MAINTA
 import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
 import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, isSkillsLoaded, } from "../../utils/featureFlags.js";
 import { fixErrorsInstruction } from "../../skills/fixTestImportErrorsSkill.js";
+import { runTestCall } from "../../skills/runTestSkill.js";
 import { alignAssertionsStep } from "../../skills/validateAssertionAlignmentSkill.js";
 import { ASSERTION_ENHANCEABLE_TEST_TYPES } from "../../types/TestTypes.js";
 import { enhanceAssertionsCall } from "../../skills/enhanceAssertionsSkill.js";
@@ -307,16 +308,19 @@ export function getTestbotPrompt(opts) {
 
 `;
     }
-    // The Task 1 maintenance baseline sub-step branches in plan-only eval runs (SKYR-3879
-    // plan-only lane): the SUT is not running, so the pre-edit baseline
-    // execution is replaced by static-analysis-only verdicts. Defined before
-    // task1Section, which interpolates it.
+    // The Task 1 maintenance sub-steps that run tests branch in plan-only eval runs
+    // (SKYR-3879 plan-only lane): the SUT is not running, so those runs are replaced
+    // by static-analysis-only verdicts. Defined before task1Section, which
+    // interpolates them.
     let maintenanceBeforeExecStep;
+    let verifyExternalStep;
     if (planOnly) {
+        verifyExternalStep = `   ${MAINTAIN.VERIFY_EXTERNAL}. Plan-only run: do not run the external tests you edited.`;
         maintenanceBeforeExecStep = `   ${MAINTAIN.BASELINE}. Plan-only run: skip the pre-edit baseline execution — the application is not running. Record every maintenance verdict from static drift analysis alone; execution statuses simply remain unrecorded.`;
     }
     else {
-        maintenanceBeforeExecStep = `   ${MAINTAIN.BASELINE}. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every UPDATE/REGENERATE/DELETE test. Exclude tests marked \`[external]\` — those are baselined in ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.CONFIRM_EXTERNAL)} via \`skyramp_run_existing_tests\`. Run them sequentially, not in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail. Never pass \`rebaselineSnapshots\` here: a \`Screenshot comparison failed\` on this run is the evidence that a visual baseline is stale, and the refresh belongs to the final execution.`;
+        verifyExternalStep = `   ${MAINTAIN.VERIFY_EXTERNAL}. Run the external tests you edited again. Do this step right after ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.APPLY_ACTIONS)}. It applies to each \`[external]\` test file that you edited in ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.APPLY_ACTIONS)} and that ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.BASELINE)} ran. Run each such file with \`skyramp_execute_test\`, \`phase: "after"\` and \`stateFile\`, and use the same command as in ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.BASELINE)}. Record each result as the file's \`afterStatus\`. The final execution excludes \`[external]\` files, so this step is where they get their after-run. If no file applies, say so in your report. If a file still fails, report it — do not run it again.`;
+        maintenanceBeforeExecStep = `   ${MAINTAIN.BASELINE}. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every Skyramp test you will change, and for every test the repository owns that you will mark UPDATE: ${runTestCall()}. Run them one at a time, not in parallel. Run the command even when you expect the runner or a package to be missing: the failure output is what names what is missing, and a check you make before the run reaches no repair. On this run repair the environment only — install what is missing and run the file again — and never edit the test file, because the result of the file as the repository wrote it IS the baseline. This captures the pre-edit baseline — do not skip it even when you expect the test to fail. \`skyramp_actions\` refuses an UPDATE of the repository's own test until that test has this run. Nothing else owes one: a VERIFY changes no file, and a REGENERATE or a DELETE of the repository's own test is a recommendation you report rather than apply. Never pass \`rebaselineSnapshots\` on this run: a \`Screenshot comparison failed\` here is the evidence that a visual baseline is stale, and the refresh belongs to the final execution.`;
     }
     // For follow-up requests: emit the @skyramp-testbot header + guardrails, both stop-early.
     // For first-run prompts: emit the full Task 1 analysis + maintenance section.
@@ -346,8 +350,6 @@ ${analyzeBlock}
 
 ${TASK1.MAINTAIN}. **Maintain existing tests:**
 
-   ${MAINTAIN.CONFIRM_EXTERNAL}. Confirm external-test breakage before assessment. If any service in \`${repositoryPath}/.skyramp/workspace.yml\` declares test-env config (\`runtimeDetails.test*\`), call \`skyramp_run_existing_tests\` (\`mode: "confirm"\`, \`stateFile\`) on the tests \`skyramp_analyze_changes\` marked \`[external]\` — the repo's OWN suite, not the Skyramp-generated tests — before \`skyramp_analyze_test_health\`, so the run-confirmed failures fold into its assessment. Do NOT read the \`[external]\` test files to *guess* the change's impact — RUN them with \`skyramp_run_existing_tests\` to get real pass/fail; reading them is not a substitute for running them. These \`[external]\` suites also do not run through \`skyramp_execute_test\`, so skipping this leaves their status \`Unknown\`. Selector, health-gate, and self-skip behavior are in the tool description. (Skyramp-generated tests are handled in ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.BASELINE)}, not here.)
-
    ${MAINTAIN.TEST_HEALTH}. Call \`skyramp_analyze_test_health\` with \`stateFile\` (from \`skyramp_analyze_changes\` output). Pass \`blueprintCaptured: true\` when \`browser_blueprint\` was called successfully earlier in this session — see the parameter description for when this applies. **Do NOT read application source files** (routes, models, controllers) — all change information you need is in the \`skyramp_analyze_changes\` output and the diff. Exception: the UI drift pre-scan (\`UI_SYMBOL_PRESCAN\`) may instruct you to read changed frontend files to extract exported symbols — follow those instructions when present.
 
    ${MAINTAIN.UPDATE_INSTRUCTIONS}. Write \`updateInstructions\` for each UPDATE or REGENERATE test before calling \`skyramp_actions\` — articulating the change first prevents file content from overriding diff-based reasoning.
@@ -356,7 +358,7 @@ ${maintenanceBeforeExecStep}
 
    ${MAINTAIN.APPLY_ACTIONS}. Call \`skyramp_actions\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) and apply the edits it returns.
 
-   ${MAINTAIN.VERIFY_EXTERNAL}. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.APPLY_ACTIONS)}, so come back here before you move on to anything else.** It applies whenever ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.CONFIRM_EXTERNAL)} reported a real pass/fail result for a file you then edited. It does NOT apply when ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.CONFIRM_EXTERNAL)} returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) and record each file's result as its \`afterStatus\`. Editing an \`[external]\` file that ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.CONFIRM_EXTERNAL)} confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
+${verifyExternalStep}
 
 ${codeReviewBlock}
 
@@ -624,7 +626,7 @@ ${FINISH_CHECKS_BLOCK}
 **Execution timing:**
 - **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
 - **Probe residue**: before the final execution, delete through the API the records your own probes and recordings created. Find each one by the id its create response returned, or by the exact value you typed for it during a recording. Do not delete anything else, and do not delete by a pattern. Do not use the database or a container. Do not add the deletion to a test. List in the report the records the API could not remove.
-- **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together — passing \`stateFile\` and \`repository\` on every call: the attempt cap and the \`afterStatus\` the report reads both live in that file, and \`repository\` names the section the result lands in. Omitting \`stateFile\` does not skip the cap — the tool falls back to this run's own state file — but a wrong or missing \`repository\` fails the call. For a maintained spec whose \`skyramp_actions\` entry carried \`rebaseline_snapshots\`, pass that exact list as \`rebaselineSnapshots\` on this run and only this run. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection. Exclude tests marked \`[external]\`.
+- **Final execution**: Apart from ${stepSubRef(TASK1.MAINTAIN, MAINTAIN.VERIFY_EXTERNAL)}, do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together — passing \`stateFile\` and \`repository\` on every call: the attempt cap and the \`afterStatus\` the report reads both live in that file, and \`repository\` names the section the result lands in. Omitting \`stateFile\` does not skip the cap — the tool falls back to this run's own state file — but a wrong or missing \`repository\` fails the call. For a maintained spec whose \`skyramp_actions\` entry carried \`rebaseline_snapshots\`, pass that exact list as \`rebaselineSnapshots\` on this run and only this run. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection. Exclude tests marked \`[external]\`.
 - Only report test results for files you actually ran.
 
 ### Post-Execution: Assertion Alignment

package/build/services/TestDiscoveryService.js

@@ -4,7 +4,15 @@ import { logger } from "../utils/logger.js";
 import { TestSource } from "../types/TestAnalysis.js";
 import { TestType } from "../types/TestTypes.js";
 import fg from "fast-glob";
-import { isDiscoveredTestFile } from "../utils/testFileClassification.js";
+import { hasTestFileName, isDiscoveredTestFile, } from "../utils/testFileClassification.js";
+/**
+ * A Jest/Vitest snapshot in the colocated CRA layout. Its name never identifies
+ * it as a test (`Button.test.tsx.snap` cannot match a `.<ext>` tail across the
+ * `.tsx.` dot), but a render change leaves it stale, so it is maintenance
+ * material. It never enters the before-run gate: `detectLanguage` on `.snap`
+ * returns "unknown".
+ */
+const COLOCATED_SNAPSHOT = /[\\/]__snapshots__[\\/][^\\/]+\.snap$/;
 export class TestDiscoveryService {
     EXCLUDED_DIRS = [
         "node_modules",
@@ -143,6 +151,7 @@ export class TestDiscoveryService {
         const result = { skyramp: [], external: [] };
         const contentCache = new Map();
         const marker = this.SKYRAMP_MARKER;
+        const dropped = [];
         const batchSize = 100;
         for (let i = 0; i < testCandidates.length; i += batchSize) {
             const batch = testCandidates.slice(i, i + batchSize);
@@ -153,16 +162,27 @@ export class TestDiscoveryService {
                         result.skyramp.push(file);
                         contentCache.set(file, content);
                     }
-                    else {
+                    else if (hasTestFileName(file) || COLOCATED_SNAPSHOT.test(file)) {
                         result.external.push(file);
                         contentCache.set(file, content);
                     }
+                    else {
+                        // A candidate without the marker and without a test file name is a
+                        // helper, fixture, page object, setup file or config that only
+                        // qualified by sitting in a test directory. It cannot be run as a
+                        // test, so it is not external coverage. Skyramp's own files keep the
+                        // wide rule above -- the marker is what identifies them.
+                        dropped.push(file);
+                    }
                 }
                 catch (error) {
-                    logger.debug(`Skipping file ${file}: ${error}`);
+                    logger.warning(`Skipping file ${file}: ${error}`);
                 }
             }
         }
+        if (dropped.length > 0) {
+            logger.debug(`Dropped ${dropped.length} test-directory candidates with no test file name: ${dropped.slice(0, 10).join(", ")}`);
+        }
         logger.debug(`Classified ${result.skyramp.length} Skyramp files, ${result.external.length} external test files`);
         return { ...result, contentCache };
     }
@@ -171,7 +191,11 @@ export class TestDiscoveryService {
      * or directory placement.
      */
     isExternalTestFile(filePath) {
-        return isDiscoveredTestFile(filePath);
+        // Either rule may admit a candidate: the directory rule for a test that
+        // sits in a test tree, the name rule for one that does not. Without the
+        // second, `auth.e2e.ts` or `OrdersIT.java` outside a test directory never
+        // reaches the classification below.
+        return isDiscoveredTestFile(filePath) || hasTestFileName(filePath);
     }
     /**
      * Extract metadata from a test file
@@ -473,7 +497,11 @@ export class TestDiscoveryService {
         const languageMap = {
             ".py": "python",
             ".js": "javascript",
+            ".mjs": "javascript",
+            ".cjs": "javascript",
             ".ts": "typescript",
+            ".mts": "typescript",
+            ".cts": "typescript",
             ".tsx": "typescript",
             ".jsx": "javascript",
             ".java": "java",

package/build/skills/runTestSkill.d.ts

@@ -0,0 +1,6 @@
+/** The `run-test` skill ships in `plugin/skills/` at the package root. */
+export declare const RUN_TEST_SKILL = "run-test";
+/** How to shape a command for skyramp_execute_test without the skill. */
+export declare const RUN_TEST_COMMAND_RULES: string;
+/** How a prompt tells the agent to run one test file. */
+export declare function runTestCall(): string;

package/build/skills/runTestSkill.js

@@ -0,0 +1,17 @@
+import { isSkillsLoaded } from "../utils/featureFlags.js";
+import { fixErrorsInstruction } from "./fixTestImportErrorsSkill.js";
+/** The `run-test` skill ships in `plugin/skills/` at the package root. */
+export const RUN_TEST_SKILL = "run-test";
+/** How to shape a command for skyramp_execute_test without the skill. */
+export const RUN_TEST_COMMAND_RULES = `for a TypeScript or JavaScript Playwright UI or end-to-end test, add \`--config "$SKYRAMP_PLAYWRIGHT_CONFIG"\`; ` +
+    `find the environment variable the test reads for its base URL — \`SKYRAMP_TEST_BASE_URL\` in a generated test, or the name the repository's own config reads in an existing one — and when the run exports no value for that name, put \`<NAME>=$SKYRAMP_TEST_SERVICE_URL_<SERVICENAME>\` in front of the command, for the service the test targets`;
+/** How a prompt tells the agent to run one test file. */
+export function runTestCall() {
+    if (isSkillsLoaded()) {
+        return (`invoke the \`${RUN_TEST_SKILL}\` skill with arguments ` +
+            `\`<absolute path of the test file> <language> <framework> <test type>\``);
+    }
+    return (`call \`skyramp_execute_test\` with the repository's own command for that one test file, ` +
+        `or the framework's usual command when the repository has none (${RUN_TEST_COMMAND_RULES}). ` +
+        `If the output shows that a package is missing — the runner itself, or a module the test or its config imports — ${fixErrorsInstruction()} and run the test again`);
+}

package/build/tool-phases.js

@@ -15,7 +15,6 @@ export const TOOL_PHASE_MAP = {
     skyramp_generate_enriched_integration_test: "generating",
     skyramp_enrich_test_with_mocks: "generating",
     skyramp_execute_test: { before: "maintaining", after: "executing" },
-    skyramp_run_existing_tests: { before: "maintaining", after: "executing" },
     skyramp_analyze_test_health: "maintaining",
     skyramp_submit_report: "reporting",
 };

package/build/tools/budgetExcuse.d.ts

@@ -0,0 +1,15 @@
+/** True when this answer's reason for dropping a test is that the run was running
+ *  out. Blank answers are NOT this: an unanswered objection is published open,
+ *  which is already honest. */
+export declare function citesBudget(answer: string | undefined): boolean;
+/** How many times the report may be refused for this. The refusal is meant to
+ *  cost the agent the tests it dropped, not the whole run: past this the answer
+ *  stands and is published with the report, where a reader can weigh it. Same
+ *  bound and same reasoning as the unrecorded-UI gate. */
+export declare const MAX_BUDGET_EXCUSE_REFUSALS = 3;
+/** Count one refusal and return the new total. */
+export declare function countBudgetExcuseRefusal(): number;
+/** Refusals so far, without counting one. */
+export declare function budgetExcuseRefusals(): number;
+/** Test seam. */
+export declare function resetBudgetExcuseStoreForTests(): void;

package/build/tools/budgetExcuse.js

@@ -0,0 +1,113 @@
+import { currentRunStateFile } from "../utils/AnalysisStateManager.js";
+import { logger } from "../utils/logger.js";
+/** "The run ran out of budget" as a reason a planned test was dropped, and how
+ *  many times the report has been refused for giving it.
+ *
+ *  WHY THIS EXISTS. `deliveredMatchesPlan` raises "planned but not delivered" and
+ *  suggests "add the test to the report, or record in the report why it was
+ *  dropped". Run 9dfc8405 recorded the reason seven times — "this run exhausted
+ *  its working budget" — and shipped 2 of 9 planned tests with the report
+ *  accepted. It wrote that at 46% of a 1M context window, with no warning from
+ *  anywhere, and the word appears NOWHERE in its reasoning: it is a justification
+ *  written at submit time, not a decision reached from a reading.
+ *
+ *  So this is not a budget check. It is a check on ONE SENTENCE the run is not
+ *  entitled to write. A test blocked by something outside the run — a service
+ *  that is not running, a branch that is gone, a credential the run lacks — still
+ *  answers and still ships, exactly as before. */
+/** Phrases that say "I stopped because I was running out", and nothing else.
+ *
+ *  Deliberately narrow, and narrowed further after review: `budget` and `token
+ *  limit` are ORDINARY APPLICATION NOUNS. An unscoped `\bbudget\b` refuses "the
+ *  budget page does not render: /budgets returns 500", and an unscoped `token
+ *  limit` refuses "the API rejects the request past its token limit" — real
+ *  blockers on a customer whose product has budgets, unreachable on any eval
+ *  repo, and nothing here is gated on the gauge, so the collision is the whole
+ *  test.
+ *
+ *  Every answer run 9dfc8405 sent is run-scoped — "this run exhausted its
+ *  working budget", "for budget reasons" — so scoping costs nothing. A miss
+ *  costs a refusal that would have helped; a false positive costs a run that
+ *  cannot report a genuine blocker, which is far worse. */
+const BUDGET_EXCUSE = [
+    // "for budget reasons" / "for budgetary reasons" — no other reading.
+    /\bbudget(ary)? reasons?\b/i,
+    // A spending verb beside the budget: "exhausted its working budget", "the
+    // remaining budget". SINGULAR ONLY — a product's own budgets are a plural
+    // list ("the remaining budgets are empty") and are not this.
+    /\b(exhausted|spent|ran out of|out of|low on|remaining|no more)\b[^.;]{0,24}\bbudget\b/i,
+    // "no budget left for", "not enough context left". The trailing word is what
+    // keeps "the page has no time zone selector" out.
+    /\b(no|insufficient|not enough) (time|context|tokens?|budget) (left|remaining|available|to spare|for)\b/i,
+    // The run's own context. Not a domain noun in any application we test.
+    /\bcontext (window|limit|budget)\b/i,
+    /\b(low on|out of|ran out of|exhausted (its|the|my)?) ?(context|time)\b/i,
+    /\btime (ran out|constraints?|pressure)\b/i,
+    // A token BUDGET is the agent talking about itself; a token LIMIT is a real
+    // API concept and stays out.
+    /\btokens? budget\b/i,
+    // Tokens only with the run as the subject: a rate limiter runs out of tokens
+    // too, and says so in a sentence a blocker needs.
+    /\b(this run|the run|I|we)\b[^.;]{0,30}\b(ran out of|out of|low on|exhausted)\b[^.;]{0,12}\btokens?\b/i,
+];
+/** True when this answer's reason for dropping a test is that the run was running
+ *  out. Blank answers are NOT this: an unanswered objection is published open,
+ *  which is already honest. */
+export function citesBudget(answer) {
+    const text = typeof answer === "string" ? answer.trim() : "";
+    if (!text)
+        return false;
+    return BUDGET_EXCUSE.some((pattern) => pattern.test(text));
+}
+/** How many times the report may be refused for this. The refusal is meant to
+ *  cost the agent the tests it dropped, not the whole run: past this the answer
+ *  stands and is published with the report, where a reader can weigh it. Same
+ *  bound and same reasoning as the unrecorded-UI gate. */
+export const MAX_BUDGET_EXCUSE_REFUSALS = 3;
+/** In process and keyed by the run, for the reason `traceExportStore` gives: the
+ *  only reader shares a process with the tool, and `NO_RUN` is a slot like any
+ *  other so a run whose state path will not resolve is still bounded. */
+const NO_RUN = "<no-run>";
+let countedRunStateFile;
+let refusals = 0;
+function reseatRun() {
+    let runStateFile;
+    try {
+        runStateFile = currentRunStateFile() ?? NO_RUN;
+    }
+    catch {
+        runStateFile = NO_RUN;
+    }
+    if (runStateFile !== countedRunStateFile) {
+        countedRunStateFile = runStateFile;
+        refusals = 0;
+    }
+}
+/** Count one refusal and return the new total. */
+export function countBudgetExcuseRefusal() {
+    try {
+        reseatRun();
+        refusals += 1;
+        return refusals;
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        logger.warning(`countBudgetExcuseRefusal failed: ${detail}`);
+        return MAX_BUDGET_EXCUSE_REFUSALS;
+    }
+}
+/** Refusals so far, without counting one. */
+export function budgetExcuseRefusals() {
+    try {
+        reseatRun();
+        return refusals;
+    }
+    catch {
+        return MAX_BUDGET_EXCUSE_REFUSALS;
+    }
+}
+/** Test seam. */
+export function resetBudgetExcuseStoreForTests() {
+    countedRunStateFile = undefined;
+    refusals = 0;
+}

package/build/tools/code-refactor/utils-verify-gates.js

@@ -1,5 +1,6 @@
 import * as path from "path";
 import { allowMarkerLine, describeAssertionCount, MARKERABLE_UTILS_KINDS, HELPER_FAMILIES, } from "../../utils/utils-verify/index.js";
+import { describeActionKey } from "../../utils/utils-verify/action-key.js";
 import { utilsFileLabel } from "./reuse-state.js";
 import { STATUS_ONCE_RULE } from "../../prompts/shared-helper-policy.js";
 /** How many verify rounds the agent may spend before it must document what remains.
@@ -218,6 +219,9 @@ function moduleLabel(r) {
         ? utilsFileLabel(r.utilsFiles)
         : "the shared utils file";
 }
+/** The half of the sibling call-site advisory both families share: a site inside
+ *  the sibling's own helper, the execution owed, and that nothing blocks. */
+const inlineSiteTail = (localCopy) => `For a site in a local helper, replace ${localCopy} inside the local helper with a call to the shared helper and keep the local helper's other statements; delete the local helper only if it then does nothing but forward, moving the import and the differing literals to its call sites. Then re-verify AND run the edited file with skyramp_execute_test — an edited pre-existing test without a recorded execution cannot be reported. This does not block execution.`;
 /** SKYR-4219: the sibling call-site pass the prompt asks for and the agent skips (0/2
  *  seeded eval runs). Stated here because the verify call is the one point every run
  *  reaches. Advisory — the sibling is a pre-existing test the maintenance flow owns, so
@@ -230,12 +234,22 @@ function inlineCallSitesAdvisory(r) {
     const labels = new Map(r.utilsFiles.map((f) => [f, utilsFileLabel([f])]));
     const labelOf = (f) => labels.get(f) ?? utilsFileLabel([f]);
     const n = r.inlineCallSites.length;
+    // One family per pass: the sites' kind decides the family half of the text.
+    const family = r.inlineCallSites[0].kind === "sequence"
+        ? HELPER_FAMILIES.browser
+        : HELPER_FAMILIES.api;
+    const lines = (c) => c.kind === "sequence" && c.endLine !== c.line
+        ? `${c.line}-${c.endLine}`
+        : `${c.line}`;
+    const what = (c) => c.kind === "sequence"
+        ? describeActionKey(c.actionKey)
+        : `${c.method} ${c.path}`;
     return [
         "",
         "",
-        `ADVISORY — ${n} inline request call${n === 1 ? "" : "s"} in other Skyramp-generated tests that a shared helper already wraps:`,
-        ...r.inlineCallSites.map((c) => `- ${show(c.file, r)}:${c.line}${c.localHelper ? ` (in local helper ${c.localHelper})` : ""} · ${c.method} ${c.path} · ${labelOf(c.utilsFile)}.${c.helper}`),
-        `Each is a candidate: the request's method, path and call shape match the helper's. Replace the block with a call to the helper only if the remaining differences are literals (they become the arguments; keep the block's response variable; import the helper into that file; change nothing else there). For a site in a local helper, replace that request inside the local helper with a call to the shared helper and keep the local helper's other statements; delete the local helper only if it then does nothing but forward, moving the import and the differing literals to its call sites. Then re-verify AND run the edited file with skyramp_execute_test — an edited pre-existing test without a recorded execution cannot be reported. This does not block execution.`,
+        `ADVISORY — ${n} ${family.inlineSite.noun}${n === 1 ? "" : "s"} in other Skyramp-generated tests that a shared helper already ${family.inlineSite.already}:`,
+        ...r.inlineCallSites.map((c) => `- ${show(c.file, r)}:${lines(c)}${c.localHelper ? ` (in local helper ${c.localHelper})` : ""} · ${what(c)} · ${labelOf(c.utilsFile)}.${c.helper}`),
+        `${family.inlineSite.instruction} ${inlineSiteTail(family.inlineSite.localCopy)}`,
     ].join("\n");
 }
 /** Operations written in two or more sibling generated tests with no shared helper —

package/build/tools/executeSkyrampTestTool.d.ts

@@ -1,18 +1,18 @@
+import { z } from "zod";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { TestExecutionResult } from "../types/TestExecution.js";
-import { TestType } from "../types/TestTypes.js";
+import { TestExecutionStatus, TestExecutionResult, type TestExecutionRecord } from "../types/TestExecution.js";
+import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { MaintenanceActionCore, TestAnalysisResult } from "../types/TestAnalysis.js";
-import type { TestExecutionRecord } from "../types/TestExecution.js";
-export declare const CONTRACT_EXECUTION_MODES: readonly ["provider", "consumer"];
-export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
+export declare const TOOL_NAME = "skyramp_execute_test";
+/** Output kept per run, from its end: it is held in memory, saved to the state file, and returned to the agent. */
+export declare const MAX_OUTPUT_CHARS = 200000;
 /**
- * Resolve the effective auth token for test execution.
- * `unauthenticated: true` forces no token (empty string) so the container
- * env omits SKYRAMP_TEST_TOKEN entirely — unauthenticated endpoints won't
- * receive an empty Authorization header that triggers encoding errors (E7).
+ * `unauthenticated: true` forces no token, so the child env carries no
+ * SKYRAMP_TEST_TOKEN at all — an empty Authorization header triggers encoding
+ * errors on unauthenticated endpoints (E7). An empty `token` means "use the
+ * server's environment", as the local-dev prompt passes it.
  */
 export declare function resolveEffectiveToken(unauthenticated?: boolean, paramToken?: string, envToken?: string): string;
-export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractMode?: ContractExecutionMode): boolean;
 /**
  * Append the recorded video path to execution output.
  *
@@ -22,6 +22,67 @@ export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractM
  * its own attachment line.
  */
 export declare function withVideoInfo(output: string, videoPath?: string): string;
+/**
+ * One directory per run: a unique name leaves nothing to clean up between runs, and
+ * the random suffix keeps two runs in the same millisecond apart.
+ */
+export declare function videoSubdirName(testFile: string): string;
+/** The first `video.webm` under a run's video directory, if one was recorded. */
+export declare function collectVideoPath(videoDir: string): string | undefined;
+/**
+ * The directory whose `.skyramp/workspace.yml` describes this run: the nearest
+ * ancestor of `cwd` that has one, else the run's primary checkout (a tests repo
+ * delivered apart from the SUT has no workspace.yml of its own), else `cwd`.
+ */
+export declare function resolveWorkspaceRoot(cwd: string): string;
+/** pytest splits PYTEST_ADDOPTS with shlex, so a path with whitespace needs quotes. */
+export declare function appendPytestVideoOpts(existing: string | undefined, videoDir: string): string;
+/**
+ * The names by which a command can select `testFile`: its basename, and for Java the
+ * class name, because Maven selects a test with `-Dtest=FooTest`.
+ */
+export declare function testFileNames(testFile: string): string[];
+/**
+ * Backslashes are ignored: Playwright and Jest read the path as a regular
+ * expression, so the agent escapes it (`a\.spec\.ts`).
+ *
+ * A trailing shell comment is removed first. `npm test # a_test.py` names the
+ * file only in a comment and would run the whole suite, and the check exists
+ * to stop exactly that. This is a name check, not a parse: a command that
+ * mentions the file in some other way it does not run still passes.
+ */
+/** The config an explicit `--config <path>` names, resolved against `cwd`, or
+ *  undefined when the command carries none. `$SKYRAMP_PLAYWRIGHT_CONFIG` is
+ *  ours and is not a repository config. */
+export declare function explicitPlaywrightConfig(command: string, cwd: string): string | undefined;
+/** The command with its `--config <repo config>` pointed at the wrapper. The
+ *  wrapper imports that same config, so the run keeps the repository's testDir
+ *  and projects and gains the video overlay. Without this swap a command that
+ *  names a config runs the config directly and records nothing. */
+export declare function pointConfigAtWrapper(command: string, wrapperPath: string): string;
+export declare function commandNamesTestFile(command: string, testFile: string): boolean;
+/** pytest exits 4 on a usage error; without pytest-playwright `--video` is one. */
+export declare function pytestRejectedVideoOptions(run: SpawnedRun): boolean;
+/** The line proving no test ran, or undefined when the output does not say so. */
+export declare function noTestRanReason(output: string): string | undefined;
+export declare function verdictForExit(exitCode: number | null, timedOut: boolean, output?: string): TestExecutionStatus;
+export interface SpawnedRun {
+    exitCode: number | null;
+    output: string;
+    timedOut: boolean;
+    spawnError?: string;
+    duration: number;
+}
+/**
+ * Runs `command` through the shell in its own process group, so a timeout kills
+ * the runner's children too. stdout and stderr share one buffer in arrival order.
+ */
+export declare function spawnTestCommand(opts: {
+    command: string;
+    cwd: string;
+    env: NodeJS.ProcessEnv;
+    timeoutMs: number;
+}): Promise<SpawnedRun>;
 /**
  * Normalize the `rebaselineSnapshots` request (SKYR-4298): dedupe and drop blanks,
  * and refuse it on the pre-edit run. A refresh there would overwrite the very
@@ -32,23 +93,6 @@ export declare function resolveRebaselineSnapshots(requested: string[] | undefin
     snapshots: string[];
     error?: string;
 };
-/**
- * The failure text the agent receives. Everything it needs has to be in here:
- * only a tool's return value reaches the transcript, and Claude Code does not
- * capture an MCP server's stderr, so anything this omits is unrecoverable once
- * the run ends.
- *
- * `output` alone was not enough. When the executor produced none, the message
- * read "Test execution failed:" and stopped — so the agent inferred a cause and
- * reported it as fact. Measured in eval run 32220788735: it blamed an
- * unreachable backend while the app was answering 200 on both the host and the
- * docker bridge, and that invented cause reached `issuesFound`.
- *
- * The 401 hint is emitted only when the output carries a 401 in a status-shaped
- * position — see HTTP_401_SHAPES. It used to be unconditional, which pointed the
- * agent at authentication on a run whose output was empty, and then it keyed off
- * the bare number, which pointed it there on a test count or a line number.
- */
 /** The on-disk files a requested baseline name resolves to; empty when none matches. */
 export type BaselineFileState = Array<{
     file: string;
@@ -56,20 +100,12 @@ export type BaselineFileState = Array<{
     mtimeMs: number;
 }>;
 /**
- * Snapshot of the requested baselines under `<spec>-snapshots/` (SKYR-4298): for
- * each requested name, every PNG whose name matches the stem (a spec may hold both
- * `<stem>-linux.png` and `<stem>-chromium-linux.png`; latching onto one of them would
- * misreport the other) with its size and mtime. Taken before and after the run so
- * the tool can tell the agent which baselines were actually rewritten — SmartPlaywright
- * is the only party that knows the exact filename, and an executor image that lacks
- * SKYRAMP_UPDATE_SNAPSHOTS (or a name matching no toHaveScreenshot call) leaves
- * every file untouched.
+ * For each requested name, every PNG under `<spec>-snapshots/` whose name matches the
+ * stem, with size and mtime. A spec may hold both `<stem>-linux.png` and
+ * `<stem>-chromium-linux.png`; latching onto one would misreport the other. Taken
+ * before and after the run to tell which baselines were actually rewritten.
  */
 export declare function readBaselineState(specFile: string, requested: string[]): Record<string, BaselineFileState>;
-/**
- * Which requested baselines changed on disk between two readBaselineState calls, and
- * which files carried the change (the ones to stage).
- */
 export declare function diffBaselineState(before: Record<string, BaselineFileState>, after: Record<string, BaselineFileState>): {
     refreshed: string[];
     notRefreshed: string[];
@@ -92,19 +128,16 @@ export declare function authorizeRebaseline(stateData: {
     error?: string;
 };
 /**
- * Reconcile the persisted verdict with what the executor actually did (SKYR-4298).
- * An executor image whose @skyramp/skyramp predates SKYRAMP_UPDATE_SNAPSHOTS rewrites
- * nothing; left alone, the verdict would still promise a refresh, the report gate
- * would refuse the report, and nothing in the prompt makes the agent's way out
- * deterministic. So: names that were not refreshed are dropped from the verdict; a
- * rebaseline-only UPDATE with nothing left becomes VERIFY (the test stays red, the
- * rationale says why), and an UPDATE that also carried edits keeps UPDATE and is
- * held to its edit. The report then reflects what happened, not what was asked.
+ * Reconcile the persisted verdict with what the run actually did (SKYR-4298). A
+ * @skyramp/skyramp that predates SKYRAMP_UPDATE_SNAPSHOTS rewrites nothing; left
+ * alone, the verdict would still promise a refresh and the report gate would refuse
+ * the report. Names not refreshed are dropped; a rebaseline-only UPDATE with nothing
+ * left becomes VERIFY, and an UPDATE that also carried edits is held to its edit.
  */
 export declare function applyRefreshOutcomeToVerdicts(verdicts: MaintenanceActionCore[], testFile: string, outcome: {
     refreshed: string[];
     notRefreshed: string[];
-}, executorImage: string): {
+}): {
     verdicts: MaintenanceActionCore[];
     note?: string;
 };
@@ -118,6 +151,7 @@ export declare function describeRefreshOutcome(outcome: {
     refreshed: string[];
     notRefreshed: string[];
 }): string;
+/**
 /**
  * The fix-and-rerun attempt cap (SKYR-4460), enforced where the prompt's prose
  * cannot be: a file that has already received `cap` runs in the final phase
@@ -139,4 +173,19 @@ export declare function describeAttempt(priorAfterRuns: number, cap: number, opt
 export declare function buildExecutionFailureText(result: TestExecutionResult, opts?: {
     unchangedRerun?: boolean;
 }): string;
+export declare const inputSchema: {
+    commandOverride: z.ZodOptional<z.ZodString>;
+    cwd: z.ZodOptional<z.ZodString>;
+    workspacePath: z.ZodOptional<z.ZodString>;
+    testFile: z.ZodString;
+    language: z.ZodNativeEnum<typeof ProgrammingLanguage>;
+    testType: z.ZodNativeEnum<typeof TestType>;
+    phase: z.ZodDefault<z.ZodEnum<["before", "after"]>>;
+    stateFile: z.ZodOptional<z.ZodString>;
+    repository: z.ZodString;
+    token: z.ZodOptional<z.ZodString>;
+    unauthenticated: z.ZodOptional<z.ZodBoolean>;
+    rebaselineSnapshots: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    timeout: z.ZodOptional<z.ZodNumber>;
+};
 export declare function registerExecuteSkyrampTestTool(server: McpServer): void;