@skyramp/mcp 0.4.2-rc.1 → 0.4.2-rc.2

package/build/tools/submitReportTool.js

@@ -20,6 +20,9 @@ import { checkDefectsReported, checkIssueTraceability, } from "../recommendation
 import { checkExpectedOutcomeAfterExecution, } from "../recommendation/verifiers/expectedOutcome.js";
 import { findInvalidSourceCitations, findUnchangedFileClaims, listChangedFiles, listChangedFilesAcross, listChangedFilesAbs, } from "../utils/reportVerification.js";
 import { isPlanOnlyMode } from "../utils/planOnlyMode.js";
+import { contextGaugeSentence } from "../utils/runContextGauge.js";
+import { budgetExcuseRefusals, citesBudget, countBudgetExcuseRefusal, MAX_BUDGET_EXCUSE_REFUSALS, } from "./budgetExcuse.js";
+import { countUiDeliveryRefusal, MAX_UI_DELIVERY_REFUSALS, traceExportAttempts, uiDeliveryRefusals, } from "../playwright/traceExportStore.js";
 import { getReportLanguage, isEnforcedReportLanguage, findLanguageViolations, findLanguageNearMisses, reportLanguageDisplayName, } from "../utils/reportLanguage.js";
 import { canonicalTestPath, findAssertionRecordByFileName, rederiveAssertionOutcome, } from "./code-refactor/assertion-state.js";
 import { rederiveReuse, deriveReuseFromDelivered, REUSE_SUBMIT_MAX_REFUSALS, reuseChainSkipped, samePath, } from "./code-refactor/reuse-state.js";
@@ -1124,6 +1127,29 @@ function skippedDeliveredTests(delivered, results) {
             !matching.some((row) => row.status !== "Skipped"));
     });
 }
+/** A planned test the plan typed `ui`. The plan's `testType` is free-form text —
+ *  the unapproved-entry check above says so — so it is trimmed and lower-cased
+ *  rather than compared against the enum. */
+function plannedUiTests(plannedTests) {
+    return plannedTests.filter((test) => !!test?.plannedTestId?.trim() &&
+        String(test?.scenario?.testType ?? "")
+            .trim()
+            .toLowerCase() === TestType.UI);
+}
+/** Planned UI tests with no entry in the report: the recordings that never
+ *  happened, named by the ids the agent would otherwise only have to answer for. */
+/** Every planned test with no entry in the report, whatever its type. The UI
+ *  filter below narrows this; the budget-excuse gate does not, because a dropped
+ *  contract test is dropped the same way a dropped recording is. */
+function undeliveredPlannedTests(plannedTests, delivered) {
+    const shipped = new Set(delivered
+        .map((test) => test.plannedTestId?.trim())
+        .filter((id) => !!id));
+    return plannedTests.filter((test) => !!test?.plannedTestId?.trim() && !shipped.has(test.plannedTestId.trim()));
+}
+function undeliveredPlannedUiTests(plannedTests, delivered) {
+    return undeliveredPlannedTests(plannedUiTests(plannedTests), delivered);
+}
 /** Whether the file can be read. The written record outlives the file, and a
  *  path that exists but cannot be opened owes no run either — the objection
  *  would name a file nobody can look at. */
@@ -2166,9 +2192,111 @@ export function registerSubmitReportTool(server) {
                 .map((test) => `${test.plannedTestId?.trim() || test.testId}`)
                 .join(", ")} ${skippedTests.length === 1 ? "has" : "have"} no non-skipped execution result. ` +
                 "Do not skip because you believe you are low on context or time, or because you speculate that another attempt will not help. " +
+                contextGaugeSentence() +
                 "Run each remaining test with skyramp_execute_test, include its Pass or Fail result in testResults, then call skyramp_submit_report again.");
             return errorResult;
         }
+        // A planned UI test with no report entry is a recording that never happened.
+        // `deliveredMatchesPlan` already raises it and already accepts an answer, and
+        // that is how one run shipped a 14-test plan with ZERO UI tests: a single
+        // skyramp_export_zip, a trace that came out polluted, and an answer blaming
+        // the recorder. Refuse while the run has recorded fewer times than it planned
+        // UI tests; past that it has genuinely tried each one and the answer stands.
+        //
+        // WHY THE RETRY IS NOT FUTILE, and why the message says so: a successful
+        // skyramp_export_zip DRAINS the recorder and closes the browser, so the next
+        // recording starts from an empty buffer. Nothing the agent can read says that,
+        // and that run reasoned the opposite and stopped one call short of a clean
+        // trace.
+        const undeliveredUi = isPlanOnlyMode() || planUnreadable
+            ? []
+            : undeliveredPlannedUiTests(stateData.plan?.plannedTests ?? [], dedupedNewTests);
+        const plannedUiCount = planUnreadable
+            ? 0
+            : plannedUiTests(stateData.plan?.plannedTests ?? []).length;
+        // ATTEMPTS, so one more recording always answers a refusal and the gate opens
+        // after at most `plannedUiCount` of them. The refusal cap is the second bound,
+        // for the agent that resubmits having recorded nothing: this check is meant to
+        // cost a recording, never the whole run.
+        const recordings = traceExportAttempts();
+        if (undeliveredUi.length > 0 &&
+            recordings < plannedUiCount &&
+            uiDeliveryRefusals() < MAX_UI_DELIVERY_REFUSALS) {
+            const refusal = countUiDeliveryRefusal();
+            const ids = undeliveredUi
+                .map((test) => test.plannedTestId.trim())
+                .join(", ");
+            errorResult = toolError(`Cannot submit the report: ${undeliveredUi.length} registered UI plan test${undeliveredUi.length === 1 ? "" : "s"} ` +
+                `${ids} ${undeliveredUi.length === 1 ? "has" : "have"} no entry in newTestsCreated, and this run recorded ` +
+                `${recordings} trace${recordings === 1 ? "" : "s"} for ${plannedUiCount} planned UI test${plannedUiCount === 1 ? "" : "s"}. ` +
+                "Do not skip because you believe you are low on context or time, or because you speculate that another attempt will not help. " +
+                contextGaugeSentence() +
+                "A successful skyramp_export_zip clears the recording buffer and closes the browser, so the next recording starts clean — " +
+                "a trace that came out polluted is a reason to record again, not a reason to stop. " +
+                "Record each remaining test in its own pass (browser_navigate to the start URL, the flow, skyramp_export_zip, skyramp_ui_test_generation), " +
+                "add it to newTestsCreated with its plannedTestId, then call skyramp_submit_report again. " +
+                `The plan and every answer you have given are in the state file and survive this refusal (refusal ${refusal} of ${MAX_UI_DELIVERY_REFUSALS}).`);
+            return errorResult;
+        }
+        // A planned test dropped because the run says it was running out.
+        //
+        // `deliveredMatchesPlan` raises "planned but not delivered" and suggests
+        // "add the test to the report, or record in the report why it was dropped".
+        // Run 9dfc8405 recorded the reason seven times — "this run exhausted its
+        // working budget" — and shipped 2 of 9 planned tests with the report
+        // ACCEPTED. It wrote that at 46% of a 1M context window with no warning from
+        // anywhere, and the word appears nowhere in its reasoning: the budget was not
+        // a finding, it was a sentence that closed an objection.
+        //
+        // This refuses that ONE sentence, not a budget. A test blocked by something
+        // outside the run answers and ships exactly as before, which is why the
+        // message says what a reason that stands looks like.
+        //
+        // Answers from earlier calls count too: one that closed the objection on
+        // submit N still closes it on N+1 without being re-sent, and a gate reading
+        // only this call would be walked past by not repeating it.
+        const answerText = new Map();
+        const carriedAnswers = Array.isArray(stateData.reportObjections?.answeredObjections)
+            ? stateData.reportObjections.answeredObjections
+            : [];
+        for (const entry of carriedAnswers) {
+            const id = entry
+                ?.objection?.objectionId;
+            const answer = entry?.answer;
+            if (typeof id === "string" && typeof answer === "string")
+                answerText.set(id, answer);
+        }
+        for (const entry of params.answers ?? []) {
+            if (typeof entry?.objectionId === "string" &&
+                typeof entry?.answer === "string")
+                answerText.set(entry.objectionId, entry.answer);
+        }
+        const droppedForBudget = isPlanOnlyMode() || planUnreadable
+            ? []
+            : undeliveredPlannedTests(stateData.plan?.plannedTests ?? [], dedupedNewTests).filter((test) => citesBudget(answerText.get(`deliveredMatchesPlan:${test.plannedTestId.trim()}`)));
+        if (droppedForBudget.length > 0 &&
+            budgetExcuseRefusals() < MAX_BUDGET_EXCUSE_REFUSALS) {
+            const refusal = countBudgetExcuseRefusal();
+            const ids = droppedForBudget
+                .map((test) => test.plannedTestId.trim())
+                .join(", ");
+            const plural = droppedForBudget.length === 1;
+            errorResult = toolError(`Cannot submit the report: ${droppedForBudget.length} planned test${plural ? "" : "s"} ` +
+                `${ids} ${plural ? "is" : "are"} missing from newTestsCreated, and the answer given for ` +
+                `${plural ? "it" : "each"} is that the run was short of budget, context or time. ` +
+                "That is not a reason this report accepts. " +
+                contextGaugeSentence() +
+                "A reason that stands names something outside the run that stopped the test: a service that is not " +
+                "running, a branch that no longer exists, a credential this run does not hold, a behaviour the " +
+                "application will not reach. Running low is not one of those — it is a reason to write the test now, " +
+                "with the rest of the report already built. " +
+                `Write the missing test${plural ? "" : "s"}, add ${plural ? "it" : "them"} to newTestsCreated with the ` +
+                "plannedTestId, and call skyramp_submit_report again — or replace the answer with what actually " +
+                "stopped it. " +
+                `The plan, the report and every answer you have given are in the state file and survive this refusal ` +
+                `(refusal ${refusal} of ${MAX_BUDGET_EXCUSE_REFUSALS}).`);
+            return errorResult;
+        }
         // Report-time checks on what the customer will read. They need no plan, so a
         // run without one still gets them.
         const reportChecks = [

package/build/tools/test-management/actionsTool.js

@@ -13,6 +13,9 @@ import { REBASELINE_SNAPSHOT_NAME_RE, looksLikeOnDiskBaselineName, } from "../..
 import { buildRenameStrategy, buildFileRenameStrategy, buildUpdateStrategy, buildRebaselineStrategy, buildRegenerateStrategy, buildDeleteStrategy, buildUpdateFileInstruction, buildRegenerateFileInstruction, } from "../../prompts/test-maintenance/actionsInstructions.js";
 import { buildModularizeFirstNextSteps, isMaintenanceRegenerable, isModularizeFirstHandOffTarget, regenerateCodeReuse, } from "../../prompts/reuse-hand-off.js";
 import { recordReuseHandOff } from "../code-refactor/reuse-state.js";
+import { isPlanOnlyMode } from "../../utils/planOnlyMode.js";
+import { runTestCall } from "../../skills/runTestSkill.js";
+import { formatOwedFiles, testsOwedBeforeRun } from "./testsOwedBeforeRun.js";
 /**
  * Compute a suggested new filename when an endpoint is renamed.
  */
@@ -442,6 +445,34 @@ export function registerActionsTool(server) {
                 if (renamed)
                     r._suggestedNewFile = renamed;
             }
+            // ── Before-run gate on the tests this call will edit ──────────────
+            // Drift on the repository's own test must rest on a run, not on a
+            // reading of its source: without the pre-edit result the report cannot
+            // say what the edit changed, and the test lands at afterStatus=Unknown.
+            // The subject is what this call edits, and nothing wider: the gate used
+            // to sit in skyramp_analyze_test_health and demand a run for every
+            // external test in state, which is the first hundred files discovery
+            // found in glob order.
+            //
+            // A DELETE or a REGENERATE on an external test is report-only (the run
+            // edits nothing), and a related repo's entry is already a VERIFY by
+            // here, so neither owes a run.
+            const { owed, exemptLanguage } = testsOwedBeforeRun(recommendations, catalogByFile);
+            if (exemptLanguage > 0) {
+                logger.info(`Before-run gate: ${exemptLanguage} recommended external test(s) owe no run — skyramp_execute_test cannot run their language.`);
+            }
+            // A plan-only run has no application to run the tests against.
+            if (owed.length > 0 && !isPlanOnlyMode()) {
+                const filesList = formatOwedFiles(owed);
+                errorResult = toolError(`${owed.length} test${owed.length === 1 ? "" : "s"} you recommend editing ${owed.length === 1 ? "has" : "have"} no phase: "before" result recorded — never run, or a run was started and recorded no result: ${filesList}. ` +
+                    `Run each file through skyramp_execute_test with phase: "before", stateFile: "${args.stateFile}"${args.repository ? `, repository: "${args.repository}"` : ""}: ${runTestCall()}. Then call ${TOOL_NAME} again. Nothing was recorded by this call.`);
+                return errorResult;
+            }
+            else if (owed.length > 0) {
+                // planOnlyMode is process-global and captured at prompt-render time, so
+                // a skip must be visible: no run means no evidence behind these edits.
+                logger.warning(`Before-run gate skipped in a plan-only run: ${owed.length} test(s) you recommend editing have no phase: "before" result recorded.`);
+            }
             const recommendedPaths = new Set(recommendations.map((r) => r.testFilePath));
             const ignoredTestFiles = testAnalysisResults
                 .map((t) => t.testFile)

package/build/tools/test-management/analyzeChangesTool.d.ts

@@ -131,8 +131,8 @@ export declare const analyzeChangesOutputSchema: {
             analysedWorkspaceFile: z.ZodString;
         }, "strip", z.ZodTypeAny, {
             message: string;
-            kind: "workspace-disagreement";
             serviceName: string;
+            kind: "workspace-disagreement";
             field: "authHeader" | "authScheme" | "authType";
             primaryWorkspaceFile: string;
             analysedWorkspaceFile: string;
@@ -140,8 +140,8 @@ export declare const analyzeChangesOutputSchema: {
             analysed: string;
         }, {
             message: string;
-            kind: "workspace-disagreement";
             serviceName: string;
+            kind: "workspace-disagreement";
             field: "authHeader" | "authScheme" | "authType";
             primaryWorkspaceFile: string;
             analysedWorkspaceFile: string;
@@ -156,8 +156,8 @@ export declare const analyzeChangesOutputSchema: {
         openApiSpecPath?: string | undefined;
         authFindings?: {
             message: string;
-            kind: "workspace-disagreement";
             serviceName: string;
+            kind: "workspace-disagreement";
             field: "authHeader" | "authScheme" | "authType";
             primaryWorkspaceFile: string;
             analysedWorkspaceFile: string;
@@ -172,8 +172,8 @@ export declare const analyzeChangesOutputSchema: {
         openApiSpecPath?: string | undefined;
         authFindings?: {
             message: string;
-            kind: "workspace-disagreement";
             serviceName: string;
+            kind: "workspace-disagreement";
             field: "authHeader" | "authScheme" | "authType";
             primaryWorkspaceFile: string;
             analysedWorkspaceFile: string;

package/build/tools/test-management/analyzeTestHealthTool.d.ts

@@ -1,13 +1,2 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-/**
- * Names of services whose workspace.yml records a runnable external-test suite
- * (a reader-backed framework — playwright/pytest/jest/vitest/mocha — +
- * a `runtimeDetails.testSuites` entry with a testRunCommand) — i.e. what skyramp_run_existing_tests can
- * actually run. Used to scope the confirm gate so it only fires on repos that
- * went through Test Environment Setup. Checks each service's suites via
- * `resolveSuites` (rather than calling resolveRunConfig) so every
- * `runtimeDetails.testSuites` entry is recognized. Returns [] on any read/parse
- * error so a config problem can never block drift analysis.
- */
-export declare function runnableTestEnvServices(repositoryPath: string): Promise<string[]>;
 export declare function registerAnalyzeTestHealthTool(server: McpServer): void;

package/build/tools/test-management/analyzeTestHealthTool.js

@@ -5,9 +5,6 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { buildDriftAnalysisPrompt } from "../../prompts/test-maintenance/drift-analysis-prompt.js";
 import { TestSource } from "../../types/TestAnalysis.js";
 import { toolError } from "../../utils/utils.js";
-import { WorkspaceConfigManager } from "../../workspace/workspace.js";
-import { resolveSuites } from "../../workspace/testSuites.js";
-import { frameworkHasReader } from "../runExistingTestsTool.js";
 const TOOL_NAME = "skyramp_analyze_test_health";
 // UI test frameworks. "playwright-ui" and "cypress-ui" are the browser-interaction
 // variants detected when the file uses the page fixture (not just the request fixture).
@@ -38,27 +35,6 @@ function isUiTest(t) {
         UI_TEST_EXTENSIONS.test(t.testFile) ||
         isUiSnap);
 }
-/**
- * Names of services whose workspace.yml records a runnable external-test suite
- * (a reader-backed framework — playwright/pytest/jest/vitest/mocha — +
- * a `runtimeDetails.testSuites` entry with a testRunCommand) — i.e. what skyramp_run_existing_tests can
- * actually run. Used to scope the confirm gate so it only fires on repos that
- * went through Test Environment Setup. Checks each service's suites via
- * `resolveSuites` (rather than calling resolveRunConfig) so every
- * `runtimeDetails.testSuites` entry is recognized. Returns [] on any read/parse
- * error so a config problem can never block drift analysis.
- */
-export async function runnableTestEnvServices(repositoryPath) {
-    try {
-        const config = await new WorkspaceConfigManager(repositoryPath).read();
-        return (config.services ?? [])
-            .filter((s) => resolveSuites(s).some((suite) => frameworkHasReader(suite.framework) && !!suite.testRunCommand))
-            .map((s) => s.serviceName);
-    }
-    catch {
-        return [];
-    }
-}
 export function registerAnalyzeTestHealthTool(server) {
     server.registerTool(TOOL_NAME, {
         annotations: {
@@ -111,44 +87,12 @@ export function registerAnalyzeTestHealthTool(server) {
             const skyrampCount = existingTests.filter((t) => t.source !== TestSource.External).length;
             const externalCount = existingTests.length - skyrampCount;
             logger.info(`Loaded ${skyrampCount} Skyramp + ${externalCount} relevant external tests from state file`);
-            // ── External-test confirm gate ────────────────────────────────────
-            // When the diff touches the repo's OWN [external] tests AND the workspace
-            // has a runnable test-env config (runtimeDetails.test*), the drift
-            // assessment must be grounded in a real run of those tests
-            // (skyramp_run_existing_tests mode:confirm), not the agent's guess from
-            // source. Prompt prose alone proved model-fragile — agents read the config
-            // and still skipped the confirm, leaving the repo's own tests at
-            // afterStatus=Unknown — so enforce ordering here: refuse to produce the
-            // drift prompt until a confirm run has been recorded. Loop-safe:
-            // skyramp_run_existing_tests persists an externalTestResults record for
-            // EVERY outcome (pass/fail/skip/unhealthy), so one confirm call clears the
-            // gate. Only a confirm-mode record counts — a verify run (or any future
-            // record type) must not bypass the required confirm pass.
-            // Scoped by runnableTestEnvServices so it never fires on repos without
-            // test-env setup, and fail-open on any config error.
-            const confirmAttempted = (stateData.externalTestResults ?? []).some((r) => r.mode === "confirm");
-            const runnableServices = externalCount > 0 && !confirmAttempted
-                ? await runnableTestEnvServices(repositoryPath)
-                : [];
-            if (runnableServices.length > 0) {
-                const externalFiles = existingTests
-                    .filter((t) => t.source === TestSource.External)
-                    .map((t) => t.testFile);
-                // Cap the enumerated examples — a broad diff can mark dozens of external
-                // tests, and the agent already has the full [external] list from
-                // skyramp_analyze_changes.
-                const shown = externalFiles.slice(0, 8);
-                const filesList = shown.map((f) => `"${f}"`).join(", ") +
-                    (externalFiles.length > shown.length
-                        ? `, …(+${externalFiles.length - shown.length} more)`
-                        : "");
-                // With >1 runnable service the tool needs an explicit `service`.
-                const serviceHint = runnableServices.length > 1
-                    ? ` Multiple runnable test services are configured — pass service: "${runnableServices[0]}".`
-                    : "";
-                return toolError(`This repository has a configured test environment (runtimeDetails.test*) and ${externalCount} external test${externalCount === 1 ? "" : "s"} relevant to this diff, but skyramp_run_existing_tests has not run yet. Run the repo's OWN tests BEFORE assessing drift so the assessment is grounded in real pass/fail (otherwise these tests are left at status Unknown). ` +
-                    `Call skyramp_run_existing_tests with mode: "confirm", stateFile: "${stateManager.getStatePath()}"${args.repository ? `, repository: "${args.repository}"` : ""}, and testSelectors set to the [external] test files skyramp_analyze_changes marked (e.g. ${filesList}).${serviceHint} Then call skyramp_analyze_test_health again.`);
-            }
+            // The repository's own tests, for the run results the drift prompt
+            // renders. A file with no `phase: "before"` run reaches the prompt as
+            // not run: the gate that demanded a run for every one of them moved to
+            // skyramp_actions, where the recommendations say which tests this run
+            // will edit.
+            const externalTests = existingTests.filter((t) => t.source === TestSource.External);
             // Delete stale diff files only — state files must remain for the caller's skyramp_actions call.
             try {
                 await StateManager.cleanupOldFiles(24, undefined, []);
@@ -186,7 +130,7 @@ export function registerAnalyzeTestHealthTool(server) {
             const allRepoPaths = [
                 ...new Set((await stateManager.listRepoCheckouts()).map((c) => c.root)),
             ];
-            const promptText = buildDriftAnalysisPrompt(stateManager.getStatePath(), apiTests.map((t) => ({ testFile: t.testFile, source: t.source })), uiDriftParams, allRepoPaths, stateData?.externalTestResults);
+            const promptText = buildDriftAnalysisPrompt(stateManager.getStatePath(), apiTests.map((t) => ({ testFile: t.testFile, source: t.source })), uiDriftParams, allRepoPaths, externalTests.flatMap((t) => t.executionBefore ? [t.executionBefore] : []));
             return {
                 structuredContent: { prompt: promptText },
                 content: [

package/build/tools/test-management/testsOwedBeforeRun.d.ts

@@ -0,0 +1,28 @@
+import { type DriftRecommendation, type TestAnalysisResult } from "../../types/TestAnalysis.js";
+/** One catalogued test, narrowed to what the gate reads. */
+export type GateCatalogRow = Pick<TestAnalysisResult, "testFile" | "language" | "source" | "executionBefore">;
+/** One recommendation, narrowed to what the gate reads. */
+export type GateRecommendation = Pick<DriftRecommendation, "testFilePath" | "action" | "reportOnly">;
+export interface TestsOwedBeforeRun {
+    /** Files this call would edit that have no usable pre-edit run. */
+    owed: string[];
+    /** Owed no run because skyramp_execute_test cannot run that language. */
+    exemptLanguage: number;
+}
+/**
+ * The tests this call edits that have no `phase: "before"` result.
+ *
+ * Only UPDATE reaches here. An external REGENERATE or DELETE is turned into a
+ * report-only recommendation before the gate runs, and a report-only row edits
+ * nothing, so neither owes a run.
+ *
+ * A recommendation with no catalog row is deliberately NOT gated. `skyramp_actions`
+ * accepts a real test file discovery missed and records a verdict for it rather than
+ * dropping it (SKYR-3906/SKYR-3938, the B1 test). Refusing it here for want of a
+ * before-run would reverse that: it has no catalog row, so it never has a recorded
+ * run, and the gate would refuse it on every call. Gating it is a change to B1's
+ * contract, not to this one.
+ */
+export declare function testsOwedBeforeRun(recommendations: readonly GateRecommendation[], catalogByFile: ReadonlyMap<string, GateCatalogRow>): TestsOwedBeforeRun;
+/** Every file, quoted, one list — the agent must run all of them. */
+export declare function formatOwedFiles(owed: readonly string[]): string;

package/build/tools/test-management/testsOwedBeforeRun.js

@@ -0,0 +1,53 @@
+import { DriftAction, TestSource, } from "../../types/TestAnalysis.js";
+import { ProgrammingLanguage } from "../../types/TestTypes.js";
+import { TestExecutionStatus } from "../../types/TestExecution.js";
+/**
+ * The tests this call edits that have no `phase: "before"` result.
+ *
+ * Only UPDATE reaches here. An external REGENERATE or DELETE is turned into a
+ * report-only recommendation before the gate runs, and a report-only row edits
+ * nothing, so neither owes a run.
+ *
+ * A recommendation with no catalog row is deliberately NOT gated. `skyramp_actions`
+ * accepts a real test file discovery missed and records a verdict for it rather than
+ * dropping it (SKYR-3906/SKYR-3938, the B1 test). Refusing it here for want of a
+ * before-run would reverse that: it has no catalog row, so it never has a recorded
+ * run, and the gate would refuse it on every call. Gating it is a change to B1's
+ * contract, not to this one.
+ */
+export function testsOwedBeforeRun(recommendations, catalogByFile) {
+    const runnableLanguages = Object.values(ProgrammingLanguage);
+    let exemptLanguage = 0;
+    const owed = recommendations
+        .filter((rec) => {
+        if (rec.reportOnly)
+            return false;
+        if (rec.action !== DriftAction.Update)
+            return false;
+        const row = catalogByFile.get(rec.testFilePath);
+        if (!row || row.source !== TestSource.External)
+            return false;
+        // Unknown is the placeholder skyramp_execute_test reserves before it
+        // spawns the runner. A run that died there left no evidence, so the
+        // placeholder must not read as a recorded baseline.
+        if (row.executionBefore &&
+            row.executionBefore.status !== TestExecutionStatus.Unknown)
+            return false;
+        // A language skyramp_execute_test does not accept can never record a
+        // before-run. No name check follows it: discovery admits an external file
+        // only when its name identifies a test or it is a colocated snapshot, and
+        // a .snap is exempt by language here, so a second rule would be a copy
+        // that can go out of sync.
+        if (!runnableLanguages.includes(row.language)) {
+            exemptLanguage++;
+            return false;
+        }
+        return true;
+    })
+        .map((rec) => rec.testFilePath);
+    return { owed, exemptLanguage };
+}
+/** Every file, quoted, one list — the agent must run all of them. */
+export function formatOwedFiles(owed) {
+    return owed.map((f) => `"${f}"`).join(", ");
+}

package/build/tools/trace/stopTraceCollectionTool.js

@@ -142,7 +142,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advance
             const sessionAppendix = savedSession
                 ? `\n\nPlaywright session storage saved to: ${savedSession}\nRe-use it by:\n` +
                     `• Pass \`playwrightStoragePath: "${savedSession}"\` to skyramp_start_trace_collection for future recordings (skips login).\n` +
-                    `• Generated tests that reference \`storageState: "${savedSession}"\` will auto-mount the file when run via skyramp_execute_test.`
+                    `• Generated tests that reference \`storageState: "${savedSession}"\` read it from that path when run via skyramp_execute_test.`
                 : "";
             errorResult = {
                 content: [

package/build/types/RepositoryAnalysis.d.ts

@@ -927,15 +927,15 @@ export declare const chainingRefSchema: z.ZodObject<{
 }, "strip", z.ZodTypeAny, {
     sourceStep: number;
     sourceField: string;
-    sourceLocation: "body" | "header" | "cookie";
+    sourceLocation: "cookie" | "body" | "header";
     targetParam: string;
-    targetLocation: "path" | "body" | "header" | "cookie" | "query";
+    targetLocation: "path" | "cookie" | "body" | "header" | "query";
 }, {
     sourceStep: number;
     sourceField: string;
-    sourceLocation: "body" | "header" | "cookie";
+    sourceLocation: "cookie" | "body" | "header";
     targetParam: string;
-    targetLocation: "path" | "body" | "header" | "cookie" | "query";
+    targetLocation: "path" | "cookie" | "body" | "header" | "query";
 }>;
 export declare const scenarioStepSchema: z.ZodObject<{
     order: z.ZodNumber;
@@ -957,15 +957,15 @@ export declare const scenarioStepSchema: z.ZodObject<{
     }, "strip", z.ZodTypeAny, {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }, {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }>, z.ZodArray<z.ZodObject<{
         sourceStep: z.ZodNumber;
         sourceField: z.ZodString;
@@ -975,15 +975,15 @@ export declare const scenarioStepSchema: z.ZodObject<{
     }, "strip", z.ZodTypeAny, {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }, {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }>, "many">]>>;
     bodyMustInclude: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
 }, "strip", z.ZodTypeAny, {
@@ -1000,15 +1000,15 @@ export declare const scenarioStepSchema: z.ZodObject<{
     chainsFrom?: {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     } | {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }[] | undefined;
     bodyMustInclude?: string[] | undefined;
 }, {
@@ -1025,15 +1025,15 @@ export declare const scenarioStepSchema: z.ZodObject<{
     chainsFrom?: {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     } | {
         sourceStep: number;
         sourceField: string;
-        sourceLocation: "body" | "header" | "cookie";
+        sourceLocation: "cookie" | "body" | "header";
         targetParam: string;
-        targetLocation: "path" | "body" | "header" | "cookie" | "query";
+        targetLocation: "path" | "cookie" | "body" | "header" | "query";
     }[] | undefined;
     bodyMustInclude?: string[] | undefined;
 }>;
@@ -1184,14 +1184,14 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }, {
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }>, "many">;
         playwrightRecordings: z.ZodArray<z.ZodObject<{
             path: z.ZodString;
@@ -1225,8 +1225,8 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }[];
         playwrightRecordings: {
             path: string;
@@ -1244,8 +1244,8 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }[];
         playwrightRecordings: {
             path: string;
@@ -1389,6 +1389,9 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
         scanDepth: "quick" | "full";
         analysisScope?: AnalysisScope | undefined;
     };
+    workspace: {
+        baseUrl: string;
+    };
     projectClassification: {
         projectType: "rest-api" | "frontend" | "full-stack" | "microservices" | "library" | "cli" | "other";
         primaryLanguage: string;
@@ -1416,8 +1419,8 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }[];
         playwrightRecordings: {
             path: string;
@@ -1431,9 +1434,6 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
         }[];
         notFound: string[];
     };
-    workspace: {
-        baseUrl: string;
-    };
     authentication: {
         method: string;
         configLocation: string;
@@ -1477,6 +1477,9 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
         scanDepth: "quick" | "full";
         analysisScope?: AnalysisScope | undefined;
     };
+    workspace: {
+        baseUrl: string;
+    };
     projectClassification: {
         projectType: "rest-api" | "frontend" | "full-stack" | "microservices" | "library" | "cli" | "other";
         primaryLanguage: string;
@@ -1504,8 +1507,8 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
             path: string;
             version: string;
             authType: string;
-            endpointCount: number;
             baseUrl: string;
+            endpointCount: number;
         }[];
         playwrightRecordings: {
             path: string;
@@ -1519,9 +1522,6 @@ export declare const repositoryAnalysisSchema: z.ZodObject<{
         }[];
         notFound: string[];
     };
-    workspace: {
-        baseUrl: string;
-    };
     authentication: {
         method: string;
         configLocation: string;