@skyramp/mcp 0.3.1 → 0.3.2-rc.pom-2

package/build/prompts/testbot/testbot-prompts.js

@@ -47,7 +47,7 @@ export function parseRelatedRepositories(raw) {
     }
 }
 export function getTestbotPrompt(prTitle, prDescription, summaryOutputFile, repositoryPath, baseBranch, maxRecommendations = MAX_RECOMMENDATIONS, maxGenerate = MAX_TESTS_TO_GENERATE, _maxCritical = MAX_CRITICAL_TESTS, // Reserved — accepted for API compat but not yet wired into prompt
-prNumber, userPrompt, services, uiCredentials, testsRepoDir, relatedRepositories, primaryRepo) {
+prNumber, userPrompt, services, uiCredentials, testsRepoDir, relatedRepositories, primaryRepo, planOnly = false) {
     maxGenerate = Math.min(Math.max(maxGenerate, 0), maxRecommendations);
     // TODO(SKYR-3636 follow-up): migrate Task 1 + Task 2 step bodies to PromptPlan
     // (src/prompts/test-recommendation/promptPlan.ts) so step numbers don't have
@@ -65,6 +65,17 @@ prNumber, userPrompt, services, uiCredentials, testsRepoDir, relatedRepositories
     // testDirectory, relative to the delivery root (the configured test repo if set,
     // otherwise the primary repo).
     const hasRelatedRepos = !!relatedRepositories?.length;
+    // Task 1 maintenance step 2c branches in plan-only eval runs (SKYR-3879
+    // plan-only lane): the SUT is not running, so the pre-edit baseline
+    // execution is replaced by static-analysis-only verdicts. Defined before
+    // task1Section, which interpolates it.
+    let maintenanceBeforeExecStep;
+    if (planOnly) {
+        maintenanceBeforeExecStep = `   c. Plan-only run: skip the pre-edit baseline execution — the application is not running. Record every maintenance verdict from static drift analysis alone; execution statuses simply remain unrecorded.`;
+    }
+    else {
+        maintenanceBeforeExecStep = `   c. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every UPDATE/REGENERATE/DELETE test. Exclude tests marked \`[external]\`. Run them sequentially, not in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail.`;
+    }
     // For follow-up requests: emit the @skyramp-testbot header + guardrails + retrieve-recommendations step.
     // For first-run prompts: emit the full Task 1 analysis + maintenance section.
     const task1Section = userPrompt
@@ -131,7 +142,7 @@ ${hasRelatedRepos ? `
 
    b. Write \`updateInstructions\` for each UPDATE or REGENERATE test before calling \`skyramp_actions\` — articulating the change first prevents file content from overriding diff-based reasoning.
 
-   c. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every UPDATE/REGENERATE/DELETE test. Exclude tests marked \`[external]\`. Run them sequentially, not in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail.
+${maintenanceBeforeExecStep}
 
    d. Call \`skyramp_actions\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) and apply the edits it returns.
 
@@ -343,19 +354,37 @@ ${hasRelatedRepos ? `
     const testDirInstruction = testsRepoDir
         ? `the \`<output_dir>\` from the \`<services>\` block, rooted under the test repository at \`${testsRepoDir}\` (i.e. \`${testsRepoDir}/<output_dir>\`). Write ALL test output files to paths under \`${testsRepoDir}\`, not under \`${repositoryPath}\`. Do NOT write any test files to the app repository.`
         : `${SERVICE_REFS.testDirRef}. Do NOT create a new \`tests/\` directory at the repo root — use that path. If no \`testDirectory\` is configured, default to the language-conventional location (e.g. \`src/test/java/...\` for Java, \`tests/\` for Python).`;
-    const primaryRepoBlock = primaryRepo ? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n` : '';
-    return `<TITLE>${prTitle}</TITLE>
-<DESCRIPTION>${prDescription}</DESCRIPTION>
-${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
-${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + '\n' : ''}${uiCredentialsBlock ? uiCredentialsBlock + '\n' : ''}Use the Skyramp MCP server tools for all tasks below.
-
-${task1Section}
-
-## Task 2: Generate New Tests
+    // Task 3's non-zero-budget count rule branches in plan-only eval runs: the
+    // report DECLARES the final GENERATE selection instead of recording
+    // generated files. The zero-surface abstention rules above it are shared.
+    let task3CountRule;
+    if (planOnly) {
+        task3CountRule = `Otherwise (your Budget Plan is non-zero): this is a plan-only run — \`newTestsCreated\` DECLARES your final GENERATE selection instead of recording generated files: exactly one entry per item of your final GENERATE list (the list returned by \`skyramp_register_test_plan\` when that tool was available — it may contain fewer items than your committed budget — otherwise your Budget Plan selection, at most ${maxGenerate}), with \`fileName\` set to the file name you would have used. \`testResults\` must be \`[]\` — nothing was executed. The declaration itself is the deliverable; do not generate or backfill.`;
+    }
+    else {
+        task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if all candidates failed after retry AND the fallback single-contract test also failed.`;
+    }
+    // Task 2 branches wholesale in plan-only eval runs (SKYR-3879 plan-only
+    // lane): the standard task mandates generation and execution, which a
+    // plan-only run must not do — branching the section (rather than appending
+    // an override) means the agent never sees conflicting instructions.
+    let task2Section;
+    if (planOnly) {
+        task2Section = `## Task 2: Commit the Test Plan (plan-only run)
+
+This is a plan-only evaluation run: the application under test is NOT running, and this run evaluates test SELECTION only. Nothing is generated or executed in this task.
+
+- Draft your complete candidate list exactly as the Execution Plan directs — every API test (contract / integration / batch-scenario) you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
+- If a tool named \`skyramp_register_test_plan\` is available, call it with the full candidate union — include a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for every candidate that probes changed logic — and treat its returned GENERATE list as your final selection. If that tool is not available, commit to your Budget Plan's GENERATE selection (at most ${maxGenerate}).
+- Skip UI and E2E candidates entirely — with no running app there are no blueprints to ground them, and this lane evaluates API test selection only.
+- Take no other actions in this task: no test generation tools, no browser traces or blueprint captures, no test files written, no test executions. Proceed directly to ${taskRef(TASK_SUBMIT)}.`;
+    }
+    else {
+        task2Section = `## Task 2: Generate New Tests
 
 ${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
 
-- **MANDATORY — use the pre-ranked GENERATE list as-is**: The Execution Plan's GENERATE section governs ADD actions. You MUST generate exactly those scenarios in the exact order listed. Do NOT substitute, rename, or replace a GENERATE item. If parameter grounding uncovers a distinct bug-catching scenario not already in the GENERATE or ADDITIONAL list, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n  - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Follow that pooled, type-distributed selection instead of any single repo's GENERATE list. (In single-repo runs the GENERATE list IS final — generate it exactly as-is.)` : ""}
+- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n  - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list. (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
 - **Do not fabricate tests outside the GENERATE list.** New test files cover NEW observable surface only — a new endpoint, or a newly-integrated component/route not already covered by an existing test. Changes that only modify, delete, or add fields to an EXISTING covered endpoint or component are maintenance: handle them in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test, never by creating a new spec. If the GENERATE list is empty (deletion-only, cosmetic, or modification-of-existing PRs with no new surface), create zero new tests and proceed to ${taskRef(TASK_SUBMIT)} — do not invent a new spec to have something to report.
 - Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
 - Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
@@ -416,7 +445,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
 ${CONTRACT_MODE_GUIDANCE}
 - ${PATH_PARAM_UUID_GUIDANCE}
 - **UI**: First check for existing Playwright trace \`.zip\` files in the repo (Testbot scans recursively up to 5 directory levels — the per-service output directories, \`frontend/\`, \`public/\`, \`.skyramp/\`, or any subdirectory).
-  If a relevant trace exists (covers the UI changes in this PR), use it directly with \`skyramp_ui_test_generation\` and \`modularizeCode: false\`.
+  If a relevant trace exists (covers the UI changes in this PR), use it directly with \`skyramp_ui_test_generation\`, \`modularizeCode: false\`, and \`codeReuse: true\` (when generating a TypeScript/JavaScript Playwright test — the default; leave \`codeReuse\` unset for other languages).
   If NO relevant trace exists, **you MUST write out your full trace plan as text BEFORE calling \`browser_navigate\`**. Do not touch the browser until the plan is written.
 
   **Browser authentication (check BEFORE navigating)**: If \`<ui-credentials>\` appears in your context above, the app requires login. Parse the credentials — one per line, two supported formats:
@@ -460,7 +489,7 @@ ${CONTRACT_MODE_GUIDANCE}
   Follow the **UI Recording Workflow** section at the end of this prompt. Additional CI constraints:
   - Navigate **directly** to the deepest relevant URL (e.g. \`/orders/1/edit\` instead of \`/\` then \`/orders\` then \`/orders/1\`) — minimize multi-hop navigation so the trace stays focused on the scenario under test.
   - \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
-  - \`skyramp_ui_test_generation\`: set \`modularizeCode: false\`
+  - \`skyramp_ui_test_generation\`: set \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript Playwright only — the default; leave \`codeReuse\` unset for other languages)
   - **\`browser_assert\` — MANDATORY**: at least one per page navigated. Call multiple assertions in the same tool call batch when checking independent elements. If you navigate to 2 pages, assert on both. Each assertion should verify a business outcome (state change, computed value, error condition) — not just that an element is visible.
   - **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
   - **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
@@ -512,7 +541,7 @@ ${CONTRACT_MODE_GUIDANCE}
 
   **The Blueprint Citation Invariant applies during recording too.** Every assertion you emit cites element names — those names must come from blueprint captures, not invention. For N user-intent-level actions, the reference target is N+1 \`browser_blueprint\` calls (the first returns full, the rest return deltas). Traces that follow the pattern produce assertions grounded in observable state changes; traces that skip captures fall back to author-inferred assertions and risk citing names that don't exist in the rendered DOM.
 
-  The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), \`skyramp_enhance_assertions\` post-call. Capture-act-capture adds blueprint captures alongside the existing steps; it doesn't replace anything.
+  The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), then \`skyramp_reuse_code\` (when \`codeReuse: true\`) and \`skyramp_enhance_assertions\` post-calls. Capture-act-capture adds blueprint captures alongside the existing steps; it doesn't replace anything.
 - **E2E**: Only if BOTH a backend trace \`.json\` AND a Playwright \`.zip\` already exist in the repo. Without both, move to \`additionalRecommendations\`.
 - Skip smoke tests entirely.
 
@@ -545,13 +574,24 @@ Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than o
 **After generation, you MUST do exactly these steps — nothing more, nothing less:**
 1. **[MANDATORY] After \`skyramp_integration_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated integration test file, \`testType: "integration"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
 2. **[MANDATORY] After \`skyramp_contract_test_generation\` with \`providerMode\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
-3. **[MANDATORY] After \`skyramp_ui_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file. The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
-4. **Wait**: Do NOT proceed to test execution until steps 1–3 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
-Do not make any changes other than the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
+3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Two outcomes: (1) a response starting "No reusable POM layer detected" — this is a normal outcome, continue immediately (do NOT retry), and note "no POM layer — reuse skipped" in that test's \`testResults\` entry \`details\`; (2) a refactoring workflow — follow it to completion INCLUDING its verification loop (\`skyramp_reuse_code\` with \`verify: true\`), finish only when it reports PASSED, and include \`verification: passed\` in that test's \`testResults\` entry \`details\` in your final report. If a reused test later fails execution and the failure points at a substituted POM call, restore the saved \`<testFile>.raw.bak\` over the test file and re-run — do NOT hand-edit the customer's POM methods (this re-run counts toward the 2-attempt execution cap — prefer this restore over the generic timeout fix-up when the failing locator came from a POM substitution).
+4. **[MANDATORY] After \`skyramp_ui_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file. The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
+5. **Wait**: Do NOT proceed to test execution until steps 1–4 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
+Do not make any changes other than the code-reuse refactoring (step 3) and the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
 
 **Final execution (mandatory):** Do NOT call \`skyramp_execute_test\` until ALL maintenance edits AND ALL new test generation/enhancement are complete. Run these calls sequentially, not in parallel. Exclude tests marked \`[external]\`.
 - Only report test results for files you actually ran.
-**Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.
+**Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.`;
+    }
+    const primaryRepoBlock = primaryRepo ? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n` : '';
+    return `<TITLE>${prTitle}</TITLE>
+<DESCRIPTION>${prDescription}</DESCRIPTION>
+${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
+${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + '\n' : ''}${uiCredentialsBlock ? uiCredentialsBlock + '\n' : ''}Use the Skyramp MCP server tools for all tasks below.
+
+${task1Section}
+
+${task2Section}
 
 ## Task 3: Submit Report
 
@@ -571,7 +611,7 @@ In these cases:
 - \`businessCaseAnalysis\` must be a one-sentence summary of what the PR actually does (do NOT leave it blank)
 - \`additionalRecommendations\` must be \`[]\` — do NOT recommend tests for a no-surface PR
 
-Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if all candidates failed after retry AND the fallback single-contract test also failed.
+${task3CountRule}
 
 Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}" and \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
 
@@ -683,10 +723,16 @@ export function registerTestbotPrompt(server) {
                 .string()
                 .optional()
                 .describe("The primary repository's owner/repo slug (where the testbot workflow runs). Used verbatim as the `repository` attribution for the primary repo's analysis, tests, and report items in multi-repo runs."),
+            planOnly: z
+                // Prompt/resource URL args arrive as strings — accept "true"/"1"
+                // alongside a real boolean; anything else (incl. undefined) is false.
+                .preprocess((v) => v === true || v === "true" || v === "1", z.boolean())
+                .optional()
+                .describe("Plan-only eval mode (eval harness only, SKYR-3879): run the full recommendation phase — analysis, maintenance verdicts, candidate drafting, plan registration — but generate and execute nothing; the report declares the final plan. Used by the eval pipeline to A/B selection changes without a running SUT."),
         },
     }, async (args) => {
         const services = await readWorkspaceServices(args.repositoryPath);
-        let prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.maxCritical, args.prNumber, args.userPrompt, services.length ? services : undefined, args.uiCredentials, args.testsRepoDir, parseRelatedRepositories(args.relatedRepositories), args.primaryRepo);
+        let prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.maxCritical, args.prNumber, args.userPrompt, services.length ? services : undefined, args.uiCredentials, args.testsRepoDir, parseRelatedRepositories(args.relatedRepositories), args.primaryRepo, args.planOnly);
         if (args.workspaceValidationFailed) {
             prompt = buildWorkspaceRecoveryPrefix(args.repositoryPath) + prompt;
         }

package/build/prompts/testbot/testbot-prompts.test.js

@@ -378,3 +378,47 @@ describe("parseRelatedRepositories", () => {
         expect(parsed).toEqual([{ repo: "o/ok", repositoryPath: "/ok", baseBranch: undefined }]);
     });
 });
+describe("plan-only eval mode (via getTestbotPrompt)", () => {
+    function callWithPlanOnly(planOnly) {
+        return getTestbotPrompt(baseArgs.prTitle, baseArgs.prDescription, baseArgs.summaryOutputFile, baseArgs.repositoryPath, undefined, // baseBranch
+        undefined, // maxRecommendations
+        undefined, // maxGenerate
+        undefined, // maxCritical
+        undefined, // prNumber
+        undefined, // userPrompt
+        undefined, // services
+        undefined, // uiCredentials
+        undefined, // testsRepoDir
+        undefined, // relatedRepositories
+        undefined, // primaryRepo
+        planOnly);
+    }
+    it("branches Task 2 to the plan-only variant when planOnly is true", () => {
+        const prompt = callWithPlanOnly(true);
+        expect(prompt).toContain("## Task 2: Commit the Test Plan (plan-only run)");
+        expect(prompt).not.toContain("## Task 2: Generate New Tests");
+        // The standard generation/execution mandates must be absent, not overridden.
+        expect(prompt).not.toContain("skyramp_integration_test_generation");
+        expect(prompt).not.toContain('phase: "before"');
+        // Task 3 declares the plan instead of counting generated files.
+        expect(prompt).toContain("DECLARES your final GENERATE selection");
+    });
+    it("renders the standard prompt untouched when planOnly is false or omitted", () => {
+        const prompt = callWithPlanOnly(false);
+        expect(prompt).toContain("## Task 2: Generate New Tests");
+        expect(prompt).not.toContain("plan-only");
+        expect(prompt).toBe(callWithServices([]));
+    });
+});
+describe("UI code reuse wiring", () => {
+    it("instructs codeReuse: true for TS/JS Playwright UI generation", () => {
+        const p = callWithServices([]);
+        expect(p).toContain("codeReuse: true");
+    });
+    it("teaches both reuse outcomes and the raw.bak insurance rule", () => {
+        const p = callWithServices([]);
+        expect(p).toContain("No reusable POM layer detected");
+        expect(p).toContain("verification: passed");
+        expect(p).toContain(".raw.bak");
+    });
+});

package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts

@@ -0,0 +1,7 @@
+import { Budgeter } from "../../types/Recommendation.js";
+/**
+ * Budgeter that guarantees cross-test-type coverage in the GENERATE set.
+ * `maxGenerate` acts as an upper cap (not a hard N). Fixes the single-type skew
+ * that starves contract/error-path tests. Opt-in via strategy selection.
+ */
+export declare const diversityBalancedBudgeter: Budgeter;

package/build/recommendation/budgeters/diversityBalancedBudgeter.js

@@ -0,0 +1,71 @@
+import { runBudget } from "./shared.js";
+import { bucketByType, inferScenarioType, isProtectedCandidate, roundRobinFill, } from "../diversity.js";
+const typeOf = (c) => inferScenarioType(c.scenario);
+/**
+ * Pick `count` items guaranteeing each present test type at least one slot.
+ *
+ * Unlike roundRobinByType (which fills ALL protected items first and can
+ * starve other types at small N — the SKYR-3879 "0 contract executed" skew),
+ * this caps protected occupancy to `count - (numTypes - 1)` so at least one slot
+ * per remaining type is always reachable, then:
+ *   Phase 1 — one item for each still-uncovered present type (by rank), then
+ *   Phase 2 — round-robin the remainder across types until `count` is reached.
+ * Rank order is preserved within each type; degenerate cases match a rank slice.
+ */
+function floorBalancedPick(items, count) {
+    if (count <= 0)
+        return [];
+    if (count >= items.length)
+        return items.slice(0, count);
+    // Present types in first-appearance (rank) order.
+    const typesOrder = [];
+    const seen = new Set();
+    for (const it of items) {
+        const t = typeOf(it);
+        if (!seen.has(t)) {
+            seen.add(t);
+            typesOrder.push(t);
+        }
+    }
+    const maxProtected = Math.max(0, count - (typesOrder.length - 1));
+    const selected = [];
+    const pool = [];
+    let protectedTaken = 0;
+    for (const it of items) {
+        if (isProtectedCandidate(it.priority, it.scenario) &&
+            protectedTaken < maxProtected &&
+            selected.length < count) {
+            selected.push(it);
+            protectedTaken++;
+        }
+        else {
+            pool.push(it);
+        }
+    }
+    const { buckets } = bucketByType(pool, typeOf);
+    const covered = new Set(selected.map(typeOf));
+    // Phase 1 — floor: one item per still-uncovered present type.
+    for (const t of typesOrder) {
+        if (selected.length >= count)
+            break;
+        if (covered.has(t))
+            continue;
+        const bucket = buckets.get(t);
+        if (bucket && bucket.length > 0) {
+            selected.push(bucket.shift());
+            covered.add(t);
+        }
+    }
+    // Phase 2 — round-robin the remainder across types until full.
+    roundRobinFill(selected, typesOrder, buckets, count);
+    return selected;
+}
+/**
+ * Budgeter that guarantees cross-test-type coverage in the GENERATE set.
+ * `maxGenerate` acts as an upper cap (not a hard N). Fixes the single-type skew
+ * that starves contract/error-path tests. Opt-in via strategy selection.
+ */
+export const diversityBalancedBudgeter = {
+    name: "diversity-balanced",
+    select: (ranked, ctx) => runBudget(ranked, ctx, floorBalancedPick),
+};

package/build/recommendation/budgeters/diversityBalancedBudgeter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/recommendation/budgeters/diversityBalancedBudgeter.test.js

@@ -0,0 +1,75 @@
+import { diversityBalancedBudgeter } from "./diversityBalancedBudgeter.js";
+import { PriorityTier } from "../../types/TestRecommendation.js";
+import { fixedNBudgeter } from "./fixedNBudgeter.js";
+import { mkCandidate } from "../testFixtures.js";
+function ctx(over = {}) {
+    return {
+        maxGenerate: 3,
+        maxTotal: 20,
+        isUIOnlyPR: false,
+        hasFrontendChanges: false,
+        externalCoverage: new Set(),
+        ...over,
+    };
+}
+const countType = (r, t) => r.generate.filter((c) => c.scenario.testType === t).length;
+const typesIn = (r) => new Set(r.generate.map((c) => c.scenario.testType));
+describe("diversityBalancedBudgeter", () => {
+    it("guarantees a contract slot where fixed-n starves it (the SKYR-3879 fix)", () => {
+        // Protected (CRITICAL) integration scenarios out-rank contract ones and, at N=3,
+        // fixed-n's protected-first fill consumes every slot with integration → 0 contract.
+        const ranked = [
+            mkCandidate("i1", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i2", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i3", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const c = ctx({ maxGenerate: 3 });
+        // Baseline: fixed-n reproduces the skew.
+        expect(countType(fixedNBudgeter.select(ranked, c), "contract")).toBe(0);
+        // Fix: diversity-balanced reserves a per-type floor.
+        const div = diversityBalancedBudgeter.select(ranked, c);
+        expect(div.generate.length).toBe(3);
+        expect(countType(div, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(div, "integration")).toBeGreaterThanOrEqual(1);
+    });
+    it("gives every present test type at least one slot before any type gets a second", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("i3", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+            mkCandidate("u1", { testType: "ui" }),
+        ];
+        const div = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 3 }));
+        expect(typesIn(div)).toEqual(new Set(["integration", "contract", "ui"]));
+    });
+    it("raising maxGenerate increases per-type coverage while keeping the floor", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i2", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i3", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i4", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i5", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+            mkCandidate("u1", { testType: "ui" }),
+        ];
+        const atThree = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 3 }));
+        const atSix = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 6 }));
+        expect(countType(atThree, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(atThree, "ui")).toBeGreaterThanOrEqual(1);
+        expect(countType(atSix, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(atSix, "ui")).toBeGreaterThanOrEqual(1);
+        // more budget → more of the highest-ranked (integration) coverage
+        expect(countType(atSix, "integration")).toBeGreaterThan(countType(atThree, "integration"));
+        expect(atSix.generate.length).toBe(6);
+    });
+    it("returns everything (rank order) when maxGenerate >= candidate count", () => {
+        const ranked = [mkCandidate("i1", { testType: "integration" }), mkCandidate("c1", { testType: "contract" })];
+        const div = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 5 }));
+        expect(div.generate.map((c) => c.scenario.scenarioName)).toEqual(["i1", "c1"]);
+    });
+});

package/build/recommendation/budgeters/fixedNBudgeter.d.ts

@@ -0,0 +1,7 @@
+import { Budgeter } from "../../types/Recommendation.js";
+/**
+ * Default budgeter — reproduces the pre-refactor behavior exactly: fill up to
+ * `maxGenerate` GENERATE slots by round-robin across the present test types
+ * (protected items first), the rest become ADDITIONAL.
+ */
+export declare const fixedNBudgeter: Budgeter;

package/build/recommendation/budgeters/fixedNBudgeter.js

@@ -0,0 +1,11 @@
+import { roundRobinByType } from "../diversity.js";
+import { runBudget } from "./shared.js";
+/**
+ * Default budgeter — reproduces the pre-refactor behavior exactly: fill up to
+ * `maxGenerate` GENERATE slots by round-robin across the present test types
+ * (protected items first), the rest become ADDITIONAL.
+ */
+export const fixedNBudgeter = {
+    name: "fixed-n",
+    select: (ranked, ctx) => runBudget(ranked, ctx, roundRobinByType),
+};

package/build/recommendation/budgeters/fixedNBudgeter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/recommendation/budgeters/fixedNBudgeter.test.js

@@ -0,0 +1,66 @@
+import { fixedNBudgeter } from "./fixedNBudgeter.js";
+import { mkCandidate } from "../testFixtures.js";
+import { externalDedupKey } from "../../prompts/test-recommendation/recommendationShared.js";
+function ctx(over = {}) {
+    return {
+        maxGenerate: 3,
+        maxTotal: 20,
+        isUIOnlyPR: false,
+        hasFrontendChanges: false,
+        externalCoverage: new Set(),
+        ...over,
+    };
+}
+const gen = (r) => r.generate.map((c) => c.scenario.scenarioName);
+describe("fixedNBudgeter", () => {
+    it("backend-only PR fills maxGenerate slots via round-robin across test types", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("i3", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx());
+        expect(res.generate.length).toBe(3);
+        expect(res.reservedUISlots).toBe(0);
+        // buckets [integration, contract]; round 1 → i1, c1; round 2 → i2
+        expect(gen(res)).toEqual(["i1", "c1", "i2"]);
+    });
+    it("mixed (frontend) PR reserves one UI slot → backend count = maxGenerate - 1", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx({ hasFrontendChanges: true }));
+        expect(res.generate.length).toBe(2);
+        expect(res.reservedUISlots).toBe(1);
+    });
+    it("UI-only PR selects no backend generate items", () => {
+        const res = fixedNBudgeter.select([mkCandidate("i1", { testType: "integration" })], ctx({ isUIOnlyPR: true }));
+        expect(res.generate.length).toBe(0);
+    });
+    it("additional = leftover candidates (rank order), excluding anything generated", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx());
+        const generated = new Set(gen(res));
+        for (const a of res.additional) {
+            expect(generated.has(a.scenario.scenarioName)).toBe(false);
+        }
+        expect(res.additional.length).toBe(1);
+    });
+    it("drops an ordinary candidate already covered by an external test", () => {
+        const covered = mkCandidate("covered", { testType: "contract" });
+        const fresh = mkCandidate("fresh", { testType: "integration" });
+        const res = fixedNBudgeter.select([covered, fresh], ctx({ maxGenerate: 5, externalCoverage: new Set([externalDedupKey(covered.scenario)]) }));
+        const all = [...res.generate, ...res.additional].map((c) => c.scenario.scenarioName);
+        expect(all).toContain("fresh");
+        expect(all).not.toContain("covered");
+    });
+});

package/build/recommendation/budgeters/shared.d.ts

@@ -0,0 +1,19 @@
+import { Candidate, BudgetContext, SelectionResult } from "../../types/Recommendation.js";
+/**
+ * Backend GENERATE slot count:
+ *  - UI-only PR: 0 (all slots are UI placeholders)
+ *  - Mixed PR:   maxGenerate - 1 (last slot reserved for a UI placeholder)
+ *  - Backend-only PR: maxGenerate
+ */
+export declare function backendGenerateCount(ctx: BudgetContext): number;
+/** UI placeholder slots the render layer fills (UI-only → all; mixed → one). */
+export declare function reservedUISlots(ctx: BudgetContext): number;
+/**
+ * Shared budgeting pipeline. All Budgeters run the same external-dedup,
+ * attack-surface prioritization, and ADDITIONAL set-difference; they differ
+ * ONLY in how they pick the GENERATE items from the slot-ordered list (`pick`).
+ *
+ * With `pick = roundRobinByType` this reproduces the pre-refactor selection in
+ * diffExecutionPlan.ts exactly.
+ */
+export declare function runBudget(ranked: Candidate[], ctx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;

package/build/recommendation/budgeters/shared.js

@@ -0,0 +1,66 @@
+import { prioritizeAttackSurfaceBundles } from "../diversity.js";
+import { externalDedupKey, scenarioCoverageKey, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Backend GENERATE slot count:
+ *  - UI-only PR: 0 (all slots are UI placeholders)
+ *  - Mixed PR:   maxGenerate - 1 (last slot reserved for a UI placeholder)
+ *  - Backend-only PR: maxGenerate
+ */
+export function backendGenerateCount(ctx) {
+    if (ctx.isUIOnlyPR)
+        return 0;
+    return ctx.hasFrontendChanges ? Math.max(0, ctx.maxGenerate - 1) : ctx.maxGenerate;
+}
+/** UI placeholder slots the render layer fills (UI-only → all; mixed → one). */
+export function reservedUISlots(ctx) {
+    if (ctx.isUIOnlyPR)
+        return ctx.maxGenerate;
+    return ctx.hasFrontendChanges && ctx.maxGenerate > 0 ? 1 : 0;
+}
+/**
+ * Drop candidates whose method-aware resource+type is already covered by an
+ * external test — except protected `bug_caught` / attack-surface scenarios,
+ * which require semantic flaw coverage the external test may not provide.
+ */
+function applyExternalDedup(ranked, externalCoverage) {
+    if (externalCoverage.size === 0)
+        return ranked;
+    return ranked.filter((item) => {
+        const key = externalDedupKey(item.scenario);
+        if (externalCoverage.has(key)) {
+            if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
+                logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${key}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
+                return true;
+            }
+            logger.info(`External dedup: skipping "${item.scenario.scenarioName}" (${key}) — covered by external test`);
+            return false;
+        }
+        return true;
+    });
+}
+/**
+ * Shared budgeting pipeline. All Budgeters run the same external-dedup,
+ * attack-surface prioritization, and ADDITIONAL set-difference; they differ
+ * ONLY in how they pick the GENERATE items from the slot-ordered list (`pick`).
+ *
+ * With `pick = roundRobinByType` this reproduces the pre-refactor selection in
+ * diffExecutionPlan.ts exactly.
+ */
+export function runBudget(ranked, ctx, pick) {
+    const backend = backendGenerateCount(ctx);
+    const deduped = applyExternalDedup(ranked, ctx.externalCoverage);
+    const slotOrdered = prioritizeAttackSurfaceBundles(deduped);
+    const generate = pick(slotOrdered, Math.min(backend, slotOrdered.length));
+    // ADDITIONAL = everything not chosen for GENERATE, in rank order, capped at the
+    // remaining budget, minus any whose resource+type is already covered by GENERATE.
+    const generateSet = new Set(generate);
+    const generatedCoverage = new Set(generate.map((item) => scenarioCoverageKey(item.scenario)));
+    const additional = slotOrdered
+        .filter((it) => !generateSet.has(it))
+        .slice(0, Math.max(0, ctx.maxTotal - backend))
+        .filter((item) => !generatedCoverage.has(scenarioCoverageKey(item.scenario)));
+    // Budgeting itself produces no demotions; the register-plan selection stage
+    // (planRanker.selectPlan) fills this channel from discriminator verification.
+    return { generate, additional, reservedUISlots: reservedUISlots(ctx), demotions: [] };
+}

package/build/recommendation/discriminators.d.ts

@@ -0,0 +1,31 @@
+import { DraftedScenario } from "../types/RepositoryAnalysis.js";
+import { DiscriminatorKind } from "../types/Recommendation.js";
+/**
+ * A discriminator claim the agent (or server) attaches to a candidate: the
+ * structural bug-shape (`kind`) plus the verbatim diff snippet the test probes.
+ */
+export interface DiscriminatorClaim {
+    kind: DiscriminatorKind;
+    changedCodeAnchor: string;
+}
+/** Result of verifying a claim against a scenario's steps and the PR diff. */
+export interface DiscriminatorValidation {
+    verified: boolean;
+    /** One-sentence, human-readable reason present ONLY when `verified` is false;
+     *  it is returned to the LLM to help it fix the claim. */
+    reason?: string;
+}
+/**
+ * Verify a declared discriminator against the candidate's `steps[]` and the PR
+ * diff. Mirrors the classification precedent (`recommendationShared.ts`): the
+ * claim is checked STRUCTURALLY, never by fuzzy keyword scans of prose. A failed
+ * check returns `verified: false` with a one-sentence reason (the claim is
+ * demoted, never used to reject the candidate). NEVER throws — malformed or
+ * partial input is treated as unverified.
+ *
+ * Two gates, both required to verify:
+ *  1. `changedCodeAnchor` must be a non-trivial string (>= 8 chars after trim)
+ *     occurring verbatim in `diffText` — grounds the claim in the real change.
+ *  2. The `kind`-specific structural predicate must hold over `steps[]`.
+ */
+export declare function validateDiscriminator(scenario: DraftedScenario, declared: DiscriminatorClaim, diffText: string): DiscriminatorValidation;