@skyramp/mcp 0.2.7 → 0.2.8

package/build/prompts/sut-setup/shared.js

@@ -0,0 +1,132 @@
+/**
+ * Path of the Testbot workflow file relative to the repo root. Created by the
+ * Testbot installer and edited in-place by every SUT setup mode.
+ */
+export const TESTBOT_WORKFLOW_PATH = ".github/workflows/skyramp-testbot.yml";
+export var SutSetupMode;
+(function (SutSetupMode) {
+    SutSetupMode["None"] = "none";
+    SutSetupMode["AdaptWorkflow"] = "adapt_workflow";
+    SutSetupMode["DockerCompose"] = "docker_compose";
+})(SutSetupMode || (SutSetupMode = {}));
+/**
+ * Glossary block rendered right after the first mention of "SUT" in each
+ * setup prompt. Shared across modes so the LLM gets a single explicit
+ * definition rather than inferring it from context.
+ */
+export function buildSutDefinitionBlock() {
+    return `**System Under Test (SUT)** — the application services this workflow brings up so Testbot can exercise them end-to-end. The SUT is what runs the PR's source code: backend services (REST/gRPC APIs), their supporting infrastructure (databases, queues, caches), and any frontend UI under test. The SUT does NOT include the test runner itself or Testbot. Bringing up the SUT means building from PR source, starting all required services, seeding any data tests rely on, and exposing a health check that proves the system is ready for traffic.
+
+`;
+}
+export function buildContextBlock(args) {
+    const lines = [];
+    lines.push(`<context>`);
+    lines.push(`Repository path: ${args.repositoryPath}`);
+    lines.push(`Setup mode: ${args.sutSetupMode}`);
+    if (args.sutSourceWorkflowFile)
+        lines.push(`Source workflow file: ${args.sutSourceWorkflowFile}`);
+    if (args.sutSourceDockerComposeFile)
+        lines.push(`Source docker-compose file: ${args.sutSourceDockerComposeFile}`);
+    lines.push(`</context>`);
+    return lines.join("\n");
+}
+export function buildLocalValidationSection() {
+    return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
+Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
+1. Validate build scripts and helper programs first — run these before anything else so build and auth failures surface cheaply:
+   a. Every helper script the workflow calls (e.g., \`./scripts/*.sh\`, \`make <target>\`, \`./gradlew <task>\`) — run each and confirm exit 0.
+   b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
+   c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
+   d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
+   e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, run \`docker builder prune -af -f && docker image prune -af\` before the next build. This frees BuildKit cache + dangling images without losing the running stack. Skipping this on heavy SUTs (medusa, calcom, mattermost, supabase) exhausts the ~14 GB \`ubuntu-latest\` disk mid-validation.
+2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
+   a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
+      i. Run \`targetSetupCommand\` — confirm exit 0.
+      ii. Poll \`targetReadyCheckCommand\` until it exits 0 within \`targetReadyCheckTimeout\` seconds.
+      iii. Run \`targetTeardownCommand\` — confirm cleanup and exit 0.
+   b. If GHA steps wrap the Testbot action (skipTargetSetup: 'true'):
+      i. Extract the shell body of each SETUP step and run them in order. If a step depends entirely on a \${{ secrets.* }} value your shell cannot resolve, note it in <thinking> and skip — do not fabricate a value.
+      ii. Poll \`targetReadyCheckCommand\` until it exits 0.
+      iii. Extract and run each TEARDOWN step's body in order so the next iteration starts clean.
+Only proceed to the success report once setup → health check → teardown all pass.`;
+}
+/**
+ * Returns the canonical "add to the Testbot workflow" block: every input that
+ * controls the SUT lifecycle, auth, and workflow-level setup, followed by the
+ * shared What to do / What not to do guidance (including the sutSetupMode
+ * bootstrap guard). Shared by every SUT setup mode (adapt_workflow and
+ * docker_compose) so the input semantics and rules never drift between prompts.
+ */
+export function buildAddToTestbotWorkflowSection() {
+    return `#### Testbot action inputs - Add target lifecycle commands and auth to Testbot Workflow
+For the full Testbot action input reference (defaults, edge cases, advanced options beyond the ones called out below), see the Testbot README: https://github.com/letsramp/testbot#inputs
+
+Target lifecycle inputs (control how Testbot starts, validates, and stops the SUT):
+1. \`targetSetupCommand\` — shell command that builds and starts all SUT services. Testbot runs this before tests.
+   a. Default when unset: \`docker compose up -d\` (starts existing images, does NOT build).
+   b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
+   c. With a dedicated compose file (reused or generated), point at it explicitly: \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . up -d --build\`.
+   d. Override whenever the source workflow's start sequence differs from the default.
+2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
+   a. **CRITICAL: Always set this to a valid target readiness check command.** Required in every pattern: when \`targetSetupCommand\` is explicit, when relying on the \`docker compose up -d\` default, AND when surrounding GHA steps bring the SUT up before the Testbot action (\`skipTargetSetup: 'true'\`). Testbot's fail-early gate throws \`Unable to verify service readiness\` when no readiness signal is available (neither this nor a workspace service \`baseUrl\`) and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
+3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
+   a. Default: \`1800\` (30 min) — sized for cold \`docker compose --build\` runs on multi-service stacks. The loop exits on the first success, so fast services pay no extra cost. Override only if you want to fail faster on stuck setups (e.g. \`'300'\` for a small single-service SUT).
+4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`, or \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . down -v\` for a dedicated compose file).
+   a. Always set this so each run starts from a clean state.
+   b. Omit only when teardown is handled by surrounding GHA steps with \`if: always()\`.
+5. \`skipTargetSetup\` — when \`'true'\`, Testbot skips running \`targetSetupCommand\` entirely.
+   a. Set this only when GHA steps surrounding the Testbot action already bring the SUT up.
+   b. Leave it unset (or \`'false'\`) when Testbot should run the full lifecycle via the inputs above.
+Auth inputs (used by Testbot to authenticate against the running SUT during test recording and execution):
+6. \`authTokenCommand\` — shell command whose stdout is the auth credential for REST/gRPC API testing (e.g., a Bearer token, API key, or session cookie). Testbot captures the output and injects it as the \`Authorization\` header.
+   a. Set this when the SUT requires authentication for API calls.
+   b. If the source workflow exports a token via a step output or script, wire that same script here (e.g., \`bash .skyramp/sut/get-auth-token.sh\`).
+   c. If the workflow seeds a test user during setup, create \`.skyramp/sut/get-auth-token.sh\` that logs in with those credentials and prints the token.
+   d. Omit when the SUT APIs are unauthenticated.
+7. \`uiCredentials\` — \`username:password\` pair typed into the browser login form during UI test recording (format: \`myuser:mypassword\`).
+   a. Set for any frontend service that requires browser-based login.
+   b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
+Workflow / job level settings (set on the workflow or job, not as \`skyramp/testbot\` action inputs):
+8. Secrets / environment variables — pass every secret/env var the setup needs into the workflow via an \`env:\` block or \`secrets: inherit\`, and add any dependency-resolution steps/jobs the SUT requires.
+9. \`runs-on\` runner label — switch to a larger runner when the SUT needs more disk or memory: heavy Docker builds, large dependency installs, or a multi-service (3+) compose stack. Standard runners run out of disk and crash with \`No space left on device\`. Runner names vary by org — do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses (never downgrade to \`ubuntu-latest\`), (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`.
+
+#### What to do
+Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot installer) — do NOT recreate it. Add only what the SUT needs onto the existing \`skyramp/testbot\` action step, alongside the inputs the installer already added:
+1. The 4 target lifecycle inputs — \`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetReadyCheckTimeout\`, \`targetTeardownCommand\`.
+2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
+3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
+4. The \`runs-on\` runner label, if a larger runner is needed (input 9).
+5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
+
+#### What not to do
+1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
+2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
+3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
+4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
+5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
+`;
+}
+/**
+ * `buildAddToTestbotWorkflowSection()` output with its leading
+ * `#### Testbot action inputs ...` header stripped — for use as a PromptPlan
+ * step body where the PromptPlan step header replaces the inline `####`.
+ */
+export function buildInputsBody() {
+    const text = buildAddToTestbotWorkflowSection();
+    const firstNewline = text.indexOf("\n");
+    return text.slice(firstNewline + 1);
+}
+export function buildCommonSutErrorsSection() {
+    return `Before finishing, verify the adapted Testbot workflow setup against common errors seen during SUT setup:
+1. port_conflict — another container is already using the port. Pick a free host port or stop the conflicting container.
+2. image_not_found / build_failed — the referenced image does not exist for this SHA, or the build step has the wrong context/Dockerfile path. Build from PR source, do not rely on upstream commit-SHA tags.
+3. healthcheck_timeout — the service is slow to come up. Use \`targetReadyCheckTimeout >= 120\` and ensure the health endpoint actually exists.
+4. connection_refused — the service is not listening on the expected host/port (often binding to 127.0.0.1 inside the container; bind to 0.0.0.0).
+5. dependency_error — required service (DB, cache, queue) is missing from compose or not healthy before the app starts. Add \`depends_on\` with \`condition: service_healthy\`.
+6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
+7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
+8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
+9. no_space_left — the runner ran out of disk space (\`No space left on device\`). This happens on standard runners when the SUT involves large Docker builds, many npm packages, or multiple compose services that pull large images. Switch \`runs-on\` to a larger runner — runner names vary by org, do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses, (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`. Indicator: source workflow already specifies a large runner, or the SUT has multi-stage Docker builds / a compose file with 3+ services.
+Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
+}

package/build/prompts/test-maintenance/driftAnalysisSections.js

@@ -136,7 +136,9 @@ Actions:
 **UPDATE vs REGENERATE — the deciding question is whether the root response wrapper changed:**
 - **REGENERATE** only when a new top-level envelope object wraps the entire payload or the root key is renamed so that essentially every existing assertion must change.
 - **UPDATE** for everything else. If you can describe the fix as "patch these N assertion paths", it is UPDATE regardless of how many paths there are.
-- When every assertion in the file is invalid, it is REGENERATE. When you can still patch individual paths, it is UPDATE.`;
+- When every assertion in the file is invalid, it is REGENERATE. When you can still patch individual paths, it is UPDATE.
+
+**Non-test consumer check:** When the diff removes or renames response fields, also grep the repository for non-test source files that import or reference the changed schema type (e.g. \`grep -r "WorkQueueStatusDetail" src/\`). If any client model, SDK class, or serializer references the old field names, add a VERIFY entry for each such file in your recommendations so the developer is alerted — these are not test files but they will break at runtime.`;
 }
 export function buildCheckAuthAndAuthorization() {
     return `Has the authentication or authorization for this endpoint changed?
@@ -180,7 +182,7 @@ export function buildCheckAssignAction() {
 **Every action requires a specific rationale — including IGNORE:**
 - UPDATE / REGENERATE / DELETE: quote the specific diff line that triggered it.
 - VERIFY: name the uncertain element (e.g. "model-only change, cannot confirm field is exposed without checking the output layer").
-- IGNORE: name the specific reason the changed code cannot reach this test's endpoint (e.g. "diff only touches \`auth/session-service.js\` — this test targets \`/api/v1/orders\` which has no session dependency"). Generic "unrelated endpoint" or "service boundary" without a diff reference is not sufficient.
+- IGNORE: name the changed file(s) from the diff AND explain why they cannot reach this test's endpoint (e.g. "diff only touches \`auth/session-service.js\` — this test targets \`/api/v1/orders\` which has no session dependency"). You must reference at least one specific changed file from the diff. Generic "unrelated endpoint" or "service boundary" without naming the changed file is not sufficient.
 
 - If the Additive Fields check flagged a new field with output layer signal confirmed in the diff → action is UPDATE. If the Additive Fields check returned VERIFY (model-only signal, no output layer change) → action remains VERIFY.
 - **Scope gate:** If the changed code is clearly not reachable through the service or base URL this test targets, assign IGNORE.
@@ -222,12 +224,16 @@ export function buildDriftOutputChecklist(stateFile, existingTests) {
     const noTestsNote = !hasTests
         ? `\nNo existing tests were found. Call \`skyramp_actions\` with \`recommendations: []\` and explain in your report why no maintenance was needed (e.g. "no existing tests cover the changed endpoints" or "PR adds a new endpoint with no prior coverage").\n`
         : "";
+    const symbolDiscoveryStep = hasTests
+        ? `**Symbol import discovery (before assessing tests):** If the diff adds or modifies class, function, or type definitions, grep the repository for test files that import those symbol names. Add any matched test files not already in the list below to your assessment — these files may need updating even if their path doesn't match the changed resource name. Include all discovered files in the \`recommendations[]\` passed to \`skyramp_actions\` — do NOT edit them directly without going through \`skyramp_actions\` first.`
+        : "";
     const existingTestSection = `**Existing tests (${existingTests?.length ?? 0} total) — assess ALL of the following:**
 ${testList}
 ${noTestsNote}
 **UPDATE scope notes:**
 - UPDATE is an in-place edit — do not recommend creating a new file for an already-tested endpoint.
 - If a new HTTP method is added to a resource, emit a separate UPDATE entry for every existing test file that covers that resource (contract, integration, UI).
+- **When multiple test files cover the same changed resource, prefer the most resource-specific file** (e.g. a file whose name directly reflects the changed resource) over a generic shared test file. Only UPDATE both if the change affects base types shared across both files.
 
 For each test above, output one entry:
 - **IGNORE**: \`{testFile} — IGNORE: {reason the diff cannot reach this test's endpoint}\`
@@ -253,7 +259,7 @@ Rationale: DELETE because {quoted diff signal}; all covered endpoints removed
 \`\`\`
 Include ALL actions — IGNORE, VERIFY, UPDATE, REGENERATE, DELETE — in the \`recommendations[]\` passed to \`skyramp_actions\`. For \`[external]\` tests: \`skyramp_actions\` will apply UPDATE edits and surface REGENERATE/DELETE as report-only findings.
 Be concise — one line per IGNORE/VERIFY entry.`;
-    const sections = [existingTestSection, finalStep];
+    const sections = [symbolDiscoveryStep, existingTestSection, finalStep].filter(Boolean);
     return `<output_format>
 Complete ALL of the following:
 

package/build/prompts/test-recommendation/analysisOutputPrompt.js

@@ -127,7 +127,7 @@ An unintegrated non-route component has no DOM node in the running app and canno
         return `${canInline ? "Read the \`<diff>\` above" : `Read the diff file at \`${p.diffFilePath}\``} and identify every new or modified API endpoint — route registrations, handler methods, controller annotations. Then use the **Router Mounting / Nesting** section above to reconstruct the full URL path for each endpoint by chaining all parent router prefixes down to the handler (e.g. a handler in a file with \`prefix="/reviews"\` that is mounted at \`/{product_id}\` under a router mounted at \`/api/v1/products\` → full path \`/api/v1/products/{product_id}/reviews\`).
 ${diffHasJavaFiles ? JAVA_SPRING_NOTE : ""}
 For each endpoint found: note the HTTP method, full path, and source file.
-${preDetectedEndpoints ? "The endpoint catalog above already lists some changed endpoints — verify and augment with anything it missed." : "No endpoints were pre-detected in the changed files — extract them from the diff."}
+${preDetectedEndpoints ? "The endpoint catalog above already lists some changed endpoints — verify and augment with anything it missed. **If a changed file contains many endpoints (hub/routing files like api.go, routes.py, router.ts), cross-reference against the diff hunks to identify only the endpoints whose handler code actually changed — do not include neighboring routes whose code was untouched.**" : "No endpoints were pre-detected in the changed files — extract them from the diff."}
 **Also identify removed endpoints**: Look for deleted route annotations (lines starting with \`-\` in the diff) in modified files (files that still exist but had routes deleted). A removed endpoint is a route definition present in the base branch but absent in the current branch. Cross-reference against the scanned endpoint listing below — if a deleted route annotation's endpoint still appears there (e.g. moved to another file), it is NOT removed. Only flag endpoints that are truly gone from the codebase.
 **CRITICAL — Query params vs body:** For GET endpoints (especially search/filter/list),
 identify which parameters are URL query params vs request body. Look at framework-specific

package/build/prompts/test-recommendation/diffExecutionPlan.js

@@ -154,6 +154,69 @@ function prioritizeAttackSurfaceBundles(items) {
     }
     return reordered;
 }
+/**
+ * Select `count` items from a rank-ordered list, distributing GENERATE slots
+ * EVENLY across the test types present, with spillover.
+ *
+ * Policy (kept identical to the multi-repo prose in testbot-prompts.ts's
+ * "Cross-repo test generation" block — change both together):
+ *  - Protected items first: CRITICAL-priority and attack-surface security_boundary
+ *    scenarios always take a slot before round-robin (they must stay in GENERATE).
+ *  - Bucket the rest by inferred test type (contract vs integration — the same
+ *    inference used when rendering: `testType ?? (steps===1 ? contract : integration)`).
+ *  - Round-robin one item per non-empty bucket per round, in the buckets' order of
+ *    first appearance in the rank-ordered list (so the highest-ranked type wins
+ *    round 1), preserving rank order within each bucket.
+ *  - Spillover: an exhausted bucket is skipped on later rounds, so its freed slots
+ *    go to the next type's next-highest item.
+ *
+ * Degenerate cases match the previous pure rank-order slice exactly: a single type
+ * present, or `count >= items.length`, returns the same items in the same order —
+ * so backend-only / single-type runs are unchanged (no regression).
+ */
+function roundRobinByType(rankOrdered, count) {
+    if (count <= 0)
+        return [];
+    // Everything fits → no need to bucket; identical to the old slice.
+    if (count >= rankOrdered.length)
+        return rankOrdered.slice(0, count);
+    const inferType = (s) => s.testType ?? (s.steps.length === 1 ? "contract" : "integration");
+    // Protected items occupy GENERATE slots first, in rank order.
+    const selected = [];
+    const remaining = [];
+    for (const item of rankOrdered) {
+        if (selected.length < count &&
+            (item.priority === "CRITICAL" || isAttackSurfaceSecurityBoundary(item.scenario))) {
+            selected.push(item);
+        }
+        else {
+            remaining.push(item);
+        }
+    }
+    // Bucket the remainder by inferred type, preserving rank order and first-appearance
+    // bucket order.
+    const order = [];
+    const buckets = new Map();
+    for (const item of remaining) {
+        const t = inferType(item.scenario);
+        if (!buckets.has(t)) {
+            buckets.set(t, []);
+            order.push(t);
+        }
+        buckets.get(t).push(item);
+    }
+    // Round-robin one per non-empty bucket per round until full.
+    while (selected.length < count && order.some((t) => buckets.get(t).length > 0)) {
+        for (const t of order) {
+            if (selected.length >= count)
+                break;
+            const bucket = buckets.get(t);
+            if (bucket.length > 0)
+                selected.push(bucket.shift());
+        }
+    }
+    return selected;
+}
 export function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, seed, endpointCount, isUIOnlyPR, hasFrontendChanges = false, hasTraces = false, externalCoverage = new Set(), relevantExternalTestPaths = []) {
     const frontendUrl = "<frontend_url>";
     // Slot allocation:
@@ -183,8 +246,19 @@ export function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValu
         })
         : scored;
     const slotOrderedItems = prioritizeAttackSurfaceBundles(scoredAfterExternalDedup);
-    const generateItems = slotOrderedItems.slice(0, Math.min(backendGenerateCount, slotOrderedItems.length));
-    const rawAdditionalItems = slotOrderedItems.slice(backendGenerateCount, topN);
+    // Distribute the backend GENERATE slots evenly across the test types present
+    // (contract vs integration) with spillover, instead of a pure top-N rank slice —
+    // so a backend change isn't starved of (say) integration coverage just because
+    // contract scenarios out-ranked it. See roundRobinByType.
+    const generateItems = roundRobinByType(slotOrderedItems, Math.min(backendGenerateCount, slotOrderedItems.length));
+    // ADDITIONAL = everything not chosen for GENERATE, in original rank order, capped
+    // at the remaining budget. Computed by set-difference (not a positional slice)
+    // because round-robin may pick items that aren't the first backendGenerateCount by
+    // rank — a positional slice would double-list a generated item or drop one.
+    const generateSet = new Set(generateItems);
+    const rawAdditionalItems = slotOrderedItems
+        .filter((it) => !generateSet.has(it))
+        .slice(0, Math.max(0, topN - backendGenerateCount));
     // Filter additional items whose primary resource + test type already appear in GENERATE
     const generatedCoverage = new Set(generateItems.map((item) => scenarioCoverageKey(item.scenario)));
     const additionalItems = rawAdditionalItems.filter((item) => !generatedCoverage.has(scenarioCoverageKey(item.scenario)));

package/build/prompts/test-recommendation/test-recommendation-prompt.test.js

@@ -910,6 +910,97 @@ describe("buildRecommendationPrompt — GENERATE slot allocation", () => {
         expect(additionalBlock).toContain("DELETE /api/gadgets/{id} → 401");
     });
 });
+describe("buildExecutionPlan — even-with-spillover test-type distribution", () => {
+    // Backend-only PR (hasFrontendChanges=false) so every GENERATE slot is a backend
+    // scenario drawn from `scored` — this is where contract-vs-integration distribution
+    // applies. baseUrl/auth args mirror the existing buildExecutionPlan tests above.
+    const contract = (name, path, priority = "HIGH") => ({
+        scenario: minimalScenario({
+            scenarioName: name,
+            category: "crud",
+            testType: TestType.CONTRACT,
+            steps: [{ order: 1, method: "POST", path, description: `POST ${path}`, interactionType: "success", expectedStatusCode: 201 }],
+        }),
+        priority: priority,
+        novelty: "new",
+    });
+    const integration = (name, p1, p2, priority = "HIGH") => ({
+        scenario: minimalScenario({
+            scenarioName: name,
+            category: "data_integrity",
+            testType: TestType.INTEGRATION,
+            steps: [
+                { order: 1, method: "POST", path: p1, description: `POST ${p1}`, interactionType: "success", expectedStatusCode: 201 },
+                { order: 2, method: "PATCH", path: p2, description: `PATCH ${p2}`, interactionType: "success", expectedStatusCode: 200 },
+            ],
+        }),
+        priority: priority,
+        novelty: "new",
+    });
+    const plan = (items, maxGen, topN) => buildExecutionPlan(items, maxGen, topN, "http://localhost:3000", "Authorization", ", authScheme: \"Bearer\"", "", "seed", items.length, false);
+    it("distributes GENERATE slots across contract AND integration when budget < candidates", () => {
+        // 4 contract out-rank 4 integration; pure top-3 rank slice would be all contract.
+        const items = [
+            contract("c1", "/api/a"), contract("c2", "/api/b"), contract("c3", "/api/c"), contract("c4", "/api/d"),
+            integration("i1", "/api/x", "/api/x/{id}"), integration("i2", "/api/y", "/api/y/{id}"),
+        ];
+        const prompt = plan(items, 3, 6);
+        const generated = prompt.slice(0, prompt.indexOf("[ADDITIONAL]") >= 0 ? prompt.indexOf("[ADDITIONAL]") : prompt.length);
+        expect(generated).toMatch(/GENERATE\*\* \| contract/);
+        expect(generated).toMatch(/GENERATE\*\* \| integration/);
+    });
+    it("single backend type → unchanged top-N rank slice (no regression)", () => {
+        const items = [contract("c1", "/api/a"), contract("c2", "/api/b"), contract("c3", "/api/c"), contract("c4", "/api/d")];
+        const prompt = plan(items, 3, 6);
+        const generated = prompt.slice(0, prompt.indexOf("[ADDITIONAL]") >= 0 ? prompt.indexOf("[ADDITIONAL]") : prompt.length);
+        // Top 3 contract by rank are generated; the 4th is not.
+        expect(generated).toContain("POST /api/a → 201");
+        expect(generated).toContain("POST /api/b → 201");
+        expect(generated).toContain("POST /api/c → 201");
+        expect(generated).not.toContain("POST /api/d → 201");
+        expect(generated).not.toMatch(/GENERATE\*\* \| integration/);
+    });
+    it("keeps a CRITICAL item in GENERATE even when lower-ranked than other-type candidates", () => {
+        // A CRITICAL integration sits after several HIGH contracts in rank order.
+        const items = [
+            contract("c1", "/api/a"), contract("c2", "/api/b"), contract("c3", "/api/c"),
+            integration("crit", "/api/orders", "/api/orders/{id}", "CRITICAL"),
+        ];
+        const prompt = plan(items, 3, 6);
+        const generated = prompt.slice(0, prompt.indexOf("[ADDITIONAL]") >= 0 ? prompt.indexOf("[ADDITIONAL]") : prompt.length);
+        // The CRITICAL integration must occupy a GENERATE slot (protected-first).
+        expect(generated).toContain("PATCH /api/orders/{id}");
+    });
+    it("spills freed slots to the remaining type when one type runs dry", () => {
+        // 1 integration + 4 contract, budget 3: round 1 = 1 integration + 1 contract,
+        // integration bucket now empty → remaining 1 slot spills to contract → 2 contract total.
+        const items = [
+            integration("i1", "/api/x", "/api/x/{id}"),
+            contract("c1", "/api/a"), contract("c2", "/api/b"), contract("c3", "/api/c"), contract("c4", "/api/d"),
+        ];
+        const prompt = plan(items, 3, 6);
+        const generated = prompt.slice(0, prompt.indexOf("[ADDITIONAL]") >= 0 ? prompt.indexOf("[ADDITIONAL]") : prompt.length);
+        const contractSlots = (generated.match(/GENERATE\*\* \| contract/g) || []).length;
+        const integrationSlots = (generated.match(/GENERATE\*\* \| integration/g) || []).length;
+        expect(integrationSlots).toBe(1);
+        expect(contractSlots).toBe(2);
+    });
+    it("never lists a GENERATE item again under ADDITIONAL (set-difference recompute)", () => {
+        const items = [
+            contract("c1", "/api/a"), contract("c2", "/api/b"), contract("c3", "/api/c"), contract("c4", "/api/d"),
+            integration("i1", "/api/x", "/api/x/{id}"), integration("i2", "/api/y", "/api/y/{id}"),
+        ];
+        const prompt = plan(items, 3, 6);
+        const splitIdx = prompt.indexOf("[ADDITIONAL]");
+        expect(splitIdx).toBeGreaterThan(0);
+        const generated = prompt.slice(0, splitIdx);
+        const additional = prompt.slice(splitIdx);
+        // Each generated endpoint must not reappear in the ADDITIONAL section.
+        for (const m of generated.matchAll(/(POST|PATCH) (\/api\/[^\s]+)/g)) {
+            expect(additional).not.toContain(`${m[1]} ${m[2]}`);
+        }
+    });
+});
 // ---------------------------------------------------------------------------
 // Tests — buildTestQualityCriteria contract-test guidance (regression guard)
 // ---------------------------------------------------------------------------

package/build/prompts/testbot/testbot-prompts.d.ts

@@ -1,7 +1,22 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { type Service } from "../../workspace/workspace.js";
+/** One related repository to analyze as shared context in multi-repo mode. */
+export interface RelatedRepository {
+    repositoryPath: string;
+    baseBranch?: string;
+    /** `owner/repo` slug, resolved by the action from the checkout's git remote.
+     *  Used verbatim as the `repository` argument/attribution for this repo so the
+     *  agent never has to infer it. */
+    repo: string;
+}
+/**
+ * Parse the JSON-encoded `relatedRepositories` argument passed via the testbot
+ * prompt/resource. Returns undefined for missing/blank input or malformed JSON so the
+ * caller cleanly falls back to single-repo behavior.
+ */
+export declare function parseRelatedRepositories(raw: string | undefined): RelatedRepository[] | undefined;
 export declare function getTestbotPrompt(prTitle: string, prDescription: string, summaryOutputFile: string, repositoryPath: string, baseBranch?: string, maxRecommendations?: number, maxGenerate?: number, _maxCritical?: number, // Reserved — accepted for API compat but not yet wired into prompt
-prNumber?: number, userPrompt?: string, services?: Service[], stateOutputFile?: string, uiCredentials?: string, testsRepoDir?: string): string;
+prNumber?: number, userPrompt?: string, services?: Service[], uiCredentials?: string, testsRepoDir?: string, relatedRepositories?: RelatedRepository[], primaryRepo?: string): string;
 /**
  * Read services from .skyramp/workspace.yml. Returns empty array if
  * the workspace file doesn't exist or can't be parsed.