@skyramp/mcp 0.2.8 → 0.2.150-rc.local.de

package/build/prompts/prompt-utils.js

@@ -0,0 +1,33 @@
+import { readWorkspaceConfigRaw } from "../utils/workspaceAuth.js";
+export function escapeXml(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&apos;");
+}
+export function buildServiceContext(services) {
+    const blocks = services.map((svc) => {
+        const parts = [`<service name="${escapeXml(svc.serviceName)}">`];
+        if (svc.language)
+            parts.push(`  <language>${escapeXml(svc.language)}</language>`);
+        if (svc.framework)
+            parts.push(`  <framework>${escapeXml(svc.framework)}</framework>`);
+        if (svc.api?.baseUrl)
+            parts.push(`  <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
+        if (svc.testDirectory)
+            parts.push(`  <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
+        parts.push("</service>");
+        return parts.join("\n");
+    });
+    return `<services>\n${blocks.join("\n")}\n</services>`;
+}
+/**
+ * Read services from .skyramp/workspace.yml. Returns empty array if
+ * the workspace file doesn't exist or can't be parsed.
+ */
+export async function readWorkspaceServices(repositoryPath) {
+    const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
+    return (rawConfig?.services ?? []);
+}

package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js

@@ -1,4 +1,4 @@
-import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSutDefinitionBlock, } from "../shared.js";
+import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
 import { getPersonaPrefix } from "../../personas.js";
 import { PromptPlan } from "../../test-recommendation/promptPlan.js";
 // ── Step body builders ────────────────────────────────────────────────────────
@@ -70,6 +70,7 @@ function buildAdaptRestrictionsBody() {
 // ── PromptPlan declaration ────────────────────────────────────────────────────
 const _adaptPlan = new PromptPlan()
     .addPhase("adapt", "", { headerLevel: "hidden", stepFormat: "hash" })
+    .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
     .step("SCAN", "Understand the System Under Test in this repository", () => "")
     .subStep("READ_SOURCES", "Read workflows and infra files", buildReadSourcesBody)
     .subStep("CLASSIFY_STEPS", "Classify source workflow steps", buildClassifyStepsBody)
@@ -80,6 +81,7 @@ const _adaptPlan = new PromptPlan()
     .subStep("PATTERN_C", "Pattern C — GHA steps replace service startup + Testbot lifecycle validation", buildPatternCBody)
     .subStep("PATTERN_D", "Pattern D — Fully wrapped by GHA steps", buildPatternDBody)
     .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
+    .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
     .step("ADAPT", "Edit the Testbot workflow and generate SUT files", () => "")
     .subStep("EDIT_WORKFLOW", "Edit the workflow file", buildEditWorkflowBody)
     .subStep("CONFIGURE_AUTH", "Configure auth", buildConfigureAuthBody)

package/build/prompts/sut-setup/modes/dockerComposePrompt.js

@@ -1,4 +1,4 @@
-import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSutDefinitionBlock, } from "../shared.js";
+import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
 import { getPersonaPrefix } from "../../personas.js";
 import { PromptPlan } from "../../test-recommendation/promptPlan.js";
 // ── Step body builders ────────────────────────────────────────────────────────
@@ -260,9 +260,11 @@ ${GENERATE_DOCKER_COMPOSE_INSTRUCTIONS}`;
 // ── PromptPlan declaration ────────────────────────────────────────────────────
 const _composePlan = new PromptPlan()
     .addPhase("compose", "", { headerLevel: "hidden", stepFormat: "hash" })
+    .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
     .step("SCAN", "Understand the System Under Test in this repository", buildScanComposeBody)
     .step("DECIDE", "Decide SUT setup strategy", buildDecideStrategyBody)
     .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
+    .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
     .step("BUILD", "Edit the Testbot workflow and generate SUT files", () => "")
     .subStep("PR_ACCURATE", "Build PR-accurate SUT", buildPrAccurateSutSection)
     .subStep("APPLY", "Apply the chosen strategy", buildApplyStrategyBody)

package/build/prompts/sut-setup/shared.d.ts

@@ -21,6 +21,26 @@ export interface SutPromptArgs {
  */
 export declare function buildSutDefinitionBlock(): string;
 export declare function buildContextBlock(args: SutPromptArgs): string;
+/**
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
+ * looks for existing customer seed conventions BEFORE inventing new ones.
+ * A fresh database makes list / search / filter / pagination endpoint tests
+ * shallow (only HTTP-status assertable, never response content), which
+ * collapses the data-diversity dimension of the eval. Most repos already
+ * have a seed mechanism — wire that in instead of generating one.
+ *
+ * Rendered as a top-level PromptPlan step between INPUTS and the
+ * ADAPT/BUILD step in every SUT setup mode.
+ */
+export declare function buildSeedDataDiscoverySection(): string;
+/**
+ * Workspace file initialisation block — rendered as the first instruction
+ * in every SUT setup mode, before any local validation, workflow editing,
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
+ * inventory so testbot's SUT autocommit picks it up alongside the
+ * generated `.skyramp/sut/*` files.
+ */
+export declare function buildWorkspaceInitSection(): string;
 export declare function buildLocalValidationSection(): string;
 /**
  * Returns the canonical "add to the Testbot workflow" block: every input that

package/build/prompts/sut-setup/shared.js

@@ -31,6 +31,59 @@ export function buildContextBlock(args) {
     lines.push(`</context>`);
     return lines.join("\n");
 }
+/**
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
+ * looks for existing customer seed conventions BEFORE inventing new ones.
+ * A fresh database makes list / search / filter / pagination endpoint tests
+ * shallow (only HTTP-status assertable, never response content), which
+ * collapses the data-diversity dimension of the eval. Most repos already
+ * have a seed mechanism — wire that in instead of generating one.
+ *
+ * Rendered as a top-level PromptPlan step between INPUTS and the
+ * ADAPT/BUILD step in every SUT setup mode.
+ */
+export function buildSeedDataDiscoverySection() {
+    return `**First, decide if this step applies.** Skip it entirely (in your summary state \`SEED_DATA: N/A — <reason>\`, naming the specific bullet below that applied — e.g. "no stateful service", "no list endpoints in scope", "already seeded by init container") and move on when ANY of the following is true:
+- The SUT has no database / stateful service (it's a pure stateless API, a static asset server, a webhook receiver, etc.).
+- The compose stack contains no database container (no \`postgres\`, \`mysql\`, \`mariadb\`, \`mongodb\`, \`redis\` as the primary store, \`sqlite\`, etc.) and no application-managed persistent store.
+- The PR diff and the existing test surface focus exclusively on auth, single-resource CRUD (POST/GET-one/PUT/DELETE on \`/foo/{id}\`), or compute endpoints — no list / search / filter / pagination / aggregate endpoints will be exercised.
+- You can confirm from the SCAN step's findings that representative records already exist (e.g. seeded by an init container, by a migration, or by the auth/setup flow you already wired).
+
+Only proceed with the rest of this step when the SUT has a database AND the tests will exercise list/search/filter/pagination endpoints. Otherwise an empty database is acceptable and inventing seed data adds noise.
+
+When this step DOES apply: database services usually start empty in CI, so tests against list / search / filter / pagination endpoints will then only see \`[]\` and can assert nothing beyond HTTP status — a low data-diversity outcome. Look for an existing seeding mechanism in the repo and wire it into the SUT bootstrap.
+
+**Discover existing project conventions first** — most repos already have one of:
+1. A named script: \`scripts/seed*\`, \`db/seeds/*\`, \`prisma/seed.{ts,js}\`, \`apps/api/seed/*\`, \`tests/fixtures/seed*\`. Search: \`find . -iname '*seed*' -not -path '*/node_modules/*' -not -path '*/.git/*'\`.
+2. A package.json / Makefile target: \`npm run seed\`, \`npm run db:seed\`, \`pnpm seed\`, \`yarn seed\`, \`make seed\`, \`make fixtures\`. Check \`package.json\` \`scripts\` and the project's \`Makefile\`.
+3. A migration with sample data: Django \`loaddata\`, Rails \`db:seed\`, Knex \`seed:run\`, Prisma \`db seed\`, Alembic data migrations.
+4. A docker-compose init service: existing compose files sometimes have a \`*-seed\`, \`*-fixtures\`, or \`init-*\` service that runs once on bring-up.
+
+**Wire it in:**
+- **Compose init service** — add it (or its dependency) to the testbot compose so \`targetSetupCommand\` brings it up alongside the rest. Use \`depends_on\` with \`condition: service_completed_successfully\` on dependent services so they only start after seeding finishes.
+- **Script needing an admin/API token** (most common) — chain it INTO \`authTokenCommand\` AFTER the token is created and BEFORE the script prints the token to stdout. Route seed stdout/stderr to stderr so the captured token isn't corrupted. Make the seed call best-effort (\`|| echo "seed failed" >&2\`) so a flaky seed doesn't block the run — the token is still returned and PRs whose tests don't need seed data keep working.
+- **Migration with sample data** — ensure the migration runs as part of \`targetSetupCommand\` (some ORMs auto-run on container start, others need an explicit \`migrate\` / \`db:seed\` step before tests can hit the API).
+
+**If NO existing convention is found**, generate a minimal seed step (≥3 entities per primary resource the tests will query, e.g., ≥3 repos / ≥3 issues / ≥3 users for a forge; ≥3 products / ≥3 orders for a shop). Call this out explicitly in your summary so the user knows a richer fixture may be needed for production-quality tests.
+
+**Never** hard-code production data, real customer credentials, or PII in seeds. Seed identifiers should be obviously test-only (\`acme-co\`, \`alpha-svc\`, \`testbot-admin\`).`;
+}
+/**
+ * Workspace file initialisation block — rendered as the first instruction
+ * in every SUT setup mode, before any local validation, workflow editing,
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
+ * inventory so testbot's SUT autocommit picks it up alongside the
+ * generated `.skyramp/sut/*` files.
+ */
+export function buildWorkspaceInitSection() {
+    return `Before generating any SUT setup files, check whether \`.skyramp/workspace.yml\` exists at the root of the repository being bootstrapped (the path shown next to "Repository path:" in the context block above — the repo whose PR triggered Testbot; in multi-repo runs this is the primary repo, not the sibling repos checked out as related repos). If it doesn't:
+1. Call \`skyramp_init_scan\` with \`workspacePath\` set to that repository root → follow the returned instructions to discover all services.
+2. Call \`skyramp_init_workspace\` with the same \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
+
+This produces a baseline \`.skyramp/workspace.yml\` listing the services Testbot will test. Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside it. You might have to update the per-service \`runtimeDetails.serverStartCommand\` to match the new SUT files created, so Testbot brings up the services correctly.
+
+Also verify each service's \`api.baseUrl\` includes the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.`;
+}
 export function buildLocalValidationSection() {
     return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
 Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
@@ -39,7 +92,7 @@ Run commands one at a time in your shell — individual execution pinpoints fail
    b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
    c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
    d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
-   e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, run \`docker builder prune -af -f && docker image prune -af\` before the next build. This frees BuildKit cache + dangling images without losing the running stack. Skipping this on heavy SUTs (medusa, calcom, mattermost, supabase) exhausts the ~14 GB \`ubuntu-latest\` disk mid-validation.
+   e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, prune docker BuildKit cache and dangling images before the next build (keep the running stack intact). Skipping this on heavy SUTs exhausts the standard runner disk mid-validation.
 2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
    a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
       i. Run \`targetSetupCommand\` — confirm exit 0.
@@ -68,8 +121,16 @@ Target lifecycle inputs (control how Testbot starts, validates, and stops the SU
    b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
    c. With a dedicated compose file (reused or generated), point at it explicitly: \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . up -d --build\`.
    d. Override whenever the source workflow's start sequence differs from the default.
-2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
-   a. **CRITICAL: Always set this to a valid target readiness check command.** Required in every pattern: when \`targetSetupCommand\` is explicit, when relying on the \`docker compose up -d\` default, AND when surrounding GHA steps bring the SUT up before the Testbot action (\`skipTargetSetup: 'true'\`). Testbot's fail-early gate throws \`Unable to verify service readiness\` when no readiness signal is available (neither this nor a workspace service \`baseUrl\`) and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
+2. \`targetReadyCheckCommand\` — polling command Testbot runs repeatedly after setup until it exits 0 (service ready) or the timeout fires.
+   a. **CRITICAL: must cover EVERY service the tests will hit, not just the backend.** Chain checks with \`&&\` so the command exits 0 only when ALL services respond. A single-service check on a multi-service SUT is the leading cause of "element not visible" / "heading not visible" UI test failures — Playwright lands on a partially-rendered page because only the API was up when the gate passed. Inventory of services to include:
+      - Every API service in workspace.yml with an \`api.baseUrl\` → curl an unauthenticated endpoint (health / version / status).
+      - Every UI service with a UI \`baseUrl\` (or a frontend port in compose) → curl \`/\` or a known asset like \`/assets/index.js\` (more reliable than \`/\` for SPAs that hydrate lazily).
+      - Skip auth-required endpoints — they 401 before the SUT is testable and mask the readiness signal.
+      Examples:
+      - API only: \`curl -sf http://localhost:8080/api/health\`
+      - API + UI:  \`curl -sf http://localhost:8080/api/health && curl -sf http://localhost:3000/\`
+      Required in every pattern (explicit \`targetSetupCommand\`, the \`docker compose up -d\` default, AND \`skipTargetSetup: 'true'\`). Without a real readiness signal (neither this nor a workspace service \`baseUrl\`), testbot's fail-early gate throws \`Unable to verify service readiness\` and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
+   b. Set \`targetReadyCheckDiagnosticsCommand\` to a command that explains why the SUT isn't healthy — output is surfaced in the failure PR comment when the ready check times out. For docker-compose SUTs, scope it to the project's compose file (container state + recent service logs). For non-docker SUTs, a tail of the server log.
 3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
    a. Default: \`1800\` (30 min) — sized for cold \`docker compose --build\` runs on multi-service stacks. The loop exits on the first success, so fast services pay no extra cost. Override only if you want to fail faster on stuck setups (e.g. \`'300'\` for a small single-service SUT).
 4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`, or \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . down -v\` for a dedicated compose file).
@@ -89,19 +150,20 @@ Auth inputs (used by Testbot to authenticate against the running SUT during test
    b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
 Workflow / job level settings (set on the workflow or job, not as \`skyramp/testbot\` action inputs):
 8. Secrets / environment variables — pass every secret/env var the setup needs into the workflow via an \`env:\` block or \`secrets: inherit\`, and add any dependency-resolution steps/jobs the SUT requires.
-9. \`runs-on\` runner label — switch to a larger runner when the SUT needs more disk or memory: heavy Docker builds, large dependency installs, or a multi-service (3+) compose stack. Standard runners run out of disk and crash with \`No space left on device\`. Runner names vary by org — do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses (never downgrade to \`ubuntu-latest\`), (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`.
+9. \`runs-on\` runner label — leave whatever value the Testbot installer wrote and the source workflow used. Don't override it. Picking a runner SKU is an account-level decision that depends on provisioning the agent can't see from inside the repo; a wrong choice queues the job forever with no runner ever picking it up. If the SUT genuinely doesn't fit on the chosen runner, the disk-pressure mitigation in "Common failure modes" #9 handles it; an unprovisioned-runner stall does not.
 
 #### What to do
 Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot installer) — do NOT recreate it. Add only what the SUT needs onto the existing \`skyramp/testbot\` action step, alongside the inputs the installer already added:
 1. The 4 target lifecycle inputs — \`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetReadyCheckTimeout\`, \`targetTeardownCommand\`.
 2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
 3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
-4. The \`runs-on\` runner label, if a larger runner is needed (input 9).
+4. Leave the existing \`runs-on\` value untouched (input 9).
 5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
+6. Stack-specific testability flags — some frameworks gate test-harness affordances behind a container env var. Common cases: Flutter web needs \`IS_TESTING=true\` so \`ensureSemantics()\` exposes the widget tree to Playwright (without it every selector finds nothing even though the app is up); some React/Angular builds need \`NODE_ENV\` / framework-specific build flags for \`testID\` propagation. Set these in the compose \`environment:\` block on the affected service (or as a \`Dockerfile ENV\`) — NOT as a shell var in \`targetSetupCommand\`. The var must be in the container's env at boot, not just the shell that runs the setup command. If selectors find nothing but the app is up, check for a missing flag like this first.
 
 #### What not to do
 1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
-2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
+2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code. In case of multi-mode containers (\`Dockerfile.bdd\`, \`Dockerfile.e2e\`, \`Dockerfile.integration\`, \`Dockerfile.test\`), create the necessary file in \`.skyramp/sut/\` with a PR-sourced build (e.g. \`.skyramp/sut/Dockerfile.bdd\` referenced from the testbot compose's \`build:\`). Do not edit existing Dockerfiles or compose files in the repo. If the multi-mode container's bootstrap step depends on a seeded user, ensure SEED_DATA creates it before this container tries to log in.
 3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
 4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
 5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
@@ -127,6 +189,6 @@ export function buildCommonSutErrorsSection() {
 6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
 7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
 8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
-9. no_space_left — the runner ran out of disk space (\`No space left on device\`). This happens on standard runners when the SUT involves large Docker builds, many npm packages, or multiple compose services that pull large images. Switch \`runs-on\` to a larger runner — runner names vary by org, do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses, (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`. Indicator: source workflow already specifies a large runner, or the SUT has multi-stage Docker builds / a compose file with 3+ services.
+9. no_space_left — the runner ran out of disk space (\`No space left on device\`). Prune the docker BuildKit cache and dangling images at the start of both \`targetSetupCommand\` and \`targetTeardownCommand\` so each fix-loop retry begins with a clean slate; without it, cache from the previous attempt accumulates across retries on heavy SUTs. Do not change \`runs-on\` — see input 9 above.
 Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
 }

package/build/prompts/test-recommendation/analysisOutputPrompt.js

@@ -288,11 +288,11 @@ The ranked test recommendation catalog is pre-built and shown below (after the s
 **If** Steps 1–2 revealed additional scenarios the catalog does not cover (e.g. a computed formula or foreign-key relationship that was missed), you may optionally call \`skyramp_recommend_tests\` with \`stateFile: "${p.stateFile ?? p.sessionId}"\` and \`enrichedScenarios\` to regenerate a more complete catalog — but only after presenting the current one.`;
         const hasJavaFiles = p.candidateRouteFiles?.some((f) => /\.(java|kt)$/.test(f)) ?? false;
         const routeFilesSection = p.candidateRouteFiles && p.candidateRouteFiles.length > 0
-            ? `\nRoute/controller files found by static scan (read these to discover endpoints — the regex-based catalog below may be incomplete for your framework):\n${p.candidateRouteFiles.map((f) => `- ${f}`).join("\n")}\n`
+            ? `\nCandidate route/controller files for LLM inspection (read these to discover endpoints; static parser output is only a hint):\n${p.candidateRouteFiles.map((f) => `- ${f}`).join("\n")}\n`
             : "";
         const resolvePathsNote = p.routerMountContext.length
-            ? `**Resolve nested paths** using your Step ${ANALYSIS_STEP_RESOLVE_PATHS} table — a router in the table with prefix \`/api/v1/products/{product_id}/reviews\` means every endpoint in that file lives under that full path.`
-            : `**Resolve full paths** using the prefixes you identified in Step ${ANALYSIS_STEP_READ_FILES} (e.g. Java Spring class-level \`@RequestMapping\` prefix + method-level path).`;
+            ? `**Resolve nested paths** using your Step ${ANALYSIS_STEP_RESOLVE_PATHS} table. The table you build from source/router context is authoritative over static parser hints.`
+            : `**Resolve full paths** using the prefixes, mounts, annotations, decorators, or routing DSL you identify in Step ${ANALYSIS_STEP_READ_FILES}.`;
         return `## Your Task — Fill in and Present the Catalog (full repo)
 
 ### Step ${ANALYSIS_STEP_READ_FILES}: Read key files
@@ -335,36 +335,28 @@ ${p.routerFileContents?.length
                 p.routerMountContext.map((f) => `- \`${f}\``).join("\n")}`
         : "";
     const enrichment = buildEnrichmentInstructions(p);
-    // LLM fallback: when heuristic scanning may have missed backend route files.
-    const scannerFallbackHasCatalog = p.scannedEndpoints.length > 0;
-    const scannerFallbackReason = scannerFallbackHasCatalog
-        ? "The heuristic endpoint scanner may have missed route/controller files in this diff, even though endpoint data is present from another source"
-        : "The heuristic endpoint scanner found **0 endpoints**";
-    const scannerFallbackInstruction = scannerFallbackHasCatalog
-        ? "Treat this as a supplemental gap check: merge only HTTP endpoints that are visibly missing from the existing catalog/spec data, and keep the existing catalog as the primary endpoint source."
-        : "Build a table of discovered endpoints (method, full path, source file) and use it as the authoritative endpoint list for all subsequent steps.";
-    const llmFallbackSection = (isDiffScope || p.scannedEndpoints.length === 0) &&
-        p.candidateRouteFiles &&
-        p.candidateRouteFiles.length > 0
+    const staticHintCount = p.scannedEndpoints.length;
+    const routeDiscoverySection = isDiffScope ||
+        staticHintCount > 0 ||
+        (p.candidateRouteFiles?.length ?? 0) > 0
         ? `
-## Scanner Fallback — Manual Endpoint Discovery Required
+## LLM Route Discovery Inputs
 
-${scannerFallbackReason}, and this repository contains ${p.candidateRouteFiles.length} file(s) that appear to define HTTP routes. The scanner likely failed due to multi-line definitions, framework-specific patterns, indirection the regex does not cover, or endpoint data being supplied by an OpenAPI spec instead of source scanning.
+Static endpoint scan results are **best-effort hints only**. Do not assume the scanner supports this repository's language, framework, routing DSL, or helper abstractions.
 
-**Read the following controller/router files and identify all HTTP route registrations (method + path).** Include these discovered endpoints when executing the steps above.
+${staticHintCount > 0 ? `Static hints available: ${staticHintCount}. Verify every hinted method/path against source before using it.` : "Static hints available: 0. Build the endpoint list from source, router context, spec, and diff."}
 
-${p.candidateRouteFiles.slice(0, 15).map((f) => `- \`${f}\``).join("\n")}
-${p.candidateRouteFiles.length > 15 ? `\n_(${p.candidateRouteFiles.length - 15} more files not shown)_` : ""}
+${p.candidateRouteFiles && p.candidateRouteFiles.length > 0
+            ? `Candidate files to inspect:\n${p.candidateRouteFiles
+                .slice(0, 15)
+                .map((f) => `- \`${f}\``)
+                .join("\n")}${p.candidateRouteFiles.length > 15 ? `\n_(${p.candidateRouteFiles.length - 15} more files not shown)_` : ""}`
+            : staticHintCount > 0
+                ? "Candidate files to inspect: none identified by static scanning. Verify the static hints against their source files, changed files, router context, and specs above."
+                : "Candidate files to inspect: use the changed files and routing entry-point files above."}
 
-For each file, look for:
-- Express/Fastify/Koa/Hapi: \`router.<method>('/path', ...)\` or \`app.<method>('/path', ...)\`
-- FastAPI/Flask: \`@router.<method>('/path')\` or \`@app.route('/path', ...)\`
-- Spring/NestJS: \`@GetMapping\`, \`@PostMapping\`, \`@Controller\`, etc.
-- Go (Gin/Echo/Chi): \`r.GET("/path", ...)\`, \`r.Group("/prefix")\`
-- GraphQL: schema/resolver artifacts are unsupported for REST test generation; do not invent REST endpoints from them
-- Any other framework-specific route registration pattern
-
-${scannerFallbackInstruction}`
+For each candidate file, infer route registrations from the source's actual framework or DSL. Record method, full path, and source file. Resolve mount prefixes through the routing entry-point files. If a file is GraphQL-only, do not invent REST endpoints from it.
+`
         : "";
     return `# Repository Analysis
 
@@ -373,7 +365,7 @@ ${scannerFallbackInstruction}`
 **Analysis Scope**: \`${p.analysisScope}\`
 ${isDiffScope ? `**Diff endpoints**: ${(p.parsedDiff?.newEndpoints.length ?? 0) + (p.parsedDiff?.modifiedEndpoints.length ?? 0) + (p.parsedDiff?.removedEndpoints?.length ?? 0)}` : `**Pre-scanned endpoints**: ${p.scannedEndpoints.length}`}
 ${routerSection}
-${llmFallbackSection}
+${routeDiscoverySection}
 ${enrichment}
 
 **CRITICAL**: No .json/.md file creation. Prioritize cross-resource workflows.`;

package/build/prompts/test-recommendation/analysisOutputPrompt.test.js

@@ -99,7 +99,10 @@ describe("buildAnalysisOutputText — unmatchedFiles / trace-callers sub-step",
     });
     it("lists each unmatched file as a bullet in the trace-callers block", () => {
         const params = baseParams({
-            unmatchedFiles: ["src/services/OrderService.ts", "src/utils/pricingHelper.ts"],
+            unmatchedFiles: [
+                "src/services/OrderService.ts",
+                "src/utils/pricingHelper.ts",
+            ],
         });
         const output = buildAnalysisOutputText(params);
         expect(output).toContain("- `src/services/OrderService.ts`");
@@ -139,12 +142,21 @@ describe("buildAnalysisOutputText — unmatchedFiles / trace-callers sub-step",
     });
     it("critical-patterns checklist is present in Step 2 regardless of diff availability (phase 4b)", () => {
         // Step 2.5 was merged into Step 2 — patterns appear inline in ALL diff-scope branches.
-        const withDiff = baseParams({ unmatchedFiles: ["src/utils/foo.ts"], diffFilePath: "/tmp/pr.diff" });
-        const withDiffNoUnmatched = baseParams({ unmatchedFiles: [], diffFilePath: "/tmp/pr.diff" });
+        const withDiff = baseParams({
+            unmatchedFiles: ["src/utils/foo.ts"],
+            diffFilePath: "/tmp/pr.diff",
+        });
+        const withDiffNoUnmatched = baseParams({
+            unmatchedFiles: [],
+            diffFilePath: "/tmp/pr.diff",
+        });
         const noDiff = baseParams({ unmatchedFiles: [] });
         const uiOnly = baseParams({
             parsedDiff: {
-                changedFiles: ["src/components/Cart.tsx", "src/components/OrderList.vue"],
+                changedFiles: [
+                    "src/components/Cart.tsx",
+                    "src/components/OrderList.vue",
+                ],
                 newEndpoints: [],
                 modifiedEndpoints: [],
             },
@@ -234,31 +246,82 @@ describe("buildAnalysisOutputText — unmatchedFiles / trace-callers sub-step",
         expect(output).toContain("Trace callers of changed non-route files");
     });
 });
-describe("buildAnalysisOutputText — scanner fallback", () => {
-    it("includes manual endpoint discovery in diff scope when candidate route files are present despite scanned endpoints", () => {
+describe("buildAnalysisOutputText — LLM route discovery inputs", () => {
+    it("labels static endpoint data as hints and includes candidate files", () => {
         const params = baseParams({
             analysisScope: AnalysisScope.CurrentBranchDiff,
-            scannedEndpoints: [{ path: "/openapi-only", methods: ["GET"], sourceFile: "" }],
+            scannedEndpoints: [
+                { path: "/openapi-only", methods: ["GET"], sourceFile: "" },
+            ],
             candidateRouteFiles: ["src/routes/items.ts"],
         });
         const output = buildAnalysisOutputText(params);
-        expect(output).toContain("Scanner Fallback — Manual Endpoint Discovery Required");
-        expect(output).toContain("may have missed route/controller files");
+        expect(output).toContain("LLM Route Discovery Inputs");
+        expect(output).toContain("Static endpoint scan results are **best-effort hints only**");
+        expect(output).toContain("Static hints available: 1");
         expect(output).toContain("src/routes/items.ts");
-        expect(output).toContain("supplemental gap check");
-        expect(output).toContain("keep the existing catalog as the primary endpoint source");
-        expect(output).not.toContain("use it as the authoritative endpoint list");
+        expect(output).toContain("Record method, full path, and source file");
+        expect(output).not.toContain("primary endpoint source");
     });
-    it("tells fallback discovery not to turn GraphQL artifacts into REST endpoints", () => {
+    it("uses static-hint wording when no candidate route files are available", () => {
+        const params = baseParams({
+            analysisScope: AnalysisScope.FullRepo,
+            scannedEndpoints: [
+                { path: "/openapi-only", methods: ["GET"], sourceFile: "openapi.yaml" },
+            ],
+            candidateRouteFiles: [],
+        });
+        const output = buildAnalysisOutputText(params);
+        expect(output).toContain("LLM Route Discovery Inputs");
+        expect(output).toContain("Static hints available: 1");
+        expect(output).toContain("Candidate files to inspect: none identified by static scanning");
+        expect(output).not.toContain("Candidate files to inspect: use the changed files and routing entry-point files above.");
+    });
+    it("includes route discovery inputs for diff scope with zero hints or candidates", () => {
+        const params = baseParams({
+            analysisScope: AnalysisScope.CurrentBranchDiff,
+            scannedEndpoints: [],
+            candidateRouteFiles: [],
+        });
+        const output = buildAnalysisOutputText(params);
+        expect(output).toContain("LLM Route Discovery Inputs");
+        expect(output).toContain("Static hints available: 0");
+        expect(output).toContain("Candidate files to inspect: use the changed files and routing entry-point files above.");
+    });
+    it("tells LLM route discovery not to turn GraphQL artifacts into REST endpoints", () => {
         const params = baseParams({
             analysisScope: AnalysisScope.CurrentBranchDiff,
             scannedEndpoints: [],
             candidateRouteFiles: ["src/graphql/users.resolver.ts"],
         });
         const output = buildAnalysisOutputText(params);
-        expect(output).toContain("GraphQL: schema/resolver artifacts are unsupported for REST test generation");
-        expect(output).toContain("do not invent REST endpoints from them");
-        expect(output).toContain("use it as the authoritative endpoint list");
+        expect(output).toContain("Static hints available: 0");
+        expect(output).toContain("src/graphql/users.resolver.ts");
+        expect(output).toContain("If a file is GraphQL-only, do not invent REST endpoints from it");
         expect(output).not.toContain("Query/Mutation resolvers mapping to endpoints");
     });
+    it("omits route discovery section for full-repo scans without route data", () => {
+        const params = baseParams({
+            analysisScope: AnalysisScope.FullRepo,
+            scannedEndpoints: [],
+            candidateRouteFiles: [],
+        });
+        const output = buildAnalysisOutputText(params);
+        expect(output).not.toContain("LLM Route Discovery Inputs");
+        expect(output).not.toContain("use the changed files and routing entry-point files above");
+    });
+    it("surfaces router context as source to build an authoritative path table", () => {
+        const params = baseParams({
+            routerMountContext: ["src/app/router.ts"],
+            routerFileContents: [
+                { file: "src/app/router.ts", content: "app.use('/api', router)" },
+            ],
+            candidateRouteFiles: ["src/routes/orders.ts"],
+        });
+        const output = buildAnalysisOutputText(params);
+        expect(output).toContain("Routing entry-point files");
+        expect(output).toContain("src/app/router.ts");
+        expect(output).toContain("Candidate files to inspect");
+        expect(output).toContain("Resolve mount prefixes through the routing entry-point files");
+    });
 });

package/build/prompts/test-recommendation/test-recommendation-prompt.js

@@ -123,6 +123,9 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
         }
     }
     const fmtEndpoint = (m, ep) => `  ${m.method} ${ep.path}${m.authRequired ? " [auth]" : ""} (${(m.interactions ?? []).length} interactions)`;
+    // In diff scope, cap the reference endpoint list to prevent context overflow.
+    // Changed endpoints are always shown in full; only the "other" list is capped.
+    const DIFF_SCOPE_OTHER_ENDPOINT_CAP = 20;
     let endpointLines;
     if (isDiffScope && changedEndpointKeys.size > 0) {
         const changedLines = [];
@@ -146,7 +149,25 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
                 changedLines.push(`  ${m.method} ${ep.path} [removed]`);
             }
         }
-        endpointLines = `**Likely changed in this PR (from static file→endpoint mapping — verify against diff in Step ${ANALYSIS_STEP_EXTRACT}):**\n${changedLines.join("\n") || "  none"}\n\n**Other endpoints (reference only):**\n${otherLines.join("\n") || "  none"}`;
+        const cappedOther = otherLines.slice(0, DIFF_SCOPE_OTHER_ENDPOINT_CAP);
+        const hiddenOtherCount = otherLines.length - cappedOther.length;
+        const otherSuffix = hiddenOtherCount > 0
+            ? `\n  ... and ${hiddenOtherCount} more (full endpoint list in state file)`
+            : "";
+        const otherLabel = hiddenOtherCount > 0
+            ? `Other endpoints (reference only, ${cappedOther.length} of ${otherLines.length} shown)`
+            : "Other endpoints (reference only)";
+        endpointLines = `**Likely changed in this PR (from static file→endpoint mapping — verify against diff in Step ${ANALYSIS_STEP_EXTRACT}):**\n${changedLines.join("\n") || "  none"}\n\n**${otherLabel}:**\n${cappedOther.join("\n") || "  none"}${otherSuffix}`;
+    }
+    else if (isDiffScope) {
+        // Diff scope but no changed endpoints detected — cap to avoid dumping the full catalog.
+        const allMethodLines = allEndpoints.flatMap((ep) => (ep.methods ?? []).map((m) => fmtEndpoint(m, ep)));
+        const cappedLines = allMethodLines.slice(0, DIFF_SCOPE_OTHER_ENDPOINT_CAP);
+        const hiddenCount = allMethodLines.length - cappedLines.length;
+        const suffix = hiddenCount > 0
+            ? `\n  ... and ${hiddenCount} more (full endpoint list in state file — trace changed files directly from the diff to find affected endpoints)`
+            : "";
+        endpointLines = `${cappedLines.join("\n") || "  none"}${suffix}`;
     }
     else {
         endpointLines = allEndpoints
@@ -175,9 +196,22 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
         }
     }
     const { authSchemeSnippet } = getAuthSnippets(authHeaderValue, authTypeValue, workspaceAuthScheme);
+    const routeDiscovery = analysis.routeDiscovery;
+    const routeDiscoverySection = routeDiscovery
+        ? `
+## LLM Route Discovery Inputs
+Static endpoint data below is a best-effort hint, not a complete parser for every framework.
+Authoritative endpoint extraction must come from reading the changed source files, router/module context, OpenAPI paths when available, and the diff.
+Candidate files to inspect: ${routeDiscovery.candidateFiles.length > 0 ? routeDiscovery.candidateFiles.join(", ") : "none"}
+Router/module context files: ${routeDiscovery.routerMountContext.length > 0 ? routeDiscovery.routerMountContext.join(", ") : "none"}
+OpenAPI paths available: ${routeDiscovery.openApiPaths.length}
+Static hints available: ${routeDiscovery.staticHints.length}
+${routeDiscovery.diffFilePath ? `Diff file: ${routeDiscovery.diffFilePath}` : ""}
+`.trim()
+        : "";
     const sourcePriority = `
 ## Source Priority
-When information conflicts, prefer: **Traces** (actual behavior) > **Code** (implemented behavior) > **Spec/Docs** (documented behavior).
+When information conflicts, prefer: **Traces** (actual behavior) > **Source code read by the LLM** (implemented behavior) > **OpenAPI spec/docs** (documented behavior) > **Static parser hints** (best-effort, may be incomplete or framework-blind).
 `;
     // Compact fingerprint of tests already covering endpoints in this repo (Skyramp + external).
     // Re-derived fresh each run from test files on disk — no separate persistence needed.
@@ -219,8 +253,11 @@ Framework: ${analysis.projectClassification.primaryFramework} (${analysis.projec
 Project type: ${analysis.projectClassification.projectType}
 Auth: ${authMethod} (header: ${authHeaderValue}${authTypeValue ? `, type: ${authTypeValue}` : ""})
 Base URL: ${analysis.apiEndpoints.baseUrl}
-Candidate endpoints from static scan — unverified, confirm paths against spec or source before use (${analysis.apiEndpoints.totalCount}):
+Candidate endpoint hints from static scan — unverified and non-exhaustive; confirm paths by reading source/router context before use (${analysis.apiEndpoints.totalCount}):
 ${endpointLines}${testFingerprint}
+${routeDiscoverySection ? `
+
+${routeDiscoverySection}` : ""}
 `.trim();
     // ── Branch diff ──
     let diffSection = "";
@@ -240,7 +277,7 @@ Affected services: ${diffContext.affectedServices.join(", ") || "N/A"}
 
 Focus on tests that validate these changes and how they interact with existing resources.
 For removed endpoints: verify they now return 404 or the appropriate deprecation status code.
-Allocate your test budget to endpoints listed under "Likely changed in this PR". Use other endpoints only as setup steps (e.g. creating a resource before testing its deletion).
+Treat the endpoint lists above as static hints. If source/diff inspection finds a different changed endpoint set, prefer the source-grounded set and use other endpoints only as setup steps.
 `;
     }
     // ── Interactions ──

package/build/prompts/test-recommendation/test-recommendation-prompt.test.js

@@ -6,7 +6,7 @@ jest.unstable_mockModule("@skyramp/skyramp", () => ({
 const { buildRecommendationPrompt, buildExternalCoverageSet, externalDedupKey } = await import("./test-recommendation-prompt.js");
 const { buildExecutionPlan } = await import("./diffExecutionPlan.js");
 const { PATH_PARAM_UUID_GUIDANCE, MAX_TESTS_TO_GENERATE, buildTestQualityCriteria, buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildFewShotExamples, buildVerificationChecklist, } = await import("./recommendationSections.js");
-import { AnalysisScope } from "../../types/RepositoryAnalysis.js";
+import { AnalysisScope, repositoryAnalysisSchema } from "../../types/RepositoryAnalysis.js";
 // ---------------------------------------------------------------------------
 // Minimal fixtures
 // ---------------------------------------------------------------------------
@@ -1341,6 +1341,39 @@ describe("buildRecommendationPrompt — data-before-instructions ordering", () =
         expect(interactionsIdx).toBeLessThan(instructionsIdx);
     });
 });
+describe("buildRecommendationPrompt — LLM route discovery contract", () => {
+    it("surfaces route discovery inputs and labels static parser output as hints", () => {
+        const analysis = minimalAnalysis({
+            routeDiscovery: {
+                candidateFiles: ["src/routes/items.ts", "src/app/router.ts"],
+                staticHints: [{ path: "/api/items", methods: ["GET"], sourceFile: "src/routes/items.ts" }],
+                openApiPaths: ["/api/items"],
+                routerMountContext: ["src/app/router.ts"],
+                diffFilePath: "/tmp/skyramp-diff-123.diff",
+            },
+        });
+        const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
+        expect(prompt).toContain("LLM Route Discovery Inputs");
+        expect(prompt).toContain("Static endpoint data below is a best-effort hint");
+        expect(prompt).toContain("Candidate files to inspect: src/routes/items.ts, src/app/router.ts");
+        expect(prompt).toContain("Router/module context files: src/app/router.ts");
+        expect(prompt).toContain("Static hints available: 1");
+        expect(prompt).toContain("Static parser hints");
+    });
+    it("repositoryAnalysisSchema preserves routeDiscovery", () => {
+        const analysis = minimalAnalysis({
+            routeDiscovery: {
+                candidateFiles: ["src/routes/items.ts"],
+                staticHints: [{ path: "/api/items", methods: ["GET"], sourceFile: "src/routes/items.ts" }],
+                openApiPaths: ["/api/items"],
+                routerMountContext: ["src/app/router.ts"],
+                diffFilePath: "/tmp/skyramp-diff-123.diff",
+            },
+        });
+        const parsed = repositoryAnalysisSchema.parse(analysis);
+        expect(parsed.routeDiscovery).toEqual(analysis.routeDiscovery);
+    });
+});
 // ---------------------------------------------------------------------------
 // Tests — Few-shot examples
 // ---------------------------------------------------------------------------