@skyramp/mcp 0.2.8-rc.1 → 0.2.9

package/build/index.js

@@ -59,13 +59,6 @@ const server = new McpServer({
     },
     instructions: `Skyramp MCP Server — generates and executes API tests (fuzz, contract, integration, E2E, UI).
 
-## Workspace Initialization (REQUIRED before the first Skyramp tool call)
-If the workspace root is a git repo AND \`.skyramp/workspace.yml\` does not exist:
-1. Call \`skyramp_init_scan\` with \`workspacePath\` → follow the returned instructions to discover all services.
-2. Call \`skyramp_init_workspace\` with \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
-3. Proceed with the originally requested tool.
-Skip only if: not a git repo, \`.skyramp/workspace.yml\` already exists, or user explicitly declines.
-  
 ## Rules
 - NEVER show CLI commands. NEVER attempt to install or configure the Skyramp CLI. ALWAYS use the MCP tools provided.
 - For UI and E2E tests, there are TWO recording modes:

package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js

@@ -1,4 +1,4 @@
-import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSutDefinitionBlock, } from "../shared.js";
+import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
 import { getPersonaPrefix } from "../../personas.js";
 import { PromptPlan } from "../../test-recommendation/promptPlan.js";
 // ── Step body builders ────────────────────────────────────────────────────────
@@ -70,6 +70,7 @@ function buildAdaptRestrictionsBody() {
 // ── PromptPlan declaration ────────────────────────────────────────────────────
 const _adaptPlan = new PromptPlan()
     .addPhase("adapt", "", { headerLevel: "hidden", stepFormat: "hash" })
+    .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
     .step("SCAN", "Understand the System Under Test in this repository", () => "")
     .subStep("READ_SOURCES", "Read workflows and infra files", buildReadSourcesBody)
     .subStep("CLASSIFY_STEPS", "Classify source workflow steps", buildClassifyStepsBody)
@@ -80,6 +81,7 @@ const _adaptPlan = new PromptPlan()
     .subStep("PATTERN_C", "Pattern C — GHA steps replace service startup + Testbot lifecycle validation", buildPatternCBody)
     .subStep("PATTERN_D", "Pattern D — Fully wrapped by GHA steps", buildPatternDBody)
     .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
+    .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
     .step("ADAPT", "Edit the Testbot workflow and generate SUT files", () => "")
     .subStep("EDIT_WORKFLOW", "Edit the workflow file", buildEditWorkflowBody)
     .subStep("CONFIGURE_AUTH", "Configure auth", buildConfigureAuthBody)

package/build/prompts/sut-setup/modes/dockerComposePrompt.js

@@ -1,4 +1,4 @@
-import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSutDefinitionBlock, } from "../shared.js";
+import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
 import { getPersonaPrefix } from "../../personas.js";
 import { PromptPlan } from "../../test-recommendation/promptPlan.js";
 // ── Step body builders ────────────────────────────────────────────────────────
@@ -260,9 +260,11 @@ ${GENERATE_DOCKER_COMPOSE_INSTRUCTIONS}`;
 // ── PromptPlan declaration ────────────────────────────────────────────────────
 const _composePlan = new PromptPlan()
     .addPhase("compose", "", { headerLevel: "hidden", stepFormat: "hash" })
+    .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
     .step("SCAN", "Understand the System Under Test in this repository", buildScanComposeBody)
     .step("DECIDE", "Decide SUT setup strategy", buildDecideStrategyBody)
     .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
+    .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
     .step("BUILD", "Edit the Testbot workflow and generate SUT files", () => "")
     .subStep("PR_ACCURATE", "Build PR-accurate SUT", buildPrAccurateSutSection)
     .subStep("APPLY", "Apply the chosen strategy", buildApplyStrategyBody)

package/build/prompts/sut-setup/shared.d.ts

@@ -21,6 +21,26 @@ export interface SutPromptArgs {
  */
 export declare function buildSutDefinitionBlock(): string;
 export declare function buildContextBlock(args: SutPromptArgs): string;
+/**
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
+ * looks for existing customer seed conventions BEFORE inventing new ones.
+ * A fresh database makes list / search / filter / pagination endpoint tests
+ * shallow (only HTTP-status assertable, never response content), which
+ * collapses the data-diversity dimension of the eval. Most repos already
+ * have a seed mechanism — wire that in instead of generating one.
+ *
+ * Rendered as a top-level PromptPlan step between INPUTS and the
+ * ADAPT/BUILD step in every SUT setup mode.
+ */
+export declare function buildSeedDataDiscoverySection(): string;
+/**
+ * Workspace file initialisation block — rendered as the first instruction
+ * in every SUT setup mode, before any local validation, workflow editing,
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
+ * inventory so testbot's SUT autocommit picks it up alongside the
+ * generated `.skyramp/sut/*` files.
+ */
+export declare function buildWorkspaceInitSection(): string;
 export declare function buildLocalValidationSection(): string;
 /**
  * Returns the canonical "add to the Testbot workflow" block: every input that

package/build/prompts/sut-setup/shared.js

@@ -31,6 +31,59 @@ export function buildContextBlock(args) {
     lines.push(`</context>`);
     return lines.join("\n");
 }
+/**
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
+ * looks for existing customer seed conventions BEFORE inventing new ones.
+ * A fresh database makes list / search / filter / pagination endpoint tests
+ * shallow (only HTTP-status assertable, never response content), which
+ * collapses the data-diversity dimension of the eval. Most repos already
+ * have a seed mechanism — wire that in instead of generating one.
+ *
+ * Rendered as a top-level PromptPlan step between INPUTS and the
+ * ADAPT/BUILD step in every SUT setup mode.
+ */
+export function buildSeedDataDiscoverySection() {
+    return `**First, decide if this step applies.** Skip it entirely (in your summary state \`SEED_DATA: N/A — <reason>\`, naming the specific bullet below that applied — e.g. "no stateful service", "no list endpoints in scope", "already seeded by init container") and move on when ANY of the following is true:
+- The SUT has no database / stateful service (it's a pure stateless API, a static asset server, a webhook receiver, etc.).
+- The compose stack contains no database container (no \`postgres\`, \`mysql\`, \`mariadb\`, \`mongodb\`, \`redis\` as the primary store, \`sqlite\`, etc.) and no application-managed persistent store.
+- The PR diff and the existing test surface focus exclusively on auth, single-resource CRUD (POST/GET-one/PUT/DELETE on \`/foo/{id}\`), or compute endpoints — no list / search / filter / pagination / aggregate endpoints will be exercised.
+- You can confirm from the SCAN step's findings that representative records already exist (e.g. seeded by an init container, by a migration, or by the auth/setup flow you already wired).
+
+Only proceed with the rest of this step when the SUT has a database AND the tests will exercise list/search/filter/pagination endpoints. Otherwise an empty database is acceptable and inventing seed data adds noise.
+
+When this step DOES apply: database services usually start empty in CI, so tests against list / search / filter / pagination endpoints will then only see \`[]\` and can assert nothing beyond HTTP status — a low data-diversity outcome. Look for an existing seeding mechanism in the repo and wire it into the SUT bootstrap.
+
+**Discover existing project conventions first** — most repos already have one of:
+1. A named script: \`scripts/seed*\`, \`db/seeds/*\`, \`prisma/seed.{ts,js}\`, \`apps/api/seed/*\`, \`tests/fixtures/seed*\`. Search: \`find . -iname '*seed*' -not -path '*/node_modules/*' -not -path '*/.git/*'\`.
+2. A package.json / Makefile target: \`npm run seed\`, \`npm run db:seed\`, \`pnpm seed\`, \`yarn seed\`, \`make seed\`, \`make fixtures\`. Check \`package.json\` \`scripts\` and the project's \`Makefile\`.
+3. A migration with sample data: Django \`loaddata\`, Rails \`db:seed\`, Knex \`seed:run\`, Prisma \`db seed\`, Alembic data migrations.
+4. A docker-compose init service: existing compose files sometimes have a \`*-seed\`, \`*-fixtures\`, or \`init-*\` service that runs once on bring-up.
+
+**Wire it in:**
+- **Compose init service** — add it (or its dependency) to the testbot compose so \`targetSetupCommand\` brings it up alongside the rest. Use \`depends_on\` with \`condition: service_completed_successfully\` on dependent services so they only start after seeding finishes.
+- **Script needing an admin/API token** (most common) — chain it INTO \`authTokenCommand\` AFTER the token is created and BEFORE the script prints the token to stdout. Route seed stdout/stderr to stderr so the captured token isn't corrupted. Make the seed call best-effort (\`|| echo "seed failed" >&2\`) so a flaky seed doesn't block the run — the token is still returned and PRs whose tests don't need seed data keep working.
+- **Migration with sample data** — ensure the migration runs as part of \`targetSetupCommand\` (some ORMs auto-run on container start, others need an explicit \`migrate\` / \`db:seed\` step before tests can hit the API).
+
+**If NO existing convention is found**, generate a minimal seed step (≥3 entities per primary resource the tests will query, e.g., ≥3 repos / ≥3 issues / ≥3 users for a forge; ≥3 products / ≥3 orders for a shop). Call this out explicitly in your summary so the user knows a richer fixture may be needed for production-quality tests.
+
+**Never** hard-code production data, real customer credentials, or PII in seeds. Seed identifiers should be obviously test-only (\`acme-co\`, \`alpha-svc\`, \`testbot-admin\`).`;
+}
+/**
+ * Workspace file initialisation block — rendered as the first instruction
+ * in every SUT setup mode, before any local validation, workflow editing,
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
+ * inventory so testbot's SUT autocommit picks it up alongside the
+ * generated `.skyramp/sut/*` files.
+ */
+export function buildWorkspaceInitSection() {
+    return `Before generating any SUT setup files, check whether \`.skyramp/workspace.yml\` exists at the root of the repository being bootstrapped (the path shown next to "Repository path:" in the context block above — the repo whose PR triggered Testbot; in multi-repo runs this is the primary repo, not the sibling repos checked out as related repos). If it doesn't:
+1. Call \`skyramp_init_scan\` with \`workspacePath\` set to that repository root → follow the returned instructions to discover all services.
+2. Call \`skyramp_init_workspace\` with the same \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
+
+This produces a baseline \`.skyramp/workspace.yml\` listing the services Testbot will test. Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside it. You might have to update the per-service \`runtimeDetails.serverStartCommand\` to match the new SUT files created, so Testbot brings up the services correctly.
+
+Also verify each service's \`api.baseUrl\` includes the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.`;
+}
 export function buildLocalValidationSection() {
     return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
 Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
@@ -39,7 +92,7 @@ Run commands one at a time in your shell — individual execution pinpoints fail
    b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
    c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
    d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
-   e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, run \`docker builder prune -af -f && docker image prune -af\` before the next build. This frees BuildKit cache + dangling images without losing the running stack. Skipping this on heavy SUTs (medusa, calcom, mattermost, supabase) exhausts the ~14 GB \`ubuntu-latest\` disk mid-validation.
+   e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, prune docker BuildKit cache and dangling images before the next build (keep the running stack intact). Skipping this on heavy SUTs exhausts the standard runner disk mid-validation.
 2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
    a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
       i. Run \`targetSetupCommand\` — confirm exit 0.
@@ -68,8 +121,16 @@ Target lifecycle inputs (control how Testbot starts, validates, and stops the SU
    b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
    c. With a dedicated compose file (reused or generated), point at it explicitly: \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . up -d --build\`.
    d. Override whenever the source workflow's start sequence differs from the default.
-2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
-   a. **CRITICAL: Always set this to a valid target readiness check command.** Required in every pattern: when \`targetSetupCommand\` is explicit, when relying on the \`docker compose up -d\` default, AND when surrounding GHA steps bring the SUT up before the Testbot action (\`skipTargetSetup: 'true'\`). Testbot's fail-early gate throws \`Unable to verify service readiness\` when no readiness signal is available (neither this nor a workspace service \`baseUrl\`) and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
+2. \`targetReadyCheckCommand\` — polling command Testbot runs repeatedly after setup until it exits 0 (service ready) or the timeout fires.
+   a. **CRITICAL: must cover EVERY service the tests will hit, not just the backend.** Chain checks with \`&&\` so the command exits 0 only when ALL services respond. A single-service check on a multi-service SUT is the leading cause of "element not visible" / "heading not visible" UI test failures — Playwright lands on a partially-rendered page because only the API was up when the gate passed. Inventory of services to include:
+      - Every API service in workspace.yml with an \`api.baseUrl\` → curl an unauthenticated endpoint (health / version / status).
+      - Every UI service with a UI \`baseUrl\` (or a frontend port in compose) → curl \`/\` or a known asset like \`/assets/index.js\` (more reliable than \`/\` for SPAs that hydrate lazily).
+      - Skip auth-required endpoints — they 401 before the SUT is testable and mask the readiness signal.
+      Examples:
+      - API only: \`curl -sf http://localhost:8080/api/health\`
+      - API + UI:  \`curl -sf http://localhost:8080/api/health && curl -sf http://localhost:3000/\`
+      Required in every pattern (explicit \`targetSetupCommand\`, the \`docker compose up -d\` default, AND \`skipTargetSetup: 'true'\`). Without a real readiness signal (neither this nor a workspace service \`baseUrl\`), testbot's fail-early gate throws \`Unable to verify service readiness\` and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
+   b. Set \`targetReadyCheckDiagnosticsCommand\` to a command that explains why the SUT isn't healthy — output is surfaced in the failure PR comment when the ready check times out. For docker-compose SUTs, scope it to the project's compose file (container state + recent service logs). For non-docker SUTs, a tail of the server log.
 3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
    a. Default: \`1800\` (30 min) — sized for cold \`docker compose --build\` runs on multi-service stacks. The loop exits on the first success, so fast services pay no extra cost. Override only if you want to fail faster on stuck setups (e.g. \`'300'\` for a small single-service SUT).
 4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`, or \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . down -v\` for a dedicated compose file).
@@ -89,19 +150,20 @@ Auth inputs (used by Testbot to authenticate against the running SUT during test
    b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
 Workflow / job level settings (set on the workflow or job, not as \`skyramp/testbot\` action inputs):
 8. Secrets / environment variables — pass every secret/env var the setup needs into the workflow via an \`env:\` block or \`secrets: inherit\`, and add any dependency-resolution steps/jobs the SUT requires.
-9. \`runs-on\` runner label — switch to a larger runner when the SUT needs more disk or memory: heavy Docker builds, large dependency installs, or a multi-service (3+) compose stack. Standard runners run out of disk and crash with \`No space left on device\`. Runner names vary by org — do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses (never downgrade to \`ubuntu-latest\`), (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`.
+9. \`runs-on\` runner label — leave whatever value the Testbot installer wrote and the source workflow used. Don't override it. Picking a runner SKU is an account-level decision that depends on provisioning the agent can't see from inside the repo; a wrong choice queues the job forever with no runner ever picking it up. If the SUT genuinely doesn't fit on the chosen runner, the disk-pressure mitigation in "Common failure modes" #9 handles it; an unprovisioned-runner stall does not.
 
 #### What to do
 Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot installer) — do NOT recreate it. Add only what the SUT needs onto the existing \`skyramp/testbot\` action step, alongside the inputs the installer already added:
 1. The 4 target lifecycle inputs — \`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetReadyCheckTimeout\`, \`targetTeardownCommand\`.
 2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
 3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
-4. The \`runs-on\` runner label, if a larger runner is needed (input 9).
+4. Leave the existing \`runs-on\` value untouched (input 9).
 5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
+6. Stack-specific testability flags — some frameworks gate test-harness affordances behind a container env var. Common cases: Flutter web needs \`IS_TESTING=true\` so \`ensureSemantics()\` exposes the widget tree to Playwright (without it every selector finds nothing even though the app is up); some React/Angular builds need \`NODE_ENV\` / framework-specific build flags for \`testID\` propagation. Set these in the compose \`environment:\` block on the affected service (or as a \`Dockerfile ENV\`) — NOT as a shell var in \`targetSetupCommand\`. The var must be in the container's env at boot, not just the shell that runs the setup command. If selectors find nothing but the app is up, check for a missing flag like this first.
 
 #### What not to do
 1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
-2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
+2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code. In case of multi-mode containers (\`Dockerfile.bdd\`, \`Dockerfile.e2e\`, \`Dockerfile.integration\`, \`Dockerfile.test\`), create the necessary file in \`.skyramp/sut/\` with a PR-sourced build (e.g. \`.skyramp/sut/Dockerfile.bdd\` referenced from the testbot compose's \`build:\`). Do not edit existing Dockerfiles or compose files in the repo. If the multi-mode container's bootstrap step depends on a seeded user, ensure SEED_DATA creates it before this container tries to log in.
 3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
 4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
 5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
@@ -127,6 +189,6 @@ export function buildCommonSutErrorsSection() {
 6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
 7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
 8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
-9. no_space_left — the runner ran out of disk space (\`No space left on device\`). This happens on standard runners when the SUT involves large Docker builds, many npm packages, or multiple compose services that pull large images. Switch \`runs-on\` to a larger runner — runner names vary by org, do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses, (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`. Indicator: source workflow already specifies a large runner, or the SUT has multi-stage Docker builds / a compose file with 3+ services.
+9. no_space_left — the runner ran out of disk space (\`No space left on device\`). Prune the docker BuildKit cache and dangling images at the start of both \`targetSetupCommand\` and \`targetTeardownCommand\` so each fix-loop retry begins with a clean slate; without it, cache from the previous attempt accumulates across retries on heavy SUTs. Do not change \`runs-on\` — see input 9 above.
 Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
 }

package/build/prompts/test-maintenance/actionsInstructions.js

@@ -15,7 +15,11 @@ export function buildFileRenameStrategy() {
 }
 /** Strategy string for in-place UPDATE edits. */
 export function buildUpdateStrategy() {
-    return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context. After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
+    return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context.
+
+UPDATE is an in-place edit to the existing baseline file — do not create a new test file to cover the same page or component.
+
+After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
 }
 /** Strategy string for REGENERATE — call generation tool to overwrite the file. */
 export function buildRegenerateStrategy() {

package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts

@@ -1,4 +1,17 @@
-export declare function buildDriftAnalysisPrompt(stateFile?: string, existingTests?: Array<{
+export type UiDriftParams = {
+    tests: Array<{
+        testFile: string;
+    }>;
+    changedFrontendFiles: string[];
+    blueprintCaptured?: boolean;
+};
+/**
+ * Build the drift analysis prompt.
+ * - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
+ * - ui: when provided, appends a UI drift section for component/browser tests.
+ *   Omit when no frontend files changed or no UI tests exist.
+ */
+export declare function buildDriftAnalysisPrompt(stateFile: string | undefined, apiTests: Array<{
     testFile: string;
     source?: string;
-}>): string;
+}>, ui?: UiDriftParams): string;

package/build/prompts/test-maintenance/drift-analysis-prompt.js

@@ -1,6 +1,8 @@
 import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, } from "./driftAnalysisSections.js";
+import { buildUiActionDecisionTree, buildUiCheckRouteExistence, buildUiCheckSelectors, buildUiCheckPageObjects, buildUiCheckBehavioralChanges, buildUiCheckAssignAction, buildUiCheckExportedSymbols, buildUiDriftOutputChecklist, } from "./uiDriftAnalysisSections.js";
+import { RECOMMENDATIONS_INSTRUCTION } from "./driftAnalysisShared.js";
 import { PromptPlan } from "../test-recommendation/promptPlan.js";
-const _plan = new PromptPlan()
+const _apiPlan = new PromptPlan()
     .addPhase("maintenance", "Test Maintenance Assessment", {
     headerLevel: "##",
     stepFormat: "hash",
@@ -14,6 +16,59 @@ const _plan = new PromptPlan()
     .subStep("ASSIGN_ACTION", "Assign action", () => buildCheckAssignAction())
     .step("CALL_TOOL", "Submit recommendations", (p) => buildDriftOutputChecklist(p.stateFile, p.existingTests))
     .done();
-export function buildDriftAnalysisPrompt(stateFile, existingTests) {
-    return `<drift_analysis_rules>\n${_plan.render({ stateFile, existingTests })}\n</drift_analysis_rules>`;
+const _uiPlan = new PromptPlan()
+    .addPhase("ui_maintenance", "UI Test Drift Assessment", {
+    headerLevel: "##",
+    stepFormat: "hash",
+})
+    .step("UI_SYMBOL_PRESCAN", "Exported symbol pre-scan — find additional impacted tests via grep", (p) => buildUiCheckExportedSymbols(p.changedFrontendFiles))
+    .step("UI_ASSESS", "UI Action Decision Tree — assess each UI test against the diff and blueprints", () => buildUiActionDecisionTree())
+    .subStep("UI_ROUTE_EXISTENCE", "Route existence", () => buildUiCheckRouteExistence())
+    .subStep("UI_SELECTORS", "Selector validity", () => buildUiCheckSelectors())
+    .subStep("UI_PAGE_OBJECTS", "Page object/fixture existence", () => buildUiCheckPageObjects())
+    .subStep("UI_BEHAVIOR", "Component behavior changes", () => buildUiCheckBehavioralChanges())
+    .subStep("UI_ASSIGN_ACTION", "Assign action", () => buildUiCheckAssignAction())
+    .step("UI_CALL_TOOL", "Submit recommendations", (p) => buildUiDriftOutputChecklist(p.stateFile, p.tests))
+    .done();
+/**
+ * Build the drift analysis prompt.
+ * - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
+ * - ui: when provided, appends a UI drift section for component/browser tests.
+ *   Omit when no frontend files changed or no UI tests exist.
+ */
+export function buildDriftAnalysisPrompt(stateFile, apiTests, ui) {
+    const parts = [];
+    // Include API drift when there are API tests, or when UI drift is not running
+    // (ensures skyramp_actions is always reachable even if apiTests is empty).
+    if (apiTests.length > 0 || !ui) {
+        parts.push(`<drift_analysis_rules>\n${_apiPlan.render({ stateFile, existingTests: apiTests })}\n</drift_analysis_rules>`);
+    }
+    if (ui) {
+        const signalNote = ui.blueprintCaptured
+            ? `blueprintCaptured=true — use \`browser_blueprint_diff\` for tests that navigate pages.`
+            : `blueprintCaptured=false — use source diff only. Read each test to determine what it exercises.`;
+        const contextHeader = `<ui_analysis_context>
+**UI Test Drift Analysis Context**
+
+**Signal:** ${signalNote}
+
+**Changed Frontend Files:**
+${ui.changedFrontendFiles.length > 0
+            ? ui.changedFrontendFiles.map(f => `- ${f}`).join("\n")
+            : "(none)"}
+
+**Tests to Assess:**
+${ui.tests.length > 0
+            ? ui.tests.map(t => `- ${t.testFile}`).join("\n")
+            : "(none)"}
+
+**Instructions:**
+For each test above, read it to determine its type, then apply the scope gate from the decision tree. Assign the appropriate action (DELETE, REGENERATE, UPDATE, VERIFY, IGNORE).
+</ui_analysis_context>
+
+`;
+        parts.push(contextHeader + `<ui_drift_analysis_rules>\n${_uiPlan.render({ stateFile, tests: ui.tests, changedFrontendFiles: ui.changedFrontendFiles, blueprintCaptured: ui.blueprintCaptured })}\n</ui_drift_analysis_rules>`);
+    }
+    parts.push(RECOMMENDATIONS_INSTRUCTION);
+    return parts.join("\n\n");
 }

package/build/prompts/test-maintenance/driftAnalysisSections.js

@@ -1,8 +1,9 @@
 /**
- * Modular section builders for the Drift Analysis prompt,
+ * Modular section builders for the API Drift Analysis prompt,
  * mirroring the recommendationSections.ts pattern.
  */
 import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
+import { buildApiAssignAction } from "./driftAnalysisShared.js";
 // TODO: Replace the open-ended diff-line categories below with a two-tier structure:
 // Tier 1 — mechanical patterns (specific token + file-path criteria, zero judgment needed).
 // Tier 2 — residual open-ended scan for anything observable not covered by tier 1.
@@ -177,24 +178,9 @@ Diff signals and actions:
 **Reachability for behavioral changes:** The service/interface scope gate still applies — if the test targets a completely different service or protocol, IGNORE. However, do NOT use the absence of a route or output layer file in the diff as grounds for IGNORE. Behavioral changes (new error branches, new validation, status-code changes) are observable from any test calling the same endpoint, even when the logic lives in an internal handler, middleware, or utility file rather than the route file itself.`;
 }
 export function buildCheckAssignAction() {
-    return `Based on the above checks, choose the action (IGNORE / VERIFY / UPDATE / REGENERATE / DELETE) and provide a 1-2 sentence rationale.
-
-**Every action requires a specific rationale — including IGNORE:**
-- UPDATE / REGENERATE / DELETE: quote the specific diff line that triggered it.
-- VERIFY: name the uncertain element (e.g. "model-only change, cannot confirm field is exposed without checking the output layer").
-- IGNORE: name the changed file(s) from the diff AND explain why they cannot reach this test's endpoint (e.g. "diff only touches \`auth/session-service.js\` — this test targets \`/api/v1/orders\` which has no session dependency"). You must reference at least one specific changed file from the diff. Generic "unrelated endpoint" or "service boundary" without naming the changed file is not sufficient.
-
-- If the Additive Fields check flagged a new field with output layer signal confirmed in the diff → action is UPDATE. If the Additive Fields check returned VERIFY (model-only signal, no output layer change) → action remains VERIFY.
+    return buildApiAssignAction(`- If the Additive Fields check flagged a new field with output layer signal confirmed in the diff → action is UPDATE. If the Additive Fields check returned VERIFY (model-only signal, no output layer change) → action remains VERIFY.
 - **Scope gate:** If the changed code is clearly not reachable through the service or base URL this test targets, assign IGNORE.
-- **When uncertain, use VERIFY not IGNORE:** If the diff touches a model or migration and this test's endpoint reads from that model, assign VERIFY — you cannot confirm from the diff alone whether the field is exposed.
-- **Pre-commit verification — confirm all three before finalizing UPDATE/REGENERATE/DELETE:**
-  1. You can quote a specific diff line this test's endpoint observes that triggered the action.
-  2. The changed code is reachable through this test's service and base URL.
-  3. For REGENERATE: every assertion in the file is invalid, not just some — if you can patch N paths, it is UPDATE.
-  If any check fails, downgrade to VERIFY or IGNORE.
-- **For user-written (external) tests** marked \`[external]\` in the test list:
-  - UPDATE is permitted — targeted edits only (fix renamed URL, add assertion for new field, add a new test case for a new scenario). Match the style of the existing file: use the same test framework, assertion helpers, and request pattern already present.
-  - REGENERATE and DELETE are **not permitted** — assign those actions in your recommendations but \`skyramp_actions\` will surface them as report-only findings for the developer to act on. Do NOT attempt to rewrite or delete a user-authored test file.`;
+- **When uncertain, use VERIFY not IGNORE:** If the diff touches a model or migration and this test's endpoint reads from that model, assign VERIFY — you cannot confirm from the diff alone whether the field is exposed.`);
 }
 export function buildCheckAdditiveFields() {
     return `Even if existing assertions still pass, new response fields on a covered endpoint need a new assertion. A field being optional or nullable does not make it IGNORE — it still needs to be covered.
@@ -213,7 +199,7 @@ To determine which applies: look at whether an explicit output layer (DTO, seria
 **Do not assign IGNORE because the field is optional or nullable** — "won't break existing assertions" is not the bar. The bar is "does the test now cover this endpoint's full contract?" A new field, even optional, means the contract changed and the test is incomplete.`;
 }
 export function buildDriftOutputChecklist(stateFile, existingTests) {
-    const finalStep = `Call \`skyramp_actions\` with \`stateFile: "${stateFile ?? "<stateFile>"}"\` and a \`recommendations\` entry for every test assessed.`;
+    const finalStep = `Call \`skyramp_actions\` with \`stateFile: "${stateFile ?? "<stateFile>"}"\`.`;
     const hasTests = (existingTests?.length ?? 0) > 0;
     const testList = hasTests
         ? existingTests.map((t) => {
@@ -225,7 +211,7 @@ export function buildDriftOutputChecklist(stateFile, existingTests) {
         ? `\nNo existing tests were found. Call \`skyramp_actions\` with \`recommendations: []\` and explain in your report why no maintenance was needed (e.g. "no existing tests cover the changed endpoints" or "PR adds a new endpoint with no prior coverage").\n`
         : "";
     const symbolDiscoveryStep = hasTests
-        ? `**Symbol import discovery (before assessing tests):** If the diff adds or modifies class, function, or type definitions, grep the repository for test files that import those symbol names. Add any matched test files not already in the list below to your assessment — these files may need updating even if their path doesn't match the changed resource name. Include all discovered files in the \`recommendations[]\` passed to \`skyramp_actions\` — do NOT edit them directly without going through \`skyramp_actions\` first.`
+        ? `**Symbol import discovery (before assessing tests):** If the diff adds or modifies class, function, or type definitions, grep the repository for test files that import those symbol names. Add any matched test files not already in the list below to your assessment — these files may need updating even if their path doesn't match the changed resource name. Include all discovered files in the \`recommendations[]\` passed to \`skyramp_actions\`.`
         : "";
     const existingTestSection = `**Existing tests (${existingTests?.length ?? 0} total) — assess ALL of the following:**
 ${testList}
@@ -257,8 +243,7 @@ Test: {testFile}
 Action: DELETE
 Rationale: DELETE because {quoted diff signal}; all covered endpoints removed
 \`\`\`
-Include ALL actions — IGNORE, VERIFY, UPDATE, REGENERATE, DELETE — in the \`recommendations[]\` passed to \`skyramp_actions\`. For \`[external]\` tests: \`skyramp_actions\` will apply UPDATE edits and surface REGENERATE/DELETE as report-only findings.
-Be concise — one line per IGNORE/VERIFY entry.`;
+Be concise — one line per VERIFY/IGNORE entry.`;
     const sections = [symbolDiscoveryStep, existingTestSection, finalStep].filter(Boolean);
     return `<output_format>
 Complete ALL of the following:

package/build/prompts/test-maintenance/driftAnalysisShared.d.ts

@@ -0,0 +1,14 @@
+/** Shared drift prompt sections (assign-action, external test policy) for API and UI prompts. */
+/** One source of truth for the recommendations[] contract passed to skyramp_actions. */
+export declare const RECOMMENDATIONS_INSTRUCTION = "Pass VERIFY, UPDATE, REGENERATE, and DELETE entries to `recommendations[]`. Omit IGNORE.";
+export declare function buildSharedExternalTestPolicy(): string;
+/**
+ * Assign-action section for the API drift prompt.
+ * @param domainNotes  API-specific bullets (additive-fields guidance, scope-gate note).
+ */
+export declare function buildApiAssignAction(domainNotes: string): string;
+/**
+ * Assign-action section for the UI drift prompt.
+ * @param domainNotes  UI-specific bullets (component/widget matching notes).
+ */
+export declare function buildUiAssignAction(domainNotes: string): string;

package/build/prompts/test-maintenance/driftAnalysisShared.js

@@ -0,0 +1,62 @@
+/** Shared drift prompt sections (assign-action, external test policy) for API and UI prompts. */
+/** One source of truth for the recommendations[] contract passed to skyramp_actions. */
+export const RECOMMENDATIONS_INSTRUCTION = `Pass VERIFY, UPDATE, REGENERATE, and DELETE entries to \`recommendations[]\`. Omit IGNORE.`;
+export function buildSharedExternalTestPolicy() {
+    return `**For user-written (external) tests** marked \`[external]\` in the test list:
+- UPDATE is permitted — pass the exact file that needs editing as \`testFile\`. This can be the test file itself, a snapshot file, a page object, or a fixture. \`skyramp_actions\` will edit that file using \`updateInstructions\`; application source files are automatically rejected.
+- REGENERATE and DELETE: assign these actions when the evidence clearly warrants them (page gone → DELETE, component structure fundamentally changed → REGENERATE). \`skyramp_actions\` handles execution restrictions at apply time — assign the action that the evidence supports. Leave user-authored test files unchanged; edits go only through \`skyramp_actions\` with explicit \`updateInstructions\`. VERIFY means "uncertain, need more context" — assign DELETE or REGENERATE when you have confirmed the failure, and VERIFY only when you genuinely cannot determine impact without more information.`;
+}
+/** Shared boilerplate: VERIFY rule, pre-commit structure, and external policy. */
+function buildAssignActionCore(opts) {
+    return `Based on the above checks, choose the action (IGNORE / VERIFY / UPDATE / REGENERATE / DELETE) and provide a 1-2 sentence rationale.
+
+**Every action requires a specific rationale — including IGNORE:**
+- UPDATE / REGENERATE / DELETE: ${opts.triggerEvidence}.
+- VERIFY: name the uncertain element (e.g. ${opts.verifyExample}).
+- IGNORE: ${opts.ignoreGuidance}.
+
+${opts.domainNotes}
+
+**When uncertain, use VERIFY not IGNORE:** If the diff touches code this test could exercise but you cannot confirm impact without reading more context, assign VERIFY — do not default to IGNORE.
+
+**Pre-commit verification — confirm before finalizing UPDATE/REGENERATE/DELETE:**
+1. You can quote a specific diff line${opts.blueprintSuffix} this test observes that triggered the action.
+2. ${opts.reachabilityCheck}
+3. ${opts.regenerateCheck}
+4. ${opts.deleteCheck}
+If any check fails, downgrade to VERIFY or IGNORE.
+
+${buildSharedExternalTestPolicy()}`;
+}
+/**
+ * Assign-action section for the API drift prompt.
+ * @param domainNotes  API-specific bullets (additive-fields guidance, scope-gate note).
+ */
+export function buildApiAssignAction(domainNotes) {
+    return buildAssignActionCore({
+        triggerEvidence: "quote the specific diff line that triggered it",
+        ignoreGuidance: `name the changed file(s) from the diff AND explain why they cannot reach this test's endpoint (e.g. "diff only touches \`auth/session-service.js\` — this test targets \`/api/v1/orders\` which has no session dependency"). You must reference at least one specific changed file. Generic "unrelated endpoint" or "service boundary" without naming the changed file is not sufficient`,
+        verifyExample: `"model-only change, cannot confirm field is exposed without checking the output layer"`,
+        blueprintSuffix: "",
+        reachabilityCheck: "The changed code is reachable through this test's service and base URL.",
+        regenerateCheck: "For REGENERATE: every assertion in the file is invalid, not just some — if you can patch N paths, it is UPDATE.",
+        deleteCheck: "For DELETE: all covered endpoints no longer exist.",
+        domainNotes,
+    });
+}
+/**
+ * Assign-action section for the UI drift prompt.
+ * @param domainNotes  UI-specific bullets (component/widget matching notes).
+ */
+export function buildUiAssignAction(domainNotes) {
+    return buildAssignActionCore({
+        triggerEvidence: "quote the specific diff line and/or blueprint element that triggered it",
+        ignoreGuidance: `name the specific route or component and why the diff cannot reach it (e.g. "diff only touches \`/api/v1/users\` — this test targets \`/dashboard\`")`,
+        verifyExample: `"component is imported but not directly rendered — cannot confirm impact without reading the parent"`,
+        blueprintSuffix: " or blueprint element",
+        reachabilityCheck: "The changed code is reachable through this test's routes (browser tests) or a render/mount call in the test that references a changed component (component/widget tests).",
+        regenerateCheck: "For REGENERATE: the interaction model changed fundamentally — the test's action sequence has no migration path. If selectors changed but the flow (navigate → interact → assert) can still be expressed with targeted edits, it is UPDATE regardless of how many selectors changed.",
+        deleteCheck: "For DELETE: the route is completely gone (browser tests) or the rendered component was deleted (component/widget tests).",
+        domainNotes,
+    });
+}

package/build/prompts/test-maintenance/uiDriftAnalysisSections.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Build the UI Action Decision Tree section.
+ * This is the main entry point for UI drift analysis.
+ */
+export declare function buildUiActionDecisionTree(): string;
+/**
+ * Pre-scan step: agent greps changed files' exported symbols across the test
+ * directory to catch tests that server-side discovery missed — aliased imports,
+ * indirect renders, or any test the blueprint diff doesn't directly reference.
+ */
+export declare function buildUiCheckExportedSymbols(changedFrontendFiles?: string[]): string;
+/**
+ * Check if routes targeted by the UI test still exist.
+ */
+export declare function buildUiCheckRouteExistence(): string;
+/**
+ * Check if selectors used by the UI test still exist in the blueprints.
+ */
+export declare function buildUiCheckSelectors(): string;
+/**
+ * Check if page objects/fixtures referenced by the test still exist.
+ */
+export declare function buildUiCheckPageObjects(): string;
+/**
+ * Check for component behavior changes that affect assertions.
+ */
+export declare function buildUiCheckBehavioralChanges(): string;
+/**
+ * Assign the final UI drift action.
+ */
+export declare function buildUiCheckAssignAction(): string;
+/**
+ * Build the UI drift output checklist.
+ */
+export declare function buildUiDriftOutputChecklist(stateFile?: string, tests?: Array<{
+    testFile: string;
+    source?: string;
+}>): string;