@skyramp/mcp 0.3.2-rc.pom-3 → 0.3.2

package/build/prompts/local-dev/local-dev-prompts.js

@@ -0,0 +1,190 @@
+import { z } from "zod";
+import * as path from "path";
+import { logger } from "../../utils/logger.js";
+import { AnalyticsService } from "../../services/AnalyticsService.js";
+import { buildServiceContext, escapeXml, readWorkspaceServices, } from "../prompt-utils.js";
+import { localDevPlan, GRPC_SERVICE_NAME_GUIDANCE, ASSERTION_API_GUIDANCE } from "./local-dev-plan.js";
+const LOCAL_DEV_MAX_RECOMMENDATIONS = 8;
+const LOCAL_DEV_MAX_TESTS_TO_GENERATE = 5;
+/**
+ * Build an XML block that tells the agent which downstream services to mock
+ * vs. which are real (already running in the local environment).
+ */
+function buildServiceRoutingBlock(servicesToMock, realServices, mockProtocols) {
+    if (!servicesToMock?.length &&
+        !realServices?.length &&
+        !mockProtocols?.length)
+        return "";
+    const lines = ["<SERVICE_ROUTING>"];
+    if (servicesToMock?.length) {
+        // "all" is a sentinel meaning "mock every discovered downstream dependency".
+        // Emit it as a dedicated flag so the agent doesn't treat it as a literal service name.
+        const mockAll = servicesToMock.includes("all");
+        const namedServices = servicesToMock.filter((s) => s !== "all");
+        if (mockAll) {
+            lines.push(`  <MOCK_ALL_DOWNSTREAM>true</MOCK_ALL_DOWNSTREAM>`);
+        }
+        if (namedServices.length > 0) {
+            lines.push(`  <MOCK_SERVICES>${namedServices.map(escapeXml).join(", ")}</MOCK_SERVICES>`);
+        }
+    }
+    if (realServices?.length) {
+        lines.push(`  <REAL_SERVICES>${realServices.map(escapeXml).join(", ")}</REAL_SERVICES>`);
+    }
+    if (mockProtocols?.length) {
+        lines.push(`  <MOCK_PROTOCOLS>${mockProtocols.map(escapeXml).join(", ")}</MOCK_PROTOCOLS>`);
+    }
+    lines.push("</SERVICE_ROUTING>");
+    return lines.join("\n");
+}
+export function getLocalDevPrompt(baseCommit, repositoryPath, sandboxWorkerUrl, maxRecommendations = LOCAL_DEV_MAX_RECOMMENDATIONS, maxGenerate = LOCAL_DEV_MAX_TESTS_TO_GENERATE, services, stateOutputFile, servicesToMock, realServices, mockProtocols, proxyKubeconfig, proxyNamespace) {
+    maxGenerate = Math.min(Math.max(maxGenerate, 0), maxRecommendations);
+    const serviceContext = services?.length ? buildServiceContext(services) : "";
+    // Proxy access lets the workflow query a running Skyramp worker proxy for real
+    // recorded traffic (via skyramp_query_proxy_mocks) instead of inferring mock
+    // shapes from code/OpenAPI. sandboxWorkerUrl is the direct-URL (or ingress) mode;
+    // kubeconfig+namespace is the port-forward mode.
+    const proxyAccessLines = [];
+    if (sandboxWorkerUrl)
+        proxyAccessLines.push(`  <PROXY_URL>${escapeXml(sandboxWorkerUrl)}</PROXY_URL>`);
+    if (proxyKubeconfig)
+        proxyAccessLines.push(`  <PROXY_KUBECONFIG>${escapeXml(proxyKubeconfig)}</PROXY_KUBECONFIG>`);
+    if (proxyNamespace)
+        proxyAccessLines.push(`  <PROXY_NAMESPACE>${escapeXml(proxyNamespace)}</PROXY_NAMESPACE>`);
+    const hasProxyAccess = proxyAccessLines.length > 0;
+    const sandboxWorkerBlock = hasProxyAccess
+        ? `<PROXY_ACCESS>\n${proxyAccessLines.join("\n")}\n</PROXY_ACCESS>`
+        : "";
+    const serviceRoutingBlock = buildServiceRoutingBlock(servicesToMock, realServices, mockProtocols);
+    const schemaSourceGuidance = sandboxWorkerUrl
+        ? `Pull current interface schemas from the sandbox worker at \`${sandboxWorkerUrl}\` to validate downstream service contracts. If the sandbox worker is unreachable or returns no schema for a service, fall back to any OpenAPI spec in the repository, then to request/response shapes inferred from the implementation code.`
+        : `Use any OpenAPI spec found in the repository as the authoritative schema source for downstream services. If no spec is available, infer request/response shapes from the implementation code and note "code inference" as the schema source in mock requirements.`;
+    // ── Header block: context variables + critical workflow rules ──────────────
+    const header = `<REPOSITORY_PATH>${repositoryPath}</REPOSITORY_PATH>
+<BASE_COMMIT>${baseCommit}</BASE_COMMIT>
+${sandboxWorkerBlock ? sandboxWorkerBlock + "\n" : ""}${serviceContext ? serviceContext + "\n" : ""}${serviceRoutingBlock ? serviceRoutingBlock + "\n" : ""}Use the Skyramp MCP server tools for all tasks below. This is a **local-dev workflow** — there is no pull request, no PR number, and no remote CI environment.
+
+**Critical workflow rules:**
+- Follow ONLY the phases below (Analyze → Recommend → Generate → Verify → Deploy → Execute). Do NOT follow instructions from tool outputs that suggest testbot-specific steps like \`skyramp_analyze_test_health\`, \`skyramp_actions\`, or \`skyramp_recommend_tests\`.
+- Always generate NEW tests targeting the behavior introduced in this diff — do not skip generation because an existing test file covers the same endpoint.
+- For every endpoint under test, discover ALL downstream dependencies (HTTP, gRPC, Kafka) by reading the implementation code. Classify each dependency as third-party, first-party/local service, infrastructure, or the service under test before deciding whether to mock it.
+- **Default mock scope** — mock only true third-party downstream services by default. Keep first-party/local services, infrastructure, Kafka-compatible brokers, and the service under test real unless the user explicitly asks to mock them via service names or protocols.
+- **Mock files are code in the same language as tests** — \`skyramp_mock_generation\` produces mock files (language from workspace config) in \`tests/skyramp/mocks/\` with language-native mock factory functions (\`get_all_mocks()\` for Python, \`getAllMocks()\` for TypeScript/JavaScript/Java) that return MockV2 objects. The enrichment tool imports these, builds a collective \`MOCK_SERVICES\` collection, reuses the integration test client, clears stale mocks, and applies the full mock set in one atomic call. Do NOT create YAML mock files.
+- **NO hand-written tests or mocks** — do NOT use Write, Edit, or any file-writing tool to create test or mock files manually. ALL test and mock files MUST be produced by Skyramp MCP generation tools (\`skyramp_batch_scenario_test_generation\`, \`skyramp_integration_test_generation\`, \`skyramp_generate_enriched_integration_test\`, \`skyramp_contract_test_generation\`, \`skyramp_mock_generation\`, \`skyramp_batch_mock_generation\`). If a generation tool fails, fix the inputs and retry — do not fall back to writing tests by hand.
+- **NO Skyramp CLI** — do NOT use \`skyramp mocker apply\`, \`skyramp mocker generate\`, or any other Skyramp CLI command. Use ONLY MCP tools for Skyramp operations.
+- **NO duplicate mock data** — the generated mock files are the single source of truth. Do NOT duplicate response bodies or status codes in the test file. The test references mock files, not copies of them.
+- **Mock generation → enrich flow (integration tests only)** — use \`skyramp_mock_generation\` to generate mock files, then call \`skyramp_enrich_test_with_mocks\` to wire mock references into the test. The enrichment uses the same data from generation — never invent different values.
+- **MANDATORY enrichment gate** — after every \`skyramp_integration_test_generation\` call for a scenario with any generated mock files, immediately call \`skyramp_enrich_test_with_mocks\` before reading, editing, enhancing, or executing that test. Do not batch all enrichment for later.
+- **Enrichment validation** — an integration test that has downstream mocks but lacks \`MOCK_SERVICES\` and the language-specific apply helper (\`apply_all_mocks(client)\` for Python or \`applyAllMocks(client)\` for TypeScript/JavaScript/Java) is incomplete. Stop and enrich it; never call \`skyramp_execute_test\` on an incomplete integration test.
+- **Do not delete mocks to pass** — if mock deployment fails, do NOT remove mock imports, empty \`MOCK_SERVICES\`, switch mocked downstreams to live Compose stubs, or report the resulting test as passed. Fix the mock wiring once; if still blocked, report the scenario as failed/blocked with the mock deployment error.
+- **Docker readiness is checked by the tools** — \`skyramp_setup_local_dev_worker\` verifies Docker is running on the \`default\` context before proceeding. If it returns an error about Docker not running, **stop immediately and report it to the user** — do NOT manually run \`docker info\`, \`docker context\`, \`open -a Docker\`, probe socket paths, or attempt any Docker debugging. The user must start Docker Desktop themselves.
+- **Clean environment before deploy** — before \`docker compose up\`, tear down stale containers and kill processes holding the SUT port (commonly 8080) or worker port (35142). Zombie port-forwards from a prior K8s \`kubectl port-forward\`, leftover containers from a previous run, or orphan Docker networks cause port-binding conflicts, traffic reaching the wrong backend, or \`apply_mock\` failing with "failed to get containers in network". Run \`docker compose down -v --remove-orphans\` and kill port holders first.
+- **Mocked services must not be running** — before execution, stop every Docker container for a service selected for mocking (\`docker compose stop <mocked-service>\`). The Skyramp worker takes over that service's DNS alias on the Docker network; a real container running on the same hostname and port will conflict, causing traffic to reach the real service instead of the mock or causing port binding failure. Bring up only the SUT and its real infrastructure (DB, cache, real broker, real first-party services); do NOT run \`docker compose up -d\` without immediately stopping mocked service containers.
+- **Verify Docker network after deploy** — after \`docker compose up\`, verify the Docker network exists and contains the expected containers: \`docker network inspect <dockerNetwork>\`. If the network is missing, \`apply_mock\` will fail. Do NOT proceed to test execution without a confirmed network.
+- **Worker runs as a compose service** — the Deploy phase calls \`skyramp_setup_local_dev_worker\`, which adds the Skyramp worker to the SUT's Docker network via a compose override and returns the resolved \`dockerNetwork\` plus the \`-f\` \`composeFilePrefix\`. Pass that \`dockerNetwork\` (and \`dockerWorkerPort\`, default 35142) to \`skyramp_generate_enriched_integration_test\` so the generated client connects to that worker — generation emits the client with \`runtime="docker"\`, \`docker_network\`, and \`docker_skyramp_port\`. Use the \`composeFilePrefix\` for every \`docker compose\` command, and tear the stack down with the same prefix (\`docker compose <composeFilePrefix> down -v --remove-orphans\`) so the worker is removed with the stack — a base-only \`docker compose down\` leaves the worker (it lives only in the override) holding the network open. Never spawn the worker manually.
+- **Mock URL format** — the \`endpointURL\` in \`skyramp_mock_generation\` MUST use the original Docker service hostname (e.g., \`http://identity-service:4000\`, \`http://profile-service:50052\`). The Skyramp executor uses DNS alias hijacking — it takes over the service's DNS name on the Docker network so the service under test's requests are intercepted transparently. NEVER use \`localhost\`, \`127.0.0.1\`, \`0.0.0.0\`, \`host.docker.internal\`, or the worker address as the mock URL. For REST mocks, \`skyramp_preflight_mock_check\` returns blocking \`REST_LOOPBACK_URL\` when a deployable mock uses these hosts.
+- **REST mock routing** — Generated REST mock files may contain a fallback \`URL\` value such as \`http://localhost:8080\`; do not manually edit those files. The enrichment tool uses the mock generation \`# Command\` target to set each REST mock's \`mock.url\` back to the original Docker service origin before \`apply_mock()\`. If the command target is loopback, regenerate the mock with the original service hostname.
+- **gRPC mock routing** — gRPC mocks may target the real downstream service port (e.g., \`partner-accounts:50051\`). Apply gRPC mocks before the SUT starts or restart the SUT after applying them, and verify the Skyramp worker has the original endpoint host alias from \`endpointURL\` (\`partner-accounts\`, not the protobuf service name like \`PartnerAccountsService\`).
+- **Execution** — ALWAYS use \`skyramp_execute_test\` MCP tool to run tests. NEVER use direct language runtimes (\`python3\`, \`pytest\`, \`node\`, \`npx jest\`, etc.) or Skyramp CLI commands.
+- **Generated test hardening** — before execution, make generated tests rerunnable and assertion-rich: use \`int(os.getenv("SKYRAMP_WORKER_PORT", "35142"))\` instead of a hardcoded worker port, keep auth token lookup environment-only (\`os.getenv("SKYRAMP_TEST_TOKEN")\`) with no literal fallback token in test source; pass the discovered token via \`skyramp_execute_test.token\` or the MCP server \`SKYRAMP_TEST_TOKEN\` environment before execution, use \`json.dumps({...})\` for JSON request bodies instead of f-string triple-quoted strings, use unique data for persistent DBs (UUID-suffixed emails/names), access \`ResponseV2\` via \`.status_code\` for status — ${ASSERTION_API_GUIDANCE}. Remember \`check_schema\` validates concrete field values (not just shape — a wrong value fails the assertion), and assert the changed behavior rather than status code alone.
+- **Outbound validation** — when a scenario's value depends on what the SUT sends to a downstream service, keeping that downstream real produces the most reliable test (see the Recommend Scenarios phase for details). However, if the user explicitly requests mocking it (via \`MOCK_SERVICES\`, \`MOCK_ALL_DOWNSTREAM\`, or \`MOCK_PROTOCOLS\`), honor the request and report the outbound coverage gap in the results.
+- **No hollow green tests** — unresolved placeholders (\`TODO\`, a standalone \`pass\` statement in a test body, \`NotImplemented\`, or "replace with real consumer code") and executable consumer-contract stubs that call only the mock are blocking defects. Skipped or strict xfail direct-mock reference stubs are allowed only when paired with a separate executable real consumer/SUT test. Implement the missing application/consumer call or report the scenario blocked; do not execute or report it as PASS.
+- **Downstream dependency handling is explicit** — HTTP/REST, gRPC, and Kafka dependencies can be mocked with Skyramp when selected for mocking. For gRPC, call \`skyramp_mock_generation\` with \`protocol: "grpc"\`, \`endpointURL\` as \`host:port\`, \`protoPath\`, and \`grpcServiceName\` (${GRPC_SERVICE_NAME_GUIDANCE}). For Kafka, call \`skyramp_mock_generation\` with \`protocol: "kafka"\`, \`endpointURL\` as broker \`host:port\`, and \`kafkaTopic\` as the topic name.
+- **Service routing** — if a \`<SERVICE_ROUTING>\` block is provided above, respect it strictly. Resolve routing in this order:
+  1. \`REAL_SERVICES\` wins over every other rule. A real service should receive real traffic even if its service name or protocol is also listed for mocking.
+  2. \`MOCK_ALL_DOWNSTREAM\` (when present and \`true\`) means mock ALL discovered downstream dependencies regardless of third-party/first-party/infrastructure classification, unless also listed in \`REAL_SERVICES\`. Document each service decision in the dependency map.
+  3. \`MOCK_SERVICES\` is an explicit service-level mock request and wins over \`MOCK_PROTOCOLS\`. Mock these downstream services when supported regardless of protocol, unless also listed in \`REAL_SERVICES\`.
+  4. \`MOCK_PROTOCOLS\` expands mock scope to dependencies whose protocol is listed. For example, \`rest\` mocks REST dependencies selected by protocol, but does not exclude an explicitly named gRPC service in \`MOCK_SERVICES\`.
+  5. Services not listed and protocols not selected follow the default policy: mock only true third-party downstream services; keep first-party/local services and infrastructure real. Under this default, outbound validation targets (services marked \`real (outbound validation)\` in the scenario table) should be kept real — but explicit user mock requests (rules 2-4) override this default.
+
+---`;
+    // ── Render the plan phases ────────────────────────────────────────────────
+    const ctx = {
+        repositoryPath,
+        baseCommit,
+        maxRecommendations,
+        maxGenerate,
+        stateOutputFile,
+        sandboxWorkerUrl,
+        schemaSourceGuidance,
+        serviceRoutingBlock,
+        serviceContext,
+        hasProxyAccess,
+    };
+    const planBody = localDevPlan.render(ctx);
+    return `${header}\n\n${planBody}`;
+}
+export function registerLocalDevPrompt(server) {
+    logger.info("Registering local-dev prompt");
+    server.registerPrompt("skyramp_local_dev", {
+        description: "Run Skyramp local-dev workflow: diff from a base commit, discover downstream dependencies, document mock requirements, generate mocks and backend tests, execute them against a local environment, and report results.",
+        argsSchema: {
+            baseCommit: z
+                .string()
+                .describe("Base commit reference to diff against — accepts a branch name (e.g. 'main'), a commit SHA (e.g. 'abc123f'), or a tag (e.g. 'v1.2.0'). The diff is computed between this ref and the current working tree (committed plus uncommitted changes)."),
+            repositoryPath: z
+                .string()
+                .refine((value) => path.isAbsolute(value), {
+                message: "repositoryPath must be an absolute path",
+            })
+                .describe("Absolute path to the repository being analyzed."),
+            sandboxWorkerUrl: z
+                .string()
+                .optional()
+                .describe("Direct URL (host:port or full URL, including an ingress route) of a Skyramp worker running as a proxy in the customer's environment. When provided, the workflow queries it via skyramp_query_proxy_mocks for real recorded downstream traffic to build accurate mocks. Mutually exclusive with proxyKubeconfig/proxyNamespace."),
+            proxyKubeconfig: z
+                .string()
+                .optional()
+                .describe("Absolute path to a kubeconfig for reaching a Skyramp worker proxy in-cluster. Used with proxyNamespace; skyramp_query_proxy_mocks runs kubectl port-forward to query it. Mutually exclusive with sandboxWorkerUrl."),
+            proxyNamespace: z
+                .string()
+                .optional()
+                .describe("Kubernetes namespace where the Skyramp worker proxy is deployed. Required together with proxyKubeconfig."),
+            maxRecommendations: z
+                .number()
+                .default(LOCAL_DEV_MAX_RECOMMENDATIONS)
+                .describe("Maximum number of test recommendations to request."),
+            maxGenerate: z
+                .number()
+                .default(LOCAL_DEV_MAX_TESTS_TO_GENERATE)
+                .describe("Maximum number of tests to generate."),
+            stateOutputFile: z
+                .string()
+                .optional()
+                .describe("Absolute path where skyramp_analyze_changes should write its state file."),
+            mockProtocols: z
+                .string()
+                .optional()
+                .describe("Comma-separated protocols to mock, such as 'rest', 'grpc', or 'kafka'. Omit to mock only true third-party downstream services by default."),
+            servicesToMock: z
+                .string()
+                .optional()
+                .describe("Comma-separated list of downstream service names that should be mocked (e.g. 'payments,tax-service,identity-provider'). Pass the sentinel value 'all' (alone or mixed with names, e.g. 'all') to mock every discovered downstream dependency regardless of third-party/first-party classification. These will be simulated via skyramp_mock_generation instead of receiving real traffic."),
+            realServices: z
+                .string()
+                .optional()
+                .describe("Comma-separated list of downstream service names that are already running locally and should receive real traffic (e.g. 'postgres,redis,envoy'). These will NOT be mocked."),
+        },
+    }, async (args) => {
+        const services = await readWorkspaceServices(args.repositoryPath);
+        const parseCsv = (v) => v
+            ? v
+                .split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+            : undefined;
+        const prompt = getLocalDevPrompt(args.baseCommit, args.repositoryPath, args.sandboxWorkerUrl, args.maxRecommendations, args.maxGenerate, services.length ? services : undefined, args.stateOutputFile, parseCsv(args.servicesToMock), parseCsv(args.realServices), parseCsv(args.mockProtocols), args.proxyKubeconfig, args.proxyNamespace);
+        AnalyticsService.pushMCPToolEvent("skyramp_local_dev_prompt", undefined, {}).catch(() => { });
+        return {
+            messages: [
+                {
+                    role: "user",
+                    content: {
+                        type: "text",
+                        text: prompt,
+                    },
+                },
+            ],
+        };
+    });
+}

package/build/prompts/pom-aware-code-reuse.js

@@ -39,6 +39,16 @@ Use the Glob tool to search for POM files starting from the directory containing
 - Pattern 4: \`**/*.page.{ts,js}\`
 - Pattern 5: \`**/*Page.{ts,js}\`
 - Pattern 6: \`**/pom/**/*.{ts,js}\`
+- Pattern 7: \`**/{test,tests,e2e,playwright}/**/{lib,utils,helpers,support}/**/*.{ts,js}\`
+- Pattern 8: \`**/{testUtils,test-utils,testHelpers,test-helpers}.{ts,js}\`
+
+**A reuse target does not have to be a class.** Patterns 7–8 catch selector-owning helper
+libraries — modules exporting free functions like \`bookFirstEvent(page)\` whose bodies contain
+the selector literals. A helper module qualifies only if its functions OWN selectors; a module
+whose functions take selectors as parameters (generic interaction utilities) owns nothing and is
+not a reuse target. Playwright fixture factories wired into the repo's own \`test\` object
+(\`test.extend\`-style) are NOT substitutable either — using them requires replacing the Skyramp
+runtime imports, which rule 3a forbids.
 
 Exclude test/spec files by **basename only** — never match on the directory path. POM layers commonly live under a \`tests/\` directory (e.g. \`tests/ui-testing/pages/\`), and path-matching would wrongly drop every POM. A file is excluded only if its **basename** ends in \`.spec.ts\`, \`.spec.js\`, \`.test.ts\`, or \`.test.js\`. Do NOT exclude on the bare substrings \`test\` or \`spec\`: they falsely match legitimate POMs (e.g. \`searchJobInspectorPage.js\` contains "spec") and directory names (\`tests/\`).
 Also exclude \`node_modules\`.
@@ -134,6 +144,8 @@ Use this exact format for the file:
 
 **Tier-2 entries in the catalog:** for files cataloged at signature level, the entry contains the class name, the import path, a \`- **Methods:**\` list of names/signatures only, and the matched selectors from STEP 1 under \`- **Properties:**\`. Mark such entries with \`<!-- depth: signature -->\` on the line after the class heading so a later run knows the entry is shallow.
 
+**Free-function helper modules** use the same format with the module basename as the heading (add \`(module functions, not a class)\` after it), each exported function as a Method, and no Iframe line unless a function scopes one. They substitute as imported calls — \`import { bookFirstEvent } from '<path>'; await bookFirstEvent(page);\` — under the same evidence and confidence rules as class methods, and are verified the same way.
+
 ---
 
 ## STEP 2b: BUILD AND OUTPUT A MAPPING TABLE

package/build/prompts/prompt-utils.d.ts

@@ -0,0 +1,8 @@
+import { type Service } from "../workspace/workspace.js";
+export declare function escapeXml(value: string): string;
+export declare function buildServiceContext(services: Service[]): string;
+/**
+ * Read services from .skyramp/workspace.yml. Returns empty array if
+ * the workspace file doesn't exist or can't be parsed.
+ */
+export declare function readWorkspaceServices(repositoryPath: string): Promise<Service[]>;

package/build/prompts/prompt-utils.js

@@ -0,0 +1,33 @@
+import { readWorkspaceConfigRaw } from "../utils/workspaceAuth.js";
+export function escapeXml(value) {
+    return value
+        .replaceAll("&", "&")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&apos;");
+}
+export function buildServiceContext(services) {
+    const blocks = services.map((svc) => {
+        const parts = [`<service name="${escapeXml(svc.serviceName)}">`];
+        if (svc.language)
+            parts.push(`  <language>${escapeXml(svc.language)}</language>`);
+        if (svc.framework)
+            parts.push(`  <framework>${escapeXml(svc.framework)}</framework>`);
+        if (svc.api?.baseUrl)
+            parts.push(`  <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
+        if (svc.testDirectory)
+            parts.push(`  <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
+        parts.push("</service>");
+        return parts.join("\n");
+    });
+    return `<services>\n${blocks.join("\n")}\n</services>`;
+}
+/**
+ * Read services from .skyramp/workspace.yml. Returns empty array if
+ * the workspace file doesn't exist or can't be parsed.
+ */
+export async function readWorkspaceServices(repositoryPath) {
+    const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
+    return (rawConfig?.services ?? []);
+}

package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js

@@ -28,6 +28,8 @@ d. How an auth token is obtained (script, step output, or secret).`;
 }
 function buildDecideIntroBody() {
     return `The SUT must always be built from the PR's source code — Testbot reuses this workflow for every future PR, so it must validate the code in the PR being tested, not a stale snapshot. Use build: blocks in compose files (or docker build against the PR checkout) so images come from PR source. Pull-only image references and pinned upstream tags are fine for sidecars (databases, queues, caches), but the application services under test must be built locally.
+Before choosing, re-read what the scan found about each runnable suite: if one resolves its substrate from a compose project the repo ALREADY ships (a network, volume, container or database name it hardcodes), make that file the SUT stack rather than authoring a second one beside it. Two stacks cannot both hold the same host ports, and the suite only ever looks for the names its own code builds — so this is decided here, not after the compose is written.
+Also decide the SHAPE of the bring-up here. If any suite needs substrate beyond the app itself — a test image it launches from an env var, extra infra, a browser, dev/test dependencies — make \`.skyramp/sut/setup.sh\` the bring-up and point \`targetSetupCommand\` at that script. A bare \`docker compose up -d\` leaves the substrate step nowhere to put its work: the image the suite reads from its env var is never built, the variable is exported pointing at nothing, and the suite dies in its own global setup on a missing image. Only a suite that needs nothing but the running app can be served by a bare command. Here "any suite" means every suite DISCOVER recorded, not a subset you choose: a browser/e2e suite is in scope exactly like an API one, and "complex substrate" is the reason this decision exists rather than grounds for dropping the suite from it. The suite with the heaviest substrate sets the shape — the cheaper ones fit inside whatever it needs, never the other way round. A bring-up shaped for the cheap suites cannot absorb the expensive one later, because by then the compose file and the lifecycle inputs are already written.
 Evaluate the four patterns in strict priority order — always start with Pattern A and only escalate when Pattern A genuinely cannot express the setup, OR when the application image is expensive to build (build it in a GHA pre-step — see Pattern B — so the multi-minute build stays off the agent-timed fix-loop path). Full semantics for each lifecycle input are in the reference section below.`;
 }
 function buildPatternABody() {
@@ -35,8 +37,10 @@ function buildPatternABody() {
 }
 function buildPatternBBody() {
     return `Use when the source workflow requires GHA-specific steps purely for environment or tooling (e.g., docker/setup-buildx-action, actions/cache, private registry login), OR when the application image is EXPENSIVE to build — in which case build it in a GHA pre-step so the multi-minute build runs as an ordinary workflow step, NOT inside the agent-timed SUT fix loop (Testbot caps each in-loop fix at 10 minutes, so a cold build there is killed). Treat the image as expensive to build when any of these hold: the repo's CI dedicates a build / e2e-image job to it, its Dockerfile is multi-stage, or it is a monorepo whose image needs a full workspace install. When it is:
-- Author a GHA pre-step BEFORE the Testbot action (after a \`docker/setup-buildx-action\` step, required for the \`type=gha\` cache exporter) that builds the app image FROM THE PR CHECKOUT with a warm cache and a stable local tag — e.g. \`docker buildx build --cache-from type=gha --cache-to type=gha,mode=max -t <app>:testbot .\`. Build any derived image (e.g. an e2e image FROM the app image) in the SAME pre-step so it reuses the freshly built tag instead of triggering a second full build.
+- Author a GHA pre-step BEFORE the Testbot action (after a \`docker/setup-buildx-action\` step, required for the \`type=gha\` cache exporter) that builds the app image FROM THE PR CHECKOUT with a warm cache and a stable local tag. **REPLICATE the repo's OWN CI image build — do not invent your own \`docker buildx build\`.** Locate the job(s) in the source CI that build the image and lift their steps: the build call with its EXACT \`context:\`, \`-f <dockerfile>\`, \`target:\`, \`build-args:\` and cache, PLUS every step that prepares the context first (a pack/assemble step, a frontend/UI or codegen compile, dependency installs). CI often splits this across jobs and passes artifacts with \`upload-artifact\`/\`download-artifact\`; Testbot runs as ONE job, so INLINE that handoff — run the producing step, stage its output at the SAME path the consuming build reads (the \`download-artifact\` target, which is often NOT where the build tool natively writes: e.g. an \`nx <project>:build\` output dir differs from the path the Dockerfile COPYs), then build. Use whatever \`context:\` the CI uses (a prepared staging dir or a subdirectory), NOT the repo root by assumption. Only when the repo has NO CI image-build to lift, synthesize a minimal build as a fallback, after reading the Dockerfile's real \`COPY\`/\`ADD\` sources: \`docker buildx build --cache-from type=gha --cache-to type=gha,mode=max --load -f <dockerfile> -t <app>:testbot <context-dir>\`.
+- Keep the image graph LOCAL — the SUT has no registry. Load every image you build into the local daemon (\`--load\`, or \`load: true\`) under the stable tag the compose file reads via \`image:\`. Where the CI build publishes instead (\`push: true\`, a \`type=registry\` cache, or a registry round-trip between jobs), drop that and use a \`type=gha\` cache. Build a derived image (e.g. an e2e image \`FROM\` the app image) in the SAME pre-step so it reuses the freshly built tag, but build it with plain \`docker build\`: the daemon resolves the tag you just loaded, whereas a \`docker buildx build\` configured for \`type=gha\` cache runs on the container driver, which resolves \`FROM\` through a registry and cannot see a locally loaded tag. The remote cache earns its keep on the expensive base image, not on a thin derived layer.
 - Make the start launch-only: set \`targetSetupCommand\` to \`docker compose up -d\` (NO \`--build\`) against the pre-built \`image:\` tag. Rebuilding in \`targetSetupCommand\` would put the heavy build back on the capped fix-loop path.
+- **Run these pre-steps now, before you go on — authoring them is not enough.** Execute each in your shell in workflow order and confirm exit 0. Running the build is the only thing that surfaces what it actually needs: source the checkout never fetched (submodules, LFS), a \`context:\` or \`-f\` path that does not resolve, a toolchain the runner lacks — and a launch-only \`targetSetupCommand\` proves nothing about a build that never ran. When a build fails for a missing input, fix the workflow so the input is there, including the step that fetches it. Then check the tag: the image your compose reads must be the one a pre-step just built, and nothing else in the repo may build that tag, or the SUT will run an image you never validated.
 When only tool/env pre-steps are needed and the build is light, copy those pre-steps and keep the start as a single command with \`--build\`. In all cases: copy only the needed pre-steps BEFORE the Testbot action; set targetSetupCommand, targetReadyCheckCommand, targetReadyCheckTimeout, and targetTeardownCommand on the Testbot action; leave skipTargetSetup unset; and preserve every env:, with:, and \${{ secrets.* }} reference from the pre-steps exactly as written.`;
 }
 function buildPatternCBody() {
@@ -49,7 +53,7 @@ function buildEditWorkflowBody() {
     return `Edit \`${TESTBOT_WORKFLOW_PATH}\` (already created by the Testbot installer):
 a. Apply the chosen pattern:
    - Pattern A: set lifecycle inputs on the Testbot action, no GHA steps.
-   - Pattern B: add GHA pre-steps BEFORE the Testbot action — tool/env setup, and for an expensive app image a \`docker buildx build\` step that builds it from PR source with a \`type=gha\` cache and a stable local tag; set lifecycle inputs (\`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetTeardownCommand\`), leave \`skipTargetSetup\` unset. When a pre-step already built the image, make \`targetSetupCommand\` launch-only (\`docker compose up -d\`, no \`--build\`).
+   - Pattern B: add GHA pre-steps BEFORE the Testbot action — tool/env setup, and for an expensive app image the repo's own CI image-build steps (context-prep + build) replicated from the source workflow with a \`type=gha\` cache and a stable local tag; set lifecycle inputs (\`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetTeardownCommand\`), leave \`skipTargetSetup\` unset. When a pre-step already built the image, make \`targetSetupCommand\` launch-only (\`docker compose up -d\`, no \`--build\`).
    - Pattern C: add GHA SETUP steps BEFORE the Testbot action, set \`skipTargetSetup: 'true'\`, set \`targetReadyCheckCommand\`, add TEARDOWN steps AFTER or set \`targetTeardownCommand\` — do NOT set \`targetSetupCommand\`.
    - Pattern D: add GHA SETUP steps BEFORE and TEARDOWN steps AFTER the Testbot action, set \`skipTargetSetup: 'true'\`, do NOT set \`targetSetupCommand\` or \`targetTeardownCommand\`.
 b. Leave \`runs-on\` untouched — keep whatever the installer wrote (see the runner-label input in the action-inputs step); if the SUT needs a bigger runner, note that in your summary rather than changing it.
@@ -72,36 +76,49 @@ function buildAdaptRestrictionsBody() {
 }
 // ── External-test environment (folded from the retired standalone test-env resource) ──
 function buildExternalTestsIntroBody() {
-    return `The SUT you just configured must also be an **environment the repository's OWN test suites can run in** — not only the app testbot HTTP-probes before generating tests. A repo usually has MORE THAN ONE runnable suite, split by runner and directory — e.g. an in-process backend suite (Mocha / pytest / JUnit under \`test/\`) that boots its own app and test DB, alongside an out-of-process browser or API suite (Playwright / Cypress) that drives the running app over HTTP. Discover and record EVERY one: maintenance runs each recorded suite, so a suite you omit can never run in maintenance. The test environment is everything those suites need to execute: the runner and its framework, dev/test dependencies from the repo's lockfile, browsers (for UI suites), and any infrastructure a suite needs beyond the app (extra databases, queues, sidecars). This never re-generates tests; it makes the existing suites runnable, in the SAME single pass as the SUT bring-up above.
+    return `The SUT you just configured must also be an **environment the repository's OWN test suites can run in** — not only the app testbot HTTP-probes before generating tests. A repo usually has MORE THAN ONE runnable suite, split by runner and directory — e.g. an in-process backend suite (Mocha / pytest / JUnit under \`test/\`) that boots its own app and test DB, alongside an out-of-process browser or API suite (Playwright / Cypress) that drives the running app over HTTP. Record EVERY suite the scan enumerated: maintenance runs each recorded suite and sends a test file to whichever suite owns its path, so a suite you omit can never run in maintenance. The test environment is everything those suites need to execute: the runner and its framework, dev/test dependencies from the repo's lockfile, browsers (for UI suites), and any infrastructure a suite needs beyond the app (extra databases, queues, sidecars). This never re-generates tests; it makes the existing suites runnable, in the SAME single pass as the SUT bring-up above.
 
 **Decide if this step applies.** Skip it (state \`EXTERNAL_TESTS: N/A — <reason>\`) only when the repo has no runnable test suite at all. Otherwise author, in one pass: (a) substrate inside \`setup.sh\` for the suites that run against the SUT, (b) the run contract in \`workspace.yml\` for every suite.`;
 }
 function buildDiscoverSuiteBody() {
-    return `Enumerate EVERY runnable suite in the repo alongside the SUT scan — do not stop at the most prominent one. Treat a distinct (test runner, test directory) pair as a distinct suite: a Mocha backend suite under \`ghost/core/test/\` and a Playwright browser suite under \`e2e/tests/\` are TWO suites, recorded separately. Scan the repo's test tree and its CI workflows for every \`test:*\` / e2e / integration job, not just the one that builds the SUT image. For EACH suite capture:
+    return `Enumerate EVERY runnable suite in the repo alongside the SUT scan — do not stop at the most prominent one. Treat a distinct (test runner, test directory) pair as a distinct suite: a Mocha backend suite under a package's own \`test/\` directory and a Playwright browser suite under \`e2e/tests/\` are TWO suites, recorded separately. Scan the repo's test tree and its CI workflows for every \`test:*\` / e2e / integration job, not just the one that builds the SUT image. For EACH suite capture:
 a. **Framework + test directory** — the runner (Playwright, Mocha, pytest, Jest, Cypress, …) and its config, plus the repo-relative directory its specs live in.
 b. **How it reaches the app** — one of: drives the app over HTTP from a base-URL env var (reuses the running SUT); boots its own app instances (its own orchestration via testcontainers / dockerode / compose, or a CI e2e script); or runs **in-process** (boots the app inside the test process and manages its own test DB, so it needs NO running SUT). Read the repo's CI recipe (each suite's setup/run scripts, whatever they are named) and copy the exact commands — do not re-implement them.
-c. **Extra infrastructure** a suite needs beyond the app under test (extra databases, redis, mailpit/queues, browsers).
+c. **Extra infrastructure** a suite needs beyond the app under test (extra databases, caches, mail or queue services, browsers).
 This per-suite classification decides which suites need substrate below and how each is recorded.`;
 }
 function buildSubstrateBody() {
-    return `Only suites that run against the SUT (over HTTP, or by spawning app instances) need substrate here. An **in-process** suite that self-provisions — boots the app and its test DB inside the test process — runs standalone: add nothing to \`setup.sh\` for it and just record its contract below. (If such a suite needs infra it cannot self-start, e.g. a real MySQL, reuse the SUT's shared infra as in the tiers below rather than standing up a duplicate.)
+    return `Only suites that run against the SUT (over HTTP, or by spawning app instances) need substrate here. An in-process suite that self-provisions — boots the app and its test DB inside the test process — runs standalone: add nothing to \`setup.sh\` for it and just record its contract below. (If such a suite needs infra it cannot self-start, e.g. a real MySQL, reuse the SUT's shared infra as in the tiers below rather than standing up a duplicate.)
 
 For each suite that DOES need the SUT, bring up its substrate INSIDE \`.skyramp/sut/setup.sh\` — the same script that brings up the preflight SUT app — so one \`targetSetupCommand\` stands up every consumer. Bring-up and teardown always live in \`setup.sh\` / \`teardown.sh\`, never in \`workspace.yml\`. Install the suite's dev/test dependencies from the repo's OWN lockfile (\`package-lock.json\`, \`uv.lock\`, \`go.sum\` — never "latest"), and pin any UI browser image to the repo's version, inside that script. Choose the most reuse the suite can reach, in order of preference:
 
-**Reuse the running SUT (prefer this).** When the suite targets the app over HTTP from a base-URL env var and does not boot its own app (no \`webServer\`, or one that defaults to \`reuseExistingServer\`, or a global-setup that only logs in / seeds against a running server), point it at the already-running SUT — stand up no second app and no separate env-start, and bake the suite's OWN base-URL env var into \`testRunCommand\` as an inline prefix referencing the SUT's exported URL. The left-hand name is whatever that repo's config reads — e.g. \`PW_BASE_URL=$SKYRAMP_TEST_BASE_URL npx playwright test\` for a Playwright config that reads \`PW_BASE_URL\`, or \`N8N_BASE_URL=$SKYRAMP_TEST_BASE_URL <run cmd>\` for n8n. Use \`$SKYRAMP_TEST_BASE_URL\` for a single-URL SUT, or \`$SKYRAMP_TEST_SERVICE_URL_<SERVICENAME>\` (service name upper-cased, \`- . : /\` replaced with \`_\`) when the SUT exposes multiple distinct service URLs, matching the service the suite targets. Inline ONLY this base-URL prefix (it must be shell-expanded); record any STATIC env the suite needs (e.g. \`NODE_ENV\`, a test-DB driver) in the suite's \`testEnv\` map when you record the contract, not as an inline prefix.
+**Reuse the running SUT (prefer this).** When the suite targets the app over HTTP from a base-URL env var and does not boot its own app (no \`webServer\`, or one that defaults to \`reuseExistingServer\`, or a global-setup that only logs in / seeds against a running server), point it at the already-running SUT — stand up no second app and no separate env-start, and bake the suite's OWN base-URL env var into \`testRunCommand\` as an inline prefix referencing the SUT's exported URL. The left-hand name is whatever that repo's config reads — e.g. \`PW_BASE_URL=$SKYRAMP_TEST_BASE_URL npx playwright test\` for a Playwright config that reads \`PW_BASE_URL\`, or \`APP_BASE_URL=$SKYRAMP_TEST_BASE_URL <run cmd>\` for one that reads \`APP_BASE_URL\`. Read the config to find the real name; never assume it. Use \`$SKYRAMP_TEST_BASE_URL\` for a single-URL SUT, or \`$SKYRAMP_TEST_SERVICE_URL_<SERVICENAME>\` (service name upper-cased, \`- . : /\` replaced with \`_\`) when the SUT exposes multiple distinct service URLs, matching the service the suite targets. Inline ONLY this base-URL prefix (it must be shell-expanded); record any STATIC env the suite needs (e.g. \`NODE_ENV\`, a test-DB driver) in the suite's \`testEnv\` map when you record the contract, not as an inline prefix.
 
-**Reuse the SUT's built image + infra — when the suite boots its own app.** When the suite MUST boot its own app instances (e.g. per-worker containers via its own orchestration), reuse what the SUT phase already produced — the built image and the running infra — instead of rebuilding from source or standing up a duplicate data tier (rebuilding wastes the SUT's work and duplicate infra desyncs from it). In \`setup.sh\`: (a) build the app image once and expose it under the env var the suite consumes — e.g. Ghost \`GHOST_E2E_MODE=build\` producing an image tagged as \`GHOST_E2E_IMAGE\`; the left-hand name is repo-specific (\`GHOST_E2E_IMAGE\` for one repo, \`E2E_APP_IMAGE\` for another); and (b) bring up the shared infra the suite connects to (e.g. \`mysql\` + \`redis\` + \`mailpit\`) so the suite reuses it instead of starting duplicates. The suite owns spawning the app instances; you own the substrate (built image + shared infra) and the handoff. Surface these substrate requirements for CI to provide rather than satisfying them yourself: a Docker socket (so the suite can spawn containers), and a runner large enough for the suite — if the suite needs a bigger runner, note that in your summary rather than changing \`runs-on\` (which stays as the installer set it).
+**Reuse the SUT's built image + infra — when the suite boots its own app.** When the suite MUST boot its own app instances (e.g. per-worker containers via its own orchestration), reuse what the SUT phase already produced — the built image and the running infra — instead of rebuilding from source or standing up a duplicate data tier (rebuilding wastes the SUT's work and duplicate infra desyncs from it). The suite owns spawning the app instances; you own the substrate and the handoff, in \`setup.sh\`. So read its orchestration code for how it FINDS each thing it needs, and satisfy that mechanism — an equivalent service under a different identity provides nothing, because the suite looks for the one its code names, does not find it, and dies in its own setup having never reached a test. The mechanism is one of:
+- A config value it reads — typically a mode flag telling it to use a prebuilt image, plus a variable naming the tag; also base URLs and DSNs. The names are repo-specific (\`E2E_APP_IMAGE\`, \`<APP>_E2E_IMAGE\`, …). Build the app image ONCE and export the tag under the name that suite reads. And a suite that spawns containers needs EVERY image it launches present in the local daemon, not just the app's — \`docker pull\` any gateway, proxy, database or browser image it names, explicitly, in \`setup.sh\`. Do not leave one to chance because it happened to be cached while you were working: that is ambient runner state, the next runner will not have it, and the suite fails in its own global setup naming an image you never provided.
+- A hardcoded identifier — a docker network, compose project, volume, database, or service/DNS name it resolves literally, often assembled in its own constants file. Match the identifier exactly; an equivalent stack under your own names is invisible to it. When those names are the ones a compose file the repo ALREADY ships produces, bring up THAT file under the project name it expects rather than authoring your own.
+- A path on disk — a state directory, a saved auth state, a generated config, a socket. Create it where the code looks for it.
+- Something only CI can grant — a Docker socket (so the suite can spawn containers), or a runner large enough for it (a suite giving each worker its own app instance needs real headroom). Name these in your summary instead of trying to satisfy them, and if it needs a bigger runner say so rather than changing \`runs-on\` (which stays as the installer set it).
+Whichever applies, check whether the suite merely CONNECTS to the data tier or ADMINISTERS it — creating, dropping or snapshotting databases needs a privileged user, not just reachability.
 
 **Hermetic — the suite builds its own (last resort).** When neither reuse path fits (a hermetic suite with no consumable image), let the suite build its own and accept the cost.`;
 }
 function buildRecordTestContractBody() {
     return `Record the run contract in \`.skyramp/workspace.yml\` under the service's \`runtimeDetails\` — the durable contract later maintenance runs read. Write it via the \`skyramp_init_workspace\` tool (see its \`force\` parameter for overwrite semantics), not by editing \`.skyramp/workspace.yml\` directly.
 
-**Record every runnable suite under \`runtimeDetails.testSuites\`** — one entry per suite, even a single suite (there is no flat single-suite form); not just the one that reaches the SUT. Each entry sets:
+**Record every suite you enumerated above under \`runtimeDetails.testSuites\`** — one entry per suite, even when there is only one (there is no flat single-suite form).
+
+Recording a suite documents that it EXISTS and how it is invoked; provisioning it is the substrate step's business and proving it is the local canary's. Maintenance runs choose what to execute from this list and cannot discover a suite missing from it, so record a suite even when its environment is one you could not build or its command needs infra you did not stand up — write a working \`testRunCommand\` and record the unmet requirement in that suite's \`canaryDetail\` (below) so CI can supply it. Some suites are hard to reduce to a single command — they want a Docker socket, their own orchestration, or a long list of env vars. Record those too: follow the \`testRunCommand\` rules below, and say in your summary what the suite needs. Each entry sets:
 - \`framework\` — the suite's runner.
-- \`testRunCommand\` — the base run command; the framework adapter appends scoped selectors at run time. Keep it minimal: the only thing to inline as a prefix is a value that must be SHELL-EXPANDED at run time — the SUT's exported base-URL env var (e.g. \`PW_BASE_URL=$SKYRAMP_TEST_BASE_URL npx playwright test\`), because the executor sets \`testEnv\` values verbatim and cannot expand \`$…\`. Static env vars belong in \`testEnv\` (next), NOT inlined here. An in-process suite is just the repo's own standalone command (e.g. \`yarn test:unit\`).
+- \`testRunCommand\` — the base run command. An in-process suite is just the repo's own standalone command (e.g. \`yarn test:unit\`). At run time the adapter appends a machine-readable reporter flag and the specific test-file paths to the END of this string, and points the reporter at an output path — so the command has to be one that can actually receive them. **The reporter itself must exist in the suite's environment, and \`setup.sh\` is what installs it**: Playwright and Mocha ship theirs, but **pytest needs the \`pytest-json-report\` plugin** — without it pytest exits on an unrecognised option and the suite reports nothing however healthy it is. The runner process also has to be able to WRITE the reporter's output path, which is the same constraint the container rule below describes. The rules below follow from all that, and breaking any of them means the suite can never report a result:
+  - It must accept appended arguments. Record the invocation that receives them (\`npx playwright test\`, \`pnpm --filter <package> test:unit\`). **Prefer invoking the runner directly** — the repo's own config via a flag or package filter, the suite's static env in \`testEnv\` — which satisfies this rule by construction. Most of what a repo's launcher script does IS just that env, so reach for the script only when it does setup you cannot express that way: values it derives at run time, files it sources, a sidecar it starts. Then a repo's own wrapper is fine but only if it FORWARDS its arguments to the runner. Open the script and check: one ending in \`exec "$@"\` (or otherwise passing \`"$@"\` on) is safe, and you record it with the runner's own command appended, e.g. \`bash <wrapper-script> playwright test\`. A script that hardcodes the test invocation inside itself is NOT safe: your appended arguments are ignored, so it silently runs the WHOLE suite and writes no report where the adapter looks. **When a repo ships several entrypoints for one suite, read them all and pick the forwarding one** — a \`…-host\` and a \`…-container\` variant commonly sit side by side, and only one of them can be driven. Only when the setup is genuinely irreducible AND no shipped entrypoint both forwards arguments and runs from the repository root, write your own thin wrapper in \`.skyramp/sut/\` — translate the appended paths if the script it calls changes directory, and \`exec\` with \`"$@"\`.
+  - The appended paths are relative to the repository root, and the command runs from the repository root. So do not \`cd\` into a subdirectory — the paths will not resolve from there. Point the runner at the subdirectory instead, with the config the repo ALREADY has (e.g. \`npx playwright test --config <path/to/playwright.config.*>\` — the repo's own file, which you have read and confirmed exists) or a package filter (e.g. \`pnpm --filter @scope/e2e test\`); for pytest, name its config and target from the root (e.g. \`pytest -c pytest.ini tests/api\`, or \`--rootdir\` when the suite needs one) rather than changing directory, since pytest resolves \`conftest.py\` and \`sys.path\` from where it runs. **Never name a path you did not write**: a command pointing at a config you invented but never created fails everywhere, and a config you write outside \`.skyramp/sut/\` is not captured with the SUT files, so it is missing on the next run even though it worked on yours.
+  - A value that must be SHELL-EXPANDED at run time goes directly in front of the runner, e.g. \`PW_BASE_URL=$SKYRAMP_TEST_BASE_URL npx playwright test\` — the executor writes \`testEnv\` values verbatim and cannot expand \`$…\`. Never put such an assignment in front of a \`cd\` (\`VAR=$X cd e2e && npx playwright test\`): the assignment applies only to the \`cd\`, so the runner never sees VAR. Static values belong in \`testEnv\` (next), not inlined here.
+  - If the repo runs this suite inside a container (a script doing \`docker run … <runner>\`), that boundary swallows the appended arguments, drops any environment the reporter is configured through, and leaves the reporter writing to an output path that does not exist inside the container — so the wrapper cannot be the \`testRunCommand\`. Record the runner invocation itself, and KEEP the wiring that script provided by moving the env it set into \`testEnv\` — a bare runner that has lost the mode flags and image tags is just as unrunnable as the wrapper. State the container requirement (docker socket, runner size) in your summary so CI can supply it. Record the suite either way, never drop it.
 - \`testEnv\` — a map of STATIC environment variables the suite needs, as literal values with no shell expansion (e.g. \`NODE_ENV: testing\`, \`DB: better-sqlite3\`). The executor injects these into the test process's environment at run time, so record them here instead of inlining them into \`testRunCommand\`: structured env is adapter-safe (selectors are appended to the command without disturbing env) and lets maintenance inspect or override it. Omit when the suite needs no static env. (Values that need runtime shell expansion, like the SUT base-URL, stay inline in \`testRunCommand\` — see above.)
-The workspace records only how to run and where to find each suite; the environment the SUT-dependent suites run in is brought up by \`setup.sh\`.`;
+- \`pathGlobs\` — the repo-relative paths that suite owns, written as globs (e.g. a package's \`test/unit/**\` for a backend unit suite, \`e2e/tests/**\` for a browser suite). **Record it on every suite, always** — including when the repo has only one. It is what sends a test file to the suite that can actually run it, and a suite with no globs accepts EVERY file, so an omitted entry means a suite gets handed specs it cannot load and its runner dies parsing them. Keep the globs disjoint, so no file matches two suites. Scope by DIRECTORY, not by filename — \`e2e/tests/**\`, not \`e2e/tests/**/*.spec.ts\`. A directory owns whatever the suite runs; narrowing by filename quietly drops every file named differently, and one suite routinely mixes conventions — \`.test.ts\` beside \`.spec.ts\`, or \`test_*.py\` beside \`*_test.py\`. Only narrow by filename when one directory genuinely holds two runners' files, and then list every pattern the suite owns after checking the real filenames on disk.
+- \`canaryStatus\` + \`canaryDetail\` — filled in AFTER the local canary, not now. \`proven\` means you watched a test from that suite execute; \`unproven\` means you did not, and \`canaryDetail\` then states what it still needs. An unproven suite stays recorded and stays maintained, so never delete an entry to avoid marking it.
+The workspace records only how to run and where to find each suite; the environment a suite needs — a built image, shared infra, a browser — is brought up by \`setup.sh\`.`;
 }
 // ── PromptPlan declaration ────────────────────────────────────────────────────
 const _adaptPlan = new PromptPlan()
@@ -111,9 +128,16 @@ const _adaptPlan = new PromptPlan()
     .subStep("READ_SOURCES", "Read workflows and infra files", buildReadSourcesBody)
     .subStep("CLASSIFY_STEPS", "Classify source workflow steps", buildClassifyStepsBody)
     .subStep("SUMMARIZE", "Summarize the SUT", buildSummarizeBody)
+    // Enumerated here, with the SUT scan, NOT later beside the substrate step it
+    // feeds. It is pure reading, so nothing forces it late — and placed late it
+    // landed AFTER the compose was authored, which is how a suite pinned to a
+    // compose file the repo already ships got a rival stack built beside it:
+    // capture 30410557140 named the pin in canaryDetail and could no longer act on
+    // it. The body already said "alongside the SUT scan"; only the plan disagreed.
+    .subStep("DISCOVER_SUITE", "Discover every runnable test suite", buildDiscoverSuiteBody)
     .step("DECIDE", "Decide SUT setup strategy", buildDecideIntroBody)
     .subStep("PATTERN_A", "Pattern A — Testbot lifecycle commands only (try this first, always)", buildPatternABody)
-    .subStep("PATTERN_B", "Pattern B — Hybrid: GHA pre-steps for tool/env setup + Testbot lifecycle commands for service start", buildPatternBBody)
+    .subStep("PATTERN_B", "Pattern B — Hybrid: GHA pre-steps for tooling / app-image build + Testbot lifecycle commands for service start", buildPatternBBody)
     .subStep("PATTERN_C", "Pattern C — GHA steps replace service startup + Testbot lifecycle validation", buildPatternCBody)
     .subStep("PATTERN_D", "Pattern D — Fully wrapped by GHA steps", buildPatternDBody)
     .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
@@ -124,8 +148,7 @@ const _adaptPlan = new PromptPlan()
     .subStep("HANDLE_SHAPE", "Handle source-workflow shape", buildHandleShapeBody)
     .subStep("RESTRICTIONS", "What not to do", buildAdaptRestrictionsBody)
     .step("EXTERNAL_TESTS", "Make the repository's own test suites runnable", () => buildExternalTestsIntroBody())
-    .subStep("DISCOVER_SUITE", "Discover every runnable test suite", buildDiscoverSuiteBody)
-    .subStep("BRING_UP_SUBSTRATE", "Bring up SUT-dependent suites' substrate in setup.sh", buildSubstrateBody)
+    .subStep("BRING_UP_SUBSTRATE", "Bring up the suites' substrate in setup.sh", buildSubstrateBody)
     .subStep("RECORD_CONTRACT", "Record the run contract in workspace.yml", buildRecordTestContractBody)
     .step("VERIFY", "Verify", () => "")
     .subStep("COMMON_ERRORS", "Common SUT errors", () => buildCommonSutErrorsSection())

package/build/prompts/sut-setup/shared.js

@@ -84,20 +84,23 @@ Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside t
 }
 export function buildLocalValidationSection() {
     return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
-Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. For any setup, build, or lifecycle command, a non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes. (The step-3 canary is the one exception — judge it on whether the test executes, not on its pass/fail result.)
+Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. For any setup, build, or lifecycle command, a non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes. (The suite canary in step 4 is the one exception — judge it on whether the test executes, not on its pass/fail result.)
 1. Validate build scripts and helper programs first — run these before anything else so build and auth failures surface cheaply:
    a. Every helper script the workflow calls (e.g., \`./scripts/*.sh\`, \`make <target>\`, \`./gradlew <task>\`) — run each and confirm exit 0.
-   b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
-   c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
+   b. \`.skyramp/sut/get-auth-token.sh\` if present — run it **twice**, and confirm the SECOND run also prints a non-empty token: it is invoked more than once per run, so a script that only works against a fresh SUT passes a single check here and fails later. Then send that token to an endpoint that REQUIRES authentication. A public one (a health, version, or site endpoint) answers 200 for an empty token too, so it cannot tell a working token from a missing one.
+   c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`, unless a pre-step in step 2 already builds it. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
    d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
    e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, prune docker BuildKit cache and dangling images before the next build (keep the running stack intact). Skipping this on heavy SUTs exhausts the standard runner disk mid-validation.
-2. Validate the SUT lifecycle comes up — choose the branch that matches the chosen pattern:
+2. Every GHA step you added before the Testbot action must have run and exited 0 by this point. Run any that has not, in workflow order. If one needs a \${{ secrets.* }} value your shell cannot resolve, note it in <thinking> and skip that one rather than fabricating a value.
+3. Validate the SUT lifecycle comes up — choose the branch that matches the chosen pattern:
    a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset): run \`targetSetupCommand\` (confirm exit 0), then poll \`targetReadyCheckCommand\` until it exits 0 within \`targetReadyCheckTimeout\` seconds.
-   b. If GHA steps wrap the Testbot action (skipTargetSetup: 'true'): extract the shell body of each SETUP step and run them in order (if a step depends entirely on a \${{ secrets.* }} value your shell cannot resolve, note it in <thinking> and skip — do not fabricate a value), then poll \`targetReadyCheckCommand\` until it exits 0.
-3. Canary the repository's own suite(s) — do this only when a suite was recorded (a \`testSuites\` entry under a service's \`runtimeDetails\` in \`workspace.yml\`); skip it when none was recorded. With the SUT still running from step 2 and before teardown, run **each recorded suite's** \`testRunCommand\` scoped to a SINGLE test — append a one-spec/one-test selector (Playwright: one spec path or \`-g '<title>'\`; pytest: one \`path::test\` node id; jest/vitest: \`-t '<name>'\`; Mocha: one \`--grep '<title>'\` or spec path). When \`testSuites\` holds more than one, **canary each SUT-dependent suite** — the ones that run against the running SUT (browser / API e2e). These are the whole point of the canary, so do NOT skip a SUT-dependent suite because it is slower or heavier than an in-process one: it is precisely the suite whose provisioned env (browser, built e2e image, base-URL wiring) needs proving. An in-process suite that self-provisions is secondary — canary it too when cheap, but if you can run only one canary, run a SUT-dependent suite's, never only the in-process suite's. HTTP readiness only shows the app answers; running one real test per suite is what proves each suite's environment is actually present — its browser, the built e2e image, dev/test dependencies, and base-URL wiring.
-   - Judge on whether the test EXECUTES, not whether it passes: a green result or a plain assertion failure both prove the environment is runnable, so proceed. Only a failure to execute — the runner won't start, a browser or dependency is missing, the suite cannot reach the SUT, or specs fail to collect/import — is a fix-needed signal; fix \`setup.sh\` / \`workspace.yml\` and re-run. Never edit, skip, or delete the repo's tests to force a pass; that is out of scope for setup.
-4. Tear down — run \`targetTeardownCommand\` (branch a), or each TEARDOWN step's body in order (branch b), so the next iteration starts clean; confirm exit 0.
-Only proceed to the success report once setup → health check → canary (when applicable) → teardown all pass.`;
+   b. If GHA steps wrap the Testbot action (skipTargetSetup: 'true'): the SETUP steps you ran in step 2 already brought the SUT up, so just poll \`targetReadyCheckCommand\` until it exits 0.
+4. Canary the repository's own suite(s) — do this only when a suite was recorded (a \`testSuites\` entry under a service's \`runtimeDetails\` in \`workspace.yml\`); skip it when none was recorded. With the SUT still running from step 3 and before teardown, call **\`skyramp_run_existing_tests\`** with \`workspacePath\` and \`testSelectors\` — one real test FILE per recorded suite. Do NOT canary by running the suite's command in your shell. **Maintenance runs every suite through this tool, so it is the only thing that proves what maintenance will actually be able to do**: a shell command you drove by hand can succeed while the same suite is unrunnable through the tool, and then the failure surfaces on a customer's first PR instead of here. It also resolves each selector to its owning suite via \`pathGlobs\`, so one call checks your routing too, and it appends the reporter flag and selector itself — you do not hand-build those.
+   - Cover **every** suite in \`runtimeDetails.testSuites\`, not a subset you choose: do not rank them or trim by cost. A suite that spawns its OWN app instances (per-worker containers via its own orchestration) belongs in the set like any other — it depends entirely on your provisioning, so its weight is a reason to run it, never a reason to defer it. HTTP readiness only shows the app answers; one real test per suite is what proves each suite's environment is present — its browser, the built e2e image, dev/test dependencies, and base-URL wiring.
+   - Read the result per suite: \`summary.ran > 0\` is the suite executing. \`environmentHealthy: false\` or \`skipped: true\` means it did not — \`healthDetail\`/\`skipReason\` says why, and that text is what belongs in the suite's \`canaryDetail\`.
+   - A suite that executed and then FAILED its assertions is fine — the environment is runnable, so proceed. A suite that did not execute is the fix-needed signal — the exit criteria below say what you must fix and when you are allowed to stop trying. Never edit, skip, or delete the repo's tests to force a pass; that is out of scope for setup.
+5. Tear down — run \`targetTeardownCommand\` (branch a), or each TEARDOWN step's body in order (branch b), so the next iteration starts clean; confirm exit 0.
+Only proceed to the success report once pre-steps → setup → health check → canary (every suite in the mandatory set) → teardown all pass, and every recorded suite carries a \`canaryStatus\`. \`proven\` requires that you watched a test execute. \`unproven\` is NOT a way to close out a suite you could have fixed — it is permitted only when the blocker is something only CI can grant (a secret, a registry credential, a hosted dependency), and then \`canaryDetail\` must name that grant. Every other cause is yours to fix before you finish, and those are the common ones: a runner binary the suite needs that is installed in the repo but absent from \`PATH\`, an image nobody built, infra the bring-up never started, a wrong path in \`testRunCommand\`. Fix the artifact — \`setup.sh\`, the workflow pre-steps, \`workspace.yml\` — and canary that suite again. Only after two genuine fix attempts have failed on the same cause may you record \`unproven\`, with \`canaryDetail\` naming both the cause and what you tried. **Keep every suite recorded either way** — an unproven suite is still maintained from source, so removing the entry is the only move that loses that coverage outright.`;
 }
 /**
  * Returns the canonical "add to the Testbot workflow" block: every input that
@@ -140,7 +143,8 @@ Auth inputs (used by Testbot to authenticate against the running SUT during test
    a. Set this when the SUT requires authentication for API calls.
    b. If the source workflow exports a token via a step output or script, wire that same script here (e.g., \`bash .skyramp/sut/get-auth-token.sh\`).
    c. If the workflow seeds a test user during setup, create \`.skyramp/sut/get-auth-token.sh\` that logs in with those credentials and prints the token.
-   d. Omit when the SUT APIs are unauthenticated.
+   d. **This command runs more than once per run, so it must be idempotent.** Anything it creates against a fresh SUT — an admin user, an integration, an API key, seed data — already exists the next time it runs, so look each one up first and create only what is missing. Make the lookup ask for whatever the API needs in order to return the credential: many APIs omit nested keys unless the request includes them, and then the second run finds nothing, tries to create a duplicate, is refused, and exits empty-handed.
+   e. Omit when the SUT APIs are unauthenticated.
 7. \`uiCredentials\` — \`username:password\` pair typed into the browser login form during UI test recording (format: \`myuser:mypassword\`).
    a. Set for any frontend service that requires browser-based login.
    b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
@@ -154,14 +158,14 @@ Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot insta
 2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
 3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
 4. Leave the existing \`runs-on\` value untouched (input 9).
-5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
+5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/<app> start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
 6. Stack-specific testability flags — some frameworks gate test-harness affordances behind a container env var. Common cases: Flutter web needs \`IS_TESTING=true\` so \`ensureSemantics()\` exposes the widget tree to Playwright (without it every selector finds nothing even though the app is up); some React/Angular builds need \`NODE_ENV\` / framework-specific build flags for \`testID\` propagation. Set these in the compose \`environment:\` block on the affected service (or as a \`Dockerfile ENV\`) — NOT as a shell var in \`targetSetupCommand\`. The var must be in the container's env at boot, not just the shell that runs the setup command. If selectors find nothing but the app is up, check for a missing flag like this first.
 
 #### What not to do
 1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
 2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code. In case of multi-mode containers (\`Dockerfile.bdd\`, \`Dockerfile.e2e\`, \`Dockerfile.integration\`, \`Dockerfile.test\`), create the necessary file in \`.skyramp/sut/\` with a PR-sourced build (e.g. \`.skyramp/sut/Dockerfile.bdd\` referenced from the testbot compose's \`build:\`). Do not edit existing Dockerfiles or compose files in the repo. If the multi-mode container's bootstrap step depends on a seeded user, ensure SEED_DATA creates it before this container tries to log in.
 3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
-4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
+4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/<app> start\`, \`python app.py\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
 5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
 `;
 }