@skyramp/mcp 0.2.8 → 0.2.150-rc.ldw

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/build/commands/commandLibrary.d.ts +1 -0
  2. package/build/commands/commandLibrary.js +20 -14
  3. package/build/commands/commandLibrary.test.d.ts +1 -0
  4. package/build/commands/commandLibrary.test.js +59 -0
  5. package/build/commands/localDevTestChangesCommand.d.ts +15 -0
  6. package/build/commands/localDevTestChangesCommand.js +188 -0
  7. package/build/index.js +77 -13
  8. package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -0
  9. package/build/prompts/enhance-assertions/sharedAssertionRules.js +17 -0
  10. package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +9 -8
  11. package/build/prompts/local-dev/local-dev-plan.d.ts +25 -0
  12. package/build/prompts/local-dev/local-dev-plan.js +419 -0
  13. package/build/prompts/local-dev/local-dev-prompts.d.ts +4 -0
  14. package/build/prompts/local-dev/local-dev-prompts.js +157 -0
  15. package/build/prompts/prompt-utils.d.ts +8 -0
  16. package/build/prompts/prompt-utils.js +33 -0
  17. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +3 -1
  18. package/build/prompts/sut-setup/modes/dockerComposePrompt.js +3 -1
  19. package/build/prompts/sut-setup/shared.d.ts +20 -0
  20. package/build/prompts/sut-setup/shared.js +69 -7
  21. package/build/prompts/test-recommendation/analysisOutputPrompt.js +21 -29
  22. package/build/prompts/test-recommendation/analysisOutputPrompt.test.js +0 -28
  23. package/build/prompts/test-recommendation/test-recommendation-prompt.js +41 -4
  24. package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +16 -1
  25. package/build/prompts/testbot/testbot-prompts.d.ts +0 -5
  26. package/build/prompts/testbot/testbot-prompts.js +4 -34
  27. package/build/resources/testbotResource.js +2 -1
  28. package/build/services/AnalyticsService.d.ts +1 -1
  29. package/build/services/TestExecutionService.d.ts +2 -1
  30. package/build/services/TestExecutionService.js +15 -10
  31. package/build/services/TestExecutionService.test.js +75 -1
  32. package/build/services/TestGenerationService.d.ts +2 -2
  33. package/build/tool-phases.js +4 -0
  34. package/build/tools/code-refactor/enhanceAssertionsTool.js +47 -18
  35. package/build/tools/enrichTestWithMocksTool.d.ts +28 -0
  36. package/build/tools/enrichTestWithMocksTool.js +726 -0
  37. package/build/tools/enrichTestWithMocksTool.test.d.ts +1 -0
  38. package/build/tools/enrichTestWithMocksTool.test.js +266 -0
  39. package/build/tools/executeSkyrampTestTool.d.ts +4 -0
  40. package/build/tools/executeSkyrampTestTool.js +48 -24
  41. package/build/tools/executeSkyrampTestTool.test.d.ts +1 -0
  42. package/build/tools/executeSkyrampTestTool.test.js +16 -0
  43. package/build/tools/generate-tests/batchMockGenerationTool.d.ts +85 -0
  44. package/build/tools/generate-tests/batchMockGenerationTool.js +432 -0
  45. package/build/tools/generate-tests/generateContractRestTool.js +2 -2
  46. package/build/tools/generate-tests/generateMockRestTool.d.ts +129 -6
  47. package/build/tools/generate-tests/generateMockRestTool.js +234 -22
  48. package/build/tools/generate-tests/loadTestSchema.d.ts +1 -1
  49. package/build/tools/generateEnrichedIntegrationTestTool.d.ts +24 -0
  50. package/build/tools/generateEnrichedIntegrationTestTool.js +225 -0
  51. package/build/tools/generateEnrichedIntegrationTestTool.test.d.ts +1 -0
  52. package/build/tools/generateEnrichedIntegrationTestTool.test.js +44 -0
  53. package/build/tools/localDevWorkflowFixes.test.d.ts +1 -0
  54. package/build/tools/localDevWorkflowFixes.test.js +255 -0
  55. package/build/tools/one-click/oneClickTool.d.ts +10 -0
  56. package/build/tools/one-click/oneClickTool.js +177 -21
  57. package/build/tools/one-click/oneClickTool.test.d.ts +1 -0
  58. package/build/tools/one-click/oneClickTool.test.js +172 -0
  59. package/build/tools/preflightMockCheckTool.d.ts +2 -0
  60. package/build/tools/preflightMockCheckTool.js +96 -0
  61. package/build/tools/submitReportTool.d.ts +14 -14
  62. package/build/tools/test-management/analyzeChangesTool.d.ts +2 -1
  63. package/build/tools/test-management/analyzeChangesTool.js +59 -38
  64. package/build/tools/trace/startTraceCollectionTool.js +3 -3
  65. package/build/tools/workspace/initializeWorkspaceTool.js +2 -9
  66. package/build/tools/workspace/initializeWorkspaceTool.test.js +9 -4
  67. package/build/types/OneClickCommands.d.ts +1 -1
  68. package/build/types/RepositoryAnalysis.d.ts +117 -0
  69. package/build/types/RepositoryAnalysis.js +16 -2
  70. package/build/types/TestExecution.d.ts +1 -0
  71. package/build/types/TestTypes.d.ts +20 -8
  72. package/build/types/TestTypes.js +19 -5
  73. package/build/utils/analyze-openapi.js +18 -1
  74. package/build/utils/analyze-openapi.test.d.ts +1 -0
  75. package/build/utils/analyze-openapi.test.js +19 -0
  76. package/build/utils/branchDiff.d.ts +17 -1
  77. package/build/utils/branchDiff.js +96 -14
  78. package/build/utils/branchDiff.test.d.ts +1 -0
  79. package/build/utils/branchDiff.test.js +109 -0
  80. package/build/utils/docker.test.js +1 -1
  81. package/build/utils/featureFlags.d.ts +20 -1
  82. package/build/utils/featureFlags.js +26 -2
  83. package/build/utils/featureFlags.test.js +57 -2
  84. package/build/utils/grpcMockValidation.d.ts +1 -0
  85. package/build/utils/grpcMockValidation.js +49 -0
  86. package/build/utils/grpcMockValidation.test.d.ts +1 -0
  87. package/build/utils/grpcMockValidation.test.js +41 -0
  88. package/build/utils/httpMethodValidation.d.ts +4 -0
  89. package/build/utils/httpMethodValidation.js +13 -0
  90. package/build/utils/mockCompatibility.d.ts +49 -0
  91. package/build/utils/mockCompatibility.js +82 -0
  92. package/build/utils/mockCompatibility.test.d.ts +1 -0
  93. package/build/utils/mockCompatibility.test.js +79 -0
  94. package/build/utils/versions.d.ts +3 -3
  95. package/build/utils/versions.js +1 -1
  96. package/build/workspace/workspace.d.ts +49 -35
  97. package/build/workspace/workspace.js +10 -5
  98. package/build/workspace/workspace.test.js +65 -6
  99. package/package.json +2 -2
@@ -0,0 +1,8 @@
1
+ import { type Service } from "../workspace/workspace.js";
2
+ export declare function escapeXml(value: string): string;
3
+ export declare function buildServiceContext(services: Service[]): string;
4
+ /**
5
+ * Read services from .skyramp/workspace.yml. Returns empty array if
6
+ * the workspace file doesn't exist or can't be parsed.
7
+ */
8
+ export declare function readWorkspaceServices(repositoryPath: string): Promise<Service[]>;
@@ -0,0 +1,33 @@
1
+ import { readWorkspaceConfigRaw } from "../utils/workspaceAuth.js";
2
+ export function escapeXml(value) {
3
+ return value
4
+ .replaceAll("&", "&amp;")
5
+ .replaceAll("<", "&lt;")
6
+ .replaceAll(">", "&gt;")
7
+ .replaceAll('"', "&quot;")
8
+ .replaceAll("'", "&apos;");
9
+ }
10
+ export function buildServiceContext(services) {
11
+ const blocks = services.map((svc) => {
12
+ const parts = [`<service name="${escapeXml(svc.serviceName)}">`];
13
+ if (svc.language)
14
+ parts.push(` <language>${escapeXml(svc.language)}</language>`);
15
+ if (svc.framework)
16
+ parts.push(` <framework>${escapeXml(svc.framework)}</framework>`);
17
+ if (svc.api?.baseUrl)
18
+ parts.push(` <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
19
+ if (svc.testDirectory)
20
+ parts.push(` <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
21
+ parts.push("</service>");
22
+ return parts.join("\n");
23
+ });
24
+ return `<services>\n${blocks.join("\n")}\n</services>`;
25
+ }
26
+ /**
27
+ * Read services from .skyramp/workspace.yml. Returns empty array if
28
+ * the workspace file doesn't exist or can't be parsed.
29
+ */
30
+ export async function readWorkspaceServices(repositoryPath) {
31
+ const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
32
+ return (rawConfig?.services ?? []);
33
+ }
@@ -1,4 +1,4 @@
1
- import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSutDefinitionBlock, } from "../shared.js";
1
+ import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
2
2
  import { getPersonaPrefix } from "../../personas.js";
3
3
  import { PromptPlan } from "../../test-recommendation/promptPlan.js";
4
4
  // ── Step body builders ────────────────────────────────────────────────────────
@@ -70,6 +70,7 @@ function buildAdaptRestrictionsBody() {
70
70
  // ── PromptPlan declaration ────────────────────────────────────────────────────
71
71
  const _adaptPlan = new PromptPlan()
72
72
  .addPhase("adapt", "", { headerLevel: "hidden", stepFormat: "hash" })
73
+ .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
73
74
  .step("SCAN", "Understand the System Under Test in this repository", () => "")
74
75
  .subStep("READ_SOURCES", "Read workflows and infra files", buildReadSourcesBody)
75
76
  .subStep("CLASSIFY_STEPS", "Classify source workflow steps", buildClassifyStepsBody)
@@ -80,6 +81,7 @@ const _adaptPlan = new PromptPlan()
80
81
  .subStep("PATTERN_C", "Pattern C — GHA steps replace service startup + Testbot lifecycle validation", buildPatternCBody)
81
82
  .subStep("PATTERN_D", "Pattern D — Fully wrapped by GHA steps", buildPatternDBody)
82
83
  .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
84
+ .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
83
85
  .step("ADAPT", "Edit the Testbot workflow and generate SUT files", () => "")
84
86
  .subStep("EDIT_WORKFLOW", "Edit the workflow file", buildEditWorkflowBody)
85
87
  .subStep("CONFIGURE_AUTH", "Configure auth", buildConfigureAuthBody)
@@ -1,4 +1,4 @@
1
- import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSutDefinitionBlock, } from "../shared.js";
1
+ import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
2
2
  import { getPersonaPrefix } from "../../personas.js";
3
3
  import { PromptPlan } from "../../test-recommendation/promptPlan.js";
4
4
  // ── Step body builders ────────────────────────────────────────────────────────
@@ -260,9 +260,11 @@ ${GENERATE_DOCKER_COMPOSE_INSTRUCTIONS}`;
260
260
  // ── PromptPlan declaration ────────────────────────────────────────────────────
261
261
  const _composePlan = new PromptPlan()
262
262
  .addPhase("compose", "", { headerLevel: "hidden", stepFormat: "hash" })
263
+ .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
263
264
  .step("SCAN", "Understand the System Under Test in this repository", buildScanComposeBody)
264
265
  .step("DECIDE", "Decide SUT setup strategy", buildDecideStrategyBody)
265
266
  .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
267
+ .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
266
268
  .step("BUILD", "Edit the Testbot workflow and generate SUT files", () => "")
267
269
  .subStep("PR_ACCURATE", "Build PR-accurate SUT", buildPrAccurateSutSection)
268
270
  .subStep("APPLY", "Apply the chosen strategy", buildApplyStrategyBody)
@@ -21,6 +21,26 @@ export interface SutPromptArgs {
21
21
  */
22
22
  export declare function buildSutDefinitionBlock(): string;
23
23
  export declare function buildContextBlock(args: SutPromptArgs): string;
24
+ /**
25
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
26
+ * looks for existing customer seed conventions BEFORE inventing new ones.
27
+ * A fresh database makes list / search / filter / pagination endpoint tests
28
+ * shallow (only HTTP-status assertable, never response content), which
29
+ * collapses the data-diversity dimension of the eval. Most repos already
30
+ * have a seed mechanism — wire that in instead of generating one.
31
+ *
32
+ * Rendered as a top-level PromptPlan step between INPUTS and the
33
+ * ADAPT/BUILD step in every SUT setup mode.
34
+ */
35
+ export declare function buildSeedDataDiscoverySection(): string;
36
+ /**
37
+ * Workspace file initialisation block — rendered as the first instruction
38
+ * in every SUT setup mode, before any local validation, workflow editing,
39
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
40
+ * inventory so testbot's SUT autocommit picks it up alongside the
41
+ * generated `.skyramp/sut/*` files.
42
+ */
43
+ export declare function buildWorkspaceInitSection(): string;
24
44
  export declare function buildLocalValidationSection(): string;
25
45
  /**
26
46
  * Returns the canonical "add to the Testbot workflow" block: every input that
@@ -31,6 +31,59 @@ export function buildContextBlock(args) {
31
31
  lines.push(`</context>`);
32
32
  return lines.join("\n");
33
33
  }
34
+ /**
35
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
36
+ * looks for existing customer seed conventions BEFORE inventing new ones.
37
+ * A fresh database makes list / search / filter / pagination endpoint tests
38
+ * shallow (only HTTP-status assertable, never response content), which
39
+ * collapses the data-diversity dimension of the eval. Most repos already
40
+ * have a seed mechanism — wire that in instead of generating one.
41
+ *
42
+ * Rendered as a top-level PromptPlan step between INPUTS and the
43
+ * ADAPT/BUILD step in every SUT setup mode.
44
+ */
45
+ export function buildSeedDataDiscoverySection() {
46
+ return `**First, decide if this step applies.** Skip it entirely (in your summary state \`SEED_DATA: N/A — <reason>\`, naming the specific bullet below that applied — e.g. "no stateful service", "no list endpoints in scope", "already seeded by init container") and move on when ANY of the following is true:
47
+ - The SUT has no database / stateful service (it's a pure stateless API, a static asset server, a webhook receiver, etc.).
48
+ - The compose stack contains no database container (no \`postgres\`, \`mysql\`, \`mariadb\`, \`mongodb\`, \`redis\` as the primary store, \`sqlite\`, etc.) and no application-managed persistent store.
49
+ - The PR diff and the existing test surface focus exclusively on auth, single-resource CRUD (POST/GET-one/PUT/DELETE on \`/foo/{id}\`), or compute endpoints — no list / search / filter / pagination / aggregate endpoints will be exercised.
50
+ - You can confirm from the SCAN step's findings that representative records already exist (e.g. seeded by an init container, by a migration, or by the auth/setup flow you already wired).
51
+
52
+ Only proceed with the rest of this step when the SUT has a database AND the tests will exercise list/search/filter/pagination endpoints. Otherwise an empty database is acceptable and inventing seed data adds noise.
53
+
54
+ When this step DOES apply: database services usually start empty in CI, so tests against list / search / filter / pagination endpoints will then only see \`[]\` and can assert nothing beyond HTTP status — a low data-diversity outcome. Look for an existing seeding mechanism in the repo and wire it into the SUT bootstrap.
55
+
56
+ **Discover existing project conventions first** — most repos already have one of:
57
+ 1. A named script: \`scripts/seed*\`, \`db/seeds/*\`, \`prisma/seed.{ts,js}\`, \`apps/api/seed/*\`, \`tests/fixtures/seed*\`. Search: \`find . -iname '*seed*' -not -path '*/node_modules/*' -not -path '*/.git/*'\`.
58
+ 2. A package.json / Makefile target: \`npm run seed\`, \`npm run db:seed\`, \`pnpm seed\`, \`yarn seed\`, \`make seed\`, \`make fixtures\`. Check \`package.json\` \`scripts\` and the project's \`Makefile\`.
59
+ 3. A migration with sample data: Django \`loaddata\`, Rails \`db:seed\`, Knex \`seed:run\`, Prisma \`db seed\`, Alembic data migrations.
60
+ 4. A docker-compose init service: existing compose files sometimes have a \`*-seed\`, \`*-fixtures\`, or \`init-*\` service that runs once on bring-up.
61
+
62
+ **Wire it in:**
63
+ - **Compose init service** — add it (or its dependency) to the testbot compose so \`targetSetupCommand\` brings it up alongside the rest. Use \`depends_on\` with \`condition: service_completed_successfully\` on dependent services so they only start after seeding finishes.
64
+ - **Script needing an admin/API token** (most common) — chain it INTO \`authTokenCommand\` AFTER the token is created and BEFORE the script prints the token to stdout. Route seed stdout/stderr to stderr so the captured token isn't corrupted. Make the seed call best-effort (\`|| echo "seed failed" >&2\`) so a flaky seed doesn't block the run — the token is still returned and PRs whose tests don't need seed data keep working.
65
+ - **Migration with sample data** — ensure the migration runs as part of \`targetSetupCommand\` (some ORMs auto-run on container start, others need an explicit \`migrate\` / \`db:seed\` step before tests can hit the API).
66
+
67
+ **If NO existing convention is found**, generate a minimal seed step (≥3 entities per primary resource the tests will query, e.g., ≥3 repos / ≥3 issues / ≥3 users for a forge; ≥3 products / ≥3 orders for a shop). Call this out explicitly in your summary so the user knows a richer fixture may be needed for production-quality tests.
68
+
69
+ **Never** hard-code production data, real customer credentials, or PII in seeds. Seed identifiers should be obviously test-only (\`acme-co\`, \`alpha-svc\`, \`testbot-admin\`).`;
70
+ }
71
+ /**
72
+ * Workspace file initialisation block — rendered as the first instruction
73
+ * in every SUT setup mode, before any local validation, workflow editing,
74
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
75
+ * inventory so testbot's SUT autocommit picks it up alongside the
76
+ * generated `.skyramp/sut/*` files.
77
+ */
78
+ export function buildWorkspaceInitSection() {
79
+ return `Before generating any SUT setup files, check whether \`.skyramp/workspace.yml\` exists at the root of the repository being bootstrapped (the path shown next to "Repository path:" in the context block above — the repo whose PR triggered Testbot; in multi-repo runs this is the primary repo, not the sibling repos checked out as related repos). If it doesn't:
80
+ 1. Call \`skyramp_init_scan\` with \`workspacePath\` set to that repository root → follow the returned instructions to discover all services.
81
+ 2. Call \`skyramp_init_workspace\` with the same \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
82
+
83
+ This produces a baseline \`.skyramp/workspace.yml\` listing the services Testbot will test. Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside it. You might have to update the per-service \`runtimeDetails.serverStartCommand\` to match the new SUT files created, so Testbot brings up the services correctly.
84
+
85
+ Also verify each service's \`api.baseUrl\` includes the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.`;
86
+ }
34
87
  export function buildLocalValidationSection() {
35
88
  return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
36
89
  Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
@@ -39,7 +92,7 @@ Run commands one at a time in your shell — individual execution pinpoints fail
39
92
  b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
40
93
  c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
41
94
  d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
42
- e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, run \`docker builder prune -af -f && docker image prune -af\` before the next build. This frees BuildKit cache + dangling images without losing the running stack. Skipping this on heavy SUTs (medusa, calcom, mattermost, supabase) exhausts the ~14 GB \`ubuntu-latest\` disk mid-validation.
95
+ e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, prune docker BuildKit cache and dangling images before the next build (keep the running stack intact). Skipping this on heavy SUTs exhausts the standard runner disk mid-validation.
43
96
  2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
44
97
  a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
45
98
  i. Run \`targetSetupCommand\` — confirm exit 0.
@@ -68,8 +121,16 @@ Target lifecycle inputs (control how Testbot starts, validates, and stops the SU
68
121
  b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
69
122
  c. With a dedicated compose file (reused or generated), point at it explicitly: \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . up -d --build\`.
70
123
  d. Override whenever the source workflow's start sequence differs from the default.
71
- 2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
72
- a. **CRITICAL: Always set this to a valid target readiness check command.** Required in every pattern: when \`targetSetupCommand\` is explicit, when relying on the \`docker compose up -d\` default, AND when surrounding GHA steps bring the SUT up before the Testbot action (\`skipTargetSetup: 'true'\`). Testbot's fail-early gate throws \`Unable to verify service readiness\` when no readiness signal is available (neither this nor a workspace service \`baseUrl\`) and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` without it Testbot can't tell when the service is actually listening.
124
+ 2. \`targetReadyCheckCommand\` — polling command Testbot runs repeatedly after setup until it exits 0 (service ready) or the timeout fires.
125
+ a. **CRITICAL: must cover EVERY service the tests will hit, not just the backend.** Chain checks with \`&&\` so the command exits 0 only when ALL services respond. A single-service check on a multi-service SUT is the leading cause of "element not visible" / "heading not visible" UI test failures Playwright lands on a partially-rendered page because only the API was up when the gate passed. Inventory of services to include:
126
+ - Every API service in workspace.yml with an \`api.baseUrl\` → curl an unauthenticated endpoint (health / version / status).
127
+ - Every UI service with a UI \`baseUrl\` (or a frontend port in compose) → curl \`/\` or a known asset like \`/assets/index.js\` (more reliable than \`/\` for SPAs that hydrate lazily).
128
+ - Skip auth-required endpoints — they 401 before the SUT is testable and mask the readiness signal.
129
+ Examples:
130
+ - API only: \`curl -sf http://localhost:8080/api/health\`
131
+ - API + UI: \`curl -sf http://localhost:8080/api/health && curl -sf http://localhost:3000/\`
132
+ Required in every pattern (explicit \`targetSetupCommand\`, the \`docker compose up -d\` default, AND \`skipTargetSetup: 'true'\`). Without a real readiness signal (neither this nor a workspace service \`baseUrl\`), testbot's fail-early gate throws \`Unable to verify service readiness\` and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
133
+ b. Set \`targetReadyCheckDiagnosticsCommand\` to a command that explains why the SUT isn't healthy — output is surfaced in the failure PR comment when the ready check times out. For docker-compose SUTs, scope it to the project's compose file (container state + recent service logs). For non-docker SUTs, a tail of the server log.
73
134
  3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
74
135
  a. Default: \`1800\` (30 min) — sized for cold \`docker compose --build\` runs on multi-service stacks. The loop exits on the first success, so fast services pay no extra cost. Override only if you want to fail faster on stuck setups (e.g. \`'300'\` for a small single-service SUT).
75
136
  4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`, or \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . down -v\` for a dedicated compose file).
@@ -89,19 +150,20 @@ Auth inputs (used by Testbot to authenticate against the running SUT during test
89
150
  b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
90
151
  Workflow / job level settings (set on the workflow or job, not as \`skyramp/testbot\` action inputs):
91
152
  8. Secrets / environment variables — pass every secret/env var the setup needs into the workflow via an \`env:\` block or \`secrets: inherit\`, and add any dependency-resolution steps/jobs the SUT requires.
92
- 9. \`runs-on\` runner label — switch to a larger runner when the SUT needs more disk or memory: heavy Docker builds, large dependency installs, or a multi-service (3+) compose stack. Standard runners run out of disk and crash with \`No space left on device\`. Runner names vary by org do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses (never downgrade to \`ubuntu-latest\`), (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`.
153
+ 9. \`runs-on\` runner label — leave whatever value the Testbot installer wrote and the source workflow used. Don't override it. Picking a runner SKU is an account-level decision that depends on provisioning the agent can't see from inside the repo; a wrong choice queues the job forever with no runner ever picking it up. If the SUT genuinely doesn't fit on the chosen runner, the disk-pressure mitigation in "Common failure modes" #9 handles it; an unprovisioned-runner stall does not.
93
154
 
94
155
  #### What to do
95
156
  Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot installer) — do NOT recreate it. Add only what the SUT needs onto the existing \`skyramp/testbot\` action step, alongside the inputs the installer already added:
96
157
  1. The 4 target lifecycle inputs — \`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetReadyCheckTimeout\`, \`targetTeardownCommand\`.
97
158
  2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
98
159
  3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
99
- 4. The \`runs-on\` runner label, if a larger runner is needed (input 9).
160
+ 4. Leave the existing \`runs-on\` value untouched (input 9).
100
161
  5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
162
+ 6. Stack-specific testability flags — some frameworks gate test-harness affordances behind a container env var. Common cases: Flutter web needs \`IS_TESTING=true\` so \`ensureSemantics()\` exposes the widget tree to Playwright (without it every selector finds nothing even though the app is up); some React/Angular builds need \`NODE_ENV\` / framework-specific build flags for \`testID\` propagation. Set these in the compose \`environment:\` block on the affected service (or as a \`Dockerfile ENV\`) — NOT as a shell var in \`targetSetupCommand\`. The var must be in the container's env at boot, not just the shell that runs the setup command. If selectors find nothing but the app is up, check for a missing flag like this first.
101
163
 
102
164
  #### What not to do
103
165
  1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
104
- 2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
166
+ 2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code. In case of multi-mode containers (\`Dockerfile.bdd\`, \`Dockerfile.e2e\`, \`Dockerfile.integration\`, \`Dockerfile.test\`), create the necessary file in \`.skyramp/sut/\` with a PR-sourced build (e.g. \`.skyramp/sut/Dockerfile.bdd\` referenced from the testbot compose's \`build:\`). Do not edit existing Dockerfiles or compose files in the repo. If the multi-mode container's bootstrap step depends on a seeded user, ensure SEED_DATA creates it before this container tries to log in.
105
167
  3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
106
168
  4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
107
169
  5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
@@ -127,6 +189,6 @@ export function buildCommonSutErrorsSection() {
127
189
  6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
128
190
  7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
129
191
  8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
130
- 9. no_space_left — the runner ran out of disk space (\`No space left on device\`). This happens on standard runners when the SUT involves large Docker builds, many npm packages, or multiple compose services that pull large images. Switch \`runs-on\` to a larger runner — runner names vary by org, do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses, (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`. Indicator: source workflow already specifies a large runner, or the SUT has multi-stage Docker builds / a compose file with 3+ services.
192
+ 9. no_space_left — the runner ran out of disk space (\`No space left on device\`). Prune the docker BuildKit cache and dangling images at the start of both \`targetSetupCommand\` and \`targetTeardownCommand\` so each fix-loop retry begins with a clean slate; without it, cache from the previous attempt accumulates across retries on heavy SUTs. Do not change \`runs-on\` see input 9 above.
131
193
  Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
132
194
  }
@@ -288,11 +288,11 @@ The ranked test recommendation catalog is pre-built and shown below (after the s
288
288
  **If** Steps 1–2 revealed additional scenarios the catalog does not cover (e.g. a computed formula or foreign-key relationship that was missed), you may optionally call \`skyramp_recommend_tests\` with \`stateFile: "${p.stateFile ?? p.sessionId}"\` and \`enrichedScenarios\` to regenerate a more complete catalog — but only after presenting the current one.`;
289
289
  const hasJavaFiles = p.candidateRouteFiles?.some((f) => /\.(java|kt)$/.test(f)) ?? false;
290
290
  const routeFilesSection = p.candidateRouteFiles && p.candidateRouteFiles.length > 0
291
- ? `\nRoute/controller files found by static scan (read these to discover endpoints the regex-based catalog below may be incomplete for your framework):\n${p.candidateRouteFiles.map((f) => `- ${f}`).join("\n")}\n`
291
+ ? `\nCandidate route/controller files for LLM inspection (read these to discover endpoints; static parser output is only a hint):\n${p.candidateRouteFiles.map((f) => `- ${f}`).join("\n")}\n`
292
292
  : "";
293
293
  const resolvePathsNote = p.routerMountContext.length
294
- ? `**Resolve nested paths** using your Step ${ANALYSIS_STEP_RESOLVE_PATHS} table a router in the table with prefix \`/api/v1/products/{product_id}/reviews\` means every endpoint in that file lives under that full path.`
295
- : `**Resolve full paths** using the prefixes you identified in Step ${ANALYSIS_STEP_READ_FILES} (e.g. Java Spring class-level \`@RequestMapping\` prefix + method-level path).`;
294
+ ? `**Resolve nested paths** using your Step ${ANALYSIS_STEP_RESOLVE_PATHS} table. The table you build from source/router context is authoritative over static parser hints.`
295
+ : `**Resolve full paths** using the prefixes, mounts, annotations, decorators, or routing DSL you identify in Step ${ANALYSIS_STEP_READ_FILES}.`;
296
296
  return `## Your Task — Fill in and Present the Catalog (full repo)
297
297
 
298
298
  ### Step ${ANALYSIS_STEP_READ_FILES}: Read key files
@@ -335,36 +335,28 @@ ${p.routerFileContents?.length
335
335
  p.routerMountContext.map((f) => `- \`${f}\``).join("\n")}`
336
336
  : "";
337
337
  const enrichment = buildEnrichmentInstructions(p);
338
- // LLM fallback: when heuristic scanning may have missed backend route files.
339
- const scannerFallbackHasCatalog = p.scannedEndpoints.length > 0;
340
- const scannerFallbackReason = scannerFallbackHasCatalog
341
- ? "The heuristic endpoint scanner may have missed route/controller files in this diff, even though endpoint data is present from another source"
342
- : "The heuristic endpoint scanner found **0 endpoints**";
343
- const scannerFallbackInstruction = scannerFallbackHasCatalog
344
- ? "Treat this as a supplemental gap check: merge only HTTP endpoints that are visibly missing from the existing catalog/spec data, and keep the existing catalog as the primary endpoint source."
345
- : "Build a table of discovered endpoints (method, full path, source file) and use it as the authoritative endpoint list for all subsequent steps.";
346
- const llmFallbackSection = (isDiffScope || p.scannedEndpoints.length === 0) &&
347
- p.candidateRouteFiles &&
348
- p.candidateRouteFiles.length > 0
338
+ const staticHintCount = p.scannedEndpoints.length;
339
+ const routeDiscoverySection = isDiffScope ||
340
+ staticHintCount > 0 ||
341
+ (p.candidateRouteFiles?.length ?? 0) > 0
349
342
  ? `
350
- ## Scanner Fallback — Manual Endpoint Discovery Required
343
+ ## LLM Route Discovery Inputs
351
344
 
352
- ${scannerFallbackReason}, and this repository contains ${p.candidateRouteFiles.length} file(s) that appear to define HTTP routes. The scanner likely failed due to multi-line definitions, framework-specific patterns, indirection the regex does not cover, or endpoint data being supplied by an OpenAPI spec instead of source scanning.
345
+ Static endpoint scan results are **best-effort hints only**. Do not assume the scanner supports this repository's language, framework, routing DSL, or helper abstractions.
353
346
 
354
- **Read the following controller/router files and identify all HTTP route registrations (method + path).** Include these discovered endpoints when executing the steps above.
347
+ ${staticHintCount > 0 ? `Static hints available: ${staticHintCount}. Verify every hinted method/path against source before using it.` : "Static hints available: 0. Build the endpoint list from source, router context, spec, and diff."}
355
348
 
356
- ${p.candidateRouteFiles.slice(0, 15).map((f) => `- \`${f}\``).join("\n")}
357
- ${p.candidateRouteFiles.length > 15 ? `\n_(${p.candidateRouteFiles.length - 15} more files not shown)_` : ""}
349
+ ${p.candidateRouteFiles && p.candidateRouteFiles.length > 0
350
+ ? `Candidate files to inspect:\n${p.candidateRouteFiles
351
+ .slice(0, 15)
352
+ .map((f) => `- \`${f}\``)
353
+ .join("\n")}${p.candidateRouteFiles.length > 15 ? `\n_(${p.candidateRouteFiles.length - 15} more files not shown)_` : ""}`
354
+ : staticHintCount > 0
355
+ ? "Candidate files to inspect: none identified by static scanning. Verify the static hints against their source files, changed files, router context, and specs above."
356
+ : "Candidate files to inspect: use the changed files and routing entry-point files above."}
358
357
 
359
- For each file, look for:
360
- - Express/Fastify/Koa/Hapi: \`router.<method>('/path', ...)\` or \`app.<method>('/path', ...)\`
361
- - FastAPI/Flask: \`@router.<method>('/path')\` or \`@app.route('/path', ...)\`
362
- - Spring/NestJS: \`@GetMapping\`, \`@PostMapping\`, \`@Controller\`, etc.
363
- - Go (Gin/Echo/Chi): \`r.GET("/path", ...)\`, \`r.Group("/prefix")\`
364
- - GraphQL: schema/resolver artifacts are unsupported for REST test generation; do not invent REST endpoints from them
365
- - Any other framework-specific route registration pattern
366
-
367
- ${scannerFallbackInstruction}`
358
+ For each candidate file, infer route registrations from the source's actual framework or DSL. Record method, full path, and source file. Resolve mount prefixes through the routing entry-point files. If a file is GraphQL-only, do not invent REST endpoints from it.
359
+ `
368
360
  : "";
369
361
  return `# Repository Analysis
370
362
 
@@ -373,7 +365,7 @@ ${scannerFallbackInstruction}`
373
365
  **Analysis Scope**: \`${p.analysisScope}\`
374
366
  ${isDiffScope ? `**Diff endpoints**: ${(p.parsedDiff?.newEndpoints.length ?? 0) + (p.parsedDiff?.modifiedEndpoints.length ?? 0) + (p.parsedDiff?.removedEndpoints?.length ?? 0)}` : `**Pre-scanned endpoints**: ${p.scannedEndpoints.length}`}
375
367
  ${routerSection}
376
- ${llmFallbackSection}
368
+ ${routeDiscoverySection}
377
369
  ${enrichment}
378
370
 
379
371
  **CRITICAL**: No .json/.md file creation. Prioritize cross-resource workflows.`;
@@ -234,31 +234,3 @@ describe("buildAnalysisOutputText — unmatchedFiles / trace-callers sub-step",
234
234
  expect(output).toContain("Trace callers of changed non-route files");
235
235
  });
236
236
  });
237
- describe("buildAnalysisOutputText — scanner fallback", () => {
238
- it("includes manual endpoint discovery in diff scope when candidate route files are present despite scanned endpoints", () => {
239
- const params = baseParams({
240
- analysisScope: AnalysisScope.CurrentBranchDiff,
241
- scannedEndpoints: [{ path: "/openapi-only", methods: ["GET"], sourceFile: "" }],
242
- candidateRouteFiles: ["src/routes/items.ts"],
243
- });
244
- const output = buildAnalysisOutputText(params);
245
- expect(output).toContain("Scanner Fallback — Manual Endpoint Discovery Required");
246
- expect(output).toContain("may have missed route/controller files");
247
- expect(output).toContain("src/routes/items.ts");
248
- expect(output).toContain("supplemental gap check");
249
- expect(output).toContain("keep the existing catalog as the primary endpoint source");
250
- expect(output).not.toContain("use it as the authoritative endpoint list");
251
- });
252
- it("tells fallback discovery not to turn GraphQL artifacts into REST endpoints", () => {
253
- const params = baseParams({
254
- analysisScope: AnalysisScope.CurrentBranchDiff,
255
- scannedEndpoints: [],
256
- candidateRouteFiles: ["src/graphql/users.resolver.ts"],
257
- });
258
- const output = buildAnalysisOutputText(params);
259
- expect(output).toContain("GraphQL: schema/resolver artifacts are unsupported for REST test generation");
260
- expect(output).toContain("do not invent REST endpoints from them");
261
- expect(output).toContain("use it as the authoritative endpoint list");
262
- expect(output).not.toContain("Query/Mutation resolvers mapping to endpoints");
263
- });
264
- });
@@ -123,6 +123,9 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
123
123
  }
124
124
  }
125
125
  const fmtEndpoint = (m, ep) => ` ${m.method} ${ep.path}${m.authRequired ? " [auth]" : ""} (${(m.interactions ?? []).length} interactions)`;
126
+ // In diff scope, cap the reference endpoint list to prevent context overflow.
127
+ // Changed endpoints are always shown in full; only the "other" list is capped.
128
+ const DIFF_SCOPE_OTHER_ENDPOINT_CAP = 20;
126
129
  let endpointLines;
127
130
  if (isDiffScope && changedEndpointKeys.size > 0) {
128
131
  const changedLines = [];
@@ -146,7 +149,25 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
146
149
  changedLines.push(` ${m.method} ${ep.path} [removed]`);
147
150
  }
148
151
  }
149
- endpointLines = `**Likely changed in this PR (from static file→endpoint mapping — verify against diff in Step ${ANALYSIS_STEP_EXTRACT}):**\n${changedLines.join("\n") || " none"}\n\n**Other endpoints (reference only):**\n${otherLines.join("\n") || " none"}`;
152
+ const cappedOther = otherLines.slice(0, DIFF_SCOPE_OTHER_ENDPOINT_CAP);
153
+ const hiddenOtherCount = otherLines.length - cappedOther.length;
154
+ const otherSuffix = hiddenOtherCount > 0
155
+ ? `\n ... and ${hiddenOtherCount} more (full endpoint list in state file)`
156
+ : "";
157
+ const otherLabel = hiddenOtherCount > 0
158
+ ? `Other endpoints (reference only, ${cappedOther.length} of ${otherLines.length} shown)`
159
+ : "Other endpoints (reference only)";
160
+ endpointLines = `**Likely changed in this PR (from static file→endpoint mapping — verify against diff in Step ${ANALYSIS_STEP_EXTRACT}):**\n${changedLines.join("\n") || " none"}\n\n**${otherLabel}:**\n${cappedOther.join("\n") || " none"}${otherSuffix}`;
161
+ }
162
+ else if (isDiffScope) {
163
+ // Diff scope but no changed endpoints detected — cap to avoid dumping the full catalog.
164
+ const allMethodLines = allEndpoints.flatMap((ep) => (ep.methods ?? []).map((m) => fmtEndpoint(m, ep)));
165
+ const cappedLines = allMethodLines.slice(0, DIFF_SCOPE_OTHER_ENDPOINT_CAP);
166
+ const hiddenCount = allMethodLines.length - cappedLines.length;
167
+ const suffix = hiddenCount > 0
168
+ ? `\n ... and ${hiddenCount} more (full endpoint list in state file — trace changed files directly from the diff to find affected endpoints)`
169
+ : "";
170
+ endpointLines = `${cappedLines.join("\n") || " none"}${suffix}`;
150
171
  }
151
172
  else {
152
173
  endpointLines = allEndpoints
@@ -175,9 +196,22 @@ Write UI recommendation \`reasoning\` fields in **natural prose** that names ele
175
196
  }
176
197
  }
177
198
  const { authSchemeSnippet } = getAuthSnippets(authHeaderValue, authTypeValue, workspaceAuthScheme);
199
+ const routeDiscovery = analysis.routeDiscovery;
200
+ const routeDiscoverySection = routeDiscovery
201
+ ? `
202
+ ## LLM Route Discovery Inputs
203
+ Static endpoint data below is a best-effort hint, not a complete parser for every framework.
204
+ Authoritative endpoint extraction must come from reading the changed source files, router/module context, OpenAPI paths when available, and the diff.
205
+ Candidate files to inspect: ${routeDiscovery.candidateFiles.length > 0 ? routeDiscovery.candidateFiles.join(", ") : "none"}
206
+ Router/module context files: ${routeDiscovery.routerMountContext.length > 0 ? routeDiscovery.routerMountContext.join(", ") : "none"}
207
+ OpenAPI paths available: ${routeDiscovery.openApiPaths.length}
208
+ Static hints available: ${routeDiscovery.staticHints.length}
209
+ ${routeDiscovery.diffFilePath ? `Diff file: ${routeDiscovery.diffFilePath}` : ""}
210
+ `.trim()
211
+ : "";
178
212
  const sourcePriority = `
179
213
  ## Source Priority
180
- When information conflicts, prefer: **Traces** (actual behavior) > **Code** (implemented behavior) > **Spec/Docs** (documented behavior).
214
+ When information conflicts, prefer: **Traces** (actual behavior) > **Source code read by the LLM** (implemented behavior) > **OpenAPI spec/docs** (documented behavior) > **Static parser hints** (best-effort, may be incomplete or framework-blind).
181
215
  `;
182
216
  // Compact fingerprint of tests already covering endpoints in this repo (Skyramp + external).
183
217
  // Re-derived fresh each run from test files on disk — no separate persistence needed.
@@ -219,8 +253,11 @@ Framework: ${analysis.projectClassification.primaryFramework} (${analysis.projec
219
253
  Project type: ${analysis.projectClassification.projectType}
220
254
  Auth: ${authMethod} (header: ${authHeaderValue}${authTypeValue ? `, type: ${authTypeValue}` : ""})
221
255
  Base URL: ${analysis.apiEndpoints.baseUrl}
222
- Candidate endpoints from static scan — unverified, confirm paths against spec or source before use (${analysis.apiEndpoints.totalCount}):
256
+ Candidate endpoint hints from static scan — unverified and non-exhaustive; confirm paths by reading source/router context before use (${analysis.apiEndpoints.totalCount}):
223
257
  ${endpointLines}${testFingerprint}
258
+ ${routeDiscoverySection ? `
259
+
260
+ ${routeDiscoverySection}` : ""}
224
261
  `.trim();
225
262
  // ── Branch diff ──
226
263
  let diffSection = "";
@@ -240,7 +277,7 @@ Affected services: ${diffContext.affectedServices.join(", ") || "N/A"}
240
277
 
241
278
  Focus on tests that validate these changes and how they interact with existing resources.
242
279
  For removed endpoints: verify they now return 404 or the appropriate deprecation status code.
243
- Allocate your test budget to endpoints listed under "Likely changed in this PR". Use other endpoints only as setup steps (e.g. creating a resource before testing its deletion).
280
+ Treat the endpoint lists above as static hints. If source/diff inspection finds a different changed endpoint set, prefer the source-grounded set and use other endpoints only as setup steps.
244
281
  `;
245
282
  }
246
283
  // ── Interactions ──
@@ -6,7 +6,7 @@ jest.unstable_mockModule("@skyramp/skyramp", () => ({
6
6
  const { buildRecommendationPrompt, buildExternalCoverageSet, externalDedupKey } = await import("./test-recommendation-prompt.js");
7
7
  const { buildExecutionPlan } = await import("./diffExecutionPlan.js");
8
8
  const { PATH_PARAM_UUID_GUIDANCE, MAX_TESTS_TO_GENERATE, buildTestQualityCriteria, buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildFewShotExamples, buildVerificationChecklist, } = await import("./recommendationSections.js");
9
- import { AnalysisScope } from "../../types/RepositoryAnalysis.js";
9
+ import { AnalysisScope, repositoryAnalysisSchema } from "../../types/RepositoryAnalysis.js";
10
10
  // ---------------------------------------------------------------------------
11
11
  // Minimal fixtures
12
12
  // ---------------------------------------------------------------------------
@@ -1341,6 +1341,21 @@ describe("buildRecommendationPrompt — data-before-instructions ordering", () =
1341
1341
  expect(interactionsIdx).toBeLessThan(instructionsIdx);
1342
1342
  });
1343
1343
  });
1344
+ describe("buildRecommendationPrompt — LLM route discovery contract", () => {
1345
+ it("repositoryAnalysisSchema preserves routeDiscovery", () => {
1346
+ const analysis = minimalAnalysis({
1347
+ routeDiscovery: {
1348
+ candidateFiles: ["src/routes/items.ts"],
1349
+ staticHints: [{ path: "/api/items", methods: ["GET"], sourceFile: "src/routes/items.ts" }],
1350
+ openApiPaths: ["/api/items"],
1351
+ routerMountContext: ["src/app/router.ts"],
1352
+ diffFilePath: "/tmp/skyramp-diff-123.diff",
1353
+ },
1354
+ });
1355
+ const parsed = repositoryAnalysisSchema.parse(analysis);
1356
+ expect(parsed.routeDiscovery).toEqual(analysis.routeDiscovery);
1357
+ });
1358
+ });
1344
1359
  // ---------------------------------------------------------------------------
1345
1360
  // Tests — Few-shot examples
1346
1361
  // ---------------------------------------------------------------------------
@@ -17,10 +17,5 @@ export interface RelatedRepository {
17
17
  export declare function parseRelatedRepositories(raw: string | undefined): RelatedRepository[] | undefined;
18
18
  export declare function getTestbotPrompt(prTitle: string, prDescription: string, summaryOutputFile: string, repositoryPath: string, baseBranch?: string, maxRecommendations?: number, maxGenerate?: number, _maxCritical?: number, // Reserved — accepted for API compat but not yet wired into prompt
19
19
  prNumber?: number, userPrompt?: string, services?: Service[], uiCredentials?: string, testsRepoDir?: string, relatedRepositories?: RelatedRepository[], primaryRepo?: string): string;
20
- /**
21
- * Read services from .skyramp/workspace.yml. Returns empty array if
22
- * the workspace file doesn't exist or can't be parsed.
23
- */
24
- export declare function readWorkspaceServices(repositoryPath: string): Promise<Service[]>;
25
20
  export declare function buildWorkspaceRecoveryPrefix(repositoryPath: string): string;
26
21
  export declare function registerTestbotPrompt(server: McpServer): void;
@@ -5,7 +5,7 @@ import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PA
5
5
  import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
6
6
  import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
7
7
  import { resolveServiceDetailsRef } from "../../utils/utils.js";
8
- import { readWorkspaceConfigRaw } from "../../utils/workspaceAuth.js";
8
+ import { buildServiceContext, readWorkspaceServices, } from "../prompt-utils.js";
9
9
  // Cached at module-load — flags are process-wide and cannot change per call.
10
10
  const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
11
11
  const SERVICE_REFS = resolveServiceDetailsRef();
@@ -547,7 +547,9 @@ Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than o
547
547
  4. **Wait**: Do NOT proceed to test execution until steps 1–3 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
548
548
  Do not make any changes other than the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
549
549
 
550
- **Final execution (mandatory):** Do NOT call \`skyramp_execute_test\` until ALL maintenance edits AND ALL new test generation/enhancement are complete.
550
+ **Execution timing:**
551
+ - **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
552
+ - **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection.
551
553
  - Only report test results for files you actually ran.
552
554
  **Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.
553
555
 
@@ -592,38 +594,6 @@ ${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modular
592
594
  // removing stateOutputFile from the prompt schema. Remove the RUNNER_TEMP branch in
593
595
  // AnalysisStateManager.ts when this is done.
594
596
  }
595
- function escapeXml(value) {
596
- return value
597
- .replaceAll('&', '&amp;')
598
- .replaceAll('<', '&lt;')
599
- .replaceAll('>', '&gt;')
600
- .replaceAll('"', '&quot;')
601
- .replaceAll("'", '&apos;');
602
- }
603
- function buildServiceContext(services) {
604
- const blocks = services.map(svc => {
605
- const parts = [`<service name="${escapeXml(svc.serviceName)}">`];
606
- if (svc.language)
607
- parts.push(` <language>${escapeXml(svc.language)}</language>`);
608
- if (svc.framework)
609
- parts.push(` <framework>${escapeXml(svc.framework)}</framework>`);
610
- if (svc.api?.baseUrl)
611
- parts.push(` <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
612
- if (svc.testDirectory)
613
- parts.push(` <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
614
- parts.push('</service>');
615
- return parts.join('\n');
616
- });
617
- return `<services>\n${blocks.join('\n')}\n</services>`;
618
- }
619
- /**
620
- * Read services from .skyramp/workspace.yml. Returns empty array if
621
- * the workspace file doesn't exist or can't be parsed.
622
- */
623
- export async function readWorkspaceServices(repositoryPath) {
624
- const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
625
- return (rawConfig?.services ?? []);
626
- }
627
597
  export function buildWorkspaceRecoveryPrefix(repositoryPath) {
628
598
  return `IMPORTANT: The existing .skyramp/workspace.yml failed to parse or validate. Before proceeding with any tasks below, you MUST call skyramp_init_scan with workspacePath "${repositoryPath}" and force: true, then call skyramp_init_workspace with workspacePath "${repositoryPath}", the discovered services, scanToken, and force: true to regenerate the workspace file.\n\n`;
629
599
  }
@@ -2,7 +2,8 @@ import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
2
2
  import { logger } from "../utils/logger.js";
3
3
  import { AnalyticsService } from "../services/AnalyticsService.js";
4
4
  import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, } from "../prompts/test-recommendation/recommendationSections.js";
5
- import { getTestbotPrompt, readWorkspaceServices, parseRelatedRepositories, } from "../prompts/testbot/testbot-prompts.js";
5
+ import { getTestbotPrompt, parseRelatedRepositories, } from "../prompts/testbot/testbot-prompts.js";
6
+ import { readWorkspaceServices } from "../prompts/prompt-utils.js";
6
7
  export function registerTestbotResource(server) {
7
8
  logger.info("Registering testbot resource");
8
9
  // RFC 6570 {+rest} (reserved expansion) captures the entire query string
@@ -1,7 +1,7 @@
1
1
  import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
2
2
  export declare class AnalyticsService {
3
3
  static pushTestGenerationToolEvent(toolName: string, result: CallToolResult, params: Record<string, any>): Promise<void>;
4
- static pushMCPToolEvent(toolName: string, result: CallToolResult | undefined, params: Record<string, string>): Promise<void>;
4
+ static pushMCPToolEvent(toolName: string, result: CallToolResult | undefined, params: Record<string, any>): Promise<void>;
5
5
  /**
6
6
  * Track server crash events
7
7
  */
@@ -1,5 +1,6 @@
1
1
  import { TestExecutionResult, BatchExecutionResult, TestExecutionOptions, ProgressCallback } from "../types/TestExecution.js";
2
- export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.29";
2
+ import { EXECUTOR_DOCKER_IMAGE } from "../utils/versions.js";
3
+ export { EXECUTOR_DOCKER_IMAGE };
3
4
  export declare const PLAYWRIGHT_CONFIG_FILES: string[];
4
5
  export declare const EXCLUDED_MOUNT_ITEMS: string[];
5
6
  export declare const MOUNT_NULL_ITEMS: string[];