@skyramp/mcp 0.2.8 → 0.2.10-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/build/index.js +1 -8
  2. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +28 -1
  3. package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +1 -0
  4. package/build/prompts/personas.d.ts +2 -2
  5. package/build/prompts/personas.js +2 -2
  6. package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +3 -1
  7. package/build/prompts/sut-setup/modes/dockerComposePrompt.js +6 -1
  8. package/build/prompts/sut-setup/shared.d.ts +20 -0
  9. package/build/prompts/sut-setup/shared.js +67 -7
  10. package/build/prompts/test-maintenance/actionsInstructions.js +5 -1
  11. package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +15 -2
  12. package/build/prompts/test-maintenance/drift-analysis-prompt.js +63 -3
  13. package/build/prompts/test-maintenance/driftAnalysisSections.js +7 -25
  14. package/build/prompts/test-maintenance/driftAnalysisShared.d.ts +20 -0
  15. package/build/prompts/test-maintenance/driftAnalysisShared.js +80 -0
  16. package/build/prompts/test-maintenance/uiDriftAnalysisSections.d.ts +32 -0
  17. package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +212 -0
  18. package/build/prompts/test-recommendation/recommendationShared.js +1 -1
  19. package/build/prompts/test-recommendation/scopeAssessment.js +12 -2
  20. package/build/prompts/testbot/testbot-prompts.js +10 -9
  21. package/build/resources/testbotResource.js +1 -1
  22. package/build/services/TestDiscoveryService.d.ts +12 -10
  23. package/build/services/TestDiscoveryService.js +137 -83
  24. package/build/services/TestDiscoveryService.test.js +370 -15
  25. package/build/services/TestExecutionService.d.ts +1 -1
  26. package/build/services/TestExecutionService.js +7 -7
  27. package/build/services/TestExecutionService.test.js +4 -1
  28. package/build/tools/executeSkyrampTestTool.js +40 -71
  29. package/build/tools/submitReportTool.d.ts +25 -25
  30. package/build/tools/submitReportTool.js +123 -34
  31. package/build/tools/submitReportTool.test.js +413 -94
  32. package/build/tools/test-management/actionsTool.js +177 -77
  33. package/build/tools/test-management/actionsTool.test.d.ts +1 -0
  34. package/build/tools/test-management/actionsTool.test.js +297 -0
  35. package/build/tools/test-management/analyzeChangesTool.d.ts +9 -0
  36. package/build/tools/test-management/analyzeChangesTool.js +74 -169
  37. package/build/tools/test-management/analyzeChangesTool.test.js +34 -3
  38. package/build/tools/test-management/analyzeTestHealthTool.js +63 -22
  39. package/build/tools/test-management/analyzeTestHealthTool.test.d.ts +1 -0
  40. package/build/tools/test-management/analyzeTestHealthTool.test.js +268 -0
  41. package/build/tools/workspace/initializeWorkspaceTool.js +37 -24
  42. package/build/tools/workspace/initializeWorkspaceTool.test.js +9 -4
  43. package/build/types/FrontendIntegration.d.ts +25 -0
  44. package/build/types/FrontendIntegration.js +19 -0
  45. package/build/types/RepositoryAnalysis.d.ts +8 -8
  46. package/build/types/TestAnalysis.d.ts +18 -49
  47. package/build/types/TestAnalysis.js +0 -28
  48. package/build/types/TestTypes.d.ts +3 -1
  49. package/build/types/TestTypes.js +3 -1
  50. package/build/types/TestbotReport.d.ts +1 -1
  51. package/build/utils/AnalysisStateManager.d.ts +16 -22
  52. package/build/utils/docker.test.js +1 -1
  53. package/build/utils/featureFlags.d.ts +1 -1
  54. package/build/utils/featureFlags.js +1 -1
  55. package/build/utils/frontendIntegration.d.ts +9 -0
  56. package/build/utils/frontendIntegration.js +237 -0
  57. package/build/utils/frontendIntegration.test.d.ts +1 -0
  58. package/build/utils/frontendIntegration.test.js +229 -0
  59. package/build/utils/pr-comment-parser.d.ts +3 -3
  60. package/build/utils/pr-comment-parser.js +6 -6
  61. package/build/utils/pr-comment-parser.test.js +3 -3
  62. package/build/utils/repoScanner.d.ts +7 -0
  63. package/build/utils/repoScanner.js +14 -6
  64. package/build/utils/repoScanner.test.js +9 -9
  65. package/build/utils/routeParsers.d.ts +32 -0
  66. package/build/utils/routeParsers.js +205 -2
  67. package/build/utils/routeParsers.test.js +279 -29
  68. package/build/utils/sourceRouteExtractor.js +174 -5
  69. package/build/utils/sourceRouteExtractor.test.js +173 -0
  70. package/build/utils/utils.d.ts +11 -0
  71. package/build/utils/utils.js +19 -0
  72. package/build/utils/utils.test.js +23 -1
  73. package/build/utils/versions.d.ts +3 -3
  74. package/build/utils/versions.js +1 -1
  75. package/build/workspace/workspace.d.ts +21 -21
  76. package/build/workspace/workspace.js +7 -4
  77. package/build/workspace/workspace.test.js +65 -6
  78. package/node_modules/playwright/lib/mcp/browser/tools/files.js +8 -3
  79. package/node_modules/playwright/lib/mcp/browser/tools/wait.js +1 -1
  80. package/node_modules/playwright/lib/mcp/skyramp/exportTool.js +10 -1
  81. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +33 -9
  82. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +7 -1
  83. package/node_modules/playwright/lib/mcp/test/skyRampExport.test.js +44 -0
  84. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.B_7ywgmr.js → index.d4gkVSou.js} +1 -1
  85. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
  86. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  87. package/node_modules/playwright/package.json +1 -1
  88. package/package.json +3 -3
package/build/index.js CHANGED
@@ -59,13 +59,6 @@ const server = new McpServer({
59
59
  },
60
60
  instructions: `Skyramp MCP Server — generates and executes API tests (fuzz, contract, integration, E2E, UI).
61
61
 
62
- ## Workspace Initialization (REQUIRED before the first Skyramp tool call)
63
- If the workspace root is a git repo AND \`.skyramp/workspace.yml\` does not exist:
64
- 1. Call \`skyramp_init_scan\` with \`workspacePath\` → follow the returned instructions to discover all services.
65
- 2. Call \`skyramp_init_workspace\` with \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
66
- 3. Proceed with the originally requested tool.
67
- Skip only if: not a git repo, \`.skyramp/workspace.yml\` already exists, or user explicitly declines.
68
-
69
62
  ## Rules
70
63
  - NEVER show CLI commands. NEVER attempt to install or configure the Skyramp CLI. ALWAYS use the MCP tools provided.
71
64
  - For UI and E2E tests, there are TWO recording modes:
@@ -103,7 +96,7 @@ Before calling ANY test generation tool, you MUST follow this flow:
103
96
  4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
104
97
  5. **CRITICAL — scenario generation**: When calling \`skyramp_batch_scenario_test_generation\`, ALWAYS pass:
105
98
  - \`baseURL\`: The full base URL from \`api.baseUrl\` (e.g., \`http://localhost:3000\`). This determines the scheme, host, and port in the generated trace. Without it, the trace defaults to https:443 which is almost always wrong for local development.
106
- - \`authHeader\`: Which HTTP header carries the auth credential. Get it from \`api.authHeader\` in workspace config. Examples: \`Authorization\` (Bearer/Token auth), \`X-Api-Key\` (API key auth), \`Cookie\` (session/cookie auth like NextAuth). Pass \`""\` to skip auth entirely (unauthenticated endpoints or \`api.authType: "none"\`).
99
+ - \`authHeader\`: Which HTTP header carries the auth credential. Get it from \`api.authHeader\` in workspace config. Examples: \`Authorization\` (Bearer/Token auth), \`X-Api-Key\` (API key auth), \`Cookie\` (session/cookie auth like NextAuth). Pass \`""\` to document the endpoint as unauthenticated in the generated test's assertions (unauthenticated endpoints or \`api.authType: "none"\`).
107
100
  - \`authScheme\`: Only when \`authHeader\` is \`Authorization\`. The prefix before the token (e.g., \`"Bearer"\` → \`Authorization: Bearer <token>\`). **Derive from**: (1) OpenAPI spec \`securitySchemes\`/\`securityDefinitions\`, (2) source code auth middleware, (3) workspace \`api.authType\`. **Do NOT guess.**
108
101
  - \`authToken\`: The full header value, used verbatim. When omitted, \`SKYRAMP_PLACEHOLDER_TOKEN\` is auto-generated. Only provide when the header needs a specific format (e.g., \`"session=${AUTH_PLACEHOLDER_TOKEN}"\` for Cookie). **Do NOT fabricate token values.**
109
102
  - \`apiSchema\` is OPTIONAL — omit it for code-first apps without OpenAPI specs.
@@ -4,6 +4,33 @@ const UI_ASSERTION_CATEGORIES = [
4
4
  {
5
5
  name: "Critical UI Assertions",
6
6
  rules: [
7
+ {
8
+ title: "Exercise the changed interaction (dynamic behavior)",
9
+ description: "When the PR introduces or changes interactive behavior (toggle, theme switch, expand/collapse, dialog open/close, focus management, optimistic update), the test MUST perform the triggering action and assert BOTH the state before AND after the action — not only the default or resting state. WHY: the PR's change IS the transition; a test that asserts only the initial/resting state would still pass even if the interaction were completely broken, so it gives zero coverage of what changed. If the test asserts only the resting state, add the action that triggers the change and assert the resulting transition — never mark this not applicable merely because the generated test lacks an action.",
10
+ subPoints: [
11
+ "Toggle / switch: click the control and assert the observable flips (e.g. data-theme 'light' -> 'dark'), then click again and assert it reverses.",
12
+ "Dialog / modal: open it and assert focus moves into the dialog; close via ESC or Cancel and assert focus returns to the trigger.",
13
+ "The after-state must differ from the before-state — a single assertion of the unchanged default value does not verify the change.",
14
+ ],
15
+ examples: [
16
+ {
17
+ language: "javascript",
18
+ code: `await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
19
+ await page.getByTestId('theme-toggle').click();
20
+ await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');`,
21
+ },
22
+ {
23
+ // Python (playwright-python) — same before/after transition pattern
24
+ language: "python",
25
+ code: `trigger = page.get_by_test_id('delete-recipe')
26
+ trigger.click()
27
+ expect(page.get_by_role('dialog')).to_be_focused()
28
+ page.keyboard.press('Escape')
29
+ # focus returns to the trigger
30
+ expect(trigger).to_be_focused()`,
31
+ },
32
+ ],
33
+ },
7
34
  {
8
35
  title: "Selector constraints",
9
36
  description: "Every assertion uses a selector already in the file. Never invent `data-testid`, role names, or classes.",
@@ -240,7 +267,7 @@ Before editing the given test file, you must output a \`<thinking>\` block. The
240
267
  3. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
241
268
  - Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
242
269
  - Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
243
- - Post-edit State applies only when the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit); otherwise NOT APPLICABLE.
270
+ - Post-edit State applies whenever EITHER (a) the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), OR (b) the PR changes interactive behavior — in case (b), add the triggering action and assert the before/after transition (see "Exercise the changed interaction"). Mark NOT APPLICABLE only when the PR changes no interactive behavior and the test has no state-changing action.
244
271
  4. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
245
272
  - \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
246
273
  - \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is an array of assertion lines you will add for this item under that rule. Use \`[]\` only when the rule does not apply to this item — every category key and every rule key must still be present. This forces you to consider every rule for every item.
@@ -97,6 +97,7 @@ Create one service entry per deployable unit. You MUST include every backend/API
97
97
  <api_fields>
98
98
  1. api.schemaPath: Path or URL to an OpenAPI or Swagger schema. Search for openapi.json, openapi.yaml, swagger.json, or swagger.yaml files. Framework defaults are: FastAPI serves /openapi.json, Express serves /api-docs, and Spring serves /v3/api-docs. For locally-run services, use a localhost URL. For cloud or externally hosted services (such as Salesforce, Vercel, or Cloudflare), use the actual deployment URL found in config or documentation.
99
99
  2. api.baseUrl (required): The base URL where the service is reachable, such as "http://localhost:3000" or "https://api.example.com". Derive from docker-compose ports, app config, README, or environment variables. Use localhost for services run locally and the actual deployment URL for cloud or externally hosted services. NEVER fabricate a URL. Only use URLs found in config files, README, or environment variables.
100
+ Include the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) when one applies — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.
100
101
  3. api.authType (required): The authentication type. Valid values are: ${AUTH_TYPES_PROMPT_LIST}
101
102
  Detect by checking in the following order (language-agnostic, apply whichever signals match):
102
103
  a. Dependencies and packages (package.json, requirements.txt, go.mod, Gemfile, composer.json, pom.xml, build.gradle):
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Skyramp personas injected into tool descriptions and prompts.
3
3
  *
4
- * In TestBot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
4
+ * In Testbot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
5
5
  * once via `claude --append-system-prompt` from the testbot GitHub Action
6
6
  * (see testbot.git `src/agents/claude.ts` `loadQaPersona()`) rather than
7
7
  * repeating it in every tool description. In that case getPersonaPrefix()
@@ -14,7 +14,7 @@
14
14
  export declare const SKYRAMP_QA_PERSONA = "You are acting as a Skyramp QA Automation Engineer. Your responsibility is to translate user test intent into precise, deterministic test artifacts \u2014 whether generating API tests from specs, recording browser interactions for UI flows, or maintaining existing test suites. Derive all parameters strictly from the codebase, workspace config, API schemas, and page snapshots. Never guess or hallucinate values.";
15
15
  /**
16
16
  * Returns the persona prefix for use in tool descriptions.
17
- * Returns an empty string when running inside TestBot — the testbot action
17
+ * Returns an empty string when running inside Testbot — the testbot action
18
18
  * appends the persona via `claude --append-system-prompt` instead, so we
19
19
  * avoid duplicating it in every tool description.
20
20
  */
@@ -2,7 +2,7 @@ import { isTestbotEnabled } from "../utils/featureFlags.js";
2
2
  /**
3
3
  * Skyramp personas injected into tool descriptions and prompts.
4
4
  *
5
- * In TestBot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
5
+ * In Testbot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
6
6
  * once via `claude --append-system-prompt` from the testbot GitHub Action
7
7
  * (see testbot.git `src/agents/claude.ts` `loadQaPersona()`) rather than
8
8
  * repeating it in every tool description. In that case getPersonaPrefix()
@@ -15,7 +15,7 @@ import { isTestbotEnabled } from "../utils/featureFlags.js";
15
15
  export const SKYRAMP_QA_PERSONA = `You are acting as a Skyramp QA Automation Engineer. Your responsibility is to translate user test intent into precise, deterministic test artifacts — whether generating API tests from specs, recording browser interactions for UI flows, or maintaining existing test suites. Derive all parameters strictly from the codebase, workspace config, API schemas, and page snapshots. Never guess or hallucinate values.`;
16
16
  /**
17
17
  * Returns the persona prefix for use in tool descriptions.
18
- * Returns an empty string when running inside TestBot — the testbot action
18
+ * Returns an empty string when running inside Testbot — the testbot action
19
19
  * appends the persona via `claude --append-system-prompt` instead, so we
20
20
  * avoid duplicating it in every tool description.
21
21
  */
@@ -1,4 +1,4 @@
1
- import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSutDefinitionBlock, } from "../shared.js";
1
+ import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildInputsBody, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
2
2
  import { getPersonaPrefix } from "../../personas.js";
3
3
  import { PromptPlan } from "../../test-recommendation/promptPlan.js";
4
4
  // ── Step body builders ────────────────────────────────────────────────────────
@@ -70,6 +70,7 @@ function buildAdaptRestrictionsBody() {
70
70
  // ── PromptPlan declaration ────────────────────────────────────────────────────
71
71
  const _adaptPlan = new PromptPlan()
72
72
  .addPhase("adapt", "", { headerLevel: "hidden", stepFormat: "hash" })
73
+ .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
73
74
  .step("SCAN", "Understand the System Under Test in this repository", () => "")
74
75
  .subStep("READ_SOURCES", "Read workflows and infra files", buildReadSourcesBody)
75
76
  .subStep("CLASSIFY_STEPS", "Classify source workflow steps", buildClassifyStepsBody)
@@ -80,6 +81,7 @@ const _adaptPlan = new PromptPlan()
80
81
  .subStep("PATTERN_C", "Pattern C — GHA steps replace service startup + Testbot lifecycle validation", buildPatternCBody)
81
82
  .subStep("PATTERN_D", "Pattern D — Fully wrapped by GHA steps", buildPatternDBody)
82
83
  .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
84
+ .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
83
85
  .step("ADAPT", "Edit the Testbot workflow and generate SUT files", () => "")
84
86
  .subStep("EDIT_WORKFLOW", "Edit the workflow file", buildEditWorkflowBody)
85
87
  .subStep("CONFIGURE_AUTH", "Configure auth", buildConfigureAuthBody)
@@ -1,4 +1,4 @@
1
- import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSutDefinitionBlock, } from "../shared.js";
1
+ import { buildContextBlock, buildInputsBody, buildCommonSutErrorsSection, buildLocalValidationSection, buildSeedDataDiscoverySection, buildSutDefinitionBlock, buildWorkspaceInitSection, } from "../shared.js";
2
2
  import { getPersonaPrefix } from "../../personas.js";
3
3
  import { PromptPlan } from "../../test-recommendation/promptPlan.js";
4
4
  // ── Step body builders ────────────────────────────────────────────────────────
@@ -93,6 +93,9 @@ Steps:
93
93
  - The \`--project-directory .\` means build context is the repository root (NOT .skyramp/sut/)
94
94
  - Healthchecks for ALL services (testbot relies on \`--wait\`)
95
95
  - Simple testbot credentials in environment (admin@testbot.com / testbot)
96
+ - Never paste cryptographic material into the \`environment:\` section of the compose file. This rule covers RSA and EC private keys, PEM blobs, JWK JSON, OAuth client secrets, and any other signing keypair. In practice, refuse anything that looks like a PEM block (text starting with \`-----BEGIN \` followed by \`PRIVATE KEY\`, \`RSA PRIVATE KEY\`, \`EC PRIVATE KEY\`, \`OPENSSH PRIVATE KEY\`, or \`ENCRYPTED PRIVATE KEY\` and the matching \`-----\`), a JWK in base64url-encoded JSON form (strings starting with \`eyJrdHkiOi\`, which decodes to \`{"kty":\`), or any base64 string longer than about 200 characters that looks like a key. The compose file is committed to the customer's repository, so anything written here lands in their PR diff and stays in git history — including the private half of any keypair. Two cases to handle:
97
+ 1. **Symmetric secrets** — when the app accepts a plain string (\`JWT_SECRET\`, \`SESSION_SECRET\`, signing salts, HMAC keys), use a clearly-marked dummy literal so a code reviewer immediately recognizes it as fake. For example: \`JWT_SECRET: "testbot-dummy-do-not-use-in-production"\`.
98
+ 2. **Asymmetric keys** — when the app requires a real keypair (RS256 JWT signing, gRPC TLS, signing keypairs), do not commit the keys. Instead, generate them inside the container at startup so they exist only for the container's lifetime. First look at the app's source or config to determine what format it expects: PEM, JWK, or raw base64. Then override the service's \`entrypoint\` with a short script that generates a keypair in that format, exports the key bodies into the env vars the app reads, and finally \`exec\`s the app's original command. Pick the tool that matches the format: \`openssl genpkey\` for PEM, \`python -m jwcrypto\` / \`node -e "require('jose')..."\` / \`step crypto jwk create\` for JWK, \`openssl rand -base64\` for a raw secret. Prefer a tool that already exists in the app's runtime image — for instance, a Django service usually has \`jwcrypto\` or \`josepy\` installed already — rather than installing one at startup. Testbot fetches its bearer token via \`get-auth-token.sh\`, which calls the app's login endpoint; the app signs the token with its in-container private key and returns it as an opaque string. The test client never reads the private key directly, so per-container ephemeral keys are enough — no GitHub repo secrets, no out-of-band key sharing.
96
99
  - Generous \`start_period\` (30-120s) for CI cold builds
97
100
  - For monorepos: may need a builder service that runs first
98
101
 
@@ -260,9 +263,11 @@ ${GENERATE_DOCKER_COMPOSE_INSTRUCTIONS}`;
260
263
  // ── PromptPlan declaration ────────────────────────────────────────────────────
261
264
  const _composePlan = new PromptPlan()
262
265
  .addPhase("compose", "", { headerLevel: "hidden", stepFormat: "hash" })
266
+ .step("INIT_WORKSPACE", "Initialise the Skyramp workspace file", () => buildWorkspaceInitSection())
263
267
  .step("SCAN", "Understand the System Under Test in this repository", buildScanComposeBody)
264
268
  .step("DECIDE", "Decide SUT setup strategy", buildDecideStrategyBody)
265
269
  .step("INPUTS", "Plan the Testbot action inputs", () => buildInputsBody())
270
+ .step("SEED_DATA", "Discover database seed-data conventions", () => buildSeedDataDiscoverySection())
266
271
  .step("BUILD", "Edit the Testbot workflow and generate SUT files", () => "")
267
272
  .subStep("PR_ACCURATE", "Build PR-accurate SUT", buildPrAccurateSutSection)
268
273
  .subStep("APPLY", "Apply the chosen strategy", buildApplyStrategyBody)
@@ -21,6 +21,26 @@ export interface SutPromptArgs {
21
21
  */
22
22
  export declare function buildSutDefinitionBlock(): string;
23
23
  export declare function buildContextBlock(args: SutPromptArgs): string;
24
+ /**
25
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
26
+ * looks for existing customer seed conventions BEFORE inventing new ones.
27
+ * A fresh database makes list / search / filter / pagination endpoint tests
28
+ * shallow (only HTTP-status assertable, never response content), which
29
+ * collapses the data-diversity dimension of the eval. Most repos already
30
+ * have a seed mechanism — wire that in instead of generating one.
31
+ *
32
+ * Rendered as a top-level PromptPlan step between INPUTS and the
33
+ * ADAPT/BUILD step in every SUT setup mode.
34
+ */
35
+ export declare function buildSeedDataDiscoverySection(): string;
36
+ /**
37
+ * Workspace file initialisation block — rendered as the first instruction
38
+ * in every SUT setup mode, before any local validation, workflow editing,
39
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
40
+ * inventory so testbot's SUT autocommit picks it up alongside the
41
+ * generated `.skyramp/sut/*` files.
42
+ */
43
+ export declare function buildWorkspaceInitSection(): string;
24
44
  export declare function buildLocalValidationSection(): string;
25
45
  /**
26
46
  * Returns the canonical "add to the Testbot workflow" block: every input that
@@ -31,6 +31,57 @@ export function buildContextBlock(args) {
31
31
  lines.push(`</context>`);
32
32
  return lines.join("\n");
33
33
  }
34
+ /**
35
+ * Seed-data discovery block — rendered as a SUT-bootstrap step so the agent
36
+ * looks for existing customer seed conventions BEFORE inventing new ones.
37
+ * A fresh database makes list / search / filter / pagination endpoint tests
38
+ * shallow (only HTTP-status assertable, never response content), which
39
+ * collapses the data-diversity dimension of the eval. Most repos already
40
+ * have a seed mechanism — wire that in instead of generating one.
41
+ *
42
+ * Rendered as a top-level PromptPlan step between INPUTS and the
43
+ * ADAPT/BUILD step in every SUT setup mode.
44
+ */
45
+ export function buildSeedDataDiscoverySection() {
46
+ return `**First, decide if this step applies.** Skip it entirely (in your summary state \`SEED_DATA: N/A — <reason>\`, naming the specific bullet below that applied — e.g. "no stateful service", "no list endpoints in scope", "already seeded by init container") and move on when ANY of the following is true:
47
+ - The SUT has no database / stateful service (it's a pure stateless API, a static asset server, a webhook receiver, etc.).
48
+ - The compose stack contains no database container (no \`postgres\`, \`mysql\`, \`mariadb\`, \`mongodb\`, \`redis\` as the primary store, \`sqlite\`, etc.) and no application-managed persistent store.
49
+ - The PR diff and the existing test surface focus exclusively on auth, single-resource CRUD (POST/GET-one/PUT/DELETE on \`/foo/{id}\`), or compute endpoints — no list / search / filter / pagination / aggregate endpoints will be exercised.
50
+ - You can confirm from the SCAN step's findings that representative records already exist (e.g. seeded by an init container, by a migration, or by the auth/setup flow you already wired).
51
+
52
+ Only proceed with the rest of this step when the SUT has a database AND the tests will exercise list/search/filter/pagination endpoints. Otherwise an empty database is acceptable and inventing seed data adds noise.
53
+
54
+ When this step DOES apply: database services usually start empty in CI, so tests against list / search / filter / pagination endpoints will then only see \`[]\` and can assert nothing beyond HTTP status — a low data-diversity outcome. Look for an existing seeding mechanism in the repo and wire it into the SUT bootstrap.
55
+
56
+ **Discover existing project conventions first** — most repos already have one of:
57
+ 1. A named script: \`scripts/seed*\`, \`db/seeds/*\`, \`prisma/seed.{ts,js}\`, \`apps/api/seed/*\`, \`tests/fixtures/seed*\`. Search: \`find . -iname '*seed*' -not -path '*/node_modules/*' -not -path '*/.git/*'\`.
58
+ 2. A package.json / Makefile target: \`npm run seed\`, \`npm run db:seed\`, \`pnpm seed\`, \`yarn seed\`, \`make seed\`, \`make fixtures\`. Check \`package.json\` \`scripts\` and the project's \`Makefile\`.
59
+ 3. A migration with sample data: Django \`loaddata\`, Rails \`db:seed\`, Knex \`seed:run\`, Prisma \`db seed\`, Alembic data migrations.
60
+ 4. A docker-compose init service: existing compose files sometimes have a \`*-seed\`, \`*-fixtures\`, or \`init-*\` service that runs once on bring-up.
61
+
62
+ **Wire it in:**
63
+ - **Compose init service** — add it (or its dependency) to the testbot compose so \`targetSetupCommand\` brings it up alongside the rest. Use \`depends_on\` with \`condition: service_completed_successfully\` on dependent services so they only start after seeding finishes.
64
+ - **Script needing an admin/API token** (most common) — chain it INTO \`authTokenCommand\` AFTER the token is created and BEFORE the script prints the token to stdout. Route seed stdout/stderr to stderr so the captured token isn't corrupted. Make the seed call best-effort (\`|| echo "seed failed" >&2\`) so a flaky seed doesn't block the run — the token is still returned and PRs whose tests don't need seed data keep working.
65
+ - **Migration with sample data** — ensure the migration runs as part of \`targetSetupCommand\` (some ORMs auto-run on container start, others need an explicit \`migrate\` / \`db:seed\` step before tests can hit the API).
66
+
67
+ **If NO existing convention is found**, generate a minimal seed step (≥3 entities per primary resource the tests will query, e.g., ≥3 repos / ≥3 issues / ≥3 users for a forge; ≥3 products / ≥3 orders for a shop). Call this out explicitly in your summary so the user knows a richer fixture may be needed for production-quality tests.
68
+
69
+ **Never** hard-code production data, real customer credentials, or PII in seeds. Seed identifiers should be obviously test-only (\`acme-co\`, \`alpha-svc\`, \`testbot-admin\`).`;
70
+ }
71
+ /**
72
+ * Workspace file initialisation block — rendered as the first instruction
73
+ * in every SUT setup mode, before any local validation, workflow editing,
74
+ * or SUT-file generation. Seeds `.skyramp/workspace.yml` with the service
75
+ * inventory so testbot's SUT autocommit picks it up alongside the
76
+ * generated `.skyramp/sut/*` files.
77
+ */
78
+ export function buildWorkspaceInitSection() {
79
+ return `Before generating any SUT setup files, check whether \`.skyramp/workspace.yml\` exists at the root of the repository being bootstrapped (the path shown next to "Repository path:" in the context block above — the repo whose PR triggered Testbot; in multi-repo runs this is the primary repo, not the sibling repos checked out as related repos). If it doesn't:
80
+ 1. Call \`skyramp_init_scan\` with \`workspacePath\` set to that repository root → follow the returned instructions to discover all services.
81
+ 2. Call \`skyramp_init_workspace\` with the same \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
82
+
83
+ Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside the workspace file. If you need to update per-service \`runtimeDetails.serverStartCommand\` to match the new SUT files, call \`skyramp_init_workspace\` again with \`force: true\` and the full updated services array — never use Edit/Write directly on \`.skyramp/workspace.yml\`. See the \`skyramp_init_workspace\` tool description for the edit-mode contract.`;
84
+ }
34
85
  export function buildLocalValidationSection() {
35
86
  return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
36
87
  Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
@@ -39,7 +90,7 @@ Run commands one at a time in your shell — individual execution pinpoints fail
39
90
  b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
40
91
  c. Validate any standalone Dockerfile referenced by GHA \`steps:\` (not by a compose service) with \`docker build\`. Do NOT also run \`docker build\` on Dockerfiles that belong to a compose service — \`docker compose build\` (next step) already builds them in one BuildKit graph; doubling up wastes ~5-10 GB of cache per service.
41
92
  d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
42
- e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, run \`docker builder prune -af -f && docker image prune -af\` before the next build. This frees BuildKit cache + dangling images without losing the running stack. Skipping this on heavy SUTs (medusa, calcom, mattermost, supabase) exhausts the ~14 GB \`ubuntu-latest\` disk mid-validation.
93
+ e. **Disk hygiene between heavy builds.** Between any two build commands (Dockerfile, compose, helper scripts that build), if the runner is under ~5 GB free or you see \`no space left on device\`, prune docker BuildKit cache and dangling images before the next build (keep the running stack intact). Skipping this on heavy SUTs exhausts the standard runner disk mid-validation.
43
94
  2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
44
95
  a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
45
96
  i. Run \`targetSetupCommand\` — confirm exit 0.
@@ -68,8 +119,16 @@ Target lifecycle inputs (control how Testbot starts, validates, and stops the SU
68
119
  b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
69
120
  c. With a dedicated compose file (reused or generated), point at it explicitly: \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . up -d --build\`.
70
121
  d. Override whenever the source workflow's start sequence differs from the default.
71
- 2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
72
- a. **CRITICAL: Always set this to a valid target readiness check command.** Required in every pattern: when \`targetSetupCommand\` is explicit, when relying on the \`docker compose up -d\` default, AND when surrounding GHA steps bring the SUT up before the Testbot action (\`skipTargetSetup: 'true'\`). Testbot's fail-early gate throws \`Unable to verify service readiness\` when no readiness signal is available (neither this nor a workspace service \`baseUrl\`) and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` without it Testbot can't tell when the service is actually listening.
122
+ 2. \`targetReadyCheckCommand\` — polling command Testbot runs repeatedly after setup until it exits 0 (service ready) or the timeout fires.
123
+ a. **CRITICAL: must cover EVERY service the tests will hit, not just the backend.** Chain checks with \`&&\` so the command exits 0 only when ALL services respond. A single-service check on a multi-service SUT is the leading cause of "element not visible" / "heading not visible" UI test failures Playwright lands on a partially-rendered page because only the API was up when the gate passed. Inventory of services to include:
124
+ - Every API service in workspace.yml with an \`api.baseUrl\` → curl an unauthenticated endpoint (health / version / status).
125
+ - Every UI service with a UI \`baseUrl\` (or a frontend port in compose) → curl \`/\` or a known asset like \`/assets/index.js\` (more reliable than \`/\` for SPAs that hydrate lazily).
126
+ - Skip auth-required endpoints — they 401 before the SUT is testable and mask the readiness signal.
127
+ Examples:
128
+ - API only: \`curl -sf http://localhost:8080/api/health\`
129
+ - API + UI: \`curl -sf http://localhost:8080/api/health && curl -sf http://localhost:3000/\`
130
+ Required in every pattern (explicit \`targetSetupCommand\`, the \`docker compose up -d\` default, AND \`skipTargetSetup: 'true'\`). Without a real readiness signal (neither this nor a workspace service \`baseUrl\`), testbot's fail-early gate throws \`Unable to verify service readiness\` and the action exits non-zero. It is also what gates a backgrounded \`targetSetupCommand\` — without it Testbot can't tell when the service is actually listening.
131
+ b. Set \`targetReadyCheckDiagnosticsCommand\` to a command that explains why the SUT isn't healthy — output is surfaced in the failure PR comment when the ready check times out. For docker-compose SUTs, scope it to the project's compose file (container state + recent service logs). For non-docker SUTs, a tail of the server log.
73
132
  3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
74
133
  a. Default: \`1800\` (30 min) — sized for cold \`docker compose --build\` runs on multi-service stacks. The loop exits on the first success, so fast services pay no extra cost. Override only if you want to fail faster on stuck setups (e.g. \`'300'\` for a small single-service SUT).
75
134
  4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`, or \`docker compose -f .skyramp/sut/docker-compose.testbot.yml --project-directory . down -v\` for a dedicated compose file).
@@ -89,19 +148,20 @@ Auth inputs (used by Testbot to authenticate against the running SUT during test
89
148
  b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.
90
149
  Workflow / job level settings (set on the workflow or job, not as \`skyramp/testbot\` action inputs):
91
150
  8. Secrets / environment variables — pass every secret/env var the setup needs into the workflow via an \`env:\` block or \`secrets: inherit\`, and add any dependency-resolution steps/jobs the SUT requires.
92
- 9. \`runs-on\` runner label — switch to a larger runner when the SUT needs more disk or memory: heavy Docker builds, large dependency installs, or a multi-service (3+) compose stack. Standard runners run out of disk and crash with \`No space left on device\`. Runner names vary by org do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses (never downgrade to \`ubuntu-latest\`), (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`.
151
+ 9. \`runs-on\` runner label — leave whatever value the Testbot installer wrote and the source workflow used. Don't override it. Picking a runner SKU is an account-level decision that depends on provisioning the agent can't see from inside the repo; a wrong choice queues the job forever with no runner ever picking it up. If the SUT genuinely doesn't fit on the chosen runner, the disk-pressure mitigation in "Common failure modes" #9 handles it; an unprovisioned-runner stall does not.
93
152
 
94
153
  #### What to do
95
154
  Edit \`${TESTBOT_WORKFLOW_PATH}\` in place (already created by the Testbot installer) — do NOT recreate it. Add only what the SUT needs onto the existing \`skyramp/testbot\` action step, alongside the inputs the installer already added:
96
155
  1. The 4 target lifecycle inputs — \`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetReadyCheckTimeout\`, \`targetTeardownCommand\`.
97
156
  2. The 2 auth inputs when the SUT needs them — \`authTokenCommand\` and/or \`uiCredentials\`.
98
157
  3. Any secrets / environment variables and dependency-resolution steps/jobs the setup needs (input 8).
99
- 4. The \`runs-on\` runner label, if a larger runner is needed (input 9).
158
+ 4. Leave the existing \`runs-on\` value untouched (input 9).
100
159
  5. Use a daemon/detached form for \`targetSetupCommand\` — \`docker compose up -d --build\`, \`nohup npm start &\`, \`./server &\`, \`pnpm --prefix ./apps/studio start &\`. The command should launch the service and exit; \`targetReadyCheckCommand\` is what gates readiness. Testbot backgrounds foreground-blocking commands (\`npm start\`, \`python app.py\`, \`./server\`) automatically after a 5-minute timeout as a safety net, so they will eventually work — but daemon mode is faster (no 5-minute wait) and surfaces startup crashes clearly via the command's exit code. If no daemon form is available and the repo has no \`docker-compose.yml\`, wrap the foreground command in compose (a \`Dockerfile\` + \`docker-compose.yml\` running \`npm start\`) so it can be launched with \`docker compose up -d --build\`. Only fall back to a GHA pre-step with \`&\` (e.g. \`run: npm start &\`) if Docker is not feasible.
160
+ 6. Stack-specific testability flags — some frameworks gate test-harness affordances behind a container env var. Common cases: Flutter web needs \`IS_TESTING=true\` so \`ensureSemantics()\` exposes the widget tree to Playwright (without it every selector finds nothing even though the app is up); some React/Angular builds need \`NODE_ENV\` / framework-specific build flags for \`testID\` propagation. Set these in the compose \`environment:\` block on the affected service (or as a \`Dockerfile ENV\`) — NOT as a shell var in \`targetSetupCommand\`. The var must be in the container's env at boot, not just the shell that runs the setup command. If selectors find nothing but the app is up, check for a missing flag like this first.
101
161
 
102
162
  #### What not to do
103
163
  1. IMPORTANT — do NOT remove or change the existing \`sutSetupMode\` input (nor \`sutSourceWorkflowFile\` / \`sutSourceDockerComposeFile\`). Leave \`sutSetupMode\` exactly as provided: Testbot flips it to \`none\` automatically after validation succeeds, and the committed \`sutSetupMode\` value is the signal that tells the next CI run to validate this adapted workflow rather than run tests. Deleting it or setting it to \`none\` breaks the bootstrap/validation cycle.
104
- 2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
164
+ 2. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code. In case of multi-mode containers (\`Dockerfile.bdd\`, \`Dockerfile.e2e\`, \`Dockerfile.integration\`, \`Dockerfile.test\`), create the necessary file in \`.skyramp/sut/\` with a PR-sourced build (e.g. \`.skyramp/sut/Dockerfile.bdd\` referenced from the testbot compose's \`build:\`). Do not edit existing Dockerfiles or compose files in the repo. If the multi-mode container's bootstrap step depends on a seeded user, ensure SEED_DATA creates it before this container tries to log in.
105
165
  3. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
106
166
  4. Do not use a foreground-blocking command as \`targetSetupCommand\` (e.g. \`npm start\`, \`pnpm --prefix ./apps/studio start\`, \`python app.py\`, \`prefect server start\`, \`./server\`) — see input 5 in "What to do" above for daemon forms and the compose-wrap fallback.
107
167
  5. Do not leave server processes running between fix-loop retries. If \`targetSetupCommand\` starts a direct process (not \`docker compose\`), always set \`targetTeardownCommand\` to kill it (e.g. \`pkill -f "node dist/index.js"\` or \`docker compose down\`). Without teardown, retried setup attempts will fail with \`EADDRINUSE\` because the previous process still holds the port.
@@ -127,6 +187,6 @@ export function buildCommonSutErrorsSection() {
127
187
  6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
128
188
  7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
129
189
  8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
130
- 9. no_space_left — the runner ran out of disk space (\`No space left on device\`). This happens on standard runners when the SUT involves large Docker builds, many npm packages, or multiple compose services that pull large images. Switch \`runs-on\` to a larger runner — runner names vary by org, do not hardcode \`large-ubuntu\`. Pick the label by (a) preserving whatever larger label the source workflow already uses, (b) reusing the label adjacent heavy or integration jobs in \`.github/workflows/\` already use, or (c) falling back to a known GitHub-hosted large label such as \`ubuntu-latest-large\`. Indicator: source workflow already specifies a large runner, or the SUT has multi-stage Docker builds / a compose file with 3+ services.
190
+ 9. no_space_left — the runner ran out of disk space (\`No space left on device\`). Prune the docker BuildKit cache and dangling images at the start of both \`targetSetupCommand\` and \`targetTeardownCommand\` so each fix-loop retry begins with a clean slate; without it, cache from the previous attempt accumulates across retries on heavy SUTs. Do not change \`runs-on\` see input 9 above.
131
191
  Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
132
192
  }
@@ -15,7 +15,11 @@ export function buildFileRenameStrategy() {
15
15
  }
16
16
  /** Strategy string for in-place UPDATE edits. */
17
17
  export function buildUpdateStrategy() {
18
- return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context. After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
18
+ return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context.
19
+
20
+ UPDATE is an in-place edit to the existing baseline file — do not create a new test file to cover the same page or component.
21
+
22
+ After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
19
23
  }
20
24
  /** Strategy string for REGENERATE — call generation tool to overwrite the file. */
21
25
  export function buildRegenerateStrategy() {
@@ -1,4 +1,17 @@
1
- export declare function buildDriftAnalysisPrompt(stateFile?: string, existingTests?: Array<{
1
+ export type UiDriftParams = {
2
+ tests: Array<{
3
+ testFile: string;
4
+ }>;
5
+ changedFrontendFiles: string[];
6
+ blueprintCaptured?: boolean;
7
+ };
8
+ /**
9
+ * Build the drift analysis prompt.
10
+ * - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
11
+ * - ui: when provided, appends a UI drift section for component/browser tests.
12
+ * Omit when no frontend files changed or no UI tests exist.
13
+ */
14
+ export declare function buildDriftAnalysisPrompt(stateFile: string | undefined, apiTests: Array<{
2
15
  testFile: string;
3
16
  source?: string;
4
- }>): string;
17
+ }>, ui?: UiDriftParams, repoPaths?: string[]): string;
@@ -1,6 +1,8 @@
1
1
  import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, } from "./driftAnalysisSections.js";
2
+ import { buildUiActionDecisionTree, buildUiCheckRouteExistence, buildUiCheckSelectors, buildUiCheckPageObjects, buildUiCheckBehavioralChanges, buildUiCheckAssignAction, buildUiDriftOutputChecklist, } from "./uiDriftAnalysisSections.js";
3
+ import { RECOMMENDATIONS_INSTRUCTION, buildSymbolDiscoveryStep } from "./driftAnalysisShared.js";
2
4
  import { PromptPlan } from "../test-recommendation/promptPlan.js";
3
- const _plan = new PromptPlan()
5
+ const _apiPlan = new PromptPlan()
4
6
  .addPhase("maintenance", "Test Maintenance Assessment", {
5
7
  headerLevel: "##",
6
8
  stepFormat: "hash",
@@ -14,6 +16,64 @@ const _plan = new PromptPlan()
14
16
  .subStep("ASSIGN_ACTION", "Assign action", () => buildCheckAssignAction())
15
17
  .step("CALL_TOOL", "Submit recommendations", (p) => buildDriftOutputChecklist(p.stateFile, p.existingTests))
16
18
  .done();
17
- export function buildDriftAnalysisPrompt(stateFile, existingTests) {
18
- return `<drift_analysis_rules>\n${_plan.render({ stateFile, existingTests })}\n</drift_analysis_rules>`;
19
+ const _uiPlan = new PromptPlan()
20
+ .addPhase("ui_maintenance", "UI Test Drift Assessment", {
21
+ headerLevel: "##",
22
+ stepFormat: "hash",
23
+ })
24
+ .step("UI_ASSESS", "UI Action Decision Tree — assess each UI test against the diff and blueprints", () => buildUiActionDecisionTree())
25
+ .subStep("UI_ROUTE_EXISTENCE", "Route existence", () => buildUiCheckRouteExistence())
26
+ .subStep("UI_SELECTORS", "Selector validity", () => buildUiCheckSelectors())
27
+ .subStep("UI_PAGE_OBJECTS", "Page object/fixture existence", () => buildUiCheckPageObjects())
28
+ .subStep("UI_BEHAVIOR", "Component behavior changes", () => buildUiCheckBehavioralChanges())
29
+ .subStep("UI_ASSIGN_ACTION", "Assign action", () => buildUiCheckAssignAction())
30
+ .step("UI_CALL_TOOL", "Submit recommendations", (p) => buildUiDriftOutputChecklist(p.stateFile, p.tests))
31
+ .done();
32
+ /**
33
+ * Build the drift analysis prompt.
34
+ * - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
35
+ * - ui: when provided, appends a UI drift section for component/browser tests.
36
+ * Omit when no frontend files changed or no UI tests exist.
37
+ */
38
+ export function buildDriftAnalysisPrompt(stateFile, apiTests, ui, repoPaths) {
39
+ const parts = [];
40
+ // Emit symbol discovery once at the top regardless of how many sections follow.
41
+ // Placing it inside each plan's checklist caused duplication on mixed diffs.
42
+ const hasAnyTests = apiTests.length > 0 || (ui?.tests?.length ?? 0) > 0;
43
+ const discovery = buildSymbolDiscoveryStep(hasAnyTests, repoPaths);
44
+ if (discovery)
45
+ parts.push(discovery);
46
+ // Include API drift when there are API tests, or when UI drift is not running
47
+ // (ensures skyramp_actions is always reachable even if apiTests is empty).
48
+ if (apiTests.length > 0 || !ui) {
49
+ parts.push(`<drift_analysis_rules>\n${_apiPlan.render({ stateFile, existingTests: apiTests })}\n</drift_analysis_rules>`);
50
+ }
51
+ if (ui) {
52
+ const signalNote = ui.blueprintCaptured
53
+ ? `blueprintCaptured=true — use \`browser_blueprint_diff\` for tests that navigate pages.`
54
+ : `blueprintCaptured=false — use source diff only. Read each test to determine what it exercises.`;
55
+ const contextHeader = `<ui_analysis_context>
56
+ **UI Test Drift Analysis Context**
57
+
58
+ **Signal:** ${signalNote}
59
+
60
+ **Changed Frontend Files:**
61
+ ${ui.changedFrontendFiles.length > 0
62
+ ? ui.changedFrontendFiles.map(f => `- ${f}`).join("\n")
63
+ : "(none)"}
64
+
65
+ **Tests to Assess:**
66
+ ${ui.tests.length > 0
67
+ ? ui.tests.map(t => `- ${t.testFile}`).join("\n")
68
+ : "(none)"}
69
+
70
+ **Instructions:**
71
+ For each test above, read it to determine its type, then apply the scope gate from the decision tree. Assign the appropriate action (DELETE, REGENERATE, UPDATE, VERIFY, IGNORE).
72
+ </ui_analysis_context>
73
+
74
+ `;
75
+ parts.push(contextHeader + `<ui_drift_analysis_rules>\n${_uiPlan.render({ stateFile, tests: ui.tests, blueprintCaptured: ui.blueprintCaptured })}\n</ui_drift_analysis_rules>`);
76
+ }
77
+ parts.push(RECOMMENDATIONS_INSTRUCTION);
78
+ return parts.join("\n\n");
19
79
  }
@@ -1,8 +1,9 @@
1
1
  /**
2
- * Modular section builders for the Drift Analysis prompt,
2
+ * Modular section builders for the API Drift Analysis prompt,
3
3
  * mirroring the recommendationSections.ts pattern.
4
4
  */
5
5
  import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
6
+ import { buildApiAssignAction } from "./driftAnalysisShared.js";
6
7
  // TODO: Replace the open-ended diff-line categories below with a two-tier structure:
7
8
  // Tier 1 — mechanical patterns (specific token + file-path criteria, zero judgment needed).
8
9
  // Tier 2 — residual open-ended scan for anything observable not covered by tier 1.
@@ -177,24 +178,9 @@ Diff signals and actions:
177
178
  **Reachability for behavioral changes:** The service/interface scope gate still applies — if the test targets a completely different service or protocol, IGNORE. However, do NOT use the absence of a route or output layer file in the diff as grounds for IGNORE. Behavioral changes (new error branches, new validation, status-code changes) are observable from any test calling the same endpoint, even when the logic lives in an internal handler, middleware, or utility file rather than the route file itself.`;
178
179
  }
179
180
  export function buildCheckAssignAction() {
180
- return `Based on the above checks, choose the action (IGNORE / VERIFY / UPDATE / REGENERATE / DELETE) and provide a 1-2 sentence rationale.
181
-
182
- **Every action requires a specific rationale — including IGNORE:**
183
- - UPDATE / REGENERATE / DELETE: quote the specific diff line that triggered it.
184
- - VERIFY: name the uncertain element (e.g. "model-only change, cannot confirm field is exposed without checking the output layer").
185
- - IGNORE: name the changed file(s) from the diff AND explain why they cannot reach this test's endpoint (e.g. "diff only touches \`auth/session-service.js\` — this test targets \`/api/v1/orders\` which has no session dependency"). You must reference at least one specific changed file from the diff. Generic "unrelated endpoint" or "service boundary" without naming the changed file is not sufficient.
186
-
187
- - If the Additive Fields check flagged a new field with output layer signal confirmed in the diff → action is UPDATE. If the Additive Fields check returned VERIFY (model-only signal, no output layer change) → action remains VERIFY.
181
+ return buildApiAssignAction(`- If the Additive Fields check flagged a new field with output layer signal confirmed in the diff → action is UPDATE. If the Additive Fields check returned VERIFY (model-only signal, no output layer change) action remains VERIFY.
188
182
  - **Scope gate:** If the changed code is clearly not reachable through the service or base URL this test targets, assign IGNORE.
189
- - **When uncertain, use VERIFY not IGNORE:** If the diff touches a model or migration and this test's endpoint reads from that model, assign VERIFY — you cannot confirm from the diff alone whether the field is exposed.
190
- - **Pre-commit verification — confirm all three before finalizing UPDATE/REGENERATE/DELETE:**
191
- 1. You can quote a specific diff line this test's endpoint observes that triggered the action.
192
- 2. The changed code is reachable through this test's service and base URL.
193
- 3. For REGENERATE: every assertion in the file is invalid, not just some — if you can patch N paths, it is UPDATE.
194
- If any check fails, downgrade to VERIFY or IGNORE.
195
- - **For user-written (external) tests** marked \`[external]\` in the test list:
196
- - UPDATE is permitted — targeted edits only (fix renamed URL, add assertion for new field, add a new test case for a new scenario). Match the style of the existing file: use the same test framework, assertion helpers, and request pattern already present.
197
- - REGENERATE and DELETE are **not permitted** — assign those actions in your recommendations but \`skyramp_actions\` will surface them as report-only findings for the developer to act on. Do NOT attempt to rewrite or delete a user-authored test file.`;
183
+ - **When uncertain, use VERIFY not IGNORE:** If the diff touches a model or migration and this test's endpoint reads from that model, assign VERIFY — you cannot confirm from the diff alone whether the field is exposed.`);
198
184
  }
199
185
  export function buildCheckAdditiveFields() {
200
186
  return `Even if existing assertions still pass, new response fields on a covered endpoint need a new assertion. A field being optional or nullable does not make it IGNORE — it still needs to be covered.
@@ -213,7 +199,7 @@ To determine which applies: look at whether an explicit output layer (DTO, seria
213
199
  **Do not assign IGNORE because the field is optional or nullable** — "won't break existing assertions" is not the bar. The bar is "does the test now cover this endpoint's full contract?" A new field, even optional, means the contract changed and the test is incomplete.`;
214
200
  }
215
201
  export function buildDriftOutputChecklist(stateFile, existingTests) {
216
- const finalStep = `Call \`skyramp_actions\` with \`stateFile: "${stateFile ?? "<stateFile>"}"\` and a \`recommendations\` entry for every test assessed.`;
202
+ const finalStep = `Call \`skyramp_actions\` with \`stateFile: "${stateFile ?? "<stateFile>"}"\`.`;
217
203
  const hasTests = (existingTests?.length ?? 0) > 0;
218
204
  const testList = hasTests
219
205
  ? existingTests.map((t) => {
@@ -224,9 +210,6 @@ export function buildDriftOutputChecklist(stateFile, existingTests) {
224
210
  const noTestsNote = !hasTests
225
211
  ? `\nNo existing tests were found. Call \`skyramp_actions\` with \`recommendations: []\` and explain in your report why no maintenance was needed (e.g. "no existing tests cover the changed endpoints" or "PR adds a new endpoint with no prior coverage").\n`
226
212
  : "";
227
- const symbolDiscoveryStep = hasTests
228
- ? `**Symbol import discovery (before assessing tests):** If the diff adds or modifies class, function, or type definitions, grep the repository for test files that import those symbol names. Add any matched test files not already in the list below to your assessment — these files may need updating even if their path doesn't match the changed resource name. Include all discovered files in the \`recommendations[]\` passed to \`skyramp_actions\` — do NOT edit them directly without going through \`skyramp_actions\` first.`
229
- : "";
230
213
  const existingTestSection = `**Existing tests (${existingTests?.length ?? 0} total) — assess ALL of the following:**
231
214
  ${testList}
232
215
  ${noTestsNote}
@@ -257,9 +240,8 @@ Test: {testFile}
257
240
  Action: DELETE
258
241
  Rationale: DELETE because {quoted diff signal}; all covered endpoints removed
259
242
  \`\`\`
260
- Include ALL actions IGNORE, VERIFY, UPDATE, REGENERATE, DELETE — in the \`recommendations[]\` passed to \`skyramp_actions\`. For \`[external]\` tests: \`skyramp_actions\` will apply UPDATE edits and surface REGENERATE/DELETE as report-only findings.
261
- Be concise one line per IGNORE/VERIFY entry.`;
262
- const sections = [symbolDiscoveryStep, existingTestSection, finalStep].filter(Boolean);
243
+ Be conciseone line per VERIFY/IGNORE entry.`;
244
+ const sections = [existingTestSection, finalStep];
263
245
  return `<output_format>
264
246
  Complete ALL of the following:
265
247
 
@@ -0,0 +1,20 @@
1
+ /** Shared drift prompt sections (assign-action, external test policy, symbol discovery) for API and UI prompts. */
2
+ /**
3
+ * Build the symbol-discovery grep instruction — unified for API and UI tests.
4
+ * Two-pass: hunk-context symbols (what changed) + full file exports (what exists),
5
+ * covering aliases, indirect renders, and shared hooks the diff doesn't mention.
6
+ */
7
+ export declare function buildSymbolDiscoveryStep(hasTests: boolean, repoPaths?: string[]): string;
8
+ /** One source of truth for the recommendations[] contract passed to skyramp_actions. */
9
+ export declare const RECOMMENDATIONS_INSTRUCTION = "Pass VERIFY, UPDATE, REGENERATE, and DELETE entries to `recommendations[]`. Omit IGNORE.";
10
+ export declare function buildSharedExternalTestPolicy(): string;
11
+ /**
12
+ * Assign-action section for the API drift prompt.
13
+ * @param domainNotes API-specific bullets (additive-fields guidance, scope-gate note).
14
+ */
15
+ export declare function buildApiAssignAction(domainNotes: string): string;
16
+ /**
17
+ * Assign-action section for the UI drift prompt.
18
+ * @param domainNotes UI-specific bullets (component/widget matching notes).
19
+ */
20
+ export declare function buildUiAssignAction(domainNotes: string): string;