@skyramp/mcp 0.3.4 → 0.3.6-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/build/playwright/registerPlaywrightTools.js +92 -30
  2. package/build/playwright/traceRecordingPrompt.d.ts +6 -0
  3. package/build/playwright/traceRecordingPrompt.js +6 -2
  4. package/build/prompts/code-reuse.d.ts +1 -2
  5. package/build/prompts/code-reuse.js +182 -77
  6. package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
  7. package/build/prompts/modularization/integration-test-modularization.js +83 -41
  8. package/build/prompts/modularization/render.d.ts +18 -0
  9. package/build/prompts/modularization/render.js +12 -0
  10. package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
  11. package/build/prompts/modularization/ui-test-modularization.js +89 -47
  12. package/build/prompts/pom-aware-code-reuse.js +3 -1
  13. package/build/prompts/shared-helper-policy.d.ts +57 -0
  14. package/build/prompts/shared-helper-policy.js +135 -0
  15. package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
  16. package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
  17. package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
  18. package/build/prompts/test-recommendation/recommendationShared.js +90 -16
  19. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
  20. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
  21. package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
  22. package/build/prompts/testbot/testbot-prompts.js +88 -33
  23. package/build/recommendation/budgeters/shared.js +105 -27
  24. package/build/recommendation/discriminators.js +13 -2
  25. package/build/recommendation/planRanker.d.ts +6 -6
  26. package/build/recommendation/planRanker.js +6 -61
  27. package/build/services/AnalyticsService.d.ts +7 -0
  28. package/build/services/AnalyticsService.js +7 -1
  29. package/build/services/ModularizationService.js +1 -3
  30. package/build/services/TestDiscoveryService.d.ts +0 -2
  31. package/build/services/TestDiscoveryService.js +2 -37
  32. package/build/services/TestGenerationService.d.ts +16 -0
  33. package/build/services/TestGenerationService.js +86 -10
  34. package/build/services/containerEnv.js +13 -12
  35. package/build/tools/code-refactor/codeReuseTool.js +279 -93
  36. package/build/tools/code-refactor/enhance-state.d.ts +49 -0
  37. package/build/tools/code-refactor/enhance-state.js +109 -0
  38. package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
  39. package/build/tools/code-refactor/modularizationTool.js +9 -2
  40. package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
  41. package/build/tools/code-refactor/reuse-outcome.js +14 -4
  42. package/build/tools/code-refactor/reuse-state.d.ts +127 -5
  43. package/build/tools/code-refactor/reuse-state.js +628 -16
  44. package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
  45. package/build/tools/code-refactor/utils-verify-gates.js +100 -0
  46. package/build/tools/code-refactor/verify-gates.d.ts +2 -1
  47. package/build/tools/code-refactor/verify-gates.js +90 -25
  48. package/build/tools/executeSkyrampTestTool.d.ts +19 -0
  49. package/build/tools/executeSkyrampTestTool.js +158 -8
  50. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
  51. package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
  52. package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
  53. package/build/tools/generate-tests/generateUIRestTool.js +22 -0
  54. package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
  55. package/build/tools/generate-tests/scenarioLint.js +127 -19
  56. package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
  57. package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
  58. package/build/tools/submitReportTool.d.ts +38 -38
  59. package/build/tools/submitReportTool.js +487 -104
  60. package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
  61. package/build/tools/test-management/analyzeChangesTool.js +75 -12
  62. package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
  63. package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
  64. package/build/tools/test-management/registerTestPlanTool.js +149 -23
  65. package/build/types/Recommendation.d.ts +34 -5
  66. package/build/types/RepositoryAnalysis.d.ts +133 -114
  67. package/build/types/RepositoryAnalysis.js +1 -1
  68. package/build/types/ReuseOutcome.d.ts +102 -6
  69. package/build/types/ReuseOutcome.js +16 -2
  70. package/build/types/TestRecommendation.js +21 -3
  71. package/build/types/TestTypes.js +14 -8
  72. package/build/types/TestbotReport.d.ts +25 -3
  73. package/build/types/index.d.ts +2 -2
  74. package/build/types/index.js +1 -1
  75. package/build/utils/AnalysisStateManager.d.ts +69 -1
  76. package/build/utils/AnalysisStateManager.js +69 -5
  77. package/build/utils/branchDiff.d.ts +10 -0
  78. package/build/utils/branchDiff.js +28 -0
  79. package/build/utils/changedRoutes.d.ts +29 -0
  80. package/build/utils/changedRoutes.js +87 -0
  81. package/build/utils/featureFlags.d.ts +21 -0
  82. package/build/utils/featureFlags.js +23 -0
  83. package/build/utils/frontendIntegration.js +34 -4
  84. package/build/utils/importerHop.d.ts +2 -8
  85. package/build/utils/importerHop.js +15 -53
  86. package/build/utils/pathMatching.d.ts +38 -0
  87. package/build/utils/pathMatching.js +71 -0
  88. package/build/utils/pathSignatures.d.ts +22 -0
  89. package/build/utils/pathSignatures.js +57 -0
  90. package/build/utils/planMatchKeys.d.ts +16 -3
  91. package/build/utils/planMatchKeys.js +26 -10
  92. package/build/utils/pluralization.d.ts +10 -0
  93. package/build/utils/pluralization.js +18 -0
  94. package/build/utils/pom-catalog-parse.d.ts +52 -0
  95. package/build/utils/pom-catalog-parse.js +141 -0
  96. package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
  97. package/build/utils/pom-scope/selector-extractor.js +34 -8
  98. package/build/utils/pom-verify/verify.d.ts +6 -5
  99. package/build/utils/pom-verify/verify.js +8 -6
  100. package/build/utils/reportLanguage.d.ts +43 -0
  101. package/build/utils/reportLanguage.js +125 -0
  102. package/build/utils/reportVerification.d.ts +74 -4
  103. package/build/utils/reportVerification.js +259 -3
  104. package/build/utils/reuseRouting.d.ts +3 -0
  105. package/build/utils/reuseRouting.js +50 -0
  106. package/build/utils/routeParsers.d.ts +2 -0
  107. package/build/utils/routeParsers.js +65 -8
  108. package/build/utils/scenarioDrafting.d.ts +1 -1
  109. package/build/utils/scenarioDrafting.js +57 -45
  110. package/build/utils/subjectEndpoints.d.ts +19 -0
  111. package/build/utils/subjectEndpoints.js +98 -0
  112. package/build/utils/testFileClassification.d.ts +11 -0
  113. package/build/utils/testFileClassification.js +47 -0
  114. package/build/utils/uiPageEnumerator.d.ts +45 -19
  115. package/build/utils/uiPageEnumerator.js +95 -51
  116. package/build/utils/utils-verify/allow.d.ts +16 -0
  117. package/build/utils/utils-verify/allow.js +68 -0
  118. package/build/utils/utils-verify/call-sites.d.ts +34 -0
  119. package/build/utils/utils-verify/call-sites.js +154 -0
  120. package/build/utils/utils-verify/index.d.ts +7 -0
  121. package/build/utils/utils-verify/index.js +7 -0
  122. package/build/utils/utils-verify/language-spec.d.ts +91 -0
  123. package/build/utils/utils-verify/language-spec.js +210 -0
  124. package/build/utils/utils-verify/locate.d.ts +39 -0
  125. package/build/utils/utils-verify/locate.js +199 -0
  126. package/build/utils/utils-verify/parse.d.ts +34 -0
  127. package/build/utils/utils-verify/parse.js +177 -0
  128. package/build/utils/utils-verify/stage.d.ts +24 -0
  129. package/build/utils/utils-verify/stage.js +107 -0
  130. package/build/utils/utils-verify/verify.d.ts +63 -0
  131. package/build/utils/utils-verify/verify.js +168 -0
  132. package/build/utils/utils.d.ts +3 -1
  133. package/build/utils/utils.js +3 -1
  134. package/build/workspace/workspace.d.ts +32 -32
  135. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
  136. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
  137. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
  138. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
  139. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
  140. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
  141. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
  142. package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
  143. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
  144. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
  145. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
  146. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
  147. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
  148. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
  149. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  150. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
  151. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  152. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  153. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  154. package/node_modules/playwright/package.json +1 -1
  155. package/package.json +2 -2
  156. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
  157. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
  158. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
  159. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
@@ -2,11 +2,12 @@ import { z } from "zod";
2
2
  import { logger } from "../../utils/logger.js";
3
3
  import { AnalyticsService } from "../../services/AnalyticsService.js";
4
4
  import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
5
- import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef } from "../test-recommendation/recommendationShared.js";
5
+ import { setReportLanguage } from "../../utils/reportLanguage.js";
6
+ import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef, } from "../test-recommendation/recommendationShared.js";
6
7
  import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
7
- import { isContractConsumerModeEnabled, isPomReuseEnabled } from "../../utils/featureFlags.js";
8
+ import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, } from "../../utils/featureFlags.js";
8
9
  import { resolveServiceDetailsRef } from "../../utils/utils.js";
9
- import { buildServiceContext, readWorkspaceServices, } from "../prompt-utils.js";
10
+ import { buildServiceContext, readWorkspaceServices } from "../prompt-utils.js";
10
11
  // Cached at module-load — flags are process-wide and cannot change per call.
11
12
  const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
12
13
  const SERVICE_REFS = resolveServiceDetailsRef();
@@ -19,13 +20,46 @@ const CONTRACT_MODE_GUIDANCE = CONSUMER_MODE_ENABLED
19
20
  Both modes (\`providerMode: true, consumerMode: true\`): For diff that contains BOTH provider signals (such as new/modified endpoint handlers, route changes this service owns) AND consumer signals (outbound HTTP client calls to another service, no new endpoint handlers).`
20
21
  : ` Always add \`providerMode: true\` — the tool generates provider-side contract tests only.`;
21
22
  const POM_REUSE_ENABLED = isPomReuseEnabled();
23
+ // Utils reuse (SkyrampUtils consolidation for integration tests, and for UI tests
24
+ // while the POM path is off) is customer-gated,
25
+ // default OFF — like the POM flag, its value is baked into the prompt at module load,
26
+ // so the MCP server must be restarted after flipping it.
27
+ const UTILS_REUSE_ENABLED = isUtilsReuseEnabled();
28
+ // The UI flow seeds a shared utils file (modularize-first) only with utils reuse
29
+ // on and the POM path off — the same condition isModularizeFirstTarget applies
30
+ // server-side. Every UI-chain statement in this prompt keys on this constant so
31
+ // no line can ban the modularization call the generation result instructs.
32
+ const UI_UTILS_REUSE = UTILS_REUSE_ENABLED && !POM_REUSE_ENABLED;
33
+ // Step 4 of the post-generation list. On the modularize-first UI flow the
34
+ // enhance call is the first sub-step of the generation result's chain — but
35
+ // that chain exists only for a TS/JS UI generation with enhanceAssertions on,
36
+ // which this prompt cannot see per call. So the step keeps the call and lets
37
+ // the agent skip it only when it already made it for that file.
38
+ const UI_ENHANCE_STEP = UI_UTILS_REUSE
39
+ ? `Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`, and apply every instruction returned to that file — UNLESS you already called it for this file as the first sub-step of step 3's chain, in which case skip it (never enhance the same file twice; never leave a generated UI test un-enhanced). The guidance below on which assertions to add applies to whichever call enhances the file.`
40
+ : `Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.`;
22
41
  // Post-generation code-reuse step for UI tests. The POM catalog, the `verify: true`
23
42
  // loop and the `.raw.bak` restore only exist on the POM-aware path — when that path
24
43
  // is flagged off, `skyramp_reuse_code` returns the SkyrampUtils workflow instead, so
25
44
  // the agent must not be sent looking for artifacts nothing produces.
26
45
  const UI_CODE_REUSE_STEP = POM_REUSE_ENABLED
27
46
  ? `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Two outcomes: (1) a response starting "No reusable POM layer detected" — this is a normal outcome, continue immediately (do NOT retry); (2) a refactoring workflow — follow it to completion INCLUDING its verification loop (\`skyramp_reuse_code\` with \`verify: true\`), finish only when it reports PASSED. If a reused test later fails execution and the failure points at a substituted POM call, restore the saved \`<testFile>.raw.bak\` over the test file and re-run — do NOT hand-edit the customer's POM methods (this re-run counts toward the 2-attempt execution cap — prefer this restore over the generic timeout fix-up when the failing locator came from a POM substitution). **Cleanup before reporting:** \`.raw.bak\` files are internal scratch — after ALL test executions are complete (pass or fail) and before calling \`skyramp_submit_report\`, delete every \`*.raw.bak\` you created so they are not committed to the customer-facing branch. The \`skyramp-pom-catalog.md\` is NOT scratch — leave it in place (later runs reuse it).`
28
- : `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Follow the returned steps exactly. If it finds no existing helper functions to reuse, that is a normal outcome — leave the test file unchanged and continue immediately (do NOT retry).`;
47
+ : UI_UTILS_REUSE
48
+ ? `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result carries three CRITICAL NEXT STEPS — \`skyramp_enhance_assertions\` (on the freshly generated file, while its selectors are still inline), then \`skyramp_modularization\`, then \`skyramp_reuse_code\` — with the exact arguments for each. Do all three, in that order and to completion, for EVERY generated UI test file — the enhance sub-step is that file's step 4 below (do not call \`skyramp_enhance_assertions\` a second time for the same file, and do not skip it for later files). Sibling tests offering nothing to reuse is a normal outcome, but the reuse steps still move this test's own helpers into the shared utils file and import them back, so the test file IS expected to change. Then continue (do NOT retry).`
49
+ : `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Follow the returned steps exactly. If it finds no existing helper functions to reuse, that is a normal outcome — leave the test file unchanged and continue immediately (do NOT retry).`;
50
+ // Generation-call clause for the integration pipeline: only ask for codeReuse when
51
+ // utils reuse is enabled — with the flag off, integration generation behaves exactly
52
+ // as it did before the feature existed (no file-rewriting post-steps).
53
+ const INTEGRATION_CODE_REUSE_GEN_CLAUSE = UTILS_REUSE_ENABLED
54
+ ? `, setting \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript/Python — leave \`codeReuse\` unset for Java: cross-file helpers do not compile in the test executor)`
55
+ : ``;
56
+ // The post-generation modularize → reuse steps are NOT listed here: the
57
+ // generation tool's own result emits them (see TestGenerationService) when called
58
+ // with codeReuse: true on a modularize-first flow (routing: isModularizeFirstTarget).
59
+ // The UI clauses below keep `modularizeCode: false` deliberately: the
60
+ // modularization happens via that hand-off, not the flag, so the instructions
61
+ // exist only on runs that actually generated that test type. This prompt's job
62
+ // is only the generation-call clauses that ask for codeReuse.
29
63
  /**
30
64
  * Parse the JSON-encoded `relatedRepositories` argument passed via the testbot
31
65
  * prompt/resource. Returns undefined for missing/blank input or malformed JSON so the
@@ -77,9 +111,18 @@ export function getTestbotPrompt(opts) {
77
111
  // SKYR-4023: user-facing report language. English display names improve
78
112
  // instruction-following vs bare codes; Intl.DisplayNames (Node >=18) names any
79
113
  // code correctly, falling back to the raw code only if it can't be resolved.
80
- let reportLanguageBlock = '';
81
- if (language && language !== 'en') {
82
- const reportLanguageName = new Intl.DisplayNames(['en'], { type: 'language' }).of(language) ?? language;
114
+ // SKYR-4185: rendering the prompt is the single capture point for the
115
+ // report language the prompt registration AND the testbot resource both
116
+ // come through here, so skyramp_submit_report's language guardrail sees it
117
+ // regardless of which surface served the prompt (no dual wiring to forget).
118
+ // Set on EVERY render, not just non-English ones: last render wins, so an
119
+ // en/argless render disarms a language captured earlier in a long-lived
120
+ // server process instead of falsely rejecting an English report.
121
+ setReportLanguage(language && language !== "en" ? language : undefined);
122
+ let reportLanguageBlock = "";
123
+ if (language && language !== "en") {
124
+ const reportLanguageName = new Intl.DisplayNames(["en"], { type: "language" }).of(language) ??
125
+ language;
83
126
  reportLanguageBlock = `**Report language: ${reportLanguageName}.** Write ALL user-facing free-text report fields in ${reportLanguageName}: \`businessCaseAnalysis\`, every \`description\` and \`reasoning\`, \`testResults[].details\`, \`issuesFound[].description\`, \`nextSteps[]\`, test-maintenance \`beforeDetails\`/\`afterDetails\`, and \`commitMessage\`. Do NOT translate: code identifiers, endpoint paths, file names, test IDs, \`scenarioName\`, enum values (\`Pass\`/\`Fail\`/\`Skipped\`, severity values, \`testType\`), or anything inside backticks.
84
127
 
85
128
  `;
@@ -134,7 +177,8 @@ Use those recommendations as your baseline. Only add or remove tests that the us
134
177
 
135
178
  **If \`skyramp_analyze_changes\` returns an error:** retry once only if the error is transient (timeout, network blip, temporary unavailability) — do NOT retry for permanent errors (invalid repository path, missing required parameter, authentication failure). If it fails again, call \`skyramp_submit_report\` with a minimal valid payload: leave all test arrays empty and add the error to \`issuesFound\`. Refer to the \`skyramp_submit_report\` schema for required fields. Do NOT attempt Task 2 without a valid stateFile.
136
179
  **If all changed files are non-application** (CI/CD, docs, lock files, config) → skip to Task 3 (Submit Report) with empty arrays. Put the one-paragraph summary in \`businessCaseAnalysis\` (always populated; that's where end-state narration belongs); leave \`issuesFound\` empty — a non-application diff is not an issue. Example narration for a Testbot onboarding PR (\`.github/workflows/skyramp-testbot.yml\` and/or files under \`.skyramp/\`): "This PR adds Skyramp Testbot GitHub Actions workflow configuration to enable automated test generation on every pull request. It also adds System Under Test (SUT) setup files under \`.skyramp/sut/\` required for the testbot workflow, to bring up services for testing. It contains no application code changes and has no testable behavioral surface."
137
- ${hasRelatedRepos ? `
180
+ ${hasRelatedRepos
181
+ ? `
138
182
  **MULTI-REPO CONTEXT (MANDATORY).** This run includes ${relatedRepositories.length} related ${relatedRepositories.length === 1 ? "repository" : "repositories"} listed in the \`<related_repositories>\` block below, each with an explicit \`repository\` (\`owner/repo\`), \`path\`, and \`base_branch\`. Use the \`repository\` value verbatim — do NOT infer it from git remotes or paths. You MUST analyze EACH related repository — exactly one \`skyramp_analyze_changes\` call per listed repo (${relatedRepositories.length} ${relatedRepositories.length === 1 ? "call" : "calls"}), in addition to the primary call in step 2.
139
183
 
140
184
  **Run the primary call (step 2) FIRST, then the related repos in listed order — not in parallel.** All calls in this run automatically share ONE run-scoped state file — you do NOT pass a state-file path; setting \`repository\` is enough. The primary writes its root section; each related repo's call upserts its own section into that same file. Concurrent calls would race on the shared file, so they must be sequential. For each related repo:
@@ -153,7 +197,8 @@ ${hasRelatedRepos ? `
153
197
  Within a type, higher score wins regardless of which repo it came from. Everything not selected becomes an ADDITIONAL recommendation. This guarantees a frontend-only primary repo cannot starve a related backend repo's contract/integration tests of GENERATE slots (and vice versa). When you generate a test for a related repo's endpoint:
154
198
  - **Execute it only if that repo's service is already running and reachable.** The workflow's setup may have started multiple services; before generating an API test for a related repo, confirm its \`base_url\` (from that repo's workspace/Execution Plan) responds. If the service is unreachable, still GENERATE the test but mark its \`testResults\` status as \`Skipped\` with details "service not running in this run" — do NOT count an unreachable service as a failure.
155
199
  - **Write the test file into that service's own \`testDirectory\`** — the one declared for the service in the unified workspace.yml (the related repo's services were registered there in step 1(a), each with its \`repository\`). The \`testDirectory\` is interpreted relative to the **single delivery root** (the configured test repo if set, otherwise the primary repo), so all generated tests are delivered together by the existing single-target delivery. Do NOT invent a per-source-repo subdirectory, and do NOT write into the related repo's own checkout — it is read-only context. (If two repos happen to declare the same \`testDirectory\`, their files coexist there; the \`repository\` field on each report item — below — is what attributes ownership, not the path.)
156
- - **Set the \`repository\` field** (\`owner/repo\`) on every such \`newTestsCreated\` / \`testResults\` item so the report attributes it to the originating repo (see Report Guidelines).` : ""}
200
+ - **Set the \`repository\` field** (\`owner/repo\`) on every such \`newTestsCreated\` / \`testResults\` item so the report attributes it to the originating repo (see Report Guidelines).`
201
+ : ""}
157
202
 
158
203
  2. **Maintain existing tests:**
159
204
 
@@ -169,7 +214,7 @@ ${maintenanceBeforeExecStep}
169
214
 
170
215
  f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) and record each file's result as its \`afterStatus\`. Editing an \`[external]\` file that step 2(a) confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
171
216
 
172
- 3. **Code review:** From the \`skyramp_analyze_changes\` output and the existing test files you read for maintenance, note any logic bugs. Do NOT read additional source files just for code reviewuse what is already available from the analysis and test file reads. Common patterns to flag:
217
+ 3. **Code review:** Find the logic bugs in the code that this change touches. Read the implementation of each changed endpoint: the route handler, and the functions that it calls to read or write data. For a changed screen, read the component and the functions that it calls. Read these files even when the diff does not contain them a defect often sits in the code that the change depends on. Report each finding in \`issuesFound\` with a severity, and say which file and line holds it. Common patterns to flag:
173
218
  - Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)
174
219
  - Incomplete CRUD: create without cleanup, update that adds new records without removing old ones
175
220
  - Missing input validation on new endpoints
@@ -186,8 +231,6 @@ ${maintenanceBeforeExecStep}
186
231
  "role": "button",
187
232
  "accessibleName": "Save changes",
188
233
  "testId": "save-changes-btn",
189
- "stableId": null,
190
- "contextText": null,
191
234
  "mutability": "mutable",
192
235
  "widgetType": "native"
193
236
  }
@@ -195,10 +238,10 @@ ${maintenanceBeforeExecStep}
195
238
  \`\`\`
196
239
  - **One element when the test has a single dominant target** (a click, a type, a single visibility check). Most tests fall here — use a length-1 array.
197
240
  - **Multiple elements when the test verifies several elements together** — render-state tests (heading + input + button on a form), workflow tests (click button A, assert state appears in element B), or form-fill tests (multiple inputs + submit button). Each element is its own array entry.
198
- - Each element's fields are lifted verbatim from a captured blueprint element. \`role\`, \`accessibleName\` are required. \`testId\`/\`stableId\`/\`contextText\` are \`null\` when not present in the capture.
241
+ - Each element's fields come from a captured blueprint element. Copy the values that are present, and never invent one. \`role\` and \`accessibleName\` are always in the capture; the capture omits \`testId\`/\`stableId\`/\`contextText\` when the element has no such value, so omit them too.
199
242
  - \`mutability\` — copy from \`blueprint.element.mutability\`. \`'mutable'\` = behavioral-test target; \`'immutable'\` = smoke target.
200
243
  - \`widgetType\` — copy from \`blueprint.element.widgetType\`. \`'custom'\` = JavaScript-composite control (Radix, MUI, etc.) requiring click-to-open interaction; \`'native'\` = standard HTML element.
201
- - \`contextText\` — only for elements inside repeating sections (table rows, list items). Lift from \`repeatingElement.items[].contextText\`. \`null\` otherwise.
244
+ - \`contextText\` — only for elements inside repeating sections (table rows, list items). Lift from \`repeatingElement.items[].contextText\`. Omit it otherwise.
202
245
 
203
246
  **Field 2 — \`pageContext\`** (where the test runs):
204
247
  \`\`\`json
@@ -350,19 +393,21 @@ ${maintenanceBeforeExecStep}
350
393
 
351
394
  **No upstream captures available?** Set \`targetElements\` to \`null\`, omit \`pageContext\`, and prefix \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Use page/feature-level prose; don't cite specific element names without grounding. Apply the marker per entry, not per PR — affected recs only. Log capture failures in \`issuesFound\` (one info-severity entry per failure mode, naming counts). Don't pre-emptively fall back without attempting capture first. Non-UI work is unaffected.
352
395
  `;
353
- const serviceContext = services?.length ? buildServiceContext(services) : '';
396
+ const serviceContext = services?.length ? buildServiceContext(services) : "";
354
397
  // The <ui-credentials> tags are framing for the agent's prompt context —
355
398
  // not real XML — so credentials pass through verbatim and the agent can
356
399
  // type them directly into login fields. Reject the only string that would
357
400
  // break the framing: a credential containing the closing tag itself.
358
401
  const trimmedCredentials = uiCredentials?.trim();
359
- if (trimmedCredentials && trimmedCredentials.includes('</ui-credentials>')) {
402
+ if (trimmedCredentials && trimmedCredentials.includes("</ui-credentials>")) {
360
403
  throw new Error("uiCredentials must not contain '</ui-credentials>'");
361
404
  }
362
405
  const uiCredentialsBlock = trimmedCredentials
363
406
  ? `<ui-credentials>\n${trimmedCredentials}\n</ui-credentials>`
364
- : '';
365
- const testsRepoDirBlock = testsRepoDir ? `<TESTS REPO DIR>${testsRepoDir}</TESTS REPO DIR>\n` : '';
407
+ : "";
408
+ const testsRepoDirBlock = testsRepoDir
409
+ ? `<TESTS REPO DIR>${testsRepoDir}</TESTS REPO DIR>\n`
410
+ : "";
366
411
  // Multi-repo context block. Each entry's `repositoryPath` is checked out at its
367
412
  // FEATURE ref; `baseBranch` is that repo's default branch (the diff base), so
368
413
  // skyramp_analyze_changes computes a real default…feature diff for the related repo.
@@ -373,7 +418,7 @@ ${maintenanceBeforeExecStep}
373
418
  .map((r) => ` <repository repository="${r.repo}" path="${r.repositoryPath}" base_branch="${r.baseBranch || "auto-detect"}" />`)
374
419
  .join("\n") +
375
420
  `\n</related_repositories>\n`
376
- : '';
421
+ : "";
377
422
  const testDirInstruction = testsRepoDir
378
423
  ? `the \`<output_dir>\` from the \`<services>\` block, rooted under the test repository at \`${testsRepoDir}\` (i.e. \`${testsRepoDir}/<output_dir>\`). Write ALL test output files to paths under \`${testsRepoDir}\`, not under \`${repositoryPath}\`. Do NOT write any test files to the app repository.`
379
424
  : `${SERVICE_REFS.testDirRef}. Do NOT create a new \`tests/\` directory at the repo root — use that path. If no \`testDirectory\` is configured, default to the language-conventional location (e.g. \`src/test/java/...\` for Java, \`tests/\` for Python).`;
@@ -408,7 +453,7 @@ This is a plan-only evaluation run: the application under test is NOT running, a
408
453
 
409
454
  ${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
410
455
 
411
- - **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list. (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
456
+ - **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
412
457
  - **Do not fabricate tests outside the GENERATE list provided by \`skyramp_analyze_changes\`.** Changes that only modify, delete, or add fields to an EXISTING covered endpoint or component are maintenance: handle them in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test, never by creating a new spec. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
413
458
  - Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
414
459
  - Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
@@ -460,7 +505,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
460
505
  4. Only pass \`authHeader: ""\` if you can confirm the endpoint is truly unauthenticated.
461
506
 
462
507
  **How to generate each type (for ADD):**
463
- - **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file.
508
+ - **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file${INTEGRATION_CODE_REUSE_GEN_CLAUSE}.
464
509
  **Use the pre-built scenario JSON from the Execution Plan** — pass the steps array directly. Do NOT read source code models to construct request bodies if the plan already provides them.
465
510
  Scenario JSON and test files go in ${testDirInstruction}
466
511
  **Pipeline for speed**: Call ALL \`skyramp_batch_scenario_test_generation\` calls in one batch. When they return, call ALL \`skyramp_integration_test_generation\` calls in the next batch. Do NOT serialize per-scenario (batch→integration→batch→integration) — batch ALL scenarios first, then generate ALL integration tests.
@@ -478,7 +523,7 @@ ${CONTRACT_MODE_GUIDANCE}
478
523
  - Legacy format: \`username:password\` — the first \`:\` splits username from password.
479
524
  These are format hints, not a strict grammar — apply judgment on ambiguous input (e.g. a \`=\` or \`;\` inside a value of the key=value form: split on the \`;\` that precedes a plausible login-field key).
480
525
 
481
- **Credential selection**: Use the first credential by default. When the scenario requires a specific role, find the credential whose \`role\` field matches (e.g. \`role=admin\`). If no credential matches the required role, use the first credential and add a note to \`issuesFound\` that no matching role was found.
526
+ **Credential selection**: When several credentials are provided, reason carefully about which one each test case needs BEFORE logging in. Multiple credentials exist so tests can exercise the app as different identities — an app with authorization levels behaves differently per account, and a test only has value when it runs as the identity it is about: an admin workflow needs the admin account, a permission-boundary test needs the restricted one, a plain user flow needs an ordinary user. Read what each credential says about itself — a labeling field (\`role\`, or whatever the customer named it: \`accessLevel\`, \`permissionLevel\`, a team/tenant name, …), the username itself, any extra fields — and match that against the test case's intent. When nothing about a test case calls for a specific identity, use the first credential. If the identity a test case needs is not among the credentials, use the closest match and add a note to \`issuesFound\` naming the identity that was missing. NEVER mix fields across credential lines — type the username, password, and every extra field from the SAME line. The exact values you type identify which credential the generated test will read from the environment at replay time, so a mixed or altered value breaks that binding.
482
527
 
483
528
  Type all values verbatim. Before navigating to ANY feature URL:
484
529
  1. \`browser_navigate\` to the login URL (e.g. \`{baseUrl}/login\`, \`/user/login\`, \`/signin\` — infer from the app's base URL and framework)
@@ -567,7 +612,7 @@ ${CONTRACT_MODE_GUIDANCE}
567
612
 
568
613
  **The Blueprint Citation Invariant applies during recording too.** Every assertion you emit cites element names — those names must come from blueprint captures, not invention. For N user-intent-level actions, the reference target is N+1 \`browser_blueprint\` calls (the first returns full, the rest return deltas). Traces that follow the pattern produce assertions grounded in observable state changes; traces that skip captures fall back to author-inferred assertions and risk citing names that don't exist in the rendered DOM.
569
614
 
570
- The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), then \`skyramp_reuse_code\` (when \`codeReuse: true\`) and \`skyramp_enhance_assertions\` post-calls. Capture-act-capture adds blueprint captures alongside the existing steps; it doesn't replace anything.
615
+ The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), then the post-calls the generation result lists (${UI_UTILS_REUSE ? "`skyramp_enhance_assertions`, `skyramp_modularization`, `skyramp_reuse_code`, in that order" : "`skyramp_reuse_code` (when `codeReuse: true`) and `skyramp_enhance_assertions`"}). Capture-act-capture adds blueprint captures alongside the existing steps; it doesn't replace anything.
571
616
  - **E2E**: Only if BOTH a backend trace \`.json\` AND a Playwright \`.zip\` already exist in the repo. Without both, move to \`additionalRecommendations\`.
572
617
  - Skip smoke tests entirely.
573
618
 
@@ -585,8 +630,14 @@ If a test **generation** tool call fails:
585
630
  If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated test:
586
631
  1. Read the error output to diagnose the root cause (4xx on prereq step, assertion mismatch, floating-point precision, 500 from app bug, timeout, etc.).
587
632
  2. **Expected failure check (no retry):** If the failure is an assertion error or HTTP error that matches the issue identified in the code analysis (e.g. the test was generated specifically to document a broken endpoint, a UI rendering bug, or a missing validation), then this is the **intended outcome** — the test is correctly catching the real bug. Report it immediately as \`status: "Fail"\` and move on. Do NOT retry.
633
+
634
+ This path also covers an assertion failure that application behavior outside this PR's diff explains — for example child records that survive the deletion of their parent, state inherited when an ID is recycled or reused, or a value that ignores a status the test set. Before you keep such a test red, confirm the cause in the source: read the handler, model, or query that should have done the work, and find the specific operation that is missing or wrong. If you find it, report the test as \`status: "Fail"\` and add an \`issuesFound\` entry for it. Do NOT retry.
635
+
636
+ If you cannot point at the missing or wrong line in application code, the app is not at fault — shared or seeded data, parallel test workers, or setup the test itself never did explain the collision. Treat that as an infrastructure failure: fix it and retry once as in step 3.
637
+
638
+ **If you did confirm the missing or wrong operation in the source, do NOT make the test pass.** Never add a reset, cleanup, or setup call for isolation. Never weaken the assertion — no \`==\` to \`>=\`, no exact value to a range. A failing test whose diagnosis names a pre-existing bug is the most valuable output of this run; a passing version of it reports nothing.
588
639
  3. Apply a targeted fix and retry **once** only for **infrastructure failures** — that means exactly **2 total \`skyramp_execute_test\` calls per test file** for these cases. Examples of infrastructure failures worth fixing:
589
- - Assertion mismatch due to floating-point precision or wrong expected value (not a real bug)
640
+ - Assertion mismatch from floating-point precision, or an expected value mis-transcribed from the observed response or computed with an arithmetic slip. If application behavior outside the diff explains the mismatch, it is not an infrastructure failure — use step 2 instead.
590
641
  - Import error, syntax error, or missing dependency in the generated test file
591
642
  - Connection refused or timeout unrelated to the app under test
592
643
  4. If it still fails after the retry, report it as \`status: "Fail"\` with the error details and move on — do NOT edit and re-run a third time. A failing test that documents a real bug is a valid outcome.
@@ -597,13 +648,13 @@ If a generated UI test fails with a timeout waiting for an element after navigat
597
648
  2. Add \`await page.locator('[data-testid="some-element"]').waitFor({ state: 'visible', timeout: 10000 });\` for the specific element the test needs.
598
649
  Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than once — if the test still fails after this fix, report it as "Fail".
599
650
 
600
- **After generation, you MUST do exactly these steps — nothing more, nothing less:**
651
+ **After generation, you MUST do exactly these steps — nothing more, nothing less** (generation results may add their own CRITICAL NEXT STEPS — e.g. modularize-then-reuse for integration or UI tests generated with \`codeReuse: true\` — follow those too, in the order the result states):
601
652
  1. **[MANDATORY] After \`skyramp_integration_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated integration test file, \`testType: "integration"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
602
653
  2. **[MANDATORY] After \`skyramp_contract_test_generation\` with \`providerMode\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
603
654
  ${UI_CODE_REUSE_STEP}
604
- 4. **[MANDATORY] After \`skyramp_ui_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file. The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
605
- 5. **Wait**: Do NOT proceed to test execution until steps 1–4 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
606
- Do not make any changes other than the code-reuse refactoring (step 3) and the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
655
+ 4. **[MANDATORY] After \`skyramp_ui_test_generation\`**: ${UI_ENHANCE_STEP} The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
656
+ 5. **Wait**: Do NOT proceed to test execution until steps 1–4 (plus any generation-result CRITICAL NEXT STEPS) are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
657
+ Do not make any changes other than the code-reuse refactoring (step 3 and the generation-result reuse steps) and the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
607
658
 
608
659
  **Execution timing:**
609
660
  - **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
@@ -611,11 +662,13 @@ Do not make any changes other than the code-reuse refactoring (step 3) and the a
611
662
  - Only report test results for files you actually ran.
612
663
  **Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.`;
613
664
  }
614
- const primaryRepoBlock = primaryRepo ? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n` : '';
665
+ const primaryRepoBlock = primaryRepo
666
+ ? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n`
667
+ : "";
615
668
  return `<TITLE>${prTitle}</TITLE>
616
669
  <DESCRIPTION>${prDescription}</DESCRIPTION>
617
670
  ${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
618
- ${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + '\n' : ''}${uiCredentialsBlock ? uiCredentialsBlock + '\n' : ''}## Goal
671
+ ${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + "\n" : ""}${uiCredentialsBlock ? uiCredentialsBlock + "\n" : ""}## Goal
619
672
 
620
673
  Every test this run delivers must be a usable functional test — one that exercises the running application through its real API or UI surface and that the user can keep running in CI. Optimize for catching real production bugs: business-rule and computed-field errors, data-integrity violations, security-boundary bypasses, broken user journeys. A test that would FAIL if the application's logic were wrong beats several that merely exercise new surface — prefer fewer, higher-signal tests over padded coverage. The tasks below define which tests are in scope for this run; use the Skyramp MCP server tools for all of them.
621
674
 
@@ -645,13 +698,15 @@ ${task3CountRule}
645
698
 
646
699
  ${reportLanguageBlock}Call \`skyramp_submit_report\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking, and the report is written beside it. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
647
700
 
648
- ${hasRelatedRepos ? `
701
+ ${hasRelatedRepos
702
+ ? `
649
703
  - **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
650
- ` : ""}
704
+ `
705
+ : ""}
651
706
  - **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
652
707
  - For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
653
708
 
654
- ${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false })}`;
709
+ ${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false, modularizeViaGenerationResult: UI_UTILS_REUSE })}`;
655
710
  // Neither path reaches the agent any more: SKYR-4147 made the report derive from the
656
711
  // state file's directory, and that directory comes from the environment via
657
712
  // runArtifactDir(). Keep it that way — a path the model retypes out of this prose is a
@@ -1,6 +1,6 @@
1
1
  import { TestType } from "../../types/TestTypes.js";
2
2
  import { inferScenarioType, prioritizeAttackSurfaceBundles } from "../diversity.js";
3
- import { externalDedupKey, scenarioCoverageKey, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
3
+ import { externalDedupKeys, scenarioCoverageKeys, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
4
4
  import { logger } from "../../utils/logger.js";
5
5
  /**
6
6
  * Backend GENERATE slot count:
@@ -20,25 +20,39 @@ export function reservedUISlots(ctx) {
20
20
  return ctx.hasFrontendChanges && ctx.maxGenerate > 0 ? 1 : 0;
21
21
  }
22
22
  /**
23
- * Drop candidates whose method-aware resource+type is already covered by an
24
- * external test — except protected `bug_caught` / attack-surface scenarios,
25
- * which require semantic flaw coverage the external test may not provide.
23
+ * Split candidates by whether an external test already covers them. Protected
24
+ * `bug_caught` / attack-surface scenarios always count as uncovered — they
25
+ * require semantic flaw coverage the external test may not provide.
26
+ *
27
+ * A candidate is covered only when the set holds EVERY one of its keys. One
28
+ * match is not coverage: a scenario exercising two changed endpoints would lose
29
+ * the endpoint the external test does not reach (SKYR-4214). An empty key list
30
+ * carries no information, so it never removes anything.
31
+ *
32
+ * The covered group is held back, not discarded. `runBudget` moves it into
33
+ * ADDITIONAL when GENERATE's backend half would otherwise be empty (SKYR-4024),
34
+ * and records it as dropped, with a reason, when it does not.
26
35
  */
27
- function applyExternalDedup(ranked, externalCoverage) {
36
+ function partitionByExternalCoverage(ranked, externalCoverage) {
28
37
  if (externalCoverage.size === 0)
29
- return ranked;
30
- return ranked.filter((item) => {
31
- const key = externalDedupKey(item.scenario);
32
- if (externalCoverage.has(key)) {
33
- if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
34
- logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${key}) protected bug/attack-surface scenario requires semantic flaw coverage`);
35
- return true;
36
- }
37
- logger.info(`External dedup: skipping "${item.scenario.scenarioName}" (${key}) — covered by external test`);
38
- return false;
38
+ return { uncovered: ranked, covered: [] };
39
+ const uncovered = [];
40
+ const covered = [];
41
+ for (const item of ranked) {
42
+ const keys = externalDedupKeys(item.scenario);
43
+ if (keys.length === 0 || !keys.every((key) => externalCoverage.has(key))) {
44
+ uncovered.push(item);
45
+ continue;
39
46
  }
40
- return true;
41
- });
47
+ if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
48
+ logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
49
+ uncovered.push(item);
50
+ continue;
51
+ }
52
+ logger.info(`External dedup: "${item.scenario.scenarioName}" (${keys.join(", ")}) covered by an external test — eligible for the reserve`);
53
+ covered.push({ item, keys });
54
+ }
55
+ return { uncovered, covered };
42
56
  }
43
57
  /**
44
58
  * Shared budgeting pipeline. All Budgeters run the same external-dedup,
@@ -49,10 +63,11 @@ function applyExternalDedup(ranked, externalCoverage) {
49
63
  * diffExecutionPlan.ts exactly.
50
64
  */
51
65
  export function runBudget(ranked, ctx, pick) {
66
+ const dropped = [];
52
67
  const backend = backendGenerateCount(ctx);
53
68
  const uiSlots = reservedUISlots(ctx);
54
- const deduped = applyExternalDedup(ranked, ctx.externalCoverage);
55
- const slotOrdered = prioritizeAttackSurfaceBundles(deduped);
69
+ const { uncovered, covered } = partitionByExternalCoverage(ranked, ctx.externalCoverage);
70
+ const slotOrdered = prioritizeAttackSurfaceBundles(uncovered);
56
71
  // When UI slots are reserved, keep UI candidates out of the backend pick so the
57
72
  // reserved slot can always be filled below. Otherwise a high-ranked (e.g.
58
73
  // CRITICAL) UI candidate could be consumed by the backend pick on a mixed PR,
@@ -77,14 +92,77 @@ export function runBudget(ranked, ctx, pick) {
77
92
  chosen.add(item);
78
93
  }
79
94
  const generate = [...backendGenerate, ...uiGenerate];
80
- // ADDITIONAL = everything not chosen for GENERATE, in rank order, capped at the
81
- // remaining budget, minus any whose resource+type is already covered by GENERATE.
82
- const generatedCoverage = new Set(generate.map((item) => scenarioCoverageKey(item.scenario)));
83
- const additional = slotOrdered
84
- .filter((it) => !chosen.has(it))
85
- .slice(0, Math.max(0, ctx.maxTotal - generate.length))
86
- .filter((item) => !generatedCoverage.has(scenarioCoverageKey(item.scenario)));
95
+ // SKYR-4024. A covered candidate is one an existing test already reaches, so
96
+ // generating a second test for it is wrong maintenance updates the existing
97
+ // one. But dropping it outright removed it from the RECOMMENDATIONS too, and an
98
+ // empty GENERATE is authoritative downstream. The reserve surfaces it as
99
+ // ADDITIONAL instead. The emptiness test reads the backend half: on a mixed PR
100
+ // one UI candidate fills the reserved slot while every covered backend
101
+ // candidate would otherwise vanish.
102
+ //
103
+ // Two conditions keep the reserve shut where recommending nothing is right.
104
+ // Backend slots must exist, so a UI-only PR is left alone. And the PR must not
105
+ // touch a test file of its own: coverage is read from the working tree, and a
106
+ // test file without the Skyramp marker counts as external, so a PR that adds
107
+ // its own tests would have that coverage used as grounds to recommend
108
+ // duplicates of it.
109
+ const spendReserve = backendGenerate.length === 0 && backend > 0 && covered.length > 0 && !ctx.diffChangesTestFiles;
110
+ if (spendReserve) {
111
+ logger.info(`External dedup: GENERATE empty after dedup — recommending ${covered.length} held-back candidate(s) as ADDITIONAL rather than approving an empty plan`);
112
+ }
113
+ else if (covered.length > 0) {
114
+ // SKYR-4214. The reserve stayed shut, so these really are dropped — say so,
115
+ // with the reason, instead of leaving the agent an unexplained short list.
116
+ const reason = backendGenerate.length > 0
117
+ ? `the backend half of GENERATE is filled (${backendGenerate.length})`
118
+ : backend === 0
119
+ ? "no backend slots (UI-only PR)"
120
+ : "the PR changes test files of its own";
121
+ logger.info(`External dedup: dropping ${covered.length} covered candidate(s) — reserve shut because ${reason}`);
122
+ for (const { item, keys } of covered) {
123
+ dropped.push({
124
+ candidateId: item.candidateId,
125
+ reason: `covered by an external test (${keys.join(", ")})`,
126
+ });
127
+ }
128
+ }
129
+ // ADDITIONAL = everything not chosen for GENERATE, in rank order, minus any
130
+ // whose subject endpoints are ALL already covered by GENERATE, then capped at
131
+ // the remaining budget. All, not any: an item that also covers an endpoint no
132
+ // GENERATE item reaches is not a duplicate. An empty key list never removes
133
+ // an item.
134
+ //
135
+ // The duplicate filter runs BEFORE the cap. In the other order a duplicate
136
+ // took a slot and then vacated it, and nothing moved up: eval run
137
+ // 32543498147, fixture cc15-org-reviewer-role, filled 19 of a maxTotal of 20
138
+ // while 16 candidates carried the reason "beyond the maxTotal budget of 20".
139
+ // A candidate that is both a duplicate and past the cap now reports the
140
+ // duplicate — the stronger reason, and the only one that does not depend on
141
+ // where the cap happened to fall.
142
+ const additionalSource = spendReserve
143
+ ? [...slotOrdered, ...prioritizeAttackSurfaceBundles(covered.map((c) => c.item))]
144
+ : slotOrdered;
145
+ const notChosen = additionalSource.filter((it) => !chosen.has(it));
146
+ const remainingBudget = Math.max(0, ctx.maxTotal - generate.length);
147
+ const generatedCoverage = new Set(generate.flatMap((item) => scenarioCoverageKeys(item.scenario)));
148
+ const notDuplicate = notChosen.filter((item) => {
149
+ const keys = scenarioCoverageKeys(item.scenario);
150
+ if (keys.length === 0 || !keys.every((key) => generatedCoverage.has(key)))
151
+ return true;
152
+ dropped.push({
153
+ candidateId: item.candidateId,
154
+ reason: `covered by GENERATE (${keys.join(", ")})`,
155
+ });
156
+ return false;
157
+ });
158
+ const additional = notDuplicate.slice(0, remainingBudget);
159
+ for (const item of notDuplicate.slice(remainingBudget)) {
160
+ dropped.push({
161
+ candidateId: item.candidateId,
162
+ reason: `beyond the maxTotal budget of ${ctx.maxTotal}`,
163
+ });
164
+ }
87
165
  // Budgeting itself produces no demotions; the register-plan selection stage
88
166
  // (planRanker.selectPlan) fills this channel from discriminator verification.
89
- return { generate, additional, reservedUISlots: uiSlots, demotions: [] };
167
+ return { generate, additional, reservedUISlots: uiSlots, demotions: [], dropped };
90
168
  }
@@ -281,9 +281,20 @@ function isNegativeAssertStep(step) {
281
281
  if (step?.expectedStatusCode === 404)
282
282
  return true;
283
283
  const assertions = [...toStringArray(step?.expectedResponseFields), ...toStringArray(step?.bodyMustInclude)];
284
- return assertsEmptyOrZero(assertions);
284
+ return assertsEmptyOrZero(assertions, step?.responseBody);
285
285
  }
286
- function assertsEmptyOrZero(strings) {
286
+ /**
287
+ * True when the step asserts an empty or zero result, reading the assertion
288
+ * strings AND `responseBody`. An EMPTY ARRAY body is itself the "no rows came
289
+ * back" assertion, and can be the only empty-result signal a step carries.
290
+ *
291
+ * `{}` deliberately does NOT count: an unspecified body is filled in as `{}`
292
+ * (ScenarioGenerationService), so it cannot be told apart from a body the agent
293
+ * never wrote. `isAssertStep` above already reads `{}` as asserting nothing.
294
+ */
295
+ function assertsEmptyOrZero(strings, responseBody) {
296
+ if (Array.isArray(responseBody) && responseBody.length === 0)
297
+ return true;
287
298
  return strings.some((raw) => {
288
299
  const s = raw.toLowerCase();
289
300
  if (/\bcount\b/.test(s) && /\b0\b/.test(s))
@@ -4,9 +4,9 @@ import { ScenarioCategory } from "../types/TestRecommendation.js";
4
4
  export interface RankOptions {
5
5
  /**
6
6
  * Categories that take the top carve-out tier ahead of everything else.
7
- * Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (new_endpoint,
8
- * bug_caught). Exposed so phase 2 can tune the carve-out WITHOUT reintroducing
9
- * the agent's priority tag as a ranking input.
7
+ * Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught only —
8
+ * new_endpoint is MEDIUM, not carved out). Exposed so phase 2 can tune the carve-out
9
+ * WITHOUT reintroducing the agent's priority tag as a ranking input.
10
10
  */
11
11
  carveOutCategories?: ScenarioCategory[];
12
12
  /**
@@ -32,9 +32,9 @@ export interface SelectPlanContext extends BudgetContext {
32
32
  * independent of input order (the final tiebreak is the stable `candidateId`).
33
33
  *
34
34
  * Ordering (highest first):
35
- * 1. Carve-out — bug_caught / CRITICAL-category scenarios (new_endpoint,
36
- * bug_caught), preserving the protected-first convention of
37
- * `roundRobinByType` / `prioritizeAttackSurfaceBundles`.
35
+ * 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
36
+ * protected-first convention of `roundRobinByType` /
37
+ * `prioritizeAttackSurfaceBundles`.
38
38
  * 2. Verified discriminators — candidates whose declared discriminator survived
39
39
  * `validateDiscriminator` (marked via `verifiedDiscriminator`) float ahead of
40
40
  * unverified peers in the same tier.
@@ -1,6 +1,6 @@
1
1
  import { CATEGORY_PRIORITY, PriorityTier } from "../types/TestRecommendation.js";
2
2
  import { diversityBalancedBudgeter } from "./budgeters/diversityBalancedBudgeter.js";
3
- import { parseRouteLine, normalizeDiffPath } from "../utils/routeParsers.js";
3
+ import { collectChangedRouteLines, findStepOnChangedRoute } from "../utils/changedRoutes.js";
4
4
  const PRIORITY_RANK = {
5
5
  CRITICAL: 0,
6
6
  HIGH: 1,
@@ -14,9 +14,9 @@ const DEFAULT_CARVE_OUT_CATEGORIES = Object.keys(CATEGORY_PRIORITY).filter((cate
14
14
  * independent of input order (the final tiebreak is the stable `candidateId`).
15
15
  *
16
16
  * Ordering (highest first):
17
- * 1. Carve-out — bug_caught / CRITICAL-category scenarios (new_endpoint,
18
- * bug_caught), preserving the protected-first convention of
19
- * `roundRobinByType` / `prioritizeAttackSurfaceBundles`.
17
+ * 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
18
+ * protected-first convention of `roundRobinByType` /
19
+ * `prioritizeAttackSurfaceBundles`.
20
20
  * 2. Verified discriminators — candidates whose declared discriminator survived
21
21
  * `validateDiscriminator` (marked via `verifiedDiscriminator`) float ahead of
22
22
  * unverified peers in the same tier.
@@ -77,62 +77,7 @@ function categoryRank(candidate) {
77
77
  const tier = CATEGORY_PRIORITY[candidate.scenario?.category] ?? PriorityTier.LOW;
78
78
  return PRIORITY_RANK[tier];
79
79
  }
80
- /**
81
- * Extract method+path from every changed (`+`/`-`, non-header) line of a raw
82
- * unified diff. Reuses `parseRouteLine`, which already strips the leading
83
- * `+`/`-` marker and matches the same route-decorator patterns the endpoint
84
- * scanner does.
85
- */
86
- function collectChangedRouteLines(diffText) {
87
- const routes = [];
88
- let currentFile = "";
89
- for (const line of diffText.split("\n")) {
90
- // Track the current file from the unified-diff header so parseRouteLine
91
- // gets the real path — its UI-component guard (UI_COMPONENT_EXT) depends
92
- // on it, or a route-shaped line inside a .tsx/.jsx file (e.g. a client
93
- // router registration) gets misparsed as a changed backend route.
94
- if (line.startsWith("+++ ")) {
95
- const spec = line.slice(4).trim().split("\t")[0];
96
- currentFile = spec === "/dev/null" ? "" : normalizeDiffPath(spec);
97
- continue;
98
- }
99
- if (line.startsWith("--- ") || line.startsWith("diff --git") || line.startsWith("index "))
100
- continue;
101
- if (!(line.startsWith("+") || line.startsWith("-")))
102
- continue;
103
- const parsed = parseRouteLine(line, currentFile);
104
- if (parsed)
105
- routes.push({ method: parsed.method, path: parsed.path });
106
- }
107
- return routes;
108
- }
109
- /**
110
- * Whether any step of `scenario` targets a method+path on the changed hunk.
111
- * Diff-extracted paths are local to the file's own router declaration (e.g.
112
- * "/suggestions"); scenario step paths are fully mounted (e.g.
113
- * "/api/recipes/suggestions") once drafted from a scanned/recovered endpoint.
114
- * A local path matches when the step path ends with it, so the cross-file
115
- * mount prefix difference (see recoverRemovedEndpointsFromBase, SKYR-4026)
116
- * doesn't prevent the match.
117
- */
80
+ /** Whether any step of `scenario` targets a method+path on the changed hunk. */
118
81
  function scenarioOnChangedHunk(scenario, changedRoutes) {
119
- if (changedRoutes.length === 0)
120
- return false;
121
- for (const step of scenario.steps ?? []) {
122
- const stepMethod = (step.method ?? "").toUpperCase();
123
- const stepPath = (step.path ?? "").replace(/\/+$/, "");
124
- for (const route of changedRoutes) {
125
- if (route.method.toUpperCase() !== stepMethod)
126
- continue;
127
- const routePath = route.path.replace(/\/+$/, "");
128
- if (routePath === "") {
129
- if (stepPath === "" || stepPath === "/")
130
- return true;
131
- continue;
132
- }
133
- if (stepPath === routePath || stepPath.endsWith(routePath))
134
- return true;
135
- }
136
- }
137
- return false;
82
+ return !!findStepOnChangedRoute(scenario.steps, changedRoutes);
138
83
  }