@skyramp/mcp 0.3.6 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/build/prompts/code-reuse.js +17 -2
  2. package/build/prompts/enhance-assertions/sharedAssertionRules.js +1 -1
  3. package/build/prompts/modularization/ui-test-modularization.js +9 -6
  4. package/build/prompts/pom-aware-code-reuse.js +1 -1
  5. package/build/prompts/shared-helper-policy.js +5 -5
  6. package/build/prompts/test-recommendation/diffExecutionPlan.js +40 -9
  7. package/build/prompts/test-recommendation/fullRepoCatalog.js +3 -2
  8. package/build/prompts/test-recommendation/recommendationSections.js +3 -3
  9. package/build/prompts/test-recommendation/scopeAssessment.d.ts +2 -2
  10. package/build/prompts/test-recommendation/scopeAssessment.js +58 -76
  11. package/build/prompts/test-recommendation/test-recommendation-prompt.js +14 -1
  12. package/build/prompts/testbot/testbot-prompts.js +25 -8
  13. package/build/recommendation/budgeters/diversityBalancedBudgeter.js +39 -5
  14. package/build/recommendation/budgeters/shared.d.ts +21 -8
  15. package/build/recommendation/budgeters/shared.js +134 -56
  16. package/build/recommendation/planRanker.d.ts +5 -3
  17. package/build/recommendation/planRanker.js +1 -1
  18. package/build/services/TestGenerationService.js +15 -1
  19. package/build/tools/code-refactor/assertion-state.d.ts +91 -0
  20. package/build/tools/code-refactor/assertion-state.js +375 -0
  21. package/build/tools/code-refactor/codeReuseTool.js +6 -4
  22. package/build/tools/code-refactor/enhanceAssertionsTool.js +73 -18
  23. package/build/tools/code-refactor/retrofit-state.d.ts +53 -0
  24. package/build/tools/code-refactor/retrofit-state.js +162 -0
  25. package/build/tools/code-refactor/reuse-outcome.d.ts +7 -0
  26. package/build/tools/code-refactor/reuse-state.d.ts +9 -0
  27. package/build/tools/code-refactor/reuse-state.js +42 -4
  28. package/build/tools/code-refactor/utils-verify-gates.js +69 -15
  29. package/build/tools/executeSkyrampTestTool.js +19 -14
  30. package/build/tools/submitReportTool.js +87 -9
  31. package/build/tools/test-management/registerTestPlanTool.d.ts +3 -3
  32. package/build/tools/test-management/registerTestPlanTool.js +29 -8
  33. package/build/types/AssertionOutcome.d.ts +68 -0
  34. package/build/types/AssertionOutcome.js +1 -0
  35. package/build/types/RepositoryAnalysis.d.ts +10 -10
  36. package/build/types/ReuseOutcome.d.ts +16 -0
  37. package/build/types/TestRecommendation.d.ts +11 -1
  38. package/build/types/TestRecommendation.js +34 -0
  39. package/build/types/TestTypes.d.ts +4 -0
  40. package/build/types/TestTypes.js +8 -0
  41. package/build/types/TestbotReport.d.ts +13 -0
  42. package/build/types/index.d.ts +1 -1
  43. package/build/utils/AnalysisStateManager.d.ts +20 -7
  44. package/build/utils/assertion-verify/api-shared-lints.d.ts +5 -0
  45. package/build/utils/assertion-verify/api-shared-lints.js +315 -0
  46. package/build/utils/assertion-verify/contract-lints.d.ts +3 -0
  47. package/build/utils/assertion-verify/contract-lints.js +87 -0
  48. package/build/utils/assertion-verify/format.d.ts +5 -0
  49. package/build/utils/assertion-verify/format.js +65 -0
  50. package/build/utils/assertion-verify/helper-imports.d.ts +6 -0
  51. package/build/utils/assertion-verify/helper-imports.js +178 -0
  52. package/build/utils/assertion-verify/index.d.ts +3 -0
  53. package/build/utils/assertion-verify/index.js +7 -0
  54. package/build/utils/assertion-verify/integration-lints.d.ts +3 -0
  55. package/build/utils/assertion-verify/integration-lints.js +36 -0
  56. package/build/utils/assertion-verify/js-regex-blank.d.ts +1 -0
  57. package/build/utils/assertion-verify/js-regex-blank.js +153 -0
  58. package/build/utils/assertion-verify/lint-types.d.ts +33 -0
  59. package/build/utils/assertion-verify/lint-types.js +57 -0
  60. package/build/utils/assertion-verify/marker.d.ts +27 -0
  61. package/build/utils/assertion-verify/marker.js +61 -0
  62. package/build/utils/assertion-verify/metrics.d.ts +30 -0
  63. package/build/utils/assertion-verify/metrics.js +341 -0
  64. package/build/utils/assertion-verify/python-strip.d.ts +6 -0
  65. package/build/utils/assertion-verify/python-strip.js +75 -0
  66. package/build/utils/assertion-verify/strip-dispatch.d.ts +19 -0
  67. package/build/utils/assertion-verify/strip-dispatch.js +42 -0
  68. package/build/utils/assertion-verify/ui-lints.d.ts +8 -0
  69. package/build/utils/assertion-verify/ui-lints.js +244 -0
  70. package/build/utils/assertion-verify/verify.d.ts +61 -0
  71. package/build/utils/assertion-verify/verify.js +215 -0
  72. package/build/utils/featureFlags.d.ts +12 -2
  73. package/build/utils/featureFlags.js +33 -3
  74. package/build/utils/pathSignatures.d.ts +4 -1
  75. package/build/utils/pathSignatures.js +14 -2
  76. package/build/utils/planOnlyMode.d.ts +33 -0
  77. package/build/utils/planOnlyMode.js +40 -0
  78. package/build/utils/reportVerification.d.ts +9 -0
  79. package/build/utils/reportVerification.js +39 -6
  80. package/build/utils/scenarioDrafting.d.ts +1 -1
  81. package/build/utils/scenarioDrafting.js +1 -1
  82. package/build/utils/utils-verify/allow.d.ts +22 -4
  83. package/build/utils/utils-verify/allow.js +8 -2
  84. package/build/utils/utils-verify/call-sites.d.ts +40 -1
  85. package/build/utils/utils-verify/call-sites.js +196 -30
  86. package/build/utils/utils-verify/importers.d.ts +31 -0
  87. package/build/utils/utils-verify/importers.js +78 -0
  88. package/build/utils/utils-verify/index.d.ts +1 -0
  89. package/build/utils/utils-verify/index.js +1 -0
  90. package/build/utils/utils-verify/language-spec.d.ts +13 -2
  91. package/build/utils/utils-verify/language-spec.js +12 -2
  92. package/build/utils/utils-verify/parse.d.ts +31 -3
  93. package/build/utils/utils-verify/parse.js +190 -9
  94. package/build/utils/utils-verify/retrofit-equivalence.d.ts +43 -0
  95. package/build/utils/utils-verify/retrofit-equivalence.js +218 -0
  96. package/build/utils/utils-verify/stage.d.ts +6 -0
  97. package/build/utils/utils-verify/stage.js +12 -2
  98. package/build/utils/utils-verify/verify.d.ts +54 -4
  99. package/build/utils/utils-verify/verify.js +224 -12
  100. package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
  101. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-CZfp96qZ.js → codeMirrorModule-LNgEKtdV.js} +1 -1
  102. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-gpLo02E0.js → defaultSettingsView-Bwr1eMKC.js} +135 -135
  103. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Bq1r1URj.js → index.-Id052Lr.js} +1 -1
  104. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  105. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.VEfqi1qN.js → uiMode.BPopbasy.js} +1 -1
  106. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  107. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  108. package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
  109. package/node_modules/playwright/package.json +1 -1
  110. package/package.json +2 -2
  111. package/build/tools/code-refactor/enhance-state.d.ts +0 -49
  112. package/build/tools/code-refactor/enhance-state.js +0 -109
@@ -2,7 +2,11 @@ import * as fs from "fs";
2
2
  import * as path from "path";
3
3
  import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "./recommendationSections.js";
4
4
  // .vue and .svelte cannot be route handlers in any framework — always frontend.
5
- const ALWAYS_FRONTEND_EXT = /\.(vue|svelte)$/i;
5
+ // Stylesheets cannot be either (SKYR-4292): `packages/excalidraw/css/styles.scss`
6
+ // classified as backend because `css/` is not in FRONTEND_DIR_PATTERN, so a
7
+ // stylesheet-only PR took the zero-classified backend branch and never saw the
8
+ // changed-behavior rule.
9
+ const ALWAYS_FRONTEND_EXT = /\.(vue|svelte|css|scss|less)$/i;
6
10
  // .dart is always frontend in Flutter projects (no server-side Dart web framework
7
11
  // in scope). Gated on hasFlutterPubspec so non-Flutter Dart files (rare, but
8
12
  // possible — e.g. Dart-on-server backends) don't accidentally classify as UI.
@@ -10,7 +14,7 @@ const FLUTTER_DART_EXT = /\.dart$/i;
10
14
  // .tsx/.jsx are usually UI components but can be Next.js API handlers (pages/api/*.tsx).
11
15
  const LIKELY_FRONTEND_EXT = /\.(tsx|jsx)$/i;
12
16
  // Ambiguous extensions need directory context to distinguish frontend from backend.
13
- const AMBIGUOUS_FRONTEND_PATTERN = /\.(tsx?|jsx?|css|scss|less|html?|erb|jsp|asp|jinja2?|twig)$/i;
17
+ const AMBIGUOUS_FRONTEND_PATTERN = /\.(tsx?|jsx?|html?|erb|jsp|asp|jinja2?|twig)$/i;
14
18
  // /api/ always signals backend — even .tsx files (e.g. Next.js pages/api/*.tsx handlers).
15
19
  const STRICT_API_DIR_PATTERN = /\/(api)\//i;
16
20
  // Framework route dirs that contain React page components in modern full-stack apps
@@ -189,67 +193,38 @@ export function isTestFile(filePath) {
189
193
  }
190
194
  // ── LLM scope assessment ──────────────────────────────────────────────────────
191
195
  /**
192
- * The zero-new-surface abstention rule, shared by every branch that can see a frontend
193
- * diff (SKYR-4099).
196
+ * The changed-behavior budget rule, shared by every branch that can see a frontend
197
+ * diff (SKYR-4292).
194
198
  *
195
- * It previously existed only on the mixed-PR branch, so a frontend-ONLY diff — which
196
- * takes the precomputed branchhad no sanctioned path to zero tests and the agent
197
- * generated unnecessary UI tests while stating in its own reasoning that the change was
198
- * cosmetic. Two copies then meant two definitions, and the mixed-PR one carved out
199
- * "changes that alter visibility, layout, or state", which classifies a spacing-token
200
- * change as non-cosmetic and made the override inert for exactly the diffs it should
201
- * catch. One definition, both branches.
199
+ * This replaces the zero-new-surface abstention override (SKYR-4099). That override
200
+ * named three abstain categoriescosmetic, deletion-only, and modification of an
201
+ * already-covered surface and its only sanctioned action was a Budget Plan of 0. On
202
+ * a 20-change UI-only diff under a 20 budget (Wipro PR #24) the agent generalized it
203
+ * into a graded cut to 3, classified option additions no test asserted (Siebel,
204
+ * Windows, .NET, GO, MQ) as "modification of an already-covered surface" because the
205
+ * existing specs reached the wizard page, and shipped zero recommendations for the
206
+ * rest. The taxonomy is gone: every observable behavior the diff changes gets a
207
+ * candidate, coverage means an existing test ASSERTS the changed value (not that it
208
+ * reaches the component), and the total tracks the count of changed behaviors — the
209
+ * ranker in `skyramp_register_test_plan` trims to the ceiling and reports the drops.
202
210
  *
203
- * Scoped to match `testbot-prompts.ts`'s "Do not fabricate tests outside the GENERATE
204
- * list", which names three zero-test cases deletion-only, cosmetic, and
205
- * modification-of-existing-with-no-new-surface under one principle: a new spec covers
206
- * NEW observable surface only. An earlier revision of this section implemented cosmetic
207
- * alone, and its keep-the-budget list contradicted the other two (an element being
208
- * removed, or a `data-testid` being renamed, both forced the budget to stand). The test
209
- * is coverage, not the kind of edit: does an existing test already reach this surface?
210
- *
211
- * `skipClause` is appended to the opening paragraph: the mixed-PR branch has a UI%
212
- * step that becomes irrelevant once the budget is 0, the precomputed branch does not.
213
- */
214
- /**
215
- * The zero-new-surface abstention rule, shared by every branch that can see a frontend
216
- * diff (SKYR-4099).
217
- *
218
- * It previously existed only on the mixed-PR branch, so a frontend-ONLY diff — which
219
- * takes the precomputed branch — had no sanctioned path to zero tests and the agent
220
- * generated unnecessary UI tests while stating in its own reasoning that the change was
221
- * cosmetic. Two copies then meant two definitions, and the mixed-PR one carved out
222
- * "changes that alter visibility, layout, or state", which classifies a spacing-token
223
- * change as non-cosmetic and made the override inert for exactly the diffs it should
224
- * catch. One definition, both branches.
225
- *
226
- * Scoped to match `testbot-prompts.ts`'s "Do not fabricate tests outside the GENERATE
227
- * list", which names three zero-test cases — deletion-only, cosmetic, and
228
- * modification-of-existing-with-no-new-surface — under one principle: a new spec covers
229
- * NEW observable surface only. The test is coverage, not the kind of edit: does an
230
- * existing test already reach this surface?
231
- *
232
- * `skipClause` is appended to the opening paragraph: the mixed-PR branch has a UI% step
233
- * that becomes irrelevant once the budget is 0, the precomputed branch does not.
211
+ * Backend-only diffs do not render this: their GENERATE list is server pre-ranked
212
+ * from classified endpoints (and SKYR-3820 handles a zero-classified diff), whereas a
213
+ * frontend GENERATE list is placeholders the agent must fill from the diff itself.
234
214
  */
235
- function zeroSurfaceSection(skipClause = "") {
236
- return `**Zero-new-surface override:** The total above is a default, not a mandate. A new spec exists to cover **new observable surface**a component, route, page or flow that no existing test reaches. If your code review finds the diff adds none, set your Budget Plan to **0 total** and abstainrecommend and generate zero tests${skipClause}.
215
+ function changedBehaviorSection(effectiveGenerate, budgetPlanTiming = "") {
216
+ return `**Fill the budget from changed behavior:** The ceiling above is the most this run may plan, not a quota to reach or a default to cut. Enumerate every observable behavior this diff changes each option value added, removed, renamed or reordered; each label, element, route, state, validation rule or computed value that renders or behaves differently — and give each one its own candidate. Treat no change as too trivial to test: a changed option value that no test asserts is a test. A changed style value counts too — a spacing, size, color, radius, font weight, icon size or class name that now renders differently is a changed behavior; its candidate asserts the new computed style or class on that element, or records a \`browser_visual_snapshot\` of it. A removed option, element or feature is a changed behavior too: its candidate asserts the removed thing is absent from the page where it used to render the removal is what this PR ships, so the removal is what gets tested.
237
217
 
238
- Abstain the diff adds no new surface:
239
- - **Cosmetic.** A styling-only value change (a spacing, size, color or font token, or a utility class swap such as \`size-4\`→\`size-5\`), or a \`.css\`/\`.scss\` reformat (property reordering, comment or whitespace edits, \`0px\`→\`0\`).
240
- - **Deletion-only.** A component, route, element or feature was removed. The work is DELETING the tests that covered it — a removed surface cannot be the subject of a new spec.
241
- - **Modification of an already-covered surface.** A renamed or moved selector, \`data-testid\`, \`aria-*\` or role; changed copy; an added field; a reordered or conditionally hidden element — where an existing test already reaches it. The work is UPDATING that test in place.
218
+ A changed behavior is something a user can observe differently after this diff. Code the diff deletes, dedupes or simplifies without changing what renders is not a changed behavior — its coverage is maintenance of the tests that reach it — and a pre-existing defect you notice nearby is an \`issuesFound\` entry, not a slot in this budget. Only a comment, whitespace or equivalent-value edit (\`0px\` → \`0\`) changes nothing observable.
242
219
 
243
- Keep the budget only for surface no existing test covers:
244
- - A newly added component, route, page or flow.
245
- - A component that was previously unintegrated and now has an integration point.
246
- - New interactive behavior, state or validation on a surface no existing test reaches.
220
+ A changed behavior is covered only when an existing test asserts the changed value (including after a maintenance UPDATE you recorded). A test that merely reaches the component does not cover it — that change still needs its own candidate.
247
221
 
248
- Two things that are NOT evidence of new surface: a frontend file appearing in the diff, and a large diff. Judge by whether an existing test already reaches the changed surface.
222
+ - Fewer changed behaviors than the ceiling your Budget Plan total is that count, split as generate first (up to ${effectiveGenerate}) and the remainder additional. Do not pad with tests of behavior this diff did not change.
223
+ - More changed behaviors than the ceiling → keep every one in the candidate list you register; \`skyramp_register_test_plan\` trims to the ceiling and reports what it dropped.
249
224
 
250
- With a 0-total Budget Plan the work this diff needs is maintenance of the tests that already cover the affected surface update the ones whose selectors or copy moved, and delete the ones that covered something this diff removed. Do NOT add a spec asserting that a removed feature is absent: the tests that covered it are the ones to delete, and an "is not present" assertion breaks the next time an unrelated sibling element changes.
225
+ **A requirement the diff did NOT implement counts too.** The enumeration above is of behaviors the diff changed; a \`requirement_conflict\` is the opposite — the PR title/description (or a requirements file it references) states a requirement and the diff failed to implement it. Give it its own candidate anyway, category \`requirement_conflict\`, asserting what the description requires so that it fails on the current code. When the description says a feature was removed and it still renders, the absence assertion IS that test it fails today, which is the finding. It counts toward your total like any changed behavior, and it is the one candidate a total of zero must never swallow: abstaining past a stated requirement reports the code back to the author as if it were the requirement.
251
226
 
252
- If nothing currently covers the changed surface, there is no maintenance to do and still no new spec to write, because a test added now would assert behavior this diff did not change. The missing coverage is a pre-existing gap, not something this PR introduced. Record it in \`additionalRecommendations\` in recommendatory voice ("would verify …") so the gap is visible without claiming a test was written.`;
227
+ Never reduce the total below the number of changed behaviors you found. State your Budget Plan in the canonical format${budgetPlanTiming} \`Budget Plan: <total> total (<generate> generate + <additional> additional), <ui_pct>% UI/E2E\` and use those numbers throughout the rest of the prompt.`;
253
228
  }
254
229
  /**
255
230
  * Builds the PR scope assessment section.
@@ -257,8 +232,8 @@ If nothing currently covers the changed surface, there is no maintenance to do
257
232
  * When `precomputedUIPct` is provided (0 = backend-only, 100 = UI-only) the server has
258
233
  * already determined the split unambiguously, so Steps A–C are skipped. Backend-only
259
234
  * (0) renders a single Budget Plan line; UI-only (100) renders that line plus the
260
- * zero-new-surface override (SKYR-4099), because the budget is a default there rather
261
- * than a mandate and a diff that adds no new surface must be able to abstain.
235
+ * changed-behavior budget rule (SKYR-4292), because a frontend GENERATE list is
236
+ * placeholders the agent fills from the diff and the total must track what changed.
262
237
  *
263
238
  * For mixed PRs (`precomputedUIPct` is undefined, `hasFrontendChanges` is true) skip
264
239
  * Steps A–C but keep Step D so the LLM can apply judgment to determine the UI%.
@@ -293,40 +268,47 @@ Budget Plan: 0 total — no new, modified, or removed endpoints were classified
293
268
 
294
269
  With a 0-total Budget Plan: generate zero tests, recommend zero tests, and follow the zero-test report path. Do NOT draft baseline or generic tests for unchanged endpoints to fill a budget — an empty diff surface is a valid, expected outcome.
295
270
 
296
- **Exception — claim the ceiling only with evidence:** if your code review of the changed files shows an observable API behavior change the classifier missed (e.g. a DTO/serializer/service change that alters a response shape, a shared library/default-value or business-rule constant change that alters the behavior of an existing, unchanged endpoint (e.g. a default schedule, threshold, or config constant imported by a route handler elsewhere in the codebase), a deployment/config change that newly exposes or removes endpoints, or a schema-defined API contract change — a CRD type/kubebuilder validation marker, GraphQL schema, or gRPC proto edit that adds, removes, or re-validates what the server accepts or returns), raise your Budget Plan to cover exactly those affected endpoints, up to ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional), 0% UI/E2E. Note: repositories whose entire API surface is schema-defined (e.g. a Kubernetes operator serving CRDs through the kube-apiserver) ALWAYS classify zero endpoints — for these, a schema change in the diff IS the endpoint change; evaluate this exception against the schema files instead of concluding there is nothing to test. Similarly, a changed file with zero classified endpoints is not by itself evidence of "no testable surface" — trace what imports the changed export (grep for its name) to check whether it feeds an existing endpoint's behavior before concluding the diff has no test value. Every test must name the changed file that justifies it. State your raised plan now in the canonical format — \`Budget Plan: <total> total (<generate> generate + <additional> additional), 0% UI/E2E\` — and use those exact numbers throughout the rest of the prompt; the raised generate count is your committed generate count.`;
271
+ **Exception — claim the ceiling only with evidence:** if your code review of the changed files shows an observable API behavior change the classifier missed (e.g. a DTO/serializer/service change that alters a response shape, a shared library/default-value or business-rule constant change that alters the behavior of an existing, unchanged endpoint (e.g. a default schedule, threshold, or config constant imported by a route handler elsewhere in the codebase), a deployment/config change that newly exposes or removes endpoints, or a schema-defined API contract change — a CRD type/kubebuilder validation marker, GraphQL schema, or gRPC proto edit that adds, removes, or re-validates what the server accepts or returns), raise your Budget Plan to cover exactly those affected endpoints, up to ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional), 0% UI/E2E. Note: repositories whose entire API surface is schema-defined (e.g. a Kubernetes operator serving CRDs through the kube-apiserver) ALWAYS classify zero endpoints — for these, a schema change in the diff IS the endpoint change; evaluate this exception against the schema files instead of concluding there is nothing to test. Similarly, a changed file with zero classified endpoints is not by itself evidence of "no testable surface" — trace what imports the changed export (grep for its name) to check whether it feeds an existing endpoint's behavior before concluding the diff has no test value. **A requirement conflict is also grounds to raise the plan:** if the PR title/description (or a requirements file it references) states a requirement the diff did not implement, claim at least one slot for a \`requirement_conflict\` test asserting that requirement — it fails on the current code, which is the finding. Every test must name the changed file that justifies it. State your raised plan now in the canonical format — \`Budget Plan: <total> total (<generate> generate + <additional> additional), 0% UI/E2E\` — and use those exact numbers throughout the rest of the prompt; the raised generate count is your committed generate count.`;
297
272
  }
298
273
  // Unambiguous backend-only or UI-only: no LLM counting needed. Backend-only emits just
299
- // the Budget Plan line; UI-only appends the zero-new-surface override (see below).
274
+ // the Budget Plan line; UI-only appends the changed-behavior budget rule (see above).
300
275
  if (precomputedUIPct !== undefined) {
301
276
  const uiSuffix = precomputedUIPct > 0 ? `, ${precomputedUIPct}% UI/E2E` : "";
302
- // Ordered ahead of the "use these exact numbers" line — that line reads as final, so
303
- // an override printed after it cannot fire. Backend-only (precomputedUIPct === 0) has
304
- // no frontend file to call cosmetic, so it gets no override.
305
- const cosmeticOverride = precomputedUIPct > 0 ? `${zeroSurfaceSection()}\n\n` : "";
277
+ // Backend-only (precomputedUIPct === 0): the server pre-ranked the GENERATE list
278
+ // from classified endpoints, so the numbers are fixed. UI-only: the list is
279
+ // placeholders the agent fills from the diff, so the line is a ceiling and the
280
+ // changed-behavior rule sets the Budget Plan a fixed "Budget Plan" plus "use
281
+ // these exact numbers" here would contradict that rule (Copilot on mcp#835).
282
+ if (precomputedUIPct === 0) {
283
+ return `### PR Scope Assessment
284
+ Budget Plan: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)
285
+
286
+ Use these exact numbers throughout the rest of the prompt.`;
287
+ }
306
288
  return `### PR Scope Assessment
307
- Budget Plan: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)${uiSuffix}
289
+ Budget ceiling: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)${uiSuffix}
308
290
 
309
- ${cosmeticOverride}Use these exact numbers throughout the rest of the prompt.`;
291
+ ${changedBehaviorSection(effectiveGenerate)}`;
310
292
  }
311
293
  // Mixed PR: server can pre-compute the total but not the UI/E2E split — keep Step D.
312
294
  if (hasFrontendChanges) {
313
295
  return `### PR Scope Assessment — determine UI% before planning recommendations
314
296
 
315
- Budget Plan (total already determined): **${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)**
297
+ Budget ceiling: **${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)**
316
298
 
317
- ${zeroSurfaceSection(", and skip Step D below")}
299
+ ${changedBehaviorSection(effectiveGenerate, ", after Step D below sets your UI%")}
318
300
 
319
- **Step D — Determine UI vs backend split for the budget above:**
301
+ **Step D — Determine UI vs backend split for your Budget Plan:**
320
302
  - Non-UI slots are backend tests; start from file-count ratio for UI%, then apply judgment:
321
- - Cosmetic CSS/style changes alongside real changes inflate the frontend file count without adding test value → reduce UI% (a whole-diff-cosmetic PR already abstained via the override above, so this only applies to the mixed-PR case)
303
+ - Cosmetic CSS/style changes alongside real changes inflate the frontend file count without adding test value → reduce UI%
322
304
  - Frontend logic bugs (state management, calculation errors, form validation) in the diff → increase UI% even if few frontend files
323
305
  - Frontend component calls a changed backend API → an E2E test covers both sides → count toward UI%
324
306
  - Frontend files only in \`__tests__/\` or \`.stories.\` → exclude from the ratio
325
307
 
326
- **Append your UI% now** update the Budget Plan to:
327
- \`Budget Plan: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional), <ui_pct>% UI/E2E\`
308
+ **Append your UI% now** and state your Budget Plan — total and split per the changed-behavior rule above, within the ceiling:
309
+ \`Budget Plan: <total> total (<generate> generate + <additional> additional), <ui_pct>% UI/E2E\`
328
310
 
329
- Use these exact numbers throughout the rest of the prompt.`;
311
+ Use those numbers throughout the rest of the prompt.`;
330
312
  }
331
313
  // Fallback (no diff context — full_repo else-branch or test): full four-step assessment.
332
314
  const minTotal = Math.min(effectiveGenerate + 1, maxTotal);
@@ -350,9 +332,9 @@ Read the Changed Files list and endpoint changes above, then work through the fo
350
332
 
351
333
  **Step A — Classify changed files:**
352
334
  Count each type from the diff context (ignore generated test files, lock files, and build artifacts):
353
- - **Frontend files**: .vue / .svelte anywhere (always UI components). .dart anywhere (always UI in a Flutter project — repo has a \`pubspec.yaml\` with \`sdk: flutter\`). .tsx / .jsx anywhere except in api/, routes/, routers/, controllers/, handlers/, endpoints/, or server/ directories. .ts / .js / .html / .css / .scss / .less / .erb / .jsp / .asp / .jinja2 / .twig only when in a frontend directory (components/, pages/, views/, layouts/, app/, frontend/, client/, styles/, templates/).
335
+ - **Frontend files**: .vue / .svelte / .css / .scss / .less anywhere (always UI). .dart anywhere (always UI in a Flutter project — repo has a \`pubspec.yaml\` with \`sdk: flutter\`). .tsx / .jsx anywhere except in api/, routes/, routers/, controllers/, handlers/, endpoints/, or server/ directories. .ts / .js / .html / .erb / .jsp / .asp / .jinja2 / .twig only when in a frontend directory (components/, pages/, views/, layouts/, app/, frontend/, client/, styles/, templates/).
354
336
  - **Backend files**: route handlers, controllers, services, models, API modules, middleware, config with business logic
355
- - **Non-application** (exclude from test value): CSS-only, copy/string changes, README, CI config with no logic
337
+ - **Non-application** (exclude from test value): README and docs, lock files, CI config with no logic
356
338
 
357
339
  **Step B — Assess semantic complexity (quality over quantity):**
358
340
  Weigh changes by their test value, not file count:
@@ -360,12 +342,12 @@ Weigh changes by their test value, not file count:
360
342
  - Modified endpoint with formula / business logic change → HIGH: edge cases matter (contributes ~1–2)
361
343
  - Auth middleware change → CRITICAL: flag for extra security tests regardless of file count
362
344
  - Frontend state / validation / calculation logic → HIGH for UI tests even if zero backend endpoints changed
363
- - CSS / copy / purely cosmetic changes LOW: may not justify any new test
345
+ - Changed style values, labels or copy each rendered difference is a changed behavior: assert the new value (computed style, class or text) — no change is too trivial to test
364
346
 
365
347
  **Step C — Determine total recommendation count (${minTotal}–${maxTotal}):**
366
348
  Start from the baseline formula: *${baselineFormula}*, then adjust:
367
349
  - **Scale up** for: critical auth/data-integrity changes (+2), complex multi-step business workflows (+1 each), new endpoints with non-trivial validation (+1 each beyond the formula)
368
- - **Scale down** for: style/copy-only changes (may reach minimum of ${minTotal}), already well-tested paths confirmed by existing test list, trivial CRUD with no validation
350
+ - **Scale down** for: already well-tested paths confirmed by existing test list, trivial CRUD with no validation
369
351
  - **Hard cap**: ${maxTotal}
370
352
 
371
353
  ${stepD}
@@ -163,6 +163,8 @@ Output should be concise and immediately actionable.`
163
163
 
164
164
  For UI recommendations you *do* emit, ground the \`reasoning\` field in elements you have actually observed via \`browser_blueprint\` calls earlier in this session. If a recommendation's target element is not in any blueprint you have observed, either rephrase the recommendation around an element that IS observed, or describe the test target in higher-level terms. Do not invent element names from the PR description, source diff, or component name. If you have not captured any blueprints yet (e.g. backend-only PR, or pre-flight skipped), UI recommendations fall back to source-grounded prose drawn from the diff alone — that is a legitimate outcome, not a reason to invent.
165
165
 
166
+ **That rule governs element *identification*, not what you may assert about an identified element.** It does NOT forbid a description-derived **negative** assertion. When the PR description says an element should be gone and a captured page still renders it, the name is blueprint-grounded — you read it off the live page, not off the description — so a failing "is not present" assertion on that page is the correct output, and the requirement conflict is what it documents. Ground \`targetElements\` against the captured page where the element DOES render (the same grounding the negative-test guidance elsewhere in this prompt requires), and cite the test as a \`requirement_conflict\`. What stays forbidden is naming an element **no** blueprint contains because the description mentioned it.
167
+
166
168
  Write UI recommendation \`reasoning\` fields in **natural prose** that names elements as a human would describe them (e.g. "the Notifications heading", "the disabled Mark all as read button"). **Do NOT mention "blueprint", "captured blueprint", "DOM analyzer", or any other internal MCP terminology in the reasoning text.** The reader of the report is a developer reviewing test recommendations on a PR; they don't know what a blueprint is and shouldn't have to. Phrases like "the captured blueprint shows X" or "visible from the captured blueprint" leak builder internals — instead just describe what the test verifies in plain product terms ("the disabled mark-all-read button in the empty state"). Likewise do NOT use internal-identifier syntax like \`role=button, accessibleName='X', logicalName=...\`.
167
169
  </ui_recommendation_authoring_rules>
168
170
  `;
@@ -264,10 +266,21 @@ Static hints available: ${routeDiscovery.staticHints.length}
264
266
  ${routeDiscovery.diffFilePath ? `Diff file: ${routeDiscovery.diffFilePath}` : ""}
265
267
  `.trim()
266
268
  : "";
269
+ // ── Source Priority ──
270
+ // The ladder ranks sources that describe what the system DOES. The PR
271
+ // title/description states what the change is SUPPOSED to do, so it is not a
272
+ // rung — resolving a description-vs-code disagreement by rung produced tests
273
+ // that asserted the code and buried the contradiction in prose (SKYR-4291).
274
+ // Only the ladder-placement rule lives here; the Execution Plan's Code Review
275
+ // step is the canonical home for how a conflict is handled, so this document
276
+ // states it once. Diff-scoped: repo mode has no PR to compare against.
277
+ const requirementConflictRules = isDiffScope ? `
278
+ That ladder ranks sources that describe what the system **does**. The **PR title and description** — plus any requirements file they reference — state what this change is **supposed to do**. They are the *intent* side; the ladder is the *implementation* side. Never settle a disagreement between the two by picking the higher rung: a requirement the implemented behavior contradicts is a \`requirement_conflict\`, NOT a case of "the description loses". The Code Review step of the Execution Plan below specifies how to detect, check, and handle one — follow it there.
279
+ ` : "";
267
280
  const sourcePriority = `
268
281
  ## Source Priority
269
282
  When information conflicts, prefer: **Traces** (actual behavior) > **Source code read by the LLM** (implemented behavior) > **OpenAPI spec/docs** (documented behavior) > **Static parser hints** (best-effort, may be incomplete or framework-blind).
270
- `;
283
+ ${requirementConflictRules}`;
271
284
  // Compact fingerprint of tests already covering endpoints in this repo (Skyramp + external).
272
285
  // Re-derived fresh each run from test files on disk — no separate persistence needed.
273
286
  const testLocations = analysis.existingTests?.testLocations ?? {};
@@ -3,6 +3,7 @@ import { logger } from "../../utils/logger.js";
3
3
  import { AnalyticsService } from "../../services/AnalyticsService.js";
4
4
  import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
5
5
  import { setReportLanguage } from "../../utils/reportLanguage.js";
6
+ import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
6
7
  import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef, } from "../test-recommendation/recommendationShared.js";
7
8
  import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
8
9
  import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, } from "../../utils/featureFlags.js";
@@ -119,6 +120,13 @@ export function getTestbotPrompt(opts) {
119
120
  // en/argless render disarms a language captured earlier in a long-lived
120
121
  // server process instead of falsely rejecting an English report.
121
122
  setReportLanguage(language && language !== "en" ? language : undefined);
123
+ // SKYR-4250: same capture point, same last-render-wins rule, for the
124
+ // plan-only lane. skyramp_submit_report's SKYR-3883 guard
125
+ // (findUnchangedFileClaims) verifies newTestsCreated against the working
126
+ // tree; a plan-only run writes no files, so without this every declaration
127
+ // is unchanged-file-flagged and the agent is forced to demote it to
128
+ // additionalRecommendations.
129
+ setPlanOnlyMode(planOnly);
122
130
  let reportLanguageBlock = "";
123
131
  if (language && language !== "en") {
124
132
  const reportLanguageName = new Intl.DisplayNames(["en"], { type: "language" }).of(language) ??
@@ -222,6 +230,9 @@ ${maintenanceBeforeExecStep}
222
230
  - Incorrect arithmetic in business logic (discount calculations, price aggregation)
223
231
  Log each finding in \`issuesFound\` with a \`severity\` (critical/high/medium/low). These bugs should inform your test design in Task 2.
224
232
 
233
+ **Requirement check — does the code do what the PR says it does?** Read the \`<TITLE>\` and \`<DESCRIPTION>\` above as a statement of *intended* behavior and check the diff against it, requirement by requirement. The description is not outranked by the code: when the two disagree, that is a \`requirement_conflict\`, not a case of "the description loses". Detection, the requirements-file rules, the four false-red pre-checks and the promotion ordering are specified in the Code Review step of the Execution Plan \`skyramp_analyze_changes\` returned — follow them there rather than improvising; read any requirements file the description names from \`${repositoryPath}\`.
234
+ - **On a mismatch, do both.** (a) Add an \`issuesFound\` entry at severity \`high\` or \`critical\` — never \`medium\` or below — quoting the requirement and naming the file and line that contradicts it. (b) Carry it into Task 2 as a \`requirement_conflict\` candidate: a test asserting **what the description requires**, which therefore FAILS on the current code. Where no such test is recordable, record the Execution Plan's VERIFY-style entry in \`additionalRecommendations\` instead — never resolve the disagreement by asserting what the code does.
235
+
225
236
  4. **Blueprint Citation Invariant** (UI test recommendations only). Every named UI element in your recommendation must correspond to an element actually captured in one of the blueprints you captured from the UI Blueprint Capture section of \`skyramp_analyze_changes\`. For **every** UI recommendation — both \`newTestsCreated[]\` entries (tests you generated) AND \`additionalRecommendations[]\` entries (deferred tests) with \`testType: "ui"\` — populate **four** fields: \`description\`, \`reasoning\`, \`targetElements\`, and \`pageContext\`. \`description\` and \`reasoning\` carry different roles — fill BOTH (see Field 4 below). The structured fields (\`targetElements\`, \`pageContext\`) are required on UI recs in BOTH arrays; non-UI recs (contract / integration / e2e / batch-scenario) MUST omit them.
226
237
 
227
238
  **Field 1 — \`targetElements\`** (the elements the test targets — array, length 1+):
@@ -431,7 +442,7 @@ ${maintenanceBeforeExecStep}
431
442
  **Self-check before calling \`skyramp_submit_report\`:** count your final GENERATE list, then confirm \`newTestsCreated\` has exactly that many entries — one per GENERATE item. A GENERATE item always goes in \`newTestsCreated\` (even though no file was generated in this run — the declaration IS the deliverable); \`additionalRecommendations\` holds ONLY candidates that are NOT in the GENERATE list. Reporting a GENERATE item as a recommendation instead of a declaration is the single most common plan-only reporting error — recheck the placement before submitting.`;
432
443
  }
433
444
  else {
434
- task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if all candidates failed after retry AND the fallback single-contract test also failed.`;
445
+ task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if every remaining candidate failed after retry and, on a PR with backend changes, the fallback single-contract test also failed. A frontend-only PR has no backend fallback: its exhausted candidates go to \`additionalRecommendations\` with the failure reason.`;
435
446
  }
436
447
  // Task 2 branches wholesale in plan-only eval runs (SKYR-3879 plan-only
437
448
  // lane): the standard task mandates generation and execution, which a
@@ -443,9 +454,9 @@ ${maintenanceBeforeExecStep}
443
454
 
444
455
  This is a plan-only evaluation run: the application under test is NOT running, and this run evaluates test SELECTION only. Nothing is generated or executed in this task.
445
456
 
446
- - Draft your complete candidate list exactly as the Execution Plan directs — every API test (contract / integration / batch-scenario) you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
457
+ - Draft your complete candidate list exactly as the Execution Plan directs — every test you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
447
458
  - If a tool named \`skyramp_register_test_plan\` is available, call it with the full candidate union — include a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for every candidate that probes changed logic — and treat its returned GENERATE list as your final selection. If that tool is not available, commit to your Budget Plan's GENERATE selection (at most ${maxGenerate}).
448
- - Skip UI and E2E candidates entirely with no running app there are no blueprints to ground them, and this lane evaluates API test selection only.
459
+ - Include UI and E2E candidates on the same footing as the API types. The app is not running, so you cannot capture a blueprint — plan them ungrounded rather than drop them. For every UI entry, set \`targetElements: null\`, omit \`pageContext\`, and prefix BOTH \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Describe the page or feature the test would exercise; do not name an element you have not seen. This lane is the one case where you fall back without attempting a capture first, so do NOT log the fallback in \`issuesFound\` — capture was never applicable here, and nothing failed.
449
460
  - Take no other actions in this task: no test generation tools, no browser traces or blueprint captures, no test files written, no test executions. Proceed directly to ${taskRef(TASK_SUBMIT)}.`;
450
461
  }
451
462
  else {
@@ -453,8 +464,8 @@ This is a plan-only evaluation run: the application under test is NOT running, a
453
464
 
454
465
  ${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
455
466
 
456
- - **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
457
- - **Do not fabricate tests outside the GENERATE list provided by \`skyramp_analyze_changes\`.** Changes that only modify, delete, or add fields to an EXISTING covered endpoint or component are maintenance: handle them in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test, never by creating a new spec. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
467
+ - **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget. **Every \`requirement_conflict\` candidate from Task 1's requirement check belongs in this registration** — register it with category \`requirement_conflict\` so it ranks in the top tier and takes the first promoted slot rather than losing one to a bug-catching candidate; omitting it from the candidate list is how a stated requirement silently loses its test.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
468
+ - **Do not fabricate tests outside the GENERATE list returned by \`skyramp_register_test_plan\`.** A change to an EXISTING covered endpoint or component is maintenance only where an existing test asserts the changed value: handle that in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test. A changed value, option, or behavior that no existing test asserts is not maintenance — it is a candidate you register (GENERATE or ADDITIONAL), however small the change. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
458
469
  - Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
459
470
  - Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
460
471
  - If the existing test source is \`[external]\`, skip the resource entirely — the external test already provides coverage. Do NOT UPDATE, REGENERATE, or DELETE external tests.
@@ -479,6 +490,9 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
479
490
  - **Always generate a test for critical bugs, even if it will fail.** When a GENERATE-tagged item targets a page or endpoint with a known bug, do NOT skip it because you expect the test to fail — a failing test that documents a bug is more valuable than a text-only description. This applies within the existing GENERATE budget; do not add extra tests beyond the plan.
480
491
  - For UI rendering bugs: navigate to the broken page and add a \`browser_assert\` that verifies the page rendered its expected content (e.g. assert the page heading is visible). The assertion will fail on the broken page, which is the correct outcome — it documents the bug as a failing test.
481
492
  - The assertion MUST target the broken page itself, not a different page that works. If \`/orders/{id}/edit\` crashes, assert on \`/orders/{id}/edit\` (e.g. "Edit Order" heading visible), NOT on \`/orders\`.
493
+ - **The same rule covers a \`requirement_conflict\` from Task 1's requirement check — and it is promoted first.** Generate the test that asserts what the PR description requires, not what the code implements. It fails today; that is the deliverable, and it turns green when the code is fixed to match the description. Do NOT re-aim it at the implemented behavior to get a passing run, and do NOT drop it because a bug-catching test already occupies a slot — requirement conflicts take the first slots of the promotion bound, so the bug-catching tests yield to it, not the other way round.
494
+ - For a "should no longer be present" requirement: navigate to the page the description names, confirm from the capture that the element still renders, then assert its **absence**. The element name comes from the live capture, so the assertion is grounded; the failure is the conflict. Assert on the page the requirement is about — not a sibling page where the element is genuinely gone.
495
+ - For a "should now do X" requirement the code does not do: assert X. Same rule — the stated requirement is the expectation.
482
496
  - **Critical categories first**: At least 1 of the generated tests MUST be from a critical category (security_boundary, business_rule, data_integrity, breaking_change) if such candidates exist in the GENERATE set.
483
497
  - **Parallel generation (IMPORTANT for speed)**: Generate **independent tests in parallel** whenever possible. Tests targeting different endpoints with different output files can be generated concurrently in the same tool call batch. Specifically:
484
498
  - Call \`skyramp_batch_scenario_test_generation\` for ALL integration scenarios AND \`skyramp_contract_test_generation\` for ALL contract tests **in the same tool call batch**.
@@ -565,7 +579,6 @@ ${CONTRACT_MODE_GUIDANCE}
565
579
  - **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
566
580
  - **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
567
581
  If \`browser_navigate\` fails (app not running / connection refused), apply skip condition (a) above: move the intended test to \`additionalRecommendations\` with the failure reason AND record the outage in \`issuesFound\`.
568
- Record at most 2-3 UI traces per run to stay within tool call budget. Quality over quantity: 1 great test is better than 3 mediocre ones — do not pad to reach the count.
569
582
  **Strategic assertions** — key checkpoints only, 3 to 5 per test:
570
583
  - **After the main action completes**: verify the outcome is visible (new item appears, form saves, confirmation shows)
571
584
  - **State transitions**: verify counts, totals, or status fields update correctly
@@ -631,6 +644,8 @@ If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated t
631
644
  1. Read the error output to diagnose the root cause (4xx on prereq step, assertion mismatch, floating-point precision, 500 from app bug, timeout, etc.).
632
645
  2. **Expected failure check (no retry):** If the failure is an assertion error or HTTP error that matches the issue identified in the code analysis (e.g. the test was generated specifically to document a broken endpoint, a UI rendering bug, or a missing validation), then this is the **intended outcome** — the test is correctly catching the real bug. Report it immediately as \`status: "Fail"\` and move on. Do NOT retry.
633
646
 
647
+ **A \`requirement_conflict\` test failing is likewise the intended outcome.** It asserts what the PR description requires and the code does not do; a red result is the finding, so report \`status: "Fail"\` with the conflict named in \`issuesFound\` and move on. Do NOT retry it, do NOT relax the assertion toward the implemented behavior, and do NOT delete the test — either of those hides the disagreement the run exists to surface.
648
+
634
649
  This path also covers an assertion failure that application behavior outside this PR's diff explains — for example child records that survive the deletion of their parent, state inherited when an ID is recycled or reused, or a value that ignores a status the test set. Before you keep such a test red, confirm the cause in the source: read the handler, model, or query that should have done the work, and find the specific operation that is missing or wrong. If you find it, report the test as \`status: "Fail"\` and add an \`issuesFound\` entry for it. Do NOT retry.
635
650
 
636
651
  If you cannot point at the missing or wrong line in application code, the app is not at fault — shared or seeded data, parallel test workers, or setup the test itself never did explain the collision. Treat that as an infrastructure failure: fix it and retry once as in step 3.
@@ -683,7 +698,7 @@ If you skipped here due to non-application changes (per Task 1), submit with emp
683
698
 
684
699
  **If you generated zero new tests because the PR has no testable behavioral surface:**
685
700
  This applies when the diff contains ONLY changes with no observable API or UI behavior change. Examples:
686
- - Cosmetic/docs/style: JSDoc updates, CSS reformats, comment-only changes
701
+ - Docs/comments only: JSDoc updates, comment-only changes
687
702
  - Dependency-only: version bumps with no API surface change
688
703
  - Dead code / unintegrated utility or component: a new helper function, utility, or UI component added to the codebase but not imported, mounted, or rendered anywhere — use this classification only after confirming the new symbol does not appear as an import or render call in any other source file; do NOT classify as dead code based solely on the diff. For UI components specifically: an unintegrated component has no DOM node in the running app and cannot be browser-tested regardless of how complex its logic is
689
704
  - Config-only: linter rules, build config, environment variable additions with no runtime behavior change
@@ -694,6 +709,8 @@ In these cases:
694
709
  - \`businessCaseAnalysis\` must be a one-sentence summary of what the PR actually does (do NOT leave it blank)
695
710
  - \`additionalRecommendations\` must be \`[]\` — do NOT recommend tests for a no-surface PR
696
711
 
712
+ **This zero-test path does NOT apply when the requirement check found a \`requirement_conflict\`.** A requirement the PR description states and the diff did not implement is testable surface: report the conflict in \`issuesFound\` at severity \`high\` or above, and carry either the failing \`requirement_conflict\` test in \`newTestsCreated\` or its VERIFY-style entry in \`additionalRecommendations\`. Abstaining there hands the author back their own code as though it were the requirement.
713
+
697
714
  ${task3CountRule}
698
715
 
699
716
  ${reportLanguageBlock}Call \`skyramp_submit_report\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking, and the report is written beside it. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
@@ -703,7 +720,7 @@ ${hasRelatedRepos
703
720
  - **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
704
721
  `
705
722
  : ""}
706
- - **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
723
+ - **additionalRecommendations**: AT MOST ${maxRecommendations} minus the number of budget-counting tests in \`newTestsCreated\` — an unused generate slot becomes a recommendation slot. The one optional discovered-scenario test does not count against the budget, so it does not reduce this allowance either.
707
724
  - For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
708
725
 
709
726
  ${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false, modularizeViaGenerationResult: UI_UTILS_REUSE })}`;
@@ -27,16 +27,50 @@ function floorBalancedPick(items, count) {
27
27
  typesOrder.push(t);
28
28
  }
29
29
  }
30
- const maxProtected = Math.max(0, count - (typesOrder.length - 1));
30
+ // Two passes, because "protected" is broader than "is a caught bug".
31
+ //
32
+ // A `bug_caught` candidate may take a type-floor slot: a caught bug is the
33
+ // strongest thing a run produces, and it outranks type spread. Every OTHER
34
+ // protected candidate — anything else CRITICAL, and attack-surface
35
+ // security_boundary items — still yields to the floor, which is what stops
36
+ // the SKYR-3879 skew where three CRITICAL integration candidates took all
37
+ // three slots and no contract test was generated.
38
+ //
39
+ // The second pass replaces `maxProtected = count - (typesOrder.length - 1)`,
40
+ // a static cap computed from the whole pool. It reserved a slot for every
41
+ // other type whether or not the picks already covered it, so at the default
42
+ // budget of 3 with three types present it evaluated to 1. Counting the types
43
+ // still uncovered keeps the same floor and drops the arithmetic that made a
44
+ // second finding unreachable.
45
+ // The same bound the prompt states (`maxGen - 1`, and the whole budget only
46
+ // at a budget of 1). Without it the first pass took every caught bug up to
47
+ // `count`, which put the SKYR-3879 skew back: at the default budget of 3, two
48
+ // same-type bugs left no slot for the type floor, and a measured 9-20% of
49
+ // backend-only pools lost a test type. It also evicted attack-surface
50
+ // security_boundary items the prompt in this same branch says to preserve
51
+ // while any non-bug slot remains.
52
+ const bugSlots = count <= 1 ? count : count - 1;
31
53
  const selected = [];
54
+ const deferred = [];
32
55
  const pool = [];
33
- let protectedTaken = 0;
56
+ let bugsTaken = 0;
34
57
  for (const it of items) {
58
+ if (it.scenario?.category === "bug_caught" && bugsTaken < bugSlots && selected.length < count) {
59
+ selected.push(it);
60
+ bugsTaken++;
61
+ }
62
+ else {
63
+ deferred.push(it);
64
+ }
65
+ }
66
+ const takenTypes = new Set(selected.map(typeOf));
67
+ for (const it of deferred) {
68
+ const stillUncovered = typesOrder.filter((t) => !takenTypes.has(t) && t !== typeOf(it)).length;
35
69
  if (isProtectedCandidate(it.priority, it.scenario) &&
36
- protectedTaken < maxProtected &&
37
- selected.length < count) {
70
+ selected.length < count &&
71
+ count - (selected.length + 1) >= stillUncovered) {
38
72
  selected.push(it);
39
- protectedTaken++;
73
+ takenTypes.add(typeOf(it));
40
74
  }
41
75
  else {
42
76
  pool.push(it);
@@ -1,13 +1,26 @@
1
1
  import { Candidate, BudgetContext, SelectionResult } from "../../types/Recommendation.js";
2
2
  /**
3
- * Backend GENERATE slot count:
4
- * - UI-only PR: 0 (all slots are UI placeholders)
5
- * - Mixed PR: maxGenerate - 1 (last slot reserved for a UI placeholder)
6
- * - Backend-only PR: maxGenerate
3
+ * What each PR class GUARANTEES in GENERATE (SKYR-4275). These are floors, not
4
+ * quotas: the slots above them are open to either kind, decided by `pick`.
5
+ * An open slot goes to whichever candidate `pick` chooses on rank, so a backend
6
+ * candidate takes one when it outranks the UI candidates, not merely because it
7
+ * exists. Measured against main on random mixed pools at `maxGenerate` 5 with no
8
+ * caught bugs: 35% get fewer backend tests than main did, average 3.53 to 3.24.
9
+ *
10
+ * Backend-only floors nothing on purpose: of 86 backend-only diffs in the eval
11
+ * corpus, ZERO generated a UI test, so a floor there would guarantee what
12
+ * already happens every time.
13
+ *
14
+ * At one slot there is no room for a floor plus an open slot, so each class
15
+ * keeps that slot for its own kind.
16
+ *
17
+ * The floors always sum to at most `maxGenerate`, which is what lets
18
+ * `applyFloors` satisfy both without one starving the other.
7
19
  */
8
- export declare function backendGenerateCount(ctx: BudgetContext): number;
9
- /** UI placeholder slots the render layer fills (UI-only → all; mixed → one). */
10
- export declare function reservedUISlots(ctx: BudgetContext): number;
20
+ export declare function generateFloors(ctx: BudgetContext): {
21
+ ui: number;
22
+ backend: number;
23
+ };
11
24
  /**
12
25
  * Shared budgeting pipeline. All Budgeters run the same external-dedup,
13
26
  * attack-surface prioritization, and ADDITIONAL set-difference; they differ
@@ -16,4 +29,4 @@ export declare function reservedUISlots(ctx: BudgetContext): number;
16
29
  * With `pick = roundRobinByType` this reproduces the pre-refactor selection in
17
30
  * diffExecutionPlan.ts exactly.
18
31
  */
19
- export declare function runBudget(ranked: Candidate[], ctx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;
32
+ export declare function runBudget(ranked: Candidate[], rawCtx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;