@skyramp/mcp 0.3.4 → 0.3.6-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/build/playwright/registerPlaywrightTools.js +92 -30
  2. package/build/playwright/traceRecordingPrompt.d.ts +6 -0
  3. package/build/playwright/traceRecordingPrompt.js +6 -2
  4. package/build/prompts/code-reuse.d.ts +1 -2
  5. package/build/prompts/code-reuse.js +182 -77
  6. package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
  7. package/build/prompts/modularization/integration-test-modularization.js +83 -41
  8. package/build/prompts/modularization/render.d.ts +18 -0
  9. package/build/prompts/modularization/render.js +12 -0
  10. package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
  11. package/build/prompts/modularization/ui-test-modularization.js +89 -47
  12. package/build/prompts/pom-aware-code-reuse.js +3 -1
  13. package/build/prompts/shared-helper-policy.d.ts +57 -0
  14. package/build/prompts/shared-helper-policy.js +135 -0
  15. package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
  16. package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
  17. package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
  18. package/build/prompts/test-recommendation/recommendationShared.js +90 -16
  19. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
  20. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
  21. package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
  22. package/build/prompts/testbot/testbot-prompts.js +88 -33
  23. package/build/recommendation/budgeters/shared.js +105 -27
  24. package/build/recommendation/discriminators.js +13 -2
  25. package/build/recommendation/planRanker.d.ts +6 -6
  26. package/build/recommendation/planRanker.js +6 -61
  27. package/build/services/AnalyticsService.d.ts +7 -0
  28. package/build/services/AnalyticsService.js +7 -1
  29. package/build/services/ModularizationService.js +1 -3
  30. package/build/services/TestDiscoveryService.d.ts +0 -2
  31. package/build/services/TestDiscoveryService.js +2 -37
  32. package/build/services/TestGenerationService.d.ts +16 -0
  33. package/build/services/TestGenerationService.js +86 -10
  34. package/build/services/containerEnv.js +13 -12
  35. package/build/tools/code-refactor/codeReuseTool.js +279 -93
  36. package/build/tools/code-refactor/enhance-state.d.ts +49 -0
  37. package/build/tools/code-refactor/enhance-state.js +109 -0
  38. package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
  39. package/build/tools/code-refactor/modularizationTool.js +9 -2
  40. package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
  41. package/build/tools/code-refactor/reuse-outcome.js +14 -4
  42. package/build/tools/code-refactor/reuse-state.d.ts +127 -5
  43. package/build/tools/code-refactor/reuse-state.js +628 -16
  44. package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
  45. package/build/tools/code-refactor/utils-verify-gates.js +100 -0
  46. package/build/tools/code-refactor/verify-gates.d.ts +2 -1
  47. package/build/tools/code-refactor/verify-gates.js +90 -25
  48. package/build/tools/executeSkyrampTestTool.d.ts +19 -0
  49. package/build/tools/executeSkyrampTestTool.js +158 -8
  50. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
  51. package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
  52. package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
  53. package/build/tools/generate-tests/generateUIRestTool.js +22 -0
  54. package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
  55. package/build/tools/generate-tests/scenarioLint.js +127 -19
  56. package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
  57. package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
  58. package/build/tools/submitReportTool.d.ts +38 -38
  59. package/build/tools/submitReportTool.js +487 -104
  60. package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
  61. package/build/tools/test-management/analyzeChangesTool.js +75 -12
  62. package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
  63. package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
  64. package/build/tools/test-management/registerTestPlanTool.js +149 -23
  65. package/build/types/Recommendation.d.ts +34 -5
  66. package/build/types/RepositoryAnalysis.d.ts +133 -114
  67. package/build/types/RepositoryAnalysis.js +1 -1
  68. package/build/types/ReuseOutcome.d.ts +102 -6
  69. package/build/types/ReuseOutcome.js +16 -2
  70. package/build/types/TestRecommendation.js +21 -3
  71. package/build/types/TestTypes.js +14 -8
  72. package/build/types/TestbotReport.d.ts +25 -3
  73. package/build/types/index.d.ts +2 -2
  74. package/build/types/index.js +1 -1
  75. package/build/utils/AnalysisStateManager.d.ts +69 -1
  76. package/build/utils/AnalysisStateManager.js +69 -5
  77. package/build/utils/branchDiff.d.ts +10 -0
  78. package/build/utils/branchDiff.js +28 -0
  79. package/build/utils/changedRoutes.d.ts +29 -0
  80. package/build/utils/changedRoutes.js +87 -0
  81. package/build/utils/featureFlags.d.ts +21 -0
  82. package/build/utils/featureFlags.js +23 -0
  83. package/build/utils/frontendIntegration.js +34 -4
  84. package/build/utils/importerHop.d.ts +2 -8
  85. package/build/utils/importerHop.js +15 -53
  86. package/build/utils/pathMatching.d.ts +38 -0
  87. package/build/utils/pathMatching.js +71 -0
  88. package/build/utils/pathSignatures.d.ts +22 -0
  89. package/build/utils/pathSignatures.js +57 -0
  90. package/build/utils/planMatchKeys.d.ts +16 -3
  91. package/build/utils/planMatchKeys.js +26 -10
  92. package/build/utils/pluralization.d.ts +10 -0
  93. package/build/utils/pluralization.js +18 -0
  94. package/build/utils/pom-catalog-parse.d.ts +52 -0
  95. package/build/utils/pom-catalog-parse.js +141 -0
  96. package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
  97. package/build/utils/pom-scope/selector-extractor.js +34 -8
  98. package/build/utils/pom-verify/verify.d.ts +6 -5
  99. package/build/utils/pom-verify/verify.js +8 -6
  100. package/build/utils/reportLanguage.d.ts +43 -0
  101. package/build/utils/reportLanguage.js +125 -0
  102. package/build/utils/reportVerification.d.ts +74 -4
  103. package/build/utils/reportVerification.js +259 -3
  104. package/build/utils/reuseRouting.d.ts +3 -0
  105. package/build/utils/reuseRouting.js +50 -0
  106. package/build/utils/routeParsers.d.ts +2 -0
  107. package/build/utils/routeParsers.js +65 -8
  108. package/build/utils/scenarioDrafting.d.ts +1 -1
  109. package/build/utils/scenarioDrafting.js +57 -45
  110. package/build/utils/subjectEndpoints.d.ts +19 -0
  111. package/build/utils/subjectEndpoints.js +98 -0
  112. package/build/utils/testFileClassification.d.ts +11 -0
  113. package/build/utils/testFileClassification.js +47 -0
  114. package/build/utils/uiPageEnumerator.d.ts +45 -19
  115. package/build/utils/uiPageEnumerator.js +95 -51
  116. package/build/utils/utils-verify/allow.d.ts +16 -0
  117. package/build/utils/utils-verify/allow.js +68 -0
  118. package/build/utils/utils-verify/call-sites.d.ts +34 -0
  119. package/build/utils/utils-verify/call-sites.js +154 -0
  120. package/build/utils/utils-verify/index.d.ts +7 -0
  121. package/build/utils/utils-verify/index.js +7 -0
  122. package/build/utils/utils-verify/language-spec.d.ts +91 -0
  123. package/build/utils/utils-verify/language-spec.js +210 -0
  124. package/build/utils/utils-verify/locate.d.ts +39 -0
  125. package/build/utils/utils-verify/locate.js +199 -0
  126. package/build/utils/utils-verify/parse.d.ts +34 -0
  127. package/build/utils/utils-verify/parse.js +177 -0
  128. package/build/utils/utils-verify/stage.d.ts +24 -0
  129. package/build/utils/utils-verify/stage.js +107 -0
  130. package/build/utils/utils-verify/verify.d.ts +63 -0
  131. package/build/utils/utils-verify/verify.js +168 -0
  132. package/build/utils/utils.d.ts +3 -1
  133. package/build/utils/utils.js +3 -1
  134. package/build/workspace/workspace.d.ts +32 -32
  135. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
  136. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
  137. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
  138. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
  139. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
  140. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
  141. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
  142. package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
  143. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
  144. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
  145. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
  146. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
  147. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
  148. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
  149. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  150. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
  151. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  152. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  153. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  154. package/node_modules/playwright/package.json +1 -1
  155. package/package.json +2 -2
  156. package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
  157. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
  158. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
  159. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
@@ -1,12 +1,12 @@
1
- import { roundRobinByType } from "../../recommendation/diversity.js";
1
+ import { CandidateSource, computeCandidateId } from "../../types/Recommendation.js";
2
+ import { fixedNBudgeter } from "../../recommendation/budgeters/fixedNBudgeter.js";
2
3
  import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
3
4
  import { resolveServiceDetailsRef } from "../../utils/utils.js";
4
- import { logger } from "../../utils/logger.js";
5
5
  import { TEST_CATEGORIES, } from "../../types/TestRecommendation.js";
6
6
  import { buildScopeAssessmentSection } from "./scopeAssessment.js";
7
7
  import { PromptPlan } from "./promptPlan.js";
8
8
  import { buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, MAX_CRITICAL_TESTS, } from "./recommendationSections.js";
9
- import { TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW, externalDedupKey, isAttackSurfaceSecurityBoundary, isOrdinaryDirectAuthBoundary, scenarioCoverageKey, taskStepRef, } from "./recommendationShared.js";
9
+ import { TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW, taskStepRef, } from "./recommendationShared.js";
10
10
  // ── Step body functions ───────────────────────────────────────────────────────
11
11
  function _execCodeReviewBody(_ctx) {
12
12
  const codeReviewRef = taskStepRef(TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW);
@@ -69,7 +69,7 @@ INSERT a non-bug source-code-derived candidate into the ranked list **only if AL
69
69
  3. It is not already covered by a structural candidate in the list below
70
70
 
71
71
  If these conditions are not met, add it to ADDITIONAL only — do NOT displace a pre-ranked GENERATE item.
72
- **CRITICAL-tier items (category: new_endpoint) and attack-surface \`security_boundary\` items should never be displaced by non-bug candidates** — they test the actual endpoints introduced in this PR or sibling destructive endpoints that could bypass the changed auth boundary. However, bug-catching tests CAN displace them only after all lower-value non-bug slots are exhausted.
72
+ **Attack-surface \`security_boundary\` items should never be displaced by non-bug candidates** — they test sibling destructive endpoints that could bypass the changed auth boundary. However, bug-catching tests CAN displace them only after all lower-value non-bug slots are exhausted.
73
73
 
74
74
  When a qualifying candidate is inserted: place it HIGH before MEDIUM before LOW; within the same priority, source-code-derived candidates go BEFORE structural ones. Re-number ranks after insertion. The top ${ctx.maxGen} ranked items become GENERATE candidates.
75
75
 
@@ -80,8 +80,8 @@ When a qualifying candidate is inserted: place it HIGH before MEDIUM before LOW;
80
80
  function _execDiversityBody(_ctx) {
81
81
  return `**Bug-coverage gate (runs BEFORE dedup):**
82
82
  Verify that the highest-severity \`<bug_found>\` flaw from Step ${EXEC_STEP_CODE_REVIEW} has exactly one GENERATE item with category \`bug_caught\` targeting it — meaning the test would FAIL on the current buggy code and PASS once the flaw is fixed. At most one promotion per run (per Step ${EXEC_STEP_CODE_REVIEW} cap). If the promoted flaw has no targeting \`bug_caught\` GENERATE item:
83
- - Check ADDITIONAL for a matching test → promote it into the lowest-priority non-bug, non-CRITICAL GENERATE slot first (lowest category rank per \`crud > error_handling > workflow > data_validation > data_integrity > business_rule\`; preserve attack-surface \`security_boundary\` and internal \`new_endpoint\` items unless no lower-priority slot exists).
84
- - If no ADDITIONAL candidate matches, create a new \`bug_caught\` test and insert it, displacing the lowest-priority non-bug, non-CRITICAL GENERATE item first; displace an attack-surface \`security_boundary\` or other CRITICAL item only when every GENERATE slot is higher priority.
83
+ - Check ADDITIONAL for a matching test → promote it into the lowest-priority non-bug, non-CRITICAL GENERATE slot first (lowest category rank per \`crud > error_handling > workflow > data_validation > data_integrity > business_rule\`; preserve attack-surface \`security_boundary\` items unless no lower-priority slot exists).
84
+ - If no ADDITIONAL candidate matches, create a new \`bug_caught\` test and insert it, displacing the lowest-priority non-bug, non-CRITICAL GENERATE item first; displace an attack-surface \`security_boundary\` item only when every GENERATE slot is higher priority.
85
85
  A \`bug_caught\` test is NEVER considered a "duplicate" of a non-bug test during the dedup below.
86
86
 
87
87
  Each GENERATE item must exercise a **distinct code path** — not just different input values on the same path.
@@ -98,7 +98,7 @@ Same step sequence with only payload differences (e.g. 10% vs 5% discount both r
98
98
  function _execRegisterBody(_ctx) {
99
99
  return `Register your complete candidate list — every test you would generate OR recommend — via \`skyramp_register_test_plan\` (\`stateFile\` required). Include a discriminator claim (\`discriminator\` field — valid kinds and anchor rules are in the tool schema) for candidates probing the changed logic identified in Step ${EXEC_STEP_CODE_REVIEW}/Step ${EXEC_STEP_ENRICH}.
100
100
 
101
- The returned GENERATE list is mandatory and final — generation tools reject unregistered scenarios. If the tool demotes a discriminator claim (returned in \`demotions\` with a reason), either strengthen the claim — a step that actually exercises the declared \`kind\`, or a verbatim anchor that occurs in the diff — or drop it; the candidate itself stays in the plan either way.`;
101
+ The returned GENERATE list is mandatory and final — generation tools reject unregistered scenarios. If the tool demotes a discriminator claim (returned in \`demotions\` with a reason), either strengthen the claim — a step that actually exercises the declared \`kind\`, or a verbatim anchor that occurs in the diff — or drop it. A demoted candidate normally stays in the plan. If it also left the plan for another recorded reason, the report lists it under "Demoted AND removed".`;
102
102
  }
103
103
  function _execExecuteBody(ctx) {
104
104
  return `Replace any scenario that pairs unrelated resources with one reflecting actual foreign-key relationships in the codebase.
@@ -149,20 +149,6 @@ export const EXEC_STEP_EXECUTE = _execPlan.labels.EXECUTE; // "4"
149
149
  /** "5" — Register test plan (SKYR-3879 Path B checkpoint) */
150
150
  export const EXEC_STEP_REGISTER = _execPlan.labels.REGISTER; // "5"
151
151
  const SERVICE_REFS = resolveServiceDetailsRef();
152
- function prioritizeAttackSurfaceBundles(items) {
153
- const reordered = [];
154
- for (const item of items) {
155
- if (isAttackSurfaceSecurityBoundary(item.scenario)) {
156
- const firstDirectAuthIndex = reordered.findIndex(candidate => isOrdinaryDirectAuthBoundary(candidate.scenario));
157
- if (firstDirectAuthIndex >= 0) {
158
- reordered.splice(firstDirectAuthIndex, 0, item);
159
- continue;
160
- }
161
- }
162
- reordered.push(item);
163
- }
164
- return reordered;
165
- }
166
152
  export function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, seed, endpointCount, isUIOnlyPR, hasFrontendChanges = false, hasTraces = false, externalCoverage = new Set(), relevantExternalTestPaths = [],
167
153
  /**
168
154
  * Whether the diff classified at least one new/modified/removed endpoint.
@@ -184,40 +170,58 @@ hasApiChanges = true) {
184
170
  : hasFrontendChanges
185
171
  ? Math.max(0, maxGen - 1)
186
172
  : maxGen;
187
- // Filter out scenarios whose primary method + resource + test type is already covered by external tests.
188
- // Method-aware: an external test covering GET /orders won't block PUT /orders scenarios.
189
- // This is the programmatic complement to the prompt-level Step ${EXEC_STEP_COVERAGE} dedup instructions.
190
- const scoredAfterExternalDedup = externalCoverage.size > 0
191
- ? scored.filter((item) => {
192
- const key = externalDedupKey(item.scenario);
193
- if (externalCoverage.has(key)) {
194
- if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
195
- logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${key}) protected bug/attack-surface scenario requires semantic flaw coverage`);
196
- return true;
197
- }
198
- logger.info(`External dedup: skipping "${item.scenario.scenarioName}" (${key}) covered by external test`);
199
- return false;
200
- }
201
- return true;
202
- })
203
- : scored;
204
- const slotOrderedItems = prioritizeAttackSurfaceBundles(scoredAfterExternalDedup);
205
- // Distribute the backend GENERATE slots evenly across the test types present
206
- // (contract vs integration) with spillover, instead of a pure top-N rank slice —
207
- // so a backend change isn't starved of (say) integration coverage just because
208
- // contract scenarios out-ranked it. See roundRobinByType.
209
- const generateItems = roundRobinByType(slotOrderedItems, Math.min(backendGenerateCount, slotOrderedItems.length));
210
- // ADDITIONAL = everything not chosen for GENERATE, in original rank order, capped
211
- // at the remaining budget. Computed by set-difference (not a positional slice)
212
- // because round-robin may pick items that aren't the first backendGenerateCount by
213
- // rank — a positional slice would double-list a generated item or drop one.
214
- const generateSet = new Set(generateItems);
215
- const rawAdditionalItems = slotOrderedItems
216
- .filter((it) => !generateSet.has(it))
217
- .slice(0, Math.max(0, topN - backendGenerateCount));
218
- // Filter additional items whose primary resource + test type already appear in GENERATE
219
- const generatedCoverage = new Set(generateItems.map((item) => scenarioCoverageKey(item.scenario)));
220
- const additionalItems = rawAdditionalItems.filter((item) => !generatedCoverage.has(scenarioCoverageKey(item.scenario)));
173
+ // Both this renderer and the register-plan path run the SAME selection
174
+ // pipeline `runBudget` instead of two copies of it (SKYR-4214 item 7).
175
+ // This file used to hold its own external dedup, attack-surface ordering,
176
+ // round-robin GENERATE pick and ADDITIONAL set-difference, so a fix in one
177
+ // copy left the other wrong.
178
+ //
179
+ // The two are NOT guaranteed to produce the same GENERATE set, and this
180
+ // comment must not claim they are: `selectPlan` re-ranks and calls
181
+ // `diversityBalancedBudgeter` (planRanker.ts:96) where this calls
182
+ // `fixedNBudgeter`, and the two differ in the `pick` step —
183
+ // diversityBalancedBudgeter.test.ts covers that divergence. What is now
184
+ // shared is the dedup rule, the ordering and the ADDITIONAL cap.
185
+ //
186
+ // `runBudget` needs a Candidate, which carries provenance and a stable id
187
+ // that only the register-plan path fills. Everything reaching this renderer
188
+ // is server-drafted, and `computeCandidateId` is a pure function of the
189
+ // scenario, so both fields are derivable here.
190
+ //
191
+ // The context deliberately reports no frontend work, whatever this PR
192
+ // contains: this renderer never had the UI-promotion loop, and the backend
193
+ // count it computed above already applied the UI reservation. Passing the
194
+ // real flags would introduce UI slots into the prompt's GENERATE list, which
195
+ // is a behaviour change this item does not carry.
196
+ const candidates = scored.map((item) => ({
197
+ ...item,
198
+ source: CandidateSource.SERVER,
199
+ candidateId: computeCandidateId(item.scenario),
200
+ }));
201
+ const selection = fixedNBudgeter.select(candidates, {
202
+ maxGenerate: backendGenerateCount,
203
+ maxTotal: topN,
204
+ isUIOnlyPR: false,
205
+ hasFrontendChanges: false,
206
+ externalCoverage,
207
+ // This renderer never receives the branch diff, so it cannot tell whether the
208
+ // PR wrote tests of its own. `true` is the same abstain direction the analyze
209
+ // end takes for an unknown (SKYR-4024): the covered-candidate reserve stays
210
+ // shut here, and those candidates are reported in the Removed section below
211
+ // rather than promoted to ADDITIONAL.
212
+ diffChangesTestFiles: true,
213
+ });
214
+ const generateItems = selection.generate;
215
+ const additionalItems = selection.additional;
216
+ // A candidate the selection stage removed — covered by an external test, or
217
+ // past the maxTotal cap — leaves no trace in the two lists above. In the
218
+ // recommendation-only flow (analyze_changes plus this prompt, with no
219
+ // register call) this section is the ONLY place the agent can read that a
220
+ // candidate existed and why it went, which is what SKYR-4214 exists to fix.
221
+ const scenarioNameById = new Map(candidates.map((c) => [c.candidateId, c.scenario.scenarioName]));
222
+ const removedLines = selection.dropped
223
+ .map((drop) => `- ${scenarioNameById.get(drop.candidateId) ?? drop.candidateId}: ${drop.reason}`)
224
+ .join("\n");
221
225
  const hasWorkspaceAuthType = !!authTypeValue && authTypeValue !== "none";
222
226
  // For skyramp_integration_test_generation with scenarioFile:
223
227
  // - If workspace has authType set: omit auth entirely — workspace handles Bearer prefix.
@@ -432,7 +436,9 @@ ${isUIOnlyPR
432
436
  ### ADDITIONAL (list in additionalRecommendations in this order after Step ${EXEC_STEP_ENRICH} insertion)
433
437
 
434
438
  ${additionalLines || " (none pre-ranked)"}
435
- ${uiGroundingGuidance}
439
+ ${removedLines
440
+ ? `\n### Removed (${selection.dropped.length}) — candidates the selection stage dropped before ranking. They are NOT in the lists above; do not re-add them.\n\n${removedLines}\n`
441
+ : ""}${uiGroundingGuidance}
436
442
  ${uiGuidance}
437
443
  ${supplementNote}
438
444
 
@@ -1,6 +1,6 @@
1
1
  import { logger } from "../../utils/logger.js";
2
2
  import { buildTestQualityCriteria } from "./recommendationSections.js";
3
- import { externalDedupKey } from "./recommendationShared.js";
3
+ import { externalDedupKeys, isAttackSurfaceSecurityBoundary } from "./recommendationShared.js";
4
4
  export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, isFrontendProject = false, isFrontendOnlyProject = false, externalCoverage = new Set()) {
5
5
  // Full-repo mode only — percentage-based UI/E2E slot targets (15% each, floor 1).
6
6
  const rawE2E = isFrontendProject ? Math.max(1, Math.round(topN * 0.15)) : 0;
@@ -17,22 +17,29 @@ export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderVa
17
17
  : authHeaderValue
18
18
  ? `, authHeader: "${authHeaderValue}"`
19
19
  : `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
20
- // Supplement count for full-repo mode
21
- const supplementCount = topN - Math.min(scored.length, topN);
22
20
  const toTitle = (name) => name.replace(/-/g, " ").replace(/\b\w/g, c => c.toUpperCase());
23
21
  const TYPE_ORDER = ["e2e", "ui", "integration", "contract"];
24
22
  const TYPE_LABEL = {
25
23
  e2e: "E2E", ui: "UI", integration: "Integration", contract: "Contract",
26
24
  };
27
25
  // Filter out scenarios already covered by external tests before slicing.
26
+ // The bug/attack-surface exemption matches applyExternalDedup in
27
+ // budgeters/shared.ts: those scenarios cover a semantic flaw that a
28
+ // same-endpoint external test does not, so an external match must not
29
+ // remove them here either.
28
30
  const scoredFiltered = externalCoverage.size > 0
29
31
  ? scored.filter(item => {
30
- const key = externalDedupKey(item.scenario);
31
- if (externalCoverage.has(key)) {
32
- logger.info(`External dedup (full-repo): skipping "${item.scenario.scenarioName}" (${key})`);
33
- return false;
32
+ const keys = externalDedupKeys(item.scenario);
33
+ if (keys.length === 0)
34
+ return true;
35
+ if (!keys.every((key) => externalCoverage.has(key)))
36
+ return true;
37
+ if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
38
+ logger.info(`External dedup (full-repo): preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
39
+ return true;
34
40
  }
35
- return true;
41
+ logger.info(`External dedup (full-repo): skipping "${item.scenario.scenarioName}" (${keys.join(", ")})`);
42
+ return false;
36
43
  })
37
44
  : scored;
38
45
  // For full-stack repos, carve out E2E and UI slots before filling with backend tests.
@@ -40,6 +47,10 @@ export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderVa
40
47
  ? Math.max(0, topN - minE2ESlots - minUISlots)
41
48
  : topN;
42
49
  const allItems = scoredFiltered.slice(0, backendSlotCount);
50
+ // Count the supplements against what this prompt actually lists. Reading the
51
+ // pre-filter `scored.length` claimed more pre-ranked items than the sections
52
+ // below render, so a pool thinned by external dedup asked for no supplements.
53
+ const supplementCount = topN - Math.min(allItems.length, topN);
43
54
  const byType = new Map();
44
55
  for (const t of TYPE_ORDER)
45
56
  byType.set(t, []);
@@ -3,14 +3,36 @@
3
3
  * full-repo mode (fullRepoCatalog.ts). Extracted here to avoid circular imports.
4
4
  */
5
5
  import { DraftedScenario } from "../../types/RepositoryAnalysis.js";
6
- export declare function scenarioCoverageKey(scenario: DraftedScenario): string;
7
6
  /**
8
- * Method-aware coverage key for external test dedup.
9
- * Unlike scenarioCoverageKey (resource::testType), this includes the HTTP method
10
- * so that e.g. an external test covering "GET /orders" doesn't block generating
11
- * a test for "PUT /orders" a different operation on the same resource.
7
+ * The one spelling of a coverage key, `METHOD::resource::testType`. Both
8
+ * operands of the dedup comparison call it: a proposed test through
9
+ * `externalDedupKeys`, and an existing test through `buildExternalCoverageSet`.
10
+ * They used to build the same string in two places from two different inputs —
11
+ * a typed step list on one side, regex-scraped prose on the other — so a fix to
12
+ * one side left the other producing keys that could never match (SKYR-4214).
12
13
  */
13
- export declare function externalDedupKey(scenario: DraftedScenario): string;
14
+ export declare function coverageKey(input: {
15
+ method: string;
16
+ path: string;
17
+ testType: string;
18
+ }): string;
19
+ /**
20
+ * Method-aware coverage keys for external test dedup — one per recorded subject
21
+ * endpoint. Method-aware so that an external test covering "GET /orders" does
22
+ * not block a test for "PUT /orders", a different operation on the same
23
+ * resource.
24
+ *
25
+ * A caller must remove a candidate ONLY when the coverage set holds EVERY key
26
+ * returned here. Removing it on one match would discard the coverage of its
27
+ * other endpoints. An empty list must never remove anything.
28
+ */
29
+ export declare function externalDedupKeys(scenario: DraftedScenario): string[];
30
+ /**
31
+ * Resource+type keys (no method) for the GENERATE/ADDITIONAL overlap filter —
32
+ * one per recorded subject endpoint. Same rule as `externalDedupKeys`: remove
33
+ * only on ALL keys, never on an empty list.
34
+ */
35
+ export declare function scenarioCoverageKeys(scenario: DraftedScenario): string[];
14
36
  export declare function isAttackSurfaceSecurityBoundary(scenario: DraftedScenario): boolean;
15
37
  export declare function isOrdinaryDirectAuthBoundary(scenario: DraftedScenario): boolean;
16
38
  /**
@@ -10,22 +10,96 @@ function resolvePrimaryStep(scenario) {
10
10
  const primaryStep = mutatingSteps[mutatingSteps.length - 1] ?? scenario.steps[scenario.steps.length - 1];
11
11
  return { primaryStep, testType };
12
12
  }
13
- export function scenarioCoverageKey(scenario) {
14
- const { primaryStep, testType } = resolvePrimaryStep(scenario);
15
- const resource = extractResourceFromPath(primaryStep?.path ?? "");
16
- return `${resource}::${testType}`;
13
+ /**
14
+ * The one spelling of a coverage key, `METHOD::resource::testType`. Both
15
+ * operands of the dedup comparison call it: a proposed test through
16
+ * `externalDedupKeys`, and an existing test through `buildExternalCoverageSet`.
17
+ * They used to build the same string in two places from two different inputs —
18
+ * a typed step list on one side, regex-scraped prose on the other — so a fix to
19
+ * one side left the other producing keys that could never match (SKYR-4214).
20
+ */
21
+ export function coverageKey(input) {
22
+ const method = (input.method ?? "GET").toUpperCase();
23
+ return `${method}::${extractResourceFromPath(input.path ?? "")}::${input.testType}`;
24
+ }
25
+ /** Test type of a scenario: its own label, else one step means a contract test. */
26
+ function resolveTestType(scenario) {
27
+ return scenario.testType ?? (scenario.steps.length === 1 ? "contract" : "integration");
28
+ }
29
+ /**
30
+ * Method-aware coverage keys for external test dedup — one per recorded subject
31
+ * endpoint. Method-aware so that an external test covering "GET /orders" does
32
+ * not block a test for "PUT /orders", a different operation on the same
33
+ * resource.
34
+ *
35
+ * A caller must remove a candidate ONLY when the coverage set holds EVERY key
36
+ * returned here. Removing it on one match would discard the coverage of its
37
+ * other endpoints. An empty list must never remove anything.
38
+ */
39
+ export function externalDedupKeys(scenario) {
40
+ const testType = resolveTestType(scenario);
41
+ const subjects = scenario.subjectEndpoints;
42
+ if (subjects && subjects.length > 0) {
43
+ return subjects.map((s) => coverageKey({ ...s, testType }));
44
+ }
45
+ if (subjects)
46
+ return [];
47
+ const { primaryStep } = resolvePrimaryStep(scenario);
48
+ if (!primaryStep)
49
+ return [];
50
+ return [coverageKey({ method: primaryStep.method, path: primaryStep.path, testType })];
51
+ }
52
+ /**
53
+ * Resource+type keys (no method) for the GENERATE/ADDITIONAL overlap filter —
54
+ * one per recorded subject endpoint. Same rule as `externalDedupKeys`: remove
55
+ * only on ALL keys, never on an empty list.
56
+ */
57
+ export function scenarioCoverageKeys(scenario) {
58
+ const testType = resolveTestType(scenario);
59
+ const steps = scenario.steps ?? [];
60
+ const subjects = scenario.subjectEndpoints;
61
+ // No recorded subject: fall back to the primary step, as externalDedupKeys does.
62
+ if (!subjects) {
63
+ const { primaryStep } = resolvePrimaryStep(scenario);
64
+ if (!primaryStep)
65
+ return [];
66
+ return [
67
+ overlapKey(primaryStep.method, extractResourceFromPath(primaryStep.path ?? ""), testType, primaryStep),
68
+ ];
69
+ }
70
+ return subjects.map((subject) => {
71
+ const step = steps.find((s) => s.method === subject.method && s.path === subject.path);
72
+ return overlapKey(subject.method, extractResourceFromPath(subject.path ?? ""), testType, step);
73
+ });
17
74
  }
18
75
  /**
19
- * Method-aware coverage key for external test dedup.
20
- * Unlike scenarioCoverageKey (resource::testType), this includes the HTTP method
21
- * so that e.g. an external test covering "GET /orders" doesn't block generating
22
- * a test for "PUT /orders" a different operation on the same resource.
76
+ * The overlap key names the endpoint AND what the test asserts about it.
77
+ *
78
+ * It deliberately does NOT reuse `coverageKey`. That key's other operand comes
79
+ * from `buildExternalCoverageSet`, which scrapes prose and can only ever know a
80
+ * method and a path — adding a status there would stop every external key from
81
+ * matching. Both operands of the OVERLAP comparison are drafted candidates with
82
+ * a step list, so both sides know the method, the interaction type and the
83
+ * status.
84
+ *
85
+ * The method is part of the key because `extractResourceFromPath` maps
86
+ * `/api/orders` and `/api/orders/{id}` to the same resource. Without it a
87
+ * GENERATE `PATCH /api/orders/{id}` (success, 200) removed an ADDITIONAL
88
+ * `GET /api/orders/{id}` (success, 200) as a duplicate — a read test and an
89
+ * update test on one resource are not duplicates.
90
+ *
91
+ * Measured on 839 nightly fixtures: 781 of them change exactly ONE route, so
92
+ * every candidate resolves to the same subject and `resource::testType` yields
93
+ * 2 keys for the whole pool. In 11 fixtures that removed EVERY ADDITIONAL
94
+ * candidate — `01-site-stats-endpoint` lost all 18. Adding the interaction type
95
+ * and the status roughly doubles the distinct keys on those pools. It does not
96
+ * make the key complete: two tests that assert different things about the same
97
+ * 200 response still collide, and no key built from a route can separate them.
23
98
  */
24
- export function externalDedupKey(scenario) {
25
- const { primaryStep, testType } = resolvePrimaryStep(scenario);
26
- const method = primaryStep?.method ?? "GET";
27
- const resource = extractResourceFromPath(primaryStep?.path ?? "");
28
- return `${method}::${resource}::${testType}`;
99
+ function overlapKey(method, resource, testType, step) {
100
+ const interaction = step?.interactionType ?? "any";
101
+ const status = step?.expectedStatusCode ?? 0;
102
+ return `${(method ?? "ANY").toUpperCase()}::${resource}::${testType}::${interaction}::${status}`;
29
103
  }
30
104
  export function isAttackSurfaceSecurityBoundary(scenario) {
31
105
  return scenario.category === "security_boundary" &&
@@ -60,11 +134,11 @@ export function buildExternalCoverageSet(testLocations) {
60
134
  const resource = extractResourceFromPath(epPath);
61
135
  if (resource !== "unknown") {
62
136
  if (testType === "unknown") {
63
- coverage.add(`${method}::${resource}::integration`);
64
- coverage.add(`${method}::${resource}::contract`);
137
+ coverage.add(coverageKey({ method, path: epPath, testType: "integration" }));
138
+ coverage.add(coverageKey({ method, path: epPath, testType: "contract" }));
65
139
  }
66
140
  else {
67
- coverage.add(`${method}::${resource}::${testType}`);
141
+ coverage.add(coverageKey({ method, path: epPath, testType }));
68
142
  }
69
143
  }
70
144
  }
@@ -5,6 +5,18 @@ import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
5
5
  import { ScenarioSource, AnalysisScope } from "../../types/RepositoryAnalysis.js";
6
6
  import { SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
7
7
  import { inferExpectedStatus } from "../../utils/httpDefaults.js";
8
+ /**
9
+ * True when two step lists name the same method+path sequence, ignoring
10
+ * everything else (body, description, ...). Used to decide whether a
11
+ * previously-resolved `subjectEndpoints` still applies to an enriched
12
+ * replacement — this function has no PR diff in scope, so it cannot call
13
+ * `resolveSubjectEndpoints` itself (SKYR-4214).
14
+ */
15
+ function sameStepRouteSequence(a, b) {
16
+ if (a.length !== b.length)
17
+ return false;
18
+ return a.every((step, i) => (step.method ?? "").toUpperCase() === (b[i].method ?? "").toUpperCase() && step.path === b[i].path);
19
+ }
8
20
  export function mergeEnrichedScenarios(serverScenarios, raw) {
9
21
  const rejectionNotes = [];
10
22
  let parsed;
@@ -72,6 +84,16 @@ export function mergeEnrichedScenarios(serverScenarios, raw) {
72
84
  }
73
85
  const merged = new Map(serverScenarios.map(s => [s.scenarioName, s]));
74
86
  for (const s of agentScenarios) {
87
+ const replaced = merged.get(s.scenarioName);
88
+ // The agent-submitted copy never carries subjectEndpoints — it is built
89
+ // fresh from the enriched JSON, not through resolveSubjectEndpoints. When
90
+ // the replaced server scenario had one and the steps still name the same
91
+ // endpoints, carry it forward so this candidate keeps the one recorded
92
+ // subject instead of falling back to a fresh last-mutating-step guess
93
+ // that could disagree with its siblings (SKYR-4214).
94
+ if (replaced?.subjectEndpoints && sameStepRouteSequence(replaced.steps, s.steps)) {
95
+ s.subjectEndpoints = replaced.subjectEndpoints;
96
+ }
75
97
  merged.set(s.scenarioName, s);
76
98
  }
77
99
  logger.info("Merged agent-enriched scenarios", {
@@ -2,8 +2,8 @@ import { RepositoryAnalysis, AnalysisScope, DraftedScenario } from "../../types/
2
2
  import { WorkspaceAuthType } from "../../utils/workspaceAuth.js";
3
3
  import { PRTestContext } from "../../utils/pr-comment-parser.js";
4
4
  import { Novelty, PriorityTier } from "../../types/TestRecommendation.js";
5
- import { buildExternalCoverageSet, externalDedupKey } from "./recommendationShared.js";
6
- export { buildExternalCoverageSet, externalDedupKey };
5
+ import { buildExternalCoverageSet, externalDedupKeys } from "./recommendationShared.js";
6
+ export { buildExternalCoverageSet, externalDedupKeys };
7
7
  /** Result of {@link computeScoredCandidates} — the scoring/classification
8
8
  * inputs shared between prompt rendering and the SKYR-3879 register-plan
9
9
  * pre-seed (analyzeChangesTool.ts), so both derive the exact same numbers. */
@@ -8,9 +8,9 @@ import { buildScopeAssessmentSection, isFrontendFile } from "./scopeAssessment.j
8
8
  import { buildExecutionPlan, EXEC_STEP_CODE_REVIEW, EXEC_STEP_ENRICH } from "./diffExecutionPlan.js";
9
9
  import { buildFullRepoRecommendations } from "./fullRepoCatalog.js";
10
10
  import { ANALYSIS_STEP_EXTRACT } from "./analysisOutputPrompt.js";
11
- import { TASK_GENERATE, buildExternalCoverageSet, externalDedupKey, isAttackSurfaceSecurityBoundary, taskRef, } from "./recommendationShared.js";
11
+ import { TASK_GENERATE, buildExternalCoverageSet, externalDedupKeys, isAttackSurfaceSecurityBoundary, taskRef, } from "./recommendationShared.js";
12
12
  // Re-export for backward compatibility (tests and external callers import these from this module)
13
- export { buildExternalCoverageSet, externalDedupKey };
13
+ export { buildExternalCoverageSet, externalDedupKeys };
14
14
  function formatTestLocations(locs) {
15
15
  const entries = Object.entries(locs || {});
16
16
  if (entries.length === 0)
@@ -23,7 +23,7 @@ function formatTestLocations(locs) {
23
23
  "**Deduplication rule (apply this table before generating anything):**\n" +
24
24
  "- `[external]` tests: if a resource is covered by an `[external]` test, do NOT create a new parallel test for the same HTTP method + resource + test type. These tests still break when the API changes — Task 1 maintenance applies to them the same as Skyramp tests (in-place UPDATE only; do not regenerate or delete).\n" +
25
25
  "- `[skyramp]` contract test: if the HTTP method + path already appears in a `[skyramp]` `covers:` entry of type `contract` → UPDATE that file, do NOT create a new one.\n" +
26
- "- `[skyramp]` integration test: if the primary (last mutating) step's method + path already appears in a `[skyramp]` `covers:` entry of type `integration` UPDATE, do NOT create a new one.\n" +
26
+ "- `[skyramp]` integration test: use the endpoints this PR changed. A setup step or a cleanup step is not an endpoint under test. A scenario can test more than one changed endpoint. UPDATE an existing `[skyramp]` `covers:` entry of type `integration` only when it already covers EVERY changed endpoint the scenario tests. If it covers some but not all of them, the scenario is not a duplicate — create it.\n" +
27
27
  "- UI/E2E test: always create a new file — traces are distinct recordings.\n" +
28
28
  "For `[skyramp]` contract and integration tests: if in doubt, prefer UPDATE over creating a duplicate.");
29
29
  }