@skyramp/mcp 0.3.4 → 0.3.6-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +92 -30
- package/build/playwright/traceRecordingPrompt.d.ts +6 -0
- package/build/playwright/traceRecordingPrompt.js +6 -2
- package/build/prompts/code-reuse.d.ts +1 -2
- package/build/prompts/code-reuse.js +182 -77
- package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
- package/build/prompts/modularization/integration-test-modularization.js +83 -41
- package/build/prompts/modularization/render.d.ts +18 -0
- package/build/prompts/modularization/render.js +12 -0
- package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
- package/build/prompts/modularization/ui-test-modularization.js +89 -47
- package/build/prompts/pom-aware-code-reuse.js +3 -1
- package/build/prompts/shared-helper-policy.d.ts +57 -0
- package/build/prompts/shared-helper-policy.js +135 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
- package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
- package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
- package/build/prompts/test-recommendation/recommendationShared.js +90 -16
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +88 -33
- package/build/recommendation/budgeters/shared.js +105 -27
- package/build/recommendation/discriminators.js +13 -2
- package/build/recommendation/planRanker.d.ts +6 -6
- package/build/recommendation/planRanker.js +6 -61
- package/build/services/AnalyticsService.d.ts +7 -0
- package/build/services/AnalyticsService.js +7 -1
- package/build/services/ModularizationService.js +1 -3
- package/build/services/TestDiscoveryService.d.ts +0 -2
- package/build/services/TestDiscoveryService.js +2 -37
- package/build/services/TestGenerationService.d.ts +16 -0
- package/build/services/TestGenerationService.js +86 -10
- package/build/services/containerEnv.js +13 -12
- package/build/tools/code-refactor/codeReuseTool.js +279 -93
- package/build/tools/code-refactor/enhance-state.d.ts +49 -0
- package/build/tools/code-refactor/enhance-state.js +109 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
- package/build/tools/code-refactor/modularizationTool.js +9 -2
- package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
- package/build/tools/code-refactor/reuse-outcome.js +14 -4
- package/build/tools/code-refactor/reuse-state.d.ts +127 -5
- package/build/tools/code-refactor/reuse-state.js +628 -16
- package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
- package/build/tools/code-refactor/utils-verify-gates.js +100 -0
- package/build/tools/code-refactor/verify-gates.d.ts +2 -1
- package/build/tools/code-refactor/verify-gates.js +90 -25
- package/build/tools/executeSkyrampTestTool.d.ts +19 -0
- package/build/tools/executeSkyrampTestTool.js +158 -8
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
- package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
- package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
- package/build/tools/generate-tests/generateUIRestTool.js +22 -0
- package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
- package/build/tools/generate-tests/scenarioLint.js +127 -19
- package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
- package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
- package/build/tools/submitReportTool.d.ts +38 -38
- package/build/tools/submitReportTool.js +487 -104
- package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
- package/build/tools/test-management/analyzeChangesTool.js +75 -12
- package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
- package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
- package/build/tools/test-management/registerTestPlanTool.js +149 -23
- package/build/types/Recommendation.d.ts +34 -5
- package/build/types/RepositoryAnalysis.d.ts +133 -114
- package/build/types/RepositoryAnalysis.js +1 -1
- package/build/types/ReuseOutcome.d.ts +102 -6
- package/build/types/ReuseOutcome.js +16 -2
- package/build/types/TestRecommendation.js +21 -3
- package/build/types/TestTypes.js +14 -8
- package/build/types/TestbotReport.d.ts +25 -3
- package/build/types/index.d.ts +2 -2
- package/build/types/index.js +1 -1
- package/build/utils/AnalysisStateManager.d.ts +69 -1
- package/build/utils/AnalysisStateManager.js +69 -5
- package/build/utils/branchDiff.d.ts +10 -0
- package/build/utils/branchDiff.js +28 -0
- package/build/utils/changedRoutes.d.ts +29 -0
- package/build/utils/changedRoutes.js +87 -0
- package/build/utils/featureFlags.d.ts +21 -0
- package/build/utils/featureFlags.js +23 -0
- package/build/utils/frontendIntegration.js +34 -4
- package/build/utils/importerHop.d.ts +2 -8
- package/build/utils/importerHop.js +15 -53
- package/build/utils/pathMatching.d.ts +38 -0
- package/build/utils/pathMatching.js +71 -0
- package/build/utils/pathSignatures.d.ts +22 -0
- package/build/utils/pathSignatures.js +57 -0
- package/build/utils/planMatchKeys.d.ts +16 -3
- package/build/utils/planMatchKeys.js +26 -10
- package/build/utils/pluralization.d.ts +10 -0
- package/build/utils/pluralization.js +18 -0
- package/build/utils/pom-catalog-parse.d.ts +52 -0
- package/build/utils/pom-catalog-parse.js +141 -0
- package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
- package/build/utils/pom-scope/selector-extractor.js +34 -8
- package/build/utils/pom-verify/verify.d.ts +6 -5
- package/build/utils/pom-verify/verify.js +8 -6
- package/build/utils/reportLanguage.d.ts +43 -0
- package/build/utils/reportLanguage.js +125 -0
- package/build/utils/reportVerification.d.ts +74 -4
- package/build/utils/reportVerification.js +259 -3
- package/build/utils/reuseRouting.d.ts +3 -0
- package/build/utils/reuseRouting.js +50 -0
- package/build/utils/routeParsers.d.ts +2 -0
- package/build/utils/routeParsers.js +65 -8
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +57 -45
- package/build/utils/subjectEndpoints.d.ts +19 -0
- package/build/utils/subjectEndpoints.js +98 -0
- package/build/utils/testFileClassification.d.ts +11 -0
- package/build/utils/testFileClassification.js +47 -0
- package/build/utils/uiPageEnumerator.d.ts +45 -19
- package/build/utils/uiPageEnumerator.js +95 -51
- package/build/utils/utils-verify/allow.d.ts +16 -0
- package/build/utils/utils-verify/allow.js +68 -0
- package/build/utils/utils-verify/call-sites.d.ts +34 -0
- package/build/utils/utils-verify/call-sites.js +154 -0
- package/build/utils/utils-verify/index.d.ts +7 -0
- package/build/utils/utils-verify/index.js +7 -0
- package/build/utils/utils-verify/language-spec.d.ts +91 -0
- package/build/utils/utils-verify/language-spec.js +210 -0
- package/build/utils/utils-verify/locate.d.ts +39 -0
- package/build/utils/utils-verify/locate.js +199 -0
- package/build/utils/utils-verify/parse.d.ts +34 -0
- package/build/utils/utils-verify/parse.js +177 -0
- package/build/utils/utils-verify/stage.d.ts +24 -0
- package/build/utils/utils-verify/stage.js +107 -0
- package/build/utils/utils-verify/verify.d.ts +63 -0
- package/build/utils/utils-verify/verify.js +168 -0
- package/build/utils/utils.d.ts +3 -1
- package/build/utils/utils.js +3 -1
- package/build/workspace/workspace.d.ts +32 -32
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CandidateSource, computeCandidateId } from "../../types/Recommendation.js";
|
|
2
|
+
import { fixedNBudgeter } from "../../recommendation/budgeters/fixedNBudgeter.js";
|
|
2
3
|
import { AUTH_MIDDLEWARE_PATTERNS_STR } from "../../utils/workspaceAuth.js";
|
|
3
4
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
4
|
-
import { logger } from "../../utils/logger.js";
|
|
5
5
|
import { TEST_CATEGORIES, } from "../../types/TestRecommendation.js";
|
|
6
6
|
import { buildScopeAssessmentSection } from "./scopeAssessment.js";
|
|
7
7
|
import { PromptPlan } from "./promptPlan.js";
|
|
8
8
|
import { buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, MAX_CRITICAL_TESTS, } from "./recommendationSections.js";
|
|
9
|
-
import { TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW,
|
|
9
|
+
import { TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW, taskStepRef, } from "./recommendationShared.js";
|
|
10
10
|
// ── Step body functions ───────────────────────────────────────────────────────
|
|
11
11
|
function _execCodeReviewBody(_ctx) {
|
|
12
12
|
const codeReviewRef = taskStepRef(TASK_ANALYZE_MAINTAIN, TESTBOT_TASK1_STEP_CODE_REVIEW);
|
|
@@ -69,7 +69,7 @@ INSERT a non-bug source-code-derived candidate into the ranked list **only if AL
|
|
|
69
69
|
3. It is not already covered by a structural candidate in the list below
|
|
70
70
|
|
|
71
71
|
If these conditions are not met, add it to ADDITIONAL only — do NOT displace a pre-ranked GENERATE item.
|
|
72
|
-
**
|
|
72
|
+
**Attack-surface \`security_boundary\` items should never be displaced by non-bug candidates** — they test sibling destructive endpoints that could bypass the changed auth boundary. However, bug-catching tests CAN displace them only after all lower-value non-bug slots are exhausted.
|
|
73
73
|
|
|
74
74
|
When a qualifying candidate is inserted: place it HIGH before MEDIUM before LOW; within the same priority, source-code-derived candidates go BEFORE structural ones. Re-number ranks after insertion. The top ${ctx.maxGen} ranked items become GENERATE candidates.
|
|
75
75
|
|
|
@@ -80,8 +80,8 @@ When a qualifying candidate is inserted: place it HIGH before MEDIUM before LOW;
|
|
|
80
80
|
function _execDiversityBody(_ctx) {
|
|
81
81
|
return `**Bug-coverage gate (runs BEFORE dedup):**
|
|
82
82
|
Verify that the highest-severity \`<bug_found>\` flaw from Step ${EXEC_STEP_CODE_REVIEW} has exactly one GENERATE item with category \`bug_caught\` targeting it — meaning the test would FAIL on the current buggy code and PASS once the flaw is fixed. At most one promotion per run (per Step ${EXEC_STEP_CODE_REVIEW} cap). If the promoted flaw has no targeting \`bug_caught\` GENERATE item:
|
|
83
|
-
- Check ADDITIONAL for a matching test → promote it into the lowest-priority non-bug, non-CRITICAL GENERATE slot first (lowest category rank per \`crud > error_handling > workflow > data_validation > data_integrity > business_rule\`; preserve attack-surface \`security_boundary\`
|
|
84
|
-
- If no ADDITIONAL candidate matches, create a new \`bug_caught\` test and insert it, displacing the lowest-priority non-bug, non-CRITICAL GENERATE item first; displace an attack-surface \`security_boundary\`
|
|
83
|
+
- Check ADDITIONAL for a matching test → promote it into the lowest-priority non-bug, non-CRITICAL GENERATE slot first (lowest category rank per \`crud > error_handling > workflow > data_validation > data_integrity > business_rule\`; preserve attack-surface \`security_boundary\` items unless no lower-priority slot exists).
|
|
84
|
+
- If no ADDITIONAL candidate matches, create a new \`bug_caught\` test and insert it, displacing the lowest-priority non-bug, non-CRITICAL GENERATE item first; displace an attack-surface \`security_boundary\` item only when every GENERATE slot is higher priority.
|
|
85
85
|
A \`bug_caught\` test is NEVER considered a "duplicate" of a non-bug test during the dedup below.
|
|
86
86
|
|
|
87
87
|
Each GENERATE item must exercise a **distinct code path** — not just different input values on the same path.
|
|
@@ -98,7 +98,7 @@ Same step sequence with only payload differences (e.g. 10% vs 5% discount both r
|
|
|
98
98
|
function _execRegisterBody(_ctx) {
|
|
99
99
|
return `Register your complete candidate list — every test you would generate OR recommend — via \`skyramp_register_test_plan\` (\`stateFile\` required). Include a discriminator claim (\`discriminator\` field — valid kinds and anchor rules are in the tool schema) for candidates probing the changed logic identified in Step ${EXEC_STEP_CODE_REVIEW}/Step ${EXEC_STEP_ENRICH}.
|
|
100
100
|
|
|
101
|
-
The returned GENERATE list is mandatory and final — generation tools reject unregistered scenarios. If the tool demotes a discriminator claim (returned in \`demotions\` with a reason), either strengthen the claim — a step that actually exercises the declared \`kind\`, or a verbatim anchor that occurs in the diff — or drop it
|
|
101
|
+
The returned GENERATE list is mandatory and final — generation tools reject unregistered scenarios. If the tool demotes a discriminator claim (returned in \`demotions\` with a reason), either strengthen the claim — a step that actually exercises the declared \`kind\`, or a verbatim anchor that occurs in the diff — or drop it. A demoted candidate normally stays in the plan. If it also left the plan for another recorded reason, the report lists it under "Demoted AND removed".`;
|
|
102
102
|
}
|
|
103
103
|
function _execExecuteBody(ctx) {
|
|
104
104
|
return `Replace any scenario that pairs unrelated resources with one reflecting actual foreign-key relationships in the codebase.
|
|
@@ -149,20 +149,6 @@ export const EXEC_STEP_EXECUTE = _execPlan.labels.EXECUTE; // "4"
|
|
|
149
149
|
/** "5" — Register test plan (SKYR-3879 Path B checkpoint) */
|
|
150
150
|
export const EXEC_STEP_REGISTER = _execPlan.labels.REGISTER; // "5"
|
|
151
151
|
const SERVICE_REFS = resolveServiceDetailsRef();
|
|
152
|
-
function prioritizeAttackSurfaceBundles(items) {
|
|
153
|
-
const reordered = [];
|
|
154
|
-
for (const item of items) {
|
|
155
|
-
if (isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
156
|
-
const firstDirectAuthIndex = reordered.findIndex(candidate => isOrdinaryDirectAuthBoundary(candidate.scenario));
|
|
157
|
-
if (firstDirectAuthIndex >= 0) {
|
|
158
|
-
reordered.splice(firstDirectAuthIndex, 0, item);
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
reordered.push(item);
|
|
163
|
-
}
|
|
164
|
-
return reordered;
|
|
165
|
-
}
|
|
166
152
|
export function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, seed, endpointCount, isUIOnlyPR, hasFrontendChanges = false, hasTraces = false, externalCoverage = new Set(), relevantExternalTestPaths = [],
|
|
167
153
|
/**
|
|
168
154
|
* Whether the diff classified at least one new/modified/removed endpoint.
|
|
@@ -184,40 +170,58 @@ hasApiChanges = true) {
|
|
|
184
170
|
: hasFrontendChanges
|
|
185
171
|
? Math.max(0, maxGen - 1)
|
|
186
172
|
: maxGen;
|
|
187
|
-
//
|
|
188
|
-
//
|
|
189
|
-
// This
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
173
|
+
// Both this renderer and the register-plan path run the SAME selection
|
|
174
|
+
// pipeline — `runBudget` — instead of two copies of it (SKYR-4214 item 7).
|
|
175
|
+
// This file used to hold its own external dedup, attack-surface ordering,
|
|
176
|
+
// round-robin GENERATE pick and ADDITIONAL set-difference, so a fix in one
|
|
177
|
+
// copy left the other wrong.
|
|
178
|
+
//
|
|
179
|
+
// The two are NOT guaranteed to produce the same GENERATE set, and this
|
|
180
|
+
// comment must not claim they are: `selectPlan` re-ranks and calls
|
|
181
|
+
// `diversityBalancedBudgeter` (planRanker.ts:96) where this calls
|
|
182
|
+
// `fixedNBudgeter`, and the two differ in the `pick` step —
|
|
183
|
+
// diversityBalancedBudgeter.test.ts covers that divergence. What is now
|
|
184
|
+
// shared is the dedup rule, the ordering and the ADDITIONAL cap.
|
|
185
|
+
//
|
|
186
|
+
// `runBudget` needs a Candidate, which carries provenance and a stable id
|
|
187
|
+
// that only the register-plan path fills. Everything reaching this renderer
|
|
188
|
+
// is server-drafted, and `computeCandidateId` is a pure function of the
|
|
189
|
+
// scenario, so both fields are derivable here.
|
|
190
|
+
//
|
|
191
|
+
// The context deliberately reports no frontend work, whatever this PR
|
|
192
|
+
// contains: this renderer never had the UI-promotion loop, and the backend
|
|
193
|
+
// count it computed above already applied the UI reservation. Passing the
|
|
194
|
+
// real flags would introduce UI slots into the prompt's GENERATE list, which
|
|
195
|
+
// is a behaviour change this item does not carry.
|
|
196
|
+
const candidates = scored.map((item) => ({
|
|
197
|
+
...item,
|
|
198
|
+
source: CandidateSource.SERVER,
|
|
199
|
+
candidateId: computeCandidateId(item.scenario),
|
|
200
|
+
}));
|
|
201
|
+
const selection = fixedNBudgeter.select(candidates, {
|
|
202
|
+
maxGenerate: backendGenerateCount,
|
|
203
|
+
maxTotal: topN,
|
|
204
|
+
isUIOnlyPR: false,
|
|
205
|
+
hasFrontendChanges: false,
|
|
206
|
+
externalCoverage,
|
|
207
|
+
// This renderer never receives the branch diff, so it cannot tell whether the
|
|
208
|
+
// PR wrote tests of its own. `true` is the same abstain direction the analyze
|
|
209
|
+
// end takes for an unknown (SKYR-4024): the covered-candidate reserve stays
|
|
210
|
+
// shut here, and those candidates are reported in the Removed section below
|
|
211
|
+
// rather than promoted to ADDITIONAL.
|
|
212
|
+
diffChangesTestFiles: true,
|
|
213
|
+
});
|
|
214
|
+
const generateItems = selection.generate;
|
|
215
|
+
const additionalItems = selection.additional;
|
|
216
|
+
// A candidate the selection stage removed — covered by an external test, or
|
|
217
|
+
// past the maxTotal cap — leaves no trace in the two lists above. In the
|
|
218
|
+
// recommendation-only flow (analyze_changes plus this prompt, with no
|
|
219
|
+
// register call) this section is the ONLY place the agent can read that a
|
|
220
|
+
// candidate existed and why it went, which is what SKYR-4214 exists to fix.
|
|
221
|
+
const scenarioNameById = new Map(candidates.map((c) => [c.candidateId, c.scenario.scenarioName]));
|
|
222
|
+
const removedLines = selection.dropped
|
|
223
|
+
.map((drop) => `- ${scenarioNameById.get(drop.candidateId) ?? drop.candidateId}: ${drop.reason}`)
|
|
224
|
+
.join("\n");
|
|
221
225
|
const hasWorkspaceAuthType = !!authTypeValue && authTypeValue !== "none";
|
|
222
226
|
// For skyramp_integration_test_generation with scenarioFile:
|
|
223
227
|
// - If workspace has authType set: omit auth entirely — workspace handles Bearer prefix.
|
|
@@ -432,7 +436,9 @@ ${isUIOnlyPR
|
|
|
432
436
|
### ADDITIONAL (list in additionalRecommendations in this order after Step ${EXEC_STEP_ENRICH} insertion)
|
|
433
437
|
|
|
434
438
|
${additionalLines || " (none pre-ranked)"}
|
|
435
|
-
${
|
|
439
|
+
${removedLines
|
|
440
|
+
? `\n### Removed (${selection.dropped.length}) — candidates the selection stage dropped before ranking. They are NOT in the lists above; do not re-add them.\n\n${removedLines}\n`
|
|
441
|
+
: ""}${uiGroundingGuidance}
|
|
436
442
|
${uiGuidance}
|
|
437
443
|
${supplementNote}
|
|
438
444
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { logger } from "../../utils/logger.js";
|
|
2
2
|
import { buildTestQualityCriteria } from "./recommendationSections.js";
|
|
3
|
-
import {
|
|
3
|
+
import { externalDedupKeys, isAttackSurfaceSecurityBoundary } from "./recommendationShared.js";
|
|
4
4
|
export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, isFrontendProject = false, isFrontendOnlyProject = false, externalCoverage = new Set()) {
|
|
5
5
|
// Full-repo mode only — percentage-based UI/E2E slot targets (15% each, floor 1).
|
|
6
6
|
const rawE2E = isFrontendProject ? Math.max(1, Math.round(topN * 0.15)) : 0;
|
|
@@ -17,22 +17,29 @@ export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderVa
|
|
|
17
17
|
: authHeaderValue
|
|
18
18
|
? `, authHeader: "${authHeaderValue}"`
|
|
19
19
|
: `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
|
|
20
|
-
// Supplement count for full-repo mode
|
|
21
|
-
const supplementCount = topN - Math.min(scored.length, topN);
|
|
22
20
|
const toTitle = (name) => name.replace(/-/g, " ").replace(/\b\w/g, c => c.toUpperCase());
|
|
23
21
|
const TYPE_ORDER = ["e2e", "ui", "integration", "contract"];
|
|
24
22
|
const TYPE_LABEL = {
|
|
25
23
|
e2e: "E2E", ui: "UI", integration: "Integration", contract: "Contract",
|
|
26
24
|
};
|
|
27
25
|
// Filter out scenarios already covered by external tests before slicing.
|
|
26
|
+
// The bug/attack-surface exemption matches applyExternalDedup in
|
|
27
|
+
// budgeters/shared.ts: those scenarios cover a semantic flaw that a
|
|
28
|
+
// same-endpoint external test does not, so an external match must not
|
|
29
|
+
// remove them here either.
|
|
28
30
|
const scoredFiltered = externalCoverage.size > 0
|
|
29
31
|
? scored.filter(item => {
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
const keys = externalDedupKeys(item.scenario);
|
|
33
|
+
if (keys.length === 0)
|
|
34
|
+
return true;
|
|
35
|
+
if (!keys.every((key) => externalCoverage.has(key)))
|
|
36
|
+
return true;
|
|
37
|
+
if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
38
|
+
logger.info(`External dedup (full-repo): preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
|
|
39
|
+
return true;
|
|
34
40
|
}
|
|
35
|
-
|
|
41
|
+
logger.info(`External dedup (full-repo): skipping "${item.scenario.scenarioName}" (${keys.join(", ")})`);
|
|
42
|
+
return false;
|
|
36
43
|
})
|
|
37
44
|
: scored;
|
|
38
45
|
// For full-stack repos, carve out E2E and UI slots before filling with backend tests.
|
|
@@ -40,6 +47,10 @@ export function buildFullRepoRecommendations(scored, topN, baseUrl, authHeaderVa
|
|
|
40
47
|
? Math.max(0, topN - minE2ESlots - minUISlots)
|
|
41
48
|
: topN;
|
|
42
49
|
const allItems = scoredFiltered.slice(0, backendSlotCount);
|
|
50
|
+
// Count the supplements against what this prompt actually lists. Reading the
|
|
51
|
+
// pre-filter `scored.length` claimed more pre-ranked items than the sections
|
|
52
|
+
// below render, so a pool thinned by external dedup asked for no supplements.
|
|
53
|
+
const supplementCount = topN - Math.min(allItems.length, topN);
|
|
43
54
|
const byType = new Map();
|
|
44
55
|
for (const t of TYPE_ORDER)
|
|
45
56
|
byType.set(t, []);
|
|
@@ -3,14 +3,36 @@
|
|
|
3
3
|
* full-repo mode (fullRepoCatalog.ts). Extracted here to avoid circular imports.
|
|
4
4
|
*/
|
|
5
5
|
import { DraftedScenario } from "../../types/RepositoryAnalysis.js";
|
|
6
|
-
export declare function scenarioCoverageKey(scenario: DraftedScenario): string;
|
|
7
6
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
7
|
+
* The one spelling of a coverage key, `METHOD::resource::testType`. Both
|
|
8
|
+
* operands of the dedup comparison call it: a proposed test through
|
|
9
|
+
* `externalDedupKeys`, and an existing test through `buildExternalCoverageSet`.
|
|
10
|
+
* They used to build the same string in two places from two different inputs —
|
|
11
|
+
* a typed step list on one side, regex-scraped prose on the other — so a fix to
|
|
12
|
+
* one side left the other producing keys that could never match (SKYR-4214).
|
|
12
13
|
*/
|
|
13
|
-
export declare function
|
|
14
|
+
export declare function coverageKey(input: {
|
|
15
|
+
method: string;
|
|
16
|
+
path: string;
|
|
17
|
+
testType: string;
|
|
18
|
+
}): string;
|
|
19
|
+
/**
|
|
20
|
+
* Method-aware coverage keys for external test dedup — one per recorded subject
|
|
21
|
+
* endpoint. Method-aware so that an external test covering "GET /orders" does
|
|
22
|
+
* not block a test for "PUT /orders", a different operation on the same
|
|
23
|
+
* resource.
|
|
24
|
+
*
|
|
25
|
+
* A caller must remove a candidate ONLY when the coverage set holds EVERY key
|
|
26
|
+
* returned here. Removing it on one match would discard the coverage of its
|
|
27
|
+
* other endpoints. An empty list must never remove anything.
|
|
28
|
+
*/
|
|
29
|
+
export declare function externalDedupKeys(scenario: DraftedScenario): string[];
|
|
30
|
+
/**
|
|
31
|
+
* Resource+type keys (no method) for the GENERATE/ADDITIONAL overlap filter —
|
|
32
|
+
* one per recorded subject endpoint. Same rule as `externalDedupKeys`: remove
|
|
33
|
+
* only on ALL keys, never on an empty list.
|
|
34
|
+
*/
|
|
35
|
+
export declare function scenarioCoverageKeys(scenario: DraftedScenario): string[];
|
|
14
36
|
export declare function isAttackSurfaceSecurityBoundary(scenario: DraftedScenario): boolean;
|
|
15
37
|
export declare function isOrdinaryDirectAuthBoundary(scenario: DraftedScenario): boolean;
|
|
16
38
|
/**
|
|
@@ -10,22 +10,96 @@ function resolvePrimaryStep(scenario) {
|
|
|
10
10
|
const primaryStep = mutatingSteps[mutatingSteps.length - 1] ?? scenario.steps[scenario.steps.length - 1];
|
|
11
11
|
return { primaryStep, testType };
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
/**
|
|
14
|
+
* The one spelling of a coverage key, `METHOD::resource::testType`. Both
|
|
15
|
+
* operands of the dedup comparison call it: a proposed test through
|
|
16
|
+
* `externalDedupKeys`, and an existing test through `buildExternalCoverageSet`.
|
|
17
|
+
* They used to build the same string in two places from two different inputs —
|
|
18
|
+
* a typed step list on one side, regex-scraped prose on the other — so a fix to
|
|
19
|
+
* one side left the other producing keys that could never match (SKYR-4214).
|
|
20
|
+
*/
|
|
21
|
+
export function coverageKey(input) {
|
|
22
|
+
const method = (input.method ?? "GET").toUpperCase();
|
|
23
|
+
return `${method}::${extractResourceFromPath(input.path ?? "")}::${input.testType}`;
|
|
24
|
+
}
|
|
25
|
+
/** Test type of a scenario: its own label, else one step means a contract test. */
|
|
26
|
+
function resolveTestType(scenario) {
|
|
27
|
+
return scenario.testType ?? (scenario.steps.length === 1 ? "contract" : "integration");
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Method-aware coverage keys for external test dedup — one per recorded subject
|
|
31
|
+
* endpoint. Method-aware so that an external test covering "GET /orders" does
|
|
32
|
+
* not block a test for "PUT /orders", a different operation on the same
|
|
33
|
+
* resource.
|
|
34
|
+
*
|
|
35
|
+
* A caller must remove a candidate ONLY when the coverage set holds EVERY key
|
|
36
|
+
* returned here. Removing it on one match would discard the coverage of its
|
|
37
|
+
* other endpoints. An empty list must never remove anything.
|
|
38
|
+
*/
|
|
39
|
+
export function externalDedupKeys(scenario) {
|
|
40
|
+
const testType = resolveTestType(scenario);
|
|
41
|
+
const subjects = scenario.subjectEndpoints;
|
|
42
|
+
if (subjects && subjects.length > 0) {
|
|
43
|
+
return subjects.map((s) => coverageKey({ ...s, testType }));
|
|
44
|
+
}
|
|
45
|
+
if (subjects)
|
|
46
|
+
return [];
|
|
47
|
+
const { primaryStep } = resolvePrimaryStep(scenario);
|
|
48
|
+
if (!primaryStep)
|
|
49
|
+
return [];
|
|
50
|
+
return [coverageKey({ method: primaryStep.method, path: primaryStep.path, testType })];
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Resource+type keys (no method) for the GENERATE/ADDITIONAL overlap filter —
|
|
54
|
+
* one per recorded subject endpoint. Same rule as `externalDedupKeys`: remove
|
|
55
|
+
* only on ALL keys, never on an empty list.
|
|
56
|
+
*/
|
|
57
|
+
export function scenarioCoverageKeys(scenario) {
|
|
58
|
+
const testType = resolveTestType(scenario);
|
|
59
|
+
const steps = scenario.steps ?? [];
|
|
60
|
+
const subjects = scenario.subjectEndpoints;
|
|
61
|
+
// No recorded subject: fall back to the primary step, as externalDedupKeys does.
|
|
62
|
+
if (!subjects) {
|
|
63
|
+
const { primaryStep } = resolvePrimaryStep(scenario);
|
|
64
|
+
if (!primaryStep)
|
|
65
|
+
return [];
|
|
66
|
+
return [
|
|
67
|
+
overlapKey(primaryStep.method, extractResourceFromPath(primaryStep.path ?? ""), testType, primaryStep),
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
return subjects.map((subject) => {
|
|
71
|
+
const step = steps.find((s) => s.method === subject.method && s.path === subject.path);
|
|
72
|
+
return overlapKey(subject.method, extractResourceFromPath(subject.path ?? ""), testType, step);
|
|
73
|
+
});
|
|
17
74
|
}
|
|
18
75
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
76
|
+
* The overlap key names the endpoint AND what the test asserts about it.
|
|
77
|
+
*
|
|
78
|
+
* It deliberately does NOT reuse `coverageKey`. That key's other operand comes
|
|
79
|
+
* from `buildExternalCoverageSet`, which scrapes prose and can only ever know a
|
|
80
|
+
* method and a path — adding a status there would stop every external key from
|
|
81
|
+
* matching. Both operands of the OVERLAP comparison are drafted candidates with
|
|
82
|
+
* a step list, so both sides know the method, the interaction type and the
|
|
83
|
+
* status.
|
|
84
|
+
*
|
|
85
|
+
* The method is part of the key because `extractResourceFromPath` maps
|
|
86
|
+
* `/api/orders` and `/api/orders/{id}` to the same resource. Without it a
|
|
87
|
+
* GENERATE `PATCH /api/orders/{id}` (success, 200) removed an ADDITIONAL
|
|
88
|
+
* `GET /api/orders/{id}` (success, 200) as a duplicate — a read test and an
|
|
89
|
+
* update test on one resource are not duplicates.
|
|
90
|
+
*
|
|
91
|
+
* Measured on 839 nightly fixtures: 781 of them change exactly ONE route, so
|
|
92
|
+
* every candidate resolves to the same subject and `resource::testType` yields
|
|
93
|
+
* 2 keys for the whole pool. In 11 fixtures that removed EVERY ADDITIONAL
|
|
94
|
+
* candidate — `01-site-stats-endpoint` lost all 18. Adding the interaction type
|
|
95
|
+
* and the status roughly doubles the distinct keys on those pools. It does not
|
|
96
|
+
* make the key complete: two tests that assert different things about the same
|
|
97
|
+
* 200 response still collide, and no key built from a route can separate them.
|
|
23
98
|
*/
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
return `${method}::${resource}::${testType}`;
|
|
99
|
+
function overlapKey(method, resource, testType, step) {
|
|
100
|
+
const interaction = step?.interactionType ?? "any";
|
|
101
|
+
const status = step?.expectedStatusCode ?? 0;
|
|
102
|
+
return `${(method ?? "ANY").toUpperCase()}::${resource}::${testType}::${interaction}::${status}`;
|
|
29
103
|
}
|
|
30
104
|
export function isAttackSurfaceSecurityBoundary(scenario) {
|
|
31
105
|
return scenario.category === "security_boundary" &&
|
|
@@ -60,11 +134,11 @@ export function buildExternalCoverageSet(testLocations) {
|
|
|
60
134
|
const resource = extractResourceFromPath(epPath);
|
|
61
135
|
if (resource !== "unknown") {
|
|
62
136
|
if (testType === "unknown") {
|
|
63
|
-
coverage.add(
|
|
64
|
-
coverage.add(
|
|
137
|
+
coverage.add(coverageKey({ method, path: epPath, testType: "integration" }));
|
|
138
|
+
coverage.add(coverageKey({ method, path: epPath, testType: "contract" }));
|
|
65
139
|
}
|
|
66
140
|
else {
|
|
67
|
-
coverage.add(
|
|
141
|
+
coverage.add(coverageKey({ method, path: epPath, testType }));
|
|
68
142
|
}
|
|
69
143
|
}
|
|
70
144
|
}
|
|
@@ -5,6 +5,18 @@ import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
|
|
|
5
5
|
import { ScenarioSource, AnalysisScope } from "../../types/RepositoryAnalysis.js";
|
|
6
6
|
import { SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
|
|
7
7
|
import { inferExpectedStatus } from "../../utils/httpDefaults.js";
|
|
8
|
+
/**
|
|
9
|
+
* True when two step lists name the same method+path sequence, ignoring
|
|
10
|
+
* everything else (body, description, ...). Used to decide whether a
|
|
11
|
+
* previously-resolved `subjectEndpoints` still applies to an enriched
|
|
12
|
+
* replacement — this function has no PR diff in scope, so it cannot call
|
|
13
|
+
* `resolveSubjectEndpoints` itself (SKYR-4214).
|
|
14
|
+
*/
|
|
15
|
+
function sameStepRouteSequence(a, b) {
|
|
16
|
+
if (a.length !== b.length)
|
|
17
|
+
return false;
|
|
18
|
+
return a.every((step, i) => (step.method ?? "").toUpperCase() === (b[i].method ?? "").toUpperCase() && step.path === b[i].path);
|
|
19
|
+
}
|
|
8
20
|
export function mergeEnrichedScenarios(serverScenarios, raw) {
|
|
9
21
|
const rejectionNotes = [];
|
|
10
22
|
let parsed;
|
|
@@ -72,6 +84,16 @@ export function mergeEnrichedScenarios(serverScenarios, raw) {
|
|
|
72
84
|
}
|
|
73
85
|
const merged = new Map(serverScenarios.map(s => [s.scenarioName, s]));
|
|
74
86
|
for (const s of agentScenarios) {
|
|
87
|
+
const replaced = merged.get(s.scenarioName);
|
|
88
|
+
// The agent-submitted copy never carries subjectEndpoints — it is built
|
|
89
|
+
// fresh from the enriched JSON, not through resolveSubjectEndpoints. When
|
|
90
|
+
// the replaced server scenario had one and the steps still name the same
|
|
91
|
+
// endpoints, carry it forward so this candidate keeps the one recorded
|
|
92
|
+
// subject instead of falling back to a fresh last-mutating-step guess
|
|
93
|
+
// that could disagree with its siblings (SKYR-4214).
|
|
94
|
+
if (replaced?.subjectEndpoints && sameStepRouteSequence(replaced.steps, s.steps)) {
|
|
95
|
+
s.subjectEndpoints = replaced.subjectEndpoints;
|
|
96
|
+
}
|
|
75
97
|
merged.set(s.scenarioName, s);
|
|
76
98
|
}
|
|
77
99
|
logger.info("Merged agent-enriched scenarios", {
|
|
@@ -2,8 +2,8 @@ import { RepositoryAnalysis, AnalysisScope, DraftedScenario } from "../../types/
|
|
|
2
2
|
import { WorkspaceAuthType } from "../../utils/workspaceAuth.js";
|
|
3
3
|
import { PRTestContext } from "../../utils/pr-comment-parser.js";
|
|
4
4
|
import { Novelty, PriorityTier } from "../../types/TestRecommendation.js";
|
|
5
|
-
import { buildExternalCoverageSet,
|
|
6
|
-
export { buildExternalCoverageSet,
|
|
5
|
+
import { buildExternalCoverageSet, externalDedupKeys } from "./recommendationShared.js";
|
|
6
|
+
export { buildExternalCoverageSet, externalDedupKeys };
|
|
7
7
|
/** Result of {@link computeScoredCandidates} — the scoring/classification
|
|
8
8
|
* inputs shared between prompt rendering and the SKYR-3879 register-plan
|
|
9
9
|
* pre-seed (analyzeChangesTool.ts), so both derive the exact same numbers. */
|
|
@@ -8,9 +8,9 @@ import { buildScopeAssessmentSection, isFrontendFile } from "./scopeAssessment.j
|
|
|
8
8
|
import { buildExecutionPlan, EXEC_STEP_CODE_REVIEW, EXEC_STEP_ENRICH } from "./diffExecutionPlan.js";
|
|
9
9
|
import { buildFullRepoRecommendations } from "./fullRepoCatalog.js";
|
|
10
10
|
import { ANALYSIS_STEP_EXTRACT } from "./analysisOutputPrompt.js";
|
|
11
|
-
import { TASK_GENERATE, buildExternalCoverageSet,
|
|
11
|
+
import { TASK_GENERATE, buildExternalCoverageSet, externalDedupKeys, isAttackSurfaceSecurityBoundary, taskRef, } from "./recommendationShared.js";
|
|
12
12
|
// Re-export for backward compatibility (tests and external callers import these from this module)
|
|
13
|
-
export { buildExternalCoverageSet,
|
|
13
|
+
export { buildExternalCoverageSet, externalDedupKeys };
|
|
14
14
|
function formatTestLocations(locs) {
|
|
15
15
|
const entries = Object.entries(locs || {});
|
|
16
16
|
if (entries.length === 0)
|
|
@@ -23,7 +23,7 @@ function formatTestLocations(locs) {
|
|
|
23
23
|
"**Deduplication rule (apply this table before generating anything):**\n" +
|
|
24
24
|
"- `[external]` tests: if a resource is covered by an `[external]` test, do NOT create a new parallel test for the same HTTP method + resource + test type. These tests still break when the API changes — Task 1 maintenance applies to them the same as Skyramp tests (in-place UPDATE only; do not regenerate or delete).\n" +
|
|
25
25
|
"- `[skyramp]` contract test: if the HTTP method + path already appears in a `[skyramp]` `covers:` entry of type `contract` → UPDATE that file, do NOT create a new one.\n" +
|
|
26
|
-
"- `[skyramp]` integration test:
|
|
26
|
+
"- `[skyramp]` integration test: use the endpoints this PR changed. A setup step or a cleanup step is not an endpoint under test. A scenario can test more than one changed endpoint. UPDATE an existing `[skyramp]` `covers:` entry of type `integration` only when it already covers EVERY changed endpoint the scenario tests. If it covers some but not all of them, the scenario is not a duplicate — create it.\n" +
|
|
27
27
|
"- UI/E2E test: always create a new file — traces are distinct recordings.\n" +
|
|
28
28
|
"For `[skyramp]` contract and integration tests: if in doubt, prefer UPDATE over creating a duplicate.");
|
|
29
29
|
}
|