@skyramp/mcp 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/code-reuse.js +17 -2
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +1 -1
- package/build/prompts/modularization/ui-test-modularization.js +9 -6
- package/build/prompts/pom-aware-code-reuse.js +1 -1
- package/build/prompts/shared-helper-policy.js +5 -5
- package/build/prompts/test-recommendation/diffExecutionPlan.js +40 -9
- package/build/prompts/test-recommendation/fullRepoCatalog.js +3 -2
- package/build/prompts/test-recommendation/recommendationSections.js +3 -3
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +2 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +58 -76
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +14 -1
- package/build/prompts/testbot/testbot-prompts.js +25 -8
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +39 -5
- package/build/recommendation/budgeters/shared.d.ts +21 -8
- package/build/recommendation/budgeters/shared.js +134 -56
- package/build/recommendation/planRanker.d.ts +5 -3
- package/build/recommendation/planRanker.js +1 -1
- package/build/services/TestGenerationService.js +15 -1
- package/build/tools/code-refactor/assertion-state.d.ts +91 -0
- package/build/tools/code-refactor/assertion-state.js +375 -0
- package/build/tools/code-refactor/codeReuseTool.js +6 -4
- package/build/tools/code-refactor/enhanceAssertionsTool.js +73 -18
- package/build/tools/code-refactor/retrofit-state.d.ts +53 -0
- package/build/tools/code-refactor/retrofit-state.js +162 -0
- package/build/tools/code-refactor/reuse-outcome.d.ts +7 -0
- package/build/tools/code-refactor/reuse-state.d.ts +9 -0
- package/build/tools/code-refactor/reuse-state.js +42 -4
- package/build/tools/code-refactor/utils-verify-gates.js +69 -15
- package/build/tools/executeSkyrampTestTool.js +19 -14
- package/build/tools/submitReportTool.js +87 -9
- package/build/tools/test-management/registerTestPlanTool.d.ts +3 -3
- package/build/tools/test-management/registerTestPlanTool.js +29 -8
- package/build/types/AssertionOutcome.d.ts +68 -0
- package/build/types/AssertionOutcome.js +1 -0
- package/build/types/RepositoryAnalysis.d.ts +10 -10
- package/build/types/ReuseOutcome.d.ts +16 -0
- package/build/types/TestRecommendation.d.ts +11 -1
- package/build/types/TestRecommendation.js +34 -0
- package/build/types/TestTypes.d.ts +4 -0
- package/build/types/TestTypes.js +8 -0
- package/build/types/TestbotReport.d.ts +13 -0
- package/build/types/index.d.ts +1 -1
- package/build/utils/AnalysisStateManager.d.ts +20 -7
- package/build/utils/assertion-verify/api-shared-lints.d.ts +5 -0
- package/build/utils/assertion-verify/api-shared-lints.js +315 -0
- package/build/utils/assertion-verify/contract-lints.d.ts +3 -0
- package/build/utils/assertion-verify/contract-lints.js +87 -0
- package/build/utils/assertion-verify/format.d.ts +5 -0
- package/build/utils/assertion-verify/format.js +65 -0
- package/build/utils/assertion-verify/helper-imports.d.ts +6 -0
- package/build/utils/assertion-verify/helper-imports.js +178 -0
- package/build/utils/assertion-verify/index.d.ts +3 -0
- package/build/utils/assertion-verify/index.js +7 -0
- package/build/utils/assertion-verify/integration-lints.d.ts +3 -0
- package/build/utils/assertion-verify/integration-lints.js +36 -0
- package/build/utils/assertion-verify/js-regex-blank.d.ts +1 -0
- package/build/utils/assertion-verify/js-regex-blank.js +153 -0
- package/build/utils/assertion-verify/lint-types.d.ts +33 -0
- package/build/utils/assertion-verify/lint-types.js +57 -0
- package/build/utils/assertion-verify/marker.d.ts +27 -0
- package/build/utils/assertion-verify/marker.js +61 -0
- package/build/utils/assertion-verify/metrics.d.ts +30 -0
- package/build/utils/assertion-verify/metrics.js +341 -0
- package/build/utils/assertion-verify/python-strip.d.ts +6 -0
- package/build/utils/assertion-verify/python-strip.js +75 -0
- package/build/utils/assertion-verify/strip-dispatch.d.ts +19 -0
- package/build/utils/assertion-verify/strip-dispatch.js +42 -0
- package/build/utils/assertion-verify/ui-lints.d.ts +8 -0
- package/build/utils/assertion-verify/ui-lints.js +244 -0
- package/build/utils/assertion-verify/verify.d.ts +61 -0
- package/build/utils/assertion-verify/verify.js +215 -0
- package/build/utils/featureFlags.d.ts +12 -2
- package/build/utils/featureFlags.js +33 -3
- package/build/utils/pathSignatures.d.ts +4 -1
- package/build/utils/pathSignatures.js +14 -2
- package/build/utils/planOnlyMode.d.ts +33 -0
- package/build/utils/planOnlyMode.js +40 -0
- package/build/utils/reportVerification.d.ts +9 -0
- package/build/utils/reportVerification.js +39 -6
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +1 -1
- package/build/utils/utils-verify/allow.d.ts +22 -4
- package/build/utils/utils-verify/allow.js +8 -2
- package/build/utils/utils-verify/call-sites.d.ts +40 -1
- package/build/utils/utils-verify/call-sites.js +196 -30
- package/build/utils/utils-verify/importers.d.ts +31 -0
- package/build/utils/utils-verify/importers.js +78 -0
- package/build/utils/utils-verify/index.d.ts +1 -0
- package/build/utils/utils-verify/index.js +1 -0
- package/build/utils/utils-verify/language-spec.d.ts +13 -2
- package/build/utils/utils-verify/language-spec.js +12 -2
- package/build/utils/utils-verify/parse.d.ts +31 -3
- package/build/utils/utils-verify/parse.js +190 -9
- package/build/utils/utils-verify/retrofit-equivalence.d.ts +43 -0
- package/build/utils/utils-verify/retrofit-equivalence.js +218 -0
- package/build/utils/utils-verify/stage.d.ts +6 -0
- package/build/utils/utils-verify/stage.js +12 -2
- package/build/utils/utils-verify/verify.d.ts +54 -4
- package/build/utils/utils-verify/verify.js +224 -12
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-CZfp96qZ.js → codeMirrorModule-LNgEKtdV.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-gpLo02E0.js → defaultSettingsView-Bwr1eMKC.js} +135 -135
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Bq1r1URj.js → index.-Id052Lr.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.VEfqi1qN.js → uiMode.BPopbasy.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/build/tools/code-refactor/enhance-state.d.ts +0 -49
- package/build/tools/code-refactor/enhance-state.js +0 -109
|
@@ -1,28 +1,76 @@
|
|
|
1
1
|
import { TestType } from "../../types/TestTypes.js";
|
|
2
2
|
import { inferScenarioType, prioritizeAttackSurfaceBundles } from "../diversity.js";
|
|
3
3
|
import { externalDedupKeys, scenarioCoverageKeys, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
|
|
4
|
+
import { isFlawTargetingCategory } from "../../types/TestRecommendation.js";
|
|
4
5
|
import { logger } from "../../utils/logger.js";
|
|
5
6
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* What each PR class GUARANTEES in GENERATE (SKYR-4275). These are floors, not
|
|
8
|
+
* quotas: the slots above them are open to either kind, decided by `pick`.
|
|
9
|
+
* An open slot goes to whichever candidate `pick` chooses on rank, so a backend
|
|
10
|
+
* candidate takes one when it outranks the UI candidates, not merely because it
|
|
11
|
+
* exists. Measured against main on random mixed pools at `maxGenerate` 5 with no
|
|
12
|
+
* caught bugs: 35% get fewer backend tests than main did, average 3.53 to 3.24.
|
|
13
|
+
*
|
|
14
|
+
* Backend-only floors nothing on purpose: of 86 backend-only diffs in the eval
|
|
15
|
+
* corpus, ZERO generated a UI test, so a floor there would guarantee what
|
|
16
|
+
* already happens every time.
|
|
17
|
+
*
|
|
18
|
+
* At one slot there is no room for a floor plus an open slot, so each class
|
|
19
|
+
* keeps that slot for its own kind.
|
|
20
|
+
*
|
|
21
|
+
* The floors always sum to at most `maxGenerate`, which is what lets
|
|
22
|
+
* `applyFloors` satisfy both without one starving the other.
|
|
10
23
|
*/
|
|
11
|
-
export function
|
|
24
|
+
export function generateFloors(ctx) {
|
|
25
|
+
// Number.isFinite, not Math.max: `Math.max(0, NaN)` is NaN, which compares
|
|
26
|
+
// false against 0 and would fall through to the class branches below.
|
|
27
|
+
const max = Number.isFinite(ctx.maxGenerate) ? Math.max(0, Math.floor(ctx.maxGenerate)) : 0;
|
|
28
|
+
if (max === 0)
|
|
29
|
+
return { ui: 0, backend: 0 };
|
|
12
30
|
if (ctx.isUIOnlyPR)
|
|
13
|
-
return 0;
|
|
14
|
-
|
|
31
|
+
return { ui: max === 1 ? 1 : max - 1, backend: 0 };
|
|
32
|
+
if (ctx.hasFrontendChanges)
|
|
33
|
+
return max === 1 ? { ui: 1, backend: 0 } : { ui: 1, backend: 1 };
|
|
34
|
+
return { ui: 0, backend: 0 };
|
|
15
35
|
}
|
|
16
|
-
/**
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Raise a chosen GENERATE set to meet the floors, by swapping rather than
|
|
38
|
+
* re-picking. `pick` runs ONCE over the whole pool, so its protected-first
|
|
39
|
+
* ordering, its one-slot-per-test-type coverage and attack-surface bundle
|
|
40
|
+
* adjacency are all settled before this runs. Picking each floor from its own
|
|
41
|
+
* slice instead would compute those guarantees per slice and collapse them —
|
|
42
|
+
* measured at 25% of mixed-PR inputs losing a test type, 8.3% losing contract
|
|
43
|
+
* coverage outright.
|
|
44
|
+
*
|
|
45
|
+
* A floor is raised only as far as the pool allows. Satisfying one floor cannot
|
|
46
|
+
* starve the other, because `generateFloors` never returns two that sum above
|
|
47
|
+
* the budget — `generateFloors.test.ts` pins that invariant.
|
|
48
|
+
*/
|
|
49
|
+
function applyFloors(generate, slotOrdered, floors, isUI) {
|
|
50
|
+
const isBackend = (item) => !isUI(item);
|
|
51
|
+
const result = [...generate];
|
|
52
|
+
for (const [want, wanted] of [
|
|
53
|
+
[floors.ui, isUI],
|
|
54
|
+
[floors.backend, isBackend],
|
|
55
|
+
]) {
|
|
56
|
+
const spare = slotOrdered.filter((item) => wanted(item) && !result.includes(item));
|
|
57
|
+
let shortfall = Math.min(want, result.filter(wanted).length + spare.length) - result.filter(wanted).length;
|
|
58
|
+
// Swap from the END: `pick` returns its slots in preference order, so the
|
|
59
|
+
// last one it chose is the one this PR class needs least.
|
|
60
|
+
for (let i = result.length - 1; i >= 0 && shortfall > 0; i--) {
|
|
61
|
+
if (wanted(result[i]))
|
|
62
|
+
continue;
|
|
63
|
+
result[i] = spare.shift();
|
|
64
|
+
shortfall--;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
21
68
|
}
|
|
22
69
|
/**
|
|
23
70
|
* Split candidates by whether an external test already covers them. Protected
|
|
24
|
-
* `bug_caught` / attack-surface scenarios always count
|
|
25
|
-
* require semantic flaw coverage the external test may not
|
|
71
|
+
* `bug_caught` / `requirement_conflict` / attack-surface scenarios always count
|
|
72
|
+
* as uncovered — they require semantic flaw coverage the external test may not
|
|
73
|
+
* provide.
|
|
26
74
|
*
|
|
27
75
|
* A candidate is covered only when the set holds EVERY one of its keys. One
|
|
28
76
|
* match is not coverage: a scenario exercising two changed endpoints would lose
|
|
@@ -44,8 +92,8 @@ function partitionByExternalCoverage(ranked, externalCoverage) {
|
|
|
44
92
|
uncovered.push(item);
|
|
45
93
|
continue;
|
|
46
94
|
}
|
|
47
|
-
if (item.scenario.category
|
|
48
|
-
logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
|
|
95
|
+
if (isFlawTargetingCategory(item.scenario.category) || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
96
|
+
logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/requirement-conflict/attack-surface scenario requires semantic flaw coverage`);
|
|
49
97
|
uncovered.push(item);
|
|
50
98
|
continue;
|
|
51
99
|
}
|
|
@@ -62,36 +110,27 @@ function partitionByExternalCoverage(ranked, externalCoverage) {
|
|
|
62
110
|
* With `pick = roundRobinByType` this reproduces the pre-refactor selection in
|
|
63
111
|
* diffExecutionPlan.ts exactly.
|
|
64
112
|
*/
|
|
65
|
-
export function runBudget(ranked,
|
|
113
|
+
export function runBudget(ranked, rawCtx, pick) {
|
|
114
|
+
// SKYR-4292: maxTotal is the budget; maxGenerate is how much of it to generate.
|
|
115
|
+
// Every producer clamps upstream (getTestbotPrompt, computeScoredCandidates,
|
|
116
|
+
// buildScopeAssessmentSection), but a state file is the tool boundary and can
|
|
117
|
+
// carry maxGenerate > maxTotal — the UI-only branch then reserved maxGenerate
|
|
118
|
+
// slots and planned more items than the total. Clamp once here so every
|
|
119
|
+
// budgeter agrees with the prompt. It must stay ABOVE generateFloors(ctx):
|
|
120
|
+
// the floors are computed from the clamped budget, and moving the clamp
|
|
121
|
+
// below them would let the floors sum past maxTotal again (review on #835).
|
|
122
|
+
const ctx = { ...rawCtx, maxGenerate: Math.min(rawCtx.maxGenerate, rawCtx.maxTotal) };
|
|
66
123
|
const dropped = [];
|
|
67
|
-
const
|
|
68
|
-
const uiSlots = reservedUISlots(ctx);
|
|
124
|
+
const floors = generateFloors(ctx);
|
|
69
125
|
const { uncovered, covered } = partitionByExternalCoverage(ranked, ctx.externalCoverage);
|
|
70
126
|
const slotOrdered = prioritizeAttackSurfaceBundles(uncovered);
|
|
71
|
-
|
|
72
|
-
//
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const backendGenerate = pick(backendSource, Math.min(backend, backendSource.length));
|
|
79
|
-
// Fill the reserved UI slots the `pick` step leaves empty: the backend count
|
|
80
|
-
// is 0 on a UI-only PR (and maxGenerate-1 on a mixed PR), so UI candidates are
|
|
81
|
-
// never chosen above even though reservedUISlots budgets for them. Promote the
|
|
82
|
-
// top-ranked UI candidates not already chosen, up to the reserved count, so
|
|
83
|
-
// GENERATE is complete instead of an empty list plus a dropped count.
|
|
84
|
-
const chosen = new Set(backendGenerate);
|
|
85
|
-
const uiGenerate = [];
|
|
86
|
-
for (const item of slotOrdered) {
|
|
87
|
-
if (uiGenerate.length >= uiSlots)
|
|
88
|
-
break;
|
|
89
|
-
if (chosen.has(item) || inferScenarioType(item.scenario) !== TestType.UI)
|
|
90
|
-
continue;
|
|
91
|
-
uiGenerate.push(item);
|
|
92
|
-
chosen.add(item);
|
|
93
|
-
}
|
|
94
|
-
const generate = [...backendGenerate, ...uiGenerate];
|
|
127
|
+
const isUICandidate = (item) => inferScenarioType(item.scenario) === TestType.UI;
|
|
128
|
+
// ONE pick over the whole pool — the shape the backend pick always had — so
|
|
129
|
+
// `pick` decides protected-first order, per-type coverage and bundle adjacency
|
|
130
|
+
// exactly once. `applyFloors` then corrects only the UI/backend mix.
|
|
131
|
+
const budget = Number.isFinite(ctx.maxGenerate) ? Math.max(0, Math.floor(ctx.maxGenerate)) : 0;
|
|
132
|
+
const picked = pick(slotOrdered, Math.min(budget, slotOrdered.length));
|
|
133
|
+
const generate = applyFloors(picked, slotOrdered, floors, isUICandidate);
|
|
95
134
|
// SKYR-4024. A covered candidate is one an existing test already reaches, so
|
|
96
135
|
// generating a second test for it is wrong — maintenance updates the existing
|
|
97
136
|
// one. But dropping it outright removed it from the RECOMMENDATIONS too, and an
|
|
@@ -100,24 +139,50 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
100
139
|
// one UI candidate fills the reserved slot while every covered backend
|
|
101
140
|
// candidate would otherwise vanish.
|
|
102
141
|
//
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
// test file
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
|
|
142
|
+
// Three conditions keep the reserve shut where recommending nothing is right.
|
|
143
|
+
// A UI-only PR is left alone. A zero budget is an explicit abstention, so it
|
|
144
|
+
// must not be talked out of recommending nothing. And the PR must not touch a
|
|
145
|
+
// test file of its own: coverage is read from the working tree, and a test
|
|
146
|
+
// file without the Skyramp marker counts as external, so a PR that adds its
|
|
147
|
+
// own tests would have that coverage used as grounds to recommend duplicates
|
|
148
|
+
// of it.
|
|
149
|
+
//
|
|
150
|
+
// One pick over the whole pool leaves no "backend half" to count, so the
|
|
151
|
+
// emptiness test reproduces what that half used to mean per class. On a mixed
|
|
152
|
+
// PR the UI slot must not read as backend coverage, or a covered backend
|
|
153
|
+
// candidate vanishes behind it. On a backend-only PR the pick always drew from
|
|
154
|
+
// the whole pool, so a non-empty GENERATE closed the reserve even when every
|
|
155
|
+
// item was UI — keep that, or an all-UI pool opens a reserve it never opened.
|
|
156
|
+
const backendChosen = generate.filter((item) => !isUICandidate(item));
|
|
157
|
+
const backendHalfEmpty = ctx.hasFrontendChanges
|
|
158
|
+
? backendChosen.length === 0
|
|
159
|
+
: generate.length === 0;
|
|
160
|
+
// One per-class behaviour DID change here, deliberately. On a mixed PR at a
|
|
161
|
+
// budget of 1 the old gate also required `backendGenerateCount > 0`, which is
|
|
162
|
+
// 0 for that class, so a covered backend candidate was dropped outright. The
|
|
163
|
+
// reserve exists because dropping removes a candidate from the
|
|
164
|
+
// RECOMMENDATIONS too (SKYR-4024), and a budget of 1 is the case where that
|
|
165
|
+
// costs most — there is no second slot to find it later. It is now
|
|
166
|
+
// recommended instead of dropped. `reserveOpensForMixedPRAtBudgetOne` in
|
|
167
|
+
// shared.test.ts pins it, so a future change has to mean it.
|
|
168
|
+
const spendReserve = backendHalfEmpty &&
|
|
169
|
+
!ctx.isUIOnlyPR &&
|
|
170
|
+
budget > 0 &&
|
|
171
|
+
covered.length > 0 &&
|
|
172
|
+
!ctx.diffChangesTestFiles;
|
|
110
173
|
if (spendReserve) {
|
|
111
174
|
logger.info(`External dedup: GENERATE empty after dedup — recommending ${covered.length} held-back candidate(s) as ADDITIONAL rather than approving an empty plan`);
|
|
112
175
|
}
|
|
113
176
|
else if (covered.length > 0) {
|
|
114
177
|
// SKYR-4214. The reserve stayed shut, so these really are dropped — say so,
|
|
115
178
|
// with the reason, instead of leaving the agent an unexplained short list.
|
|
116
|
-
const reason =
|
|
117
|
-
? `the backend half of GENERATE is filled (${
|
|
118
|
-
:
|
|
119
|
-
? "
|
|
120
|
-
:
|
|
179
|
+
const reason = !backendHalfEmpty
|
|
180
|
+
? `the backend half of GENERATE is filled (${backendChosen.length || generate.length})`
|
|
181
|
+
: ctx.isUIOnlyPR
|
|
182
|
+
? "a UI-only PR keeps its slots for UI"
|
|
183
|
+
: budget === 0
|
|
184
|
+
? "the budget is zero"
|
|
185
|
+
: "the PR changes test files of its own";
|
|
121
186
|
logger.info(`External dedup: dropping ${covered.length} covered candidate(s) — reserve shut because ${reason}`);
|
|
122
187
|
for (const { item, keys } of covered) {
|
|
123
188
|
dropped.push({
|
|
@@ -142,6 +207,7 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
142
207
|
const additionalSource = spendReserve
|
|
143
208
|
? [...slotOrdered, ...prioritizeAttackSurfaceBundles(covered.map((c) => c.item))]
|
|
144
209
|
: slotOrdered;
|
|
210
|
+
const chosen = new Set(generate);
|
|
145
211
|
const notChosen = additionalSource.filter((it) => !chosen.has(it));
|
|
146
212
|
const remainingBudget = Math.max(0, ctx.maxTotal - generate.length);
|
|
147
213
|
const generatedCoverage = new Set(generate.flatMap((item) => scenarioCoverageKeys(item.scenario)));
|
|
@@ -149,6 +215,18 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
149
215
|
const keys = scenarioCoverageKeys(item.scenario);
|
|
150
216
|
if (keys.length === 0 || !keys.every((key) => generatedCoverage.has(key)))
|
|
151
217
|
return true;
|
|
218
|
+
// Same exemption the EXTERNAL dedup above already makes, for the same
|
|
219
|
+
// reason: a flaw-targeting scenario (a caught bug, or a requirement the code
|
|
220
|
+
// contradicts) or an attack-surface boundary asserts a specific defect, and
|
|
221
|
+
// sharing an endpoint with a GENERATE item is not evidence that the defect is
|
|
222
|
+
// covered. Without it a PR with two defects in one handler had its second one
|
|
223
|
+
// deleted here — the first took a GENERATE slot, and the key is method +
|
|
224
|
+
// resource + test type + interaction + status, which two tests on one
|
|
225
|
+
// endpoint share.
|
|
226
|
+
if (isFlawTargetingCategory(item.scenario.category) || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
227
|
+
logger.info(`GENERATE dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/requirement-conflict/attack-surface scenario requires semantic flaw coverage`);
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
152
230
|
dropped.push({
|
|
153
231
|
candidateId: item.candidateId,
|
|
154
232
|
reason: `covered by GENERATE (${keys.join(", ")})`,
|
|
@@ -164,5 +242,5 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
164
242
|
}
|
|
165
243
|
// Budgeting itself produces no demotions; the register-plan selection stage
|
|
166
244
|
// (planRanker.selectPlan) fills this channel from discriminator verification.
|
|
167
|
-
return { generate, additional, reservedUISlots:
|
|
245
|
+
return { generate, additional, reservedUISlots: floors.ui, demotions: [], dropped };
|
|
168
246
|
}
|
|
@@ -4,8 +4,10 @@ import { ScenarioCategory } from "../types/TestRecommendation.js";
|
|
|
4
4
|
export interface RankOptions {
|
|
5
5
|
/**
|
|
6
6
|
* Categories that take the top carve-out tier ahead of everything else.
|
|
7
|
-
* Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught
|
|
8
|
-
* new_endpoint is MEDIUM, not carved out).
|
|
7
|
+
* Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught and
|
|
8
|
+
* requirement_conflict — new_endpoint is MEDIUM, not carved out). The two CRITICAL
|
|
9
|
+
* categories are carved out independently, so a requirement conflict never competes
|
|
10
|
+
* with a code-review bug for one slot. Exposed so phase 2 can tune the carve-out
|
|
9
11
|
* WITHOUT reintroducing the agent's priority tag as a ranking input.
|
|
10
12
|
*/
|
|
11
13
|
carveOutCategories?: ScenarioCategory[];
|
|
@@ -32,7 +34,7 @@ export interface SelectPlanContext extends BudgetContext {
|
|
|
32
34
|
* independent of input order (the final tiebreak is the stable `candidateId`).
|
|
33
35
|
*
|
|
34
36
|
* Ordering (highest first):
|
|
35
|
-
* 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
|
|
37
|
+
* 1. Carve-out — CRITICAL-category scenarios (bug_caught, requirement_conflict), preserving the
|
|
36
38
|
* protected-first convention of `roundRobinByType` /
|
|
37
39
|
* `prioritizeAttackSurfaceBundles`.
|
|
38
40
|
* 2. Verified discriminators — candidates whose declared discriminator survived
|
|
@@ -14,7 +14,7 @@ const DEFAULT_CARVE_OUT_CATEGORIES = Object.keys(CATEGORY_PRIORITY).filter((cate
|
|
|
14
14
|
* independent of input order (the final tiebreak is the stable `candidateId`).
|
|
15
15
|
*
|
|
16
16
|
* Ordering (highest first):
|
|
17
|
-
* 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
|
|
17
|
+
* 1. Carve-out — CRITICAL-category scenarios (bug_caught, requirement_conflict), preserving the
|
|
18
18
|
* protected-first convention of `roundRobinByType` /
|
|
19
19
|
* `prioritizeAttackSurfaceBundles`.
|
|
20
20
|
* 2. Verified discriminators — candidates whose declared discriminator survived
|
|
@@ -7,13 +7,14 @@ import { extractPathname, resolveQueryParamsForPath, contributesNothing, } from
|
|
|
7
7
|
import { getPathParameterValidationError, OUTPUT_DIR_FIELD_NAME, PATH_PARAMS_FIELD_NAME, QUERY_PARAMS_FIELD_NAME, FORM_PARAMS_FIELD_NAME, validateParams, validatePath, validateRequestData, } from "../utils/utils.js";
|
|
8
8
|
import { getEntryPoint } from "../utils/telemetry.js";
|
|
9
9
|
import { getLanguageSteps } from "../utils/language-helper.js";
|
|
10
|
-
import { TestType } from "../types/TestTypes.js";
|
|
10
|
+
import { ASSERTION_ENHANCEABLE_TEST_TYPES, TestType, MOCK_TYPE, } from "../types/TestTypes.js";
|
|
11
11
|
import { logger } from "../utils/logger.js";
|
|
12
12
|
import { normalizeLanguageParams } from "../utils/normalizeParams.js";
|
|
13
13
|
import { stageGeneratedPaths, resolveOutputDir } from "../utils/gitStaging.js";
|
|
14
14
|
import { isInsideExecutorWorkDir, executorWorkDirRefusal, generationTargets, } from "../utils/executorWorkDir.js";
|
|
15
15
|
import { getTestsRepoDir } from "../utils/AnalysisStateManager.js";
|
|
16
16
|
import { recordReuseHandOff } from "../tools/code-refactor/reuse-state.js";
|
|
17
|
+
import { recordAssertionBaseline } from "../tools/code-refactor/assertion-state.js";
|
|
17
18
|
import { isModularizeFirstTarget } from "../utils/reuseRouting.js";
|
|
18
19
|
/**
|
|
19
20
|
* Derive the framework actually used for generation from the (normalized)
|
|
@@ -106,7 +107,20 @@ export class TestGenerationService {
|
|
|
106
107
|
// The files this call wrote, as codegen reports them — the reuse hand-off is
|
|
107
108
|
// recorded per FILE (SKYR-4220).
|
|
108
109
|
const written = generatedFilesFrom(result);
|
|
110
|
+
// Seed the assertion-enhancement baseline SERVER-SIDE at generation time
|
|
111
|
+
// (same pattern as recordReuseHandOff below): the execute-time
|
|
112
|
+
// verification gate arms even if the agent never calls
|
|
113
|
+
// skyramp_enhance_assertions at all — the last prompt-mediated entry
|
|
114
|
+
// point. Best-effort; no run state means the gate stays open as before.
|
|
109
115
|
const testType = this.getTestType();
|
|
116
|
+
if (params.enhanceAssertions &&
|
|
117
|
+
testType !== MOCK_TYPE &&
|
|
118
|
+
ASSERTION_ENHANCEABLE_TEST_TYPES.includes(testType) &&
|
|
119
|
+
(testType !== TestType.CONTRACT || params.providerMode !== false)) {
|
|
120
|
+
for (const file of written) {
|
|
121
|
+
await recordAssertionBaseline(file, testType, "generation");
|
|
122
|
+
}
|
|
123
|
+
}
|
|
110
124
|
const languageSteps = getLanguageSteps({
|
|
111
125
|
language: params.language || "",
|
|
112
126
|
testType: testType,
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { type AssertionBaseline, type AssertionEnhanceType, type AssertionVerifyResult } from "../../utils/assertion-verify/index.js";
|
|
2
|
+
import type { AssertionOutcome, AssertionRecord } from "../../types/AssertionOutcome.js";
|
|
3
|
+
import type { TestType } from "../../types/TestTypes.js";
|
|
4
|
+
import type { RepoCheckout } from "../../utils/reportVerification.js";
|
|
5
|
+
/**
|
|
6
|
+
* Persistence for the assertion-enhancement verification loop, keyed by the
|
|
7
|
+
* spec's CANONICAL absolute path in `UnifiedAnalysisState.assertionOutcomes`.
|
|
8
|
+
* Path keying (not basename, unlike reuseOutcomes — nothing here needs to
|
|
9
|
+
* match report rows) removes the multi-repo basename-collision class outright,
|
|
10
|
+
* and canonicalization closes the alias bypass: `dir//x.spec.ts` and
|
|
11
|
+
* `dir/./x.spec.ts` must hit the same record as `dir/x.spec.ts`.
|
|
12
|
+
*
|
|
13
|
+
* Every write is best-effort: verification must never be able to fail an
|
|
14
|
+
* enhancement that otherwise succeeded, so a missing/unreadable state file
|
|
15
|
+
* degrades to stateless checks and open gates — always with a warning.
|
|
16
|
+
*/
|
|
17
|
+
/** Canonical identity of a test file — all keying and ownership comparisons
|
|
18
|
+
* go through this so a non-normalized agent-supplied path cannot fork or
|
|
19
|
+
* bypass a record. realpath also collapses symlinked directories; a path
|
|
20
|
+
* that does not (yet) exist falls back to plain resolution. */
|
|
21
|
+
export declare function canonicalTestPath(testFile: string): string;
|
|
22
|
+
/** A record only counts for `testFile` when its stored path canonicalizes to
|
|
23
|
+
* the same spec — the single home for the ownership invariant (previously
|
|
24
|
+
* copied in baselineFrom, recordAssertionVerdict, and the enhance tool). */
|
|
25
|
+
export declare function trustedRecord(record: AssertionRecord | undefined, testFile: string): AssertionRecord | undefined;
|
|
26
|
+
/** One spec's record, or undefined when there is no run state / no record. */
|
|
27
|
+
export declare function readAssertionRecord(testFile: string, explicitStateFile?: string): Promise<AssertionRecord | undefined>;
|
|
28
|
+
/**
|
|
29
|
+
* Snapshot the file at instruction hand-out time — the baseline verify measures
|
|
30
|
+
* against. Keep-first rule: while no Passed verdict exists for the same file and
|
|
31
|
+
* enhanceType, a repeat hand-out does NOT re-baseline (re-calling the tool after
|
|
32
|
+
* half-editing must not shrink the required delta). A hand-out after a Passed
|
|
33
|
+
* verdict, or with a different enhanceType, starts a new enhancement round.
|
|
34
|
+
*/
|
|
35
|
+
export declare function recordAssertionBaseline(testFile: string, testType: TestType, enhanceType: AssertionEnhanceType, explicitStateFile?: string): Promise<void>;
|
|
36
|
+
/** The baseline verify measures against, from a record; undefined when the
|
|
37
|
+
* record carries no snapshot or belongs to a different canonical path. */
|
|
38
|
+
export declare function baselineFrom(record: AssertionRecord | undefined, testFile: string): AssertionBaseline | undefined;
|
|
39
|
+
/** Record what one verify pass established. The verdict is written to the
|
|
40
|
+
* state file the record actually LIVES in (not blindly to the explicit path —
|
|
41
|
+
* a mismatched `stateFile` would otherwise split verdict from baseline).
|
|
42
|
+
* Never writes onto a record owned by a different canonical path, and never
|
|
43
|
+
* fails silently. */
|
|
44
|
+
export declare function recordAssertionVerdict(testFile: string, result: AssertionVerifyResult, explicitStateFile?: string): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Deterministic execute-time verification: skyramp_execute_test calls this
|
|
47
|
+
* itself, so the check runs whether or not the agent ever called `verify: true`
|
|
48
|
+
* — a prose instruction can be ignored, a server-side call cannot. Returns the
|
|
49
|
+
* feedback text when the assertions are insufficient (the caller returns it
|
|
50
|
+
* instead of executing), or undefined to proceed.
|
|
51
|
+
*
|
|
52
|
+
* Self-clearing: the checks run on the file's CURRENT content, so fixing the
|
|
53
|
+
* assertions and re-executing passes — no separate verify call is required.
|
|
54
|
+
* Skipped once a verdict has PASSED: sanctioned later rewrites (a `.raw.bak`
|
|
55
|
+
* restore during execution fix-up) would otherwise false-fail the differential
|
|
56
|
+
* gates. Modularization needs no special handling: the shared-helper policy
|
|
57
|
+
* keeps every value assertion in the test body, so the baseline comparison
|
|
58
|
+
* stays valid across it. Fails open on no baseline and unreadable state/file.
|
|
59
|
+
*/
|
|
60
|
+
export declare function assertionFeedbackForExecution(testFile: string, explicitStateFile?: string): Promise<string | undefined>;
|
|
61
|
+
/**
|
|
62
|
+
* Count one execution of this spec (called by skyramp_execute_test after its
|
|
63
|
+
* gates pass, best-effort). Creates a minimal record when none exists so an
|
|
64
|
+
* executed-but-never-enhanced spec is still visible to the report cross-check.
|
|
65
|
+
*/
|
|
66
|
+
export declare function recordAssertionExecution(testFile: string, testType: TestType, explicitStateFile?: string): Promise<void>;
|
|
67
|
+
/** A repo checkout as listRepoCheckouts returns it — the canonical type from
|
|
68
|
+
* reportVerification, re-exported (a local re-declaration had drifted:
|
|
69
|
+
* `root` weakened to optional, forcing a redundant guard at each use). */
|
|
70
|
+
export type RepoCheckoutRef = RepoCheckout;
|
|
71
|
+
/** The record for a report row, matched by the canonical-path keys' basenames
|
|
72
|
+
* and then GUARDED: the record's testType must match the row's, and when the
|
|
73
|
+
* run's checkouts are known, the record's path must live in the row's repo
|
|
74
|
+
* (primary when the row names none). Without the guards, a basename shared
|
|
75
|
+
* across repos published one spec's proof-of-work under another's name — and
|
|
76
|
+
* the victim was exactly the never-enhanced spec the field exists to expose.
|
|
77
|
+
* Ambiguity after filtering still fails open: wrong attribution is worse
|
|
78
|
+
* than omission. */
|
|
79
|
+
export declare function findAssertionRecordByFileName(outcomes: Record<string, AssertionRecord>, row: {
|
|
80
|
+
fileName: string;
|
|
81
|
+
testType?: string;
|
|
82
|
+
repository?: string;
|
|
83
|
+
}, checkouts: RepoCheckoutRef[]): AssertionRecord | undefined;
|
|
84
|
+
/**
|
|
85
|
+
* Report-facing proof-of-work summary re-derived from the spec as delivered
|
|
86
|
+
* NOW — final metrics come from the file, not from verify-time state, so
|
|
87
|
+
* post-verify edits cannot publish stale counts. Always returns the status
|
|
88
|
+
* fields (verification, declinedReason, executionCount — they come from the
|
|
89
|
+
* record); the metric fields are omitted when the file cannot supply them.
|
|
90
|
+
*/
|
|
91
|
+
export declare function rederiveAssertionOutcome(record: AssertionRecord): Promise<AssertionOutcome>;
|