@skyramp/mcp 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/code-reuse.js +17 -2
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +1 -1
- package/build/prompts/modularization/ui-test-modularization.js +9 -6
- package/build/prompts/pom-aware-code-reuse.js +1 -1
- package/build/prompts/shared-helper-policy.js +5 -5
- package/build/prompts/test-recommendation/diffExecutionPlan.js +40 -9
- package/build/prompts/test-recommendation/fullRepoCatalog.js +3 -2
- package/build/prompts/test-recommendation/recommendationSections.js +3 -3
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +2 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +58 -76
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +14 -1
- package/build/prompts/testbot/testbot-prompts.js +25 -8
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +39 -5
- package/build/recommendation/budgeters/shared.d.ts +21 -8
- package/build/recommendation/budgeters/shared.js +134 -56
- package/build/recommendation/planRanker.d.ts +5 -3
- package/build/recommendation/planRanker.js +1 -1
- package/build/services/TestGenerationService.js +15 -1
- package/build/tools/code-refactor/assertion-state.d.ts +91 -0
- package/build/tools/code-refactor/assertion-state.js +375 -0
- package/build/tools/code-refactor/codeReuseTool.js +6 -4
- package/build/tools/code-refactor/enhanceAssertionsTool.js +73 -18
- package/build/tools/code-refactor/retrofit-state.d.ts +53 -0
- package/build/tools/code-refactor/retrofit-state.js +162 -0
- package/build/tools/code-refactor/reuse-outcome.d.ts +7 -0
- package/build/tools/code-refactor/reuse-state.d.ts +9 -0
- package/build/tools/code-refactor/reuse-state.js +42 -4
- package/build/tools/code-refactor/utils-verify-gates.js +69 -15
- package/build/tools/executeSkyrampTestTool.js +19 -14
- package/build/tools/submitReportTool.js +87 -9
- package/build/tools/test-management/registerTestPlanTool.d.ts +3 -3
- package/build/tools/test-management/registerTestPlanTool.js +29 -8
- package/build/types/AssertionOutcome.d.ts +68 -0
- package/build/types/AssertionOutcome.js +1 -0
- package/build/types/RepositoryAnalysis.d.ts +10 -10
- package/build/types/ReuseOutcome.d.ts +16 -0
- package/build/types/TestRecommendation.d.ts +11 -1
- package/build/types/TestRecommendation.js +34 -0
- package/build/types/TestTypes.d.ts +4 -0
- package/build/types/TestTypes.js +8 -0
- package/build/types/TestbotReport.d.ts +13 -0
- package/build/types/index.d.ts +1 -1
- package/build/utils/AnalysisStateManager.d.ts +20 -7
- package/build/utils/assertion-verify/api-shared-lints.d.ts +5 -0
- package/build/utils/assertion-verify/api-shared-lints.js +315 -0
- package/build/utils/assertion-verify/contract-lints.d.ts +3 -0
- package/build/utils/assertion-verify/contract-lints.js +87 -0
- package/build/utils/assertion-verify/format.d.ts +5 -0
- package/build/utils/assertion-verify/format.js +65 -0
- package/build/utils/assertion-verify/helper-imports.d.ts +6 -0
- package/build/utils/assertion-verify/helper-imports.js +178 -0
- package/build/utils/assertion-verify/index.d.ts +3 -0
- package/build/utils/assertion-verify/index.js +7 -0
- package/build/utils/assertion-verify/integration-lints.d.ts +3 -0
- package/build/utils/assertion-verify/integration-lints.js +36 -0
- package/build/utils/assertion-verify/js-regex-blank.d.ts +1 -0
- package/build/utils/assertion-verify/js-regex-blank.js +153 -0
- package/build/utils/assertion-verify/lint-types.d.ts +33 -0
- package/build/utils/assertion-verify/lint-types.js +57 -0
- package/build/utils/assertion-verify/marker.d.ts +27 -0
- package/build/utils/assertion-verify/marker.js +61 -0
- package/build/utils/assertion-verify/metrics.d.ts +30 -0
- package/build/utils/assertion-verify/metrics.js +341 -0
- package/build/utils/assertion-verify/python-strip.d.ts +6 -0
- package/build/utils/assertion-verify/python-strip.js +75 -0
- package/build/utils/assertion-verify/strip-dispatch.d.ts +19 -0
- package/build/utils/assertion-verify/strip-dispatch.js +42 -0
- package/build/utils/assertion-verify/ui-lints.d.ts +8 -0
- package/build/utils/assertion-verify/ui-lints.js +244 -0
- package/build/utils/assertion-verify/verify.d.ts +61 -0
- package/build/utils/assertion-verify/verify.js +215 -0
- package/build/utils/featureFlags.d.ts +12 -2
- package/build/utils/featureFlags.js +33 -3
- package/build/utils/pathSignatures.d.ts +4 -1
- package/build/utils/pathSignatures.js +14 -2
- package/build/utils/planOnlyMode.d.ts +33 -0
- package/build/utils/planOnlyMode.js +40 -0
- package/build/utils/reportVerification.d.ts +9 -0
- package/build/utils/reportVerification.js +39 -6
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +1 -1
- package/build/utils/utils-verify/allow.d.ts +22 -4
- package/build/utils/utils-verify/allow.js +8 -2
- package/build/utils/utils-verify/call-sites.d.ts +40 -1
- package/build/utils/utils-verify/call-sites.js +196 -30
- package/build/utils/utils-verify/importers.d.ts +31 -0
- package/build/utils/utils-verify/importers.js +78 -0
- package/build/utils/utils-verify/index.d.ts +1 -0
- package/build/utils/utils-verify/index.js +1 -0
- package/build/utils/utils-verify/language-spec.d.ts +13 -2
- package/build/utils/utils-verify/language-spec.js +12 -2
- package/build/utils/utils-verify/parse.d.ts +31 -3
- package/build/utils/utils-verify/parse.js +190 -9
- package/build/utils/utils-verify/retrofit-equivalence.d.ts +43 -0
- package/build/utils/utils-verify/retrofit-equivalence.js +218 -0
- package/build/utils/utils-verify/stage.d.ts +6 -0
- package/build/utils/utils-verify/stage.js +12 -2
- package/build/utils/utils-verify/verify.d.ts +54 -4
- package/build/utils/utils-verify/verify.js +224 -12
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-CZfp96qZ.js → codeMirrorModule-LNgEKtdV.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-gpLo02E0.js → defaultSettingsView-Bwr1eMKC.js} +135 -135
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Bq1r1URj.js → index.-Id052Lr.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.VEfqi1qN.js → uiMode.BPopbasy.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/build/tools/code-refactor/enhance-state.d.ts +0 -49
- package/build/tools/code-refactor/enhance-state.js +0 -109
|
@@ -2,7 +2,11 @@ import * as fs from "fs";
|
|
|
2
2
|
import * as path from "path";
|
|
3
3
|
import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "./recommendationSections.js";
|
|
4
4
|
// .vue and .svelte cannot be route handlers in any framework — always frontend.
|
|
5
|
-
|
|
5
|
+
// Stylesheets cannot be either (SKYR-4292): `packages/excalidraw/css/styles.scss`
|
|
6
|
+
// classified as backend because `css/` is not in FRONTEND_DIR_PATTERN, so a
|
|
7
|
+
// stylesheet-only PR took the zero-classified backend branch and never saw the
|
|
8
|
+
// changed-behavior rule.
|
|
9
|
+
const ALWAYS_FRONTEND_EXT = /\.(vue|svelte|css|scss|less)$/i;
|
|
6
10
|
// .dart is always frontend in Flutter projects (no server-side Dart web framework
|
|
7
11
|
// in scope). Gated on hasFlutterPubspec so non-Flutter Dart files (rare, but
|
|
8
12
|
// possible — e.g. Dart-on-server backends) don't accidentally classify as UI.
|
|
@@ -10,7 +14,7 @@ const FLUTTER_DART_EXT = /\.dart$/i;
|
|
|
10
14
|
// .tsx/.jsx are usually UI components but can be Next.js API handlers (pages/api/*.tsx).
|
|
11
15
|
const LIKELY_FRONTEND_EXT = /\.(tsx|jsx)$/i;
|
|
12
16
|
// Ambiguous extensions need directory context to distinguish frontend from backend.
|
|
13
|
-
const AMBIGUOUS_FRONTEND_PATTERN = /\.(tsx?|jsx?|
|
|
17
|
+
const AMBIGUOUS_FRONTEND_PATTERN = /\.(tsx?|jsx?|html?|erb|jsp|asp|jinja2?|twig)$/i;
|
|
14
18
|
// /api/ always signals backend — even .tsx files (e.g. Next.js pages/api/*.tsx handlers).
|
|
15
19
|
const STRICT_API_DIR_PATTERN = /\/(api)\//i;
|
|
16
20
|
// Framework route dirs that contain React page components in modern full-stack apps
|
|
@@ -189,67 +193,38 @@ export function isTestFile(filePath) {
|
|
|
189
193
|
}
|
|
190
194
|
// ── LLM scope assessment ──────────────────────────────────────────────────────
|
|
191
195
|
/**
|
|
192
|
-
* The
|
|
193
|
-
* diff (SKYR-
|
|
196
|
+
* The changed-behavior budget rule, shared by every branch that can see a frontend
|
|
197
|
+
* diff (SKYR-4292).
|
|
194
198
|
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
*
|
|
199
|
+
* This replaces the zero-new-surface abstention override (SKYR-4099). That override
|
|
200
|
+
* named three abstain categories — cosmetic, deletion-only, and modification of an
|
|
201
|
+
* already-covered surface — and its only sanctioned action was a Budget Plan of 0. On
|
|
202
|
+
* a 20-change UI-only diff under a 20 budget (Wipro PR #24) the agent generalized it
|
|
203
|
+
* into a graded cut to 3, classified option additions no test asserted (Siebel,
|
|
204
|
+
* Windows, .NET, GO, MQ) as "modification of an already-covered surface" because the
|
|
205
|
+
* existing specs reached the wizard page, and shipped zero recommendations for the
|
|
206
|
+
* rest. The taxonomy is gone: every observable behavior the diff changes gets a
|
|
207
|
+
* candidate, coverage means an existing test ASSERTS the changed value (not that it
|
|
208
|
+
* reaches the component), and the total tracks the count of changed behaviors — the
|
|
209
|
+
* ranker in `skyramp_register_test_plan` trims to the ceiling and reports the drops.
|
|
202
210
|
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
* NEW observable surface only. An earlier revision of this section implemented cosmetic
|
|
207
|
-
* alone, and its keep-the-budget list contradicted the other two (an element being
|
|
208
|
-
* removed, or a `data-testid` being renamed, both forced the budget to stand). The test
|
|
209
|
-
* is coverage, not the kind of edit: does an existing test already reach this surface?
|
|
210
|
-
*
|
|
211
|
-
* `skipClause` is appended to the opening paragraph: the mixed-PR branch has a UI%
|
|
212
|
-
* step that becomes irrelevant once the budget is 0, the precomputed branch does not.
|
|
213
|
-
*/
|
|
214
|
-
/**
|
|
215
|
-
* The zero-new-surface abstention rule, shared by every branch that can see a frontend
|
|
216
|
-
* diff (SKYR-4099).
|
|
217
|
-
*
|
|
218
|
-
* It previously existed only on the mixed-PR branch, so a frontend-ONLY diff — which
|
|
219
|
-
* takes the precomputed branch — had no sanctioned path to zero tests and the agent
|
|
220
|
-
* generated unnecessary UI tests while stating in its own reasoning that the change was
|
|
221
|
-
* cosmetic. Two copies then meant two definitions, and the mixed-PR one carved out
|
|
222
|
-
* "changes that alter visibility, layout, or state", which classifies a spacing-token
|
|
223
|
-
* change as non-cosmetic and made the override inert for exactly the diffs it should
|
|
224
|
-
* catch. One definition, both branches.
|
|
225
|
-
*
|
|
226
|
-
* Scoped to match `testbot-prompts.ts`'s "Do not fabricate tests outside the GENERATE
|
|
227
|
-
* list", which names three zero-test cases — deletion-only, cosmetic, and
|
|
228
|
-
* modification-of-existing-with-no-new-surface — under one principle: a new spec covers
|
|
229
|
-
* NEW observable surface only. The test is coverage, not the kind of edit: does an
|
|
230
|
-
* existing test already reach this surface?
|
|
231
|
-
*
|
|
232
|
-
* `skipClause` is appended to the opening paragraph: the mixed-PR branch has a UI% step
|
|
233
|
-
* that becomes irrelevant once the budget is 0, the precomputed branch does not.
|
|
211
|
+
* Backend-only diffs do not render this: their GENERATE list is server pre-ranked
|
|
212
|
+
* from classified endpoints (and SKYR-3820 handles a zero-classified diff), whereas a
|
|
213
|
+
* frontend GENERATE list is placeholders the agent must fill from the diff itself.
|
|
234
214
|
*/
|
|
235
|
-
function
|
|
236
|
-
return `**
|
|
215
|
+
function changedBehaviorSection(effectiveGenerate, budgetPlanTiming = "") {
|
|
216
|
+
return `**Fill the budget from changed behavior:** The ceiling above is the most this run may plan, not a quota to reach or a default to cut. Enumerate every observable behavior this diff changes — each option value added, removed, renamed or reordered; each label, element, route, state, validation rule or computed value that renders or behaves differently — and give each one its own candidate. Treat no change as too trivial to test: a changed option value that no test asserts is a test. A changed style value counts too — a spacing, size, color, radius, font weight, icon size or class name that now renders differently is a changed behavior; its candidate asserts the new computed style or class on that element, or records a \`browser_visual_snapshot\` of it. A removed option, element or feature is a changed behavior too: its candidate asserts the removed thing is absent from the page where it used to render — the removal is what this PR ships, so the removal is what gets tested.
|
|
237
217
|
|
|
238
|
-
|
|
239
|
-
- **Cosmetic.** A styling-only value change (a spacing, size, color or font token, or a utility class swap such as \`size-4\`→\`size-5\`), or a \`.css\`/\`.scss\` reformat (property reordering, comment or whitespace edits, \`0px\`→\`0\`).
|
|
240
|
-
- **Deletion-only.** A component, route, element or feature was removed. The work is DELETING the tests that covered it — a removed surface cannot be the subject of a new spec.
|
|
241
|
-
- **Modification of an already-covered surface.** A renamed or moved selector, \`data-testid\`, \`aria-*\` or role; changed copy; an added field; a reordered or conditionally hidden element — where an existing test already reaches it. The work is UPDATING that test in place.
|
|
218
|
+
A changed behavior is something a user can observe differently after this diff. Code the diff deletes, dedupes or simplifies without changing what renders is not a changed behavior — its coverage is maintenance of the tests that reach it — and a pre-existing defect you notice nearby is an \`issuesFound\` entry, not a slot in this budget. Only a comment, whitespace or equivalent-value edit (\`0px\` → \`0\`) changes nothing observable.
|
|
242
219
|
|
|
243
|
-
|
|
244
|
-
- A newly added component, route, page or flow.
|
|
245
|
-
- A component that was previously unintegrated and now has an integration point.
|
|
246
|
-
- New interactive behavior, state or validation on a surface no existing test reaches.
|
|
220
|
+
A changed behavior is covered only when an existing test asserts the changed value (including after a maintenance UPDATE you recorded). A test that merely reaches the component does not cover it — that change still needs its own candidate.
|
|
247
221
|
|
|
248
|
-
|
|
222
|
+
- Fewer changed behaviors than the ceiling → your Budget Plan total is that count, split as generate first (up to ${effectiveGenerate}) and the remainder additional. Do not pad with tests of behavior this diff did not change.
|
|
223
|
+
- More changed behaviors than the ceiling → keep every one in the candidate list you register; \`skyramp_register_test_plan\` trims to the ceiling and reports what it dropped.
|
|
249
224
|
|
|
250
|
-
|
|
225
|
+
**A requirement the diff did NOT implement counts too.** The enumeration above is of behaviors the diff changed; a \`requirement_conflict\` is the opposite — the PR title/description (or a requirements file it references) states a requirement and the diff failed to implement it. Give it its own candidate anyway, category \`requirement_conflict\`, asserting what the description requires so that it fails on the current code. When the description says a feature was removed and it still renders, the absence assertion IS that test — it fails today, which is the finding. It counts toward your total like any changed behavior, and it is the one candidate a total of zero must never swallow: abstaining past a stated requirement reports the code back to the author as if it were the requirement.
|
|
251
226
|
|
|
252
|
-
|
|
227
|
+
Never reduce the total below the number of changed behaviors you found. State your Budget Plan in the canonical format${budgetPlanTiming} — \`Budget Plan: <total> total (<generate> generate + <additional> additional), <ui_pct>% UI/E2E\` — and use those numbers throughout the rest of the prompt.`;
|
|
253
228
|
}
|
|
254
229
|
/**
|
|
255
230
|
* Builds the PR scope assessment section.
|
|
@@ -257,8 +232,8 @@ If nothing currently covers the changed surface, there is no maintenance to do
|
|
|
257
232
|
* When `precomputedUIPct` is provided (0 = backend-only, 100 = UI-only) the server has
|
|
258
233
|
* already determined the split unambiguously, so Steps A–C are skipped. Backend-only
|
|
259
234
|
* (0) renders a single Budget Plan line; UI-only (100) renders that line plus the
|
|
260
|
-
*
|
|
261
|
-
*
|
|
235
|
+
* changed-behavior budget rule (SKYR-4292), because a frontend GENERATE list is
|
|
236
|
+
* placeholders the agent fills from the diff and the total must track what changed.
|
|
262
237
|
*
|
|
263
238
|
* For mixed PRs (`precomputedUIPct` is undefined, `hasFrontendChanges` is true) skip
|
|
264
239
|
* Steps A–C but keep Step D so the LLM can apply judgment to determine the UI%.
|
|
@@ -293,40 +268,47 @@ Budget Plan: 0 total — no new, modified, or removed endpoints were classified
|
|
|
293
268
|
|
|
294
269
|
With a 0-total Budget Plan: generate zero tests, recommend zero tests, and follow the zero-test report path. Do NOT draft baseline or generic tests for unchanged endpoints to fill a budget — an empty diff surface is a valid, expected outcome.
|
|
295
270
|
|
|
296
|
-
**Exception — claim the ceiling only with evidence:** if your code review of the changed files shows an observable API behavior change the classifier missed (e.g. a DTO/serializer/service change that alters a response shape, a shared library/default-value or business-rule constant change that alters the behavior of an existing, unchanged endpoint (e.g. a default schedule, threshold, or config constant imported by a route handler elsewhere in the codebase), a deployment/config change that newly exposes or removes endpoints, or a schema-defined API contract change — a CRD type/kubebuilder validation marker, GraphQL schema, or gRPC proto edit that adds, removes, or re-validates what the server accepts or returns), raise your Budget Plan to cover exactly those affected endpoints, up to ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional), 0% UI/E2E. Note: repositories whose entire API surface is schema-defined (e.g. a Kubernetes operator serving CRDs through the kube-apiserver) ALWAYS classify zero endpoints — for these, a schema change in the diff IS the endpoint change; evaluate this exception against the schema files instead of concluding there is nothing to test. Similarly, a changed file with zero classified endpoints is not by itself evidence of "no testable surface" — trace what imports the changed export (grep for its name) to check whether it feeds an existing endpoint's behavior before concluding the diff has no test value. Every test must name the changed file that justifies it. State your raised plan now in the canonical format — \`Budget Plan: <total> total (<generate> generate + <additional> additional), 0% UI/E2E\` — and use those exact numbers throughout the rest of the prompt; the raised generate count is your committed generate count.`;
|
|
271
|
+
**Exception — claim the ceiling only with evidence:** if your code review of the changed files shows an observable API behavior change the classifier missed (e.g. a DTO/serializer/service change that alters a response shape, a shared library/default-value or business-rule constant change that alters the behavior of an existing, unchanged endpoint (e.g. a default schedule, threshold, or config constant imported by a route handler elsewhere in the codebase), a deployment/config change that newly exposes or removes endpoints, or a schema-defined API contract change — a CRD type/kubebuilder validation marker, GraphQL schema, or gRPC proto edit that adds, removes, or re-validates what the server accepts or returns), raise your Budget Plan to cover exactly those affected endpoints, up to ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional), 0% UI/E2E. Note: repositories whose entire API surface is schema-defined (e.g. a Kubernetes operator serving CRDs through the kube-apiserver) ALWAYS classify zero endpoints — for these, a schema change in the diff IS the endpoint change; evaluate this exception against the schema files instead of concluding there is nothing to test. Similarly, a changed file with zero classified endpoints is not by itself evidence of "no testable surface" — trace what imports the changed export (grep for its name) to check whether it feeds an existing endpoint's behavior before concluding the diff has no test value. **A requirement conflict is also grounds to raise the plan:** if the PR title/description (or a requirements file it references) states a requirement the diff did not implement, claim at least one slot for a \`requirement_conflict\` test asserting that requirement — it fails on the current code, which is the finding. Every test must name the changed file that justifies it. State your raised plan now in the canonical format — \`Budget Plan: <total> total (<generate> generate + <additional> additional), 0% UI/E2E\` — and use those exact numbers throughout the rest of the prompt; the raised generate count is your committed generate count.`;
|
|
297
272
|
}
|
|
298
273
|
// Unambiguous backend-only or UI-only: no LLM counting needed. Backend-only emits just
|
|
299
|
-
// the Budget Plan line; UI-only appends the
|
|
274
|
+
// the Budget Plan line; UI-only appends the changed-behavior budget rule (see above).
|
|
300
275
|
if (precomputedUIPct !== undefined) {
|
|
301
276
|
const uiSuffix = precomputedUIPct > 0 ? `, ${precomputedUIPct}% UI/E2E` : "";
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
|
|
277
|
+
// Backend-only (precomputedUIPct === 0): the server pre-ranked the GENERATE list
|
|
278
|
+
// from classified endpoints, so the numbers are fixed. UI-only: the list is
|
|
279
|
+
// placeholders the agent fills from the diff, so the line is a ceiling and the
|
|
280
|
+
// changed-behavior rule sets the Budget Plan — a fixed "Budget Plan" plus "use
|
|
281
|
+
// these exact numbers" here would contradict that rule (Copilot on mcp#835).
|
|
282
|
+
if (precomputedUIPct === 0) {
|
|
283
|
+
return `### PR Scope Assessment
|
|
284
|
+
Budget Plan: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)
|
|
285
|
+
|
|
286
|
+
Use these exact numbers throughout the rest of the prompt.`;
|
|
287
|
+
}
|
|
306
288
|
return `### PR Scope Assessment
|
|
307
|
-
Budget
|
|
289
|
+
Budget ceiling: ${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)${uiSuffix}
|
|
308
290
|
|
|
309
|
-
${
|
|
291
|
+
${changedBehaviorSection(effectiveGenerate)}`;
|
|
310
292
|
}
|
|
311
293
|
// Mixed PR: server can pre-compute the total but not the UI/E2E split — keep Step D.
|
|
312
294
|
if (hasFrontendChanges) {
|
|
313
295
|
return `### PR Scope Assessment — determine UI% before planning recommendations
|
|
314
296
|
|
|
315
|
-
Budget
|
|
297
|
+
Budget ceiling: **${maxTotal} total (${effectiveGenerate} generate + ${additional} additional)**
|
|
316
298
|
|
|
317
|
-
${
|
|
299
|
+
${changedBehaviorSection(effectiveGenerate, ", after Step D below sets your UI%")}
|
|
318
300
|
|
|
319
|
-
**Step D — Determine UI vs backend split for
|
|
301
|
+
**Step D — Determine UI vs backend split for your Budget Plan:**
|
|
320
302
|
- Non-UI slots are backend tests; start from file-count ratio for UI%, then apply judgment:
|
|
321
|
-
- Cosmetic CSS/style changes alongside real changes inflate the frontend file count without adding test value → reduce UI%
|
|
303
|
+
- Cosmetic CSS/style changes alongside real changes inflate the frontend file count without adding test value → reduce UI%
|
|
322
304
|
- Frontend logic bugs (state management, calculation errors, form validation) in the diff → increase UI% even if few frontend files
|
|
323
305
|
- Frontend component calls a changed backend API → an E2E test covers both sides → count toward UI%
|
|
324
306
|
- Frontend files only in \`__tests__/\` or \`.stories.\` → exclude from the ratio
|
|
325
307
|
|
|
326
|
-
**Append your UI% now**
|
|
327
|
-
\`Budget Plan:
|
|
308
|
+
**Append your UI% now** and state your Budget Plan — total and split per the changed-behavior rule above, within the ceiling:
|
|
309
|
+
\`Budget Plan: <total> total (<generate> generate + <additional> additional), <ui_pct>% UI/E2E\`
|
|
328
310
|
|
|
329
|
-
Use
|
|
311
|
+
Use those numbers throughout the rest of the prompt.`;
|
|
330
312
|
}
|
|
331
313
|
// Fallback (no diff context — full_repo else-branch or test): full four-step assessment.
|
|
332
314
|
const minTotal = Math.min(effectiveGenerate + 1, maxTotal);
|
|
@@ -350,9 +332,9 @@ Read the Changed Files list and endpoint changes above, then work through the fo
|
|
|
350
332
|
|
|
351
333
|
**Step A — Classify changed files:**
|
|
352
334
|
Count each type from the diff context (ignore generated test files, lock files, and build artifacts):
|
|
353
|
-
- **Frontend files**: .vue / .svelte anywhere (always UI
|
|
335
|
+
- **Frontend files**: .vue / .svelte / .css / .scss / .less anywhere (always UI). .dart anywhere (always UI in a Flutter project — repo has a \`pubspec.yaml\` with \`sdk: flutter\`). .tsx / .jsx anywhere except in api/, routes/, routers/, controllers/, handlers/, endpoints/, or server/ directories. .ts / .js / .html / .erb / .jsp / .asp / .jinja2 / .twig only when in a frontend directory (components/, pages/, views/, layouts/, app/, frontend/, client/, styles/, templates/).
|
|
354
336
|
- **Backend files**: route handlers, controllers, services, models, API modules, middleware, config with business logic
|
|
355
|
-
- **Non-application** (exclude from test value):
|
|
337
|
+
- **Non-application** (exclude from test value): README and docs, lock files, CI config with no logic
|
|
356
338
|
|
|
357
339
|
**Step B — Assess semantic complexity (quality over quantity):**
|
|
358
340
|
Weigh changes by their test value, not file count:
|
|
@@ -360,12 +342,12 @@ Weigh changes by their test value, not file count:
|
|
|
360
342
|
- Modified endpoint with formula / business logic change → HIGH: edge cases matter (contributes ~1–2)
|
|
361
343
|
- Auth middleware change → CRITICAL: flag for extra security tests regardless of file count
|
|
362
344
|
- Frontend state / validation / calculation logic → HIGH for UI tests even if zero backend endpoints changed
|
|
363
|
-
-
|
|
345
|
+
- Changed style values, labels or copy → each rendered difference is a changed behavior: assert the new value (computed style, class or text) — no change is too trivial to test
|
|
364
346
|
|
|
365
347
|
**Step C — Determine total recommendation count (${minTotal}–${maxTotal}):**
|
|
366
348
|
Start from the baseline formula: *${baselineFormula}*, then adjust:
|
|
367
349
|
- **Scale up** for: critical auth/data-integrity changes (+2), complex multi-step business workflows (+1 each), new endpoints with non-trivial validation (+1 each beyond the formula)
|
|
368
|
-
- **Scale down** for:
|
|
350
|
+
- **Scale down** for: already well-tested paths confirmed by existing test list, trivial CRUD with no validation
|
|
369
351
|
- **Hard cap**: ${maxTotal}
|
|
370
352
|
|
|
371
353
|
${stepD}
|
|
@@ -163,6 +163,8 @@ Output should be concise and immediately actionable.`
|
|
|
163
163
|
|
|
164
164
|
For UI recommendations you *do* emit, ground the \`reasoning\` field in elements you have actually observed via \`browser_blueprint\` calls earlier in this session. If a recommendation's target element is not in any blueprint you have observed, either rephrase the recommendation around an element that IS observed, or describe the test target in higher-level terms. Do not invent element names from the PR description, source diff, or component name. If you have not captured any blueprints yet (e.g. backend-only PR, or pre-flight skipped), UI recommendations fall back to source-grounded prose drawn from the diff alone — that is a legitimate outcome, not a reason to invent.
|
|
165
165
|
|
|
166
|
+
**That rule governs element *identification*, not what you may assert about an identified element.** It does NOT forbid a description-derived **negative** assertion. When the PR description says an element should be gone and a captured page still renders it, the name is blueprint-grounded — you read it off the live page, not off the description — so a failing "is not present" assertion on that page is the correct output, and the requirement conflict is what it documents. Ground \`targetElements\` against the captured page where the element DOES render (the same grounding the negative-test guidance elsewhere in this prompt requires), and cite the test as a \`requirement_conflict\`. What stays forbidden is naming an element **no** blueprint contains because the description mentioned it.
|
|
167
|
+
|
|
166
168
|
Write UI recommendation \`reasoning\` fields in **natural prose** that names elements as a human would describe them (e.g. "the Notifications heading", "the disabled Mark all as read button"). **Do NOT mention "blueprint", "captured blueprint", "DOM analyzer", or any other internal MCP terminology in the reasoning text.** The reader of the report is a developer reviewing test recommendations on a PR; they don't know what a blueprint is and shouldn't have to. Phrases like "the captured blueprint shows X" or "visible from the captured blueprint" leak builder internals — instead just describe what the test verifies in plain product terms ("the disabled mark-all-read button in the empty state"). Likewise do NOT use internal-identifier syntax like \`role=button, accessibleName='X', logicalName=...\`.
|
|
167
169
|
</ui_recommendation_authoring_rules>
|
|
168
170
|
`;
|
|
@@ -264,10 +266,21 @@ Static hints available: ${routeDiscovery.staticHints.length}
|
|
|
264
266
|
${routeDiscovery.diffFilePath ? `Diff file: ${routeDiscovery.diffFilePath}` : ""}
|
|
265
267
|
`.trim()
|
|
266
268
|
: "";
|
|
269
|
+
// ── Source Priority ──
|
|
270
|
+
// The ladder ranks sources that describe what the system DOES. The PR
|
|
271
|
+
// title/description states what the change is SUPPOSED to do, so it is not a
|
|
272
|
+
// rung — resolving a description-vs-code disagreement by rung produced tests
|
|
273
|
+
// that asserted the code and buried the contradiction in prose (SKYR-4291).
|
|
274
|
+
// Only the ladder-placement rule lives here; the Execution Plan's Code Review
|
|
275
|
+
// step is the canonical home for how a conflict is handled, so this document
|
|
276
|
+
// states it once. Diff-scoped: repo mode has no PR to compare against.
|
|
277
|
+
const requirementConflictRules = isDiffScope ? `
|
|
278
|
+
That ladder ranks sources that describe what the system **does**. The **PR title and description** — plus any requirements file they reference — state what this change is **supposed to do**. They are the *intent* side; the ladder is the *implementation* side. Never settle a disagreement between the two by picking the higher rung: a requirement the implemented behavior contradicts is a \`requirement_conflict\`, NOT a case of "the description loses". The Code Review step of the Execution Plan below specifies how to detect, check, and handle one — follow it there.
|
|
279
|
+
` : "";
|
|
267
280
|
const sourcePriority = `
|
|
268
281
|
## Source Priority
|
|
269
282
|
When information conflicts, prefer: **Traces** (actual behavior) > **Source code read by the LLM** (implemented behavior) > **OpenAPI spec/docs** (documented behavior) > **Static parser hints** (best-effort, may be incomplete or framework-blind).
|
|
270
|
-
`;
|
|
283
|
+
${requirementConflictRules}`;
|
|
271
284
|
// Compact fingerprint of tests already covering endpoints in this repo (Skyramp + external).
|
|
272
285
|
// Re-derived fresh each run from test files on disk — no separate persistence needed.
|
|
273
286
|
const testLocations = analysis.existingTests?.testLocations ?? {};
|
|
@@ -3,6 +3,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
3
3
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
4
4
|
import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
|
|
5
5
|
import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
6
|
+
import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
|
|
6
7
|
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
7
8
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
8
9
|
import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, } from "../../utils/featureFlags.js";
|
|
@@ -119,6 +120,13 @@ export function getTestbotPrompt(opts) {
|
|
|
119
120
|
// en/argless render disarms a language captured earlier in a long-lived
|
|
120
121
|
// server process instead of falsely rejecting an English report.
|
|
121
122
|
setReportLanguage(language && language !== "en" ? language : undefined);
|
|
123
|
+
// SKYR-4250: same capture point, same last-render-wins rule, for the
|
|
124
|
+
// plan-only lane. skyramp_submit_report's SKYR-3883 guard
|
|
125
|
+
// (findUnchangedFileClaims) verifies newTestsCreated against the working
|
|
126
|
+
// tree; a plan-only run writes no files, so without this every declaration
|
|
127
|
+
// is unchanged-file-flagged and the agent is forced to demote it to
|
|
128
|
+
// additionalRecommendations.
|
|
129
|
+
setPlanOnlyMode(planOnly);
|
|
122
130
|
let reportLanguageBlock = "";
|
|
123
131
|
if (language && language !== "en") {
|
|
124
132
|
const reportLanguageName = new Intl.DisplayNames(["en"], { type: "language" }).of(language) ??
|
|
@@ -222,6 +230,9 @@ ${maintenanceBeforeExecStep}
|
|
|
222
230
|
- Incorrect arithmetic in business logic (discount calculations, price aggregation)
|
|
223
231
|
Log each finding in \`issuesFound\` with a \`severity\` (critical/high/medium/low). These bugs should inform your test design in Task 2.
|
|
224
232
|
|
|
233
|
+
**Requirement check — does the code do what the PR says it does?** Read the \`<TITLE>\` and \`<DESCRIPTION>\` above as a statement of *intended* behavior and check the diff against it, requirement by requirement. The description is not outranked by the code: when the two disagree, that is a \`requirement_conflict\`, not a case of "the description loses". Detection, the requirements-file rules, the four false-red pre-checks and the promotion ordering are specified in the Code Review step of the Execution Plan \`skyramp_analyze_changes\` returned — follow them there rather than improvising; read any requirements file the description names from \`${repositoryPath}\`.
|
|
234
|
+
- **On a mismatch, do both.** (a) Add an \`issuesFound\` entry at severity \`high\` or \`critical\` — never \`medium\` or below — quoting the requirement and naming the file and line that contradicts it. (b) Carry it into Task 2 as a \`requirement_conflict\` candidate: a test asserting **what the description requires**, which therefore FAILS on the current code. Where no such test is recordable, record the Execution Plan's VERIFY-style entry in \`additionalRecommendations\` instead — never resolve the disagreement by asserting what the code does.
|
|
235
|
+
|
|
225
236
|
4. **Blueprint Citation Invariant** (UI test recommendations only). Every named UI element in your recommendation must correspond to an element actually captured in one of the blueprints you captured from the UI Blueprint Capture section of \`skyramp_analyze_changes\`. For **every** UI recommendation — both \`newTestsCreated[]\` entries (tests you generated) AND \`additionalRecommendations[]\` entries (deferred tests) with \`testType: "ui"\` — populate **four** fields: \`description\`, \`reasoning\`, \`targetElements\`, and \`pageContext\`. \`description\` and \`reasoning\` carry different roles — fill BOTH (see Field 4 below). The structured fields (\`targetElements\`, \`pageContext\`) are required on UI recs in BOTH arrays; non-UI recs (contract / integration / e2e / batch-scenario) MUST omit them.
|
|
226
237
|
|
|
227
238
|
**Field 1 — \`targetElements\`** (the elements the test targets — array, length 1+):
|
|
@@ -431,7 +442,7 @@ ${maintenanceBeforeExecStep}
|
|
|
431
442
|
**Self-check before calling \`skyramp_submit_report\`:** count your final GENERATE list, then confirm \`newTestsCreated\` has exactly that many entries — one per GENERATE item. A GENERATE item always goes in \`newTestsCreated\` (even though no file was generated in this run — the declaration IS the deliverable); \`additionalRecommendations\` holds ONLY candidates that are NOT in the GENERATE list. Reporting a GENERATE item as a recommendation instead of a declaration is the single most common plan-only reporting error — recheck the placement before submitting.`;
|
|
432
443
|
}
|
|
433
444
|
else {
|
|
434
|
-
task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if
|
|
445
|
+
task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if every remaining candidate failed after retry — and, on a PR with backend changes, the fallback single-contract test also failed. A frontend-only PR has no backend fallback: its exhausted candidates go to \`additionalRecommendations\` with the failure reason.`;
|
|
435
446
|
}
|
|
436
447
|
// Task 2 branches wholesale in plan-only eval runs (SKYR-3879 plan-only
|
|
437
448
|
// lane): the standard task mandates generation and execution, which a
|
|
@@ -443,9 +454,9 @@ ${maintenanceBeforeExecStep}
|
|
|
443
454
|
|
|
444
455
|
This is a plan-only evaluation run: the application under test is NOT running, and this run evaluates test SELECTION only. Nothing is generated or executed in this task.
|
|
445
456
|
|
|
446
|
-
- Draft your complete candidate list exactly as the Execution Plan directs — every
|
|
457
|
+
- Draft your complete candidate list exactly as the Execution Plan directs — every test you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
|
|
447
458
|
- If a tool named \`skyramp_register_test_plan\` is available, call it with the full candidate union — include a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for every candidate that probes changed logic — and treat its returned GENERATE list as your final selection. If that tool is not available, commit to your Budget Plan's GENERATE selection (at most ${maxGenerate}).
|
|
448
|
-
-
|
|
459
|
+
- Include UI and E2E candidates on the same footing as the API types. The app is not running, so you cannot capture a blueprint — plan them ungrounded rather than drop them. For every UI entry, set \`targetElements: null\`, omit \`pageContext\`, and prefix BOTH \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Describe the page or feature the test would exercise; do not name an element you have not seen. This lane is the one case where you fall back without attempting a capture first, so do NOT log the fallback in \`issuesFound\` — capture was never applicable here, and nothing failed.
|
|
449
460
|
- Take no other actions in this task: no test generation tools, no browser traces or blueprint captures, no test files written, no test executions. Proceed directly to ${taskRef(TASK_SUBMIT)}.`;
|
|
450
461
|
}
|
|
451
462
|
else {
|
|
@@ -453,8 +464,8 @@ This is a plan-only evaluation run: the application under test is NOT running, a
|
|
|
453
464
|
|
|
454
465
|
${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
|
|
455
466
|
|
|
456
|
-
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
457
|
-
- **Do not fabricate tests outside the GENERATE list
|
|
467
|
+
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget. **Every \`requirement_conflict\` candidate from Task 1's requirement check belongs in this registration** — register it with category \`requirement_conflict\` so it ranks in the top tier and takes the first promoted slot rather than losing one to a bug-catching candidate; omitting it from the candidate list is how a stated requirement silently loses its test.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
468
|
+
- **Do not fabricate tests outside the GENERATE list returned by \`skyramp_register_test_plan\`.** A change to an EXISTING covered endpoint or component is maintenance only where an existing test asserts the changed value: handle that in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test. A changed value, option, or behavior that no existing test asserts is not maintenance — it is a candidate you register (GENERATE or ADDITIONAL), however small the change. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
|
|
458
469
|
- Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
|
|
459
470
|
- Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
|
|
460
471
|
- If the existing test source is \`[external]\`, skip the resource entirely — the external test already provides coverage. Do NOT UPDATE, REGENERATE, or DELETE external tests.
|
|
@@ -479,6 +490,9 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
479
490
|
- **Always generate a test for critical bugs, even if it will fail.** When a GENERATE-tagged item targets a page or endpoint with a known bug, do NOT skip it because you expect the test to fail — a failing test that documents a bug is more valuable than a text-only description. This applies within the existing GENERATE budget; do not add extra tests beyond the plan.
|
|
480
491
|
- For UI rendering bugs: navigate to the broken page and add a \`browser_assert\` that verifies the page rendered its expected content (e.g. assert the page heading is visible). The assertion will fail on the broken page, which is the correct outcome — it documents the bug as a failing test.
|
|
481
492
|
- The assertion MUST target the broken page itself, not a different page that works. If \`/orders/{id}/edit\` crashes, assert on \`/orders/{id}/edit\` (e.g. "Edit Order" heading visible), NOT on \`/orders\`.
|
|
493
|
+
- **The same rule covers a \`requirement_conflict\` from Task 1's requirement check — and it is promoted first.** Generate the test that asserts what the PR description requires, not what the code implements. It fails today; that is the deliverable, and it turns green when the code is fixed to match the description. Do NOT re-aim it at the implemented behavior to get a passing run, and do NOT drop it because a bug-catching test already occupies a slot — requirement conflicts take the first slots of the promotion bound, so the bug-catching tests yield to it, not the other way round.
|
|
494
|
+
- For a "should no longer be present" requirement: navigate to the page the description names, confirm from the capture that the element still renders, then assert its **absence**. The element name comes from the live capture, so the assertion is grounded; the failure is the conflict. Assert on the page the requirement is about — not a sibling page where the element is genuinely gone.
|
|
495
|
+
- For a "should now do X" requirement the code does not do: assert X. Same rule — the stated requirement is the expectation.
|
|
482
496
|
- **Critical categories first**: At least 1 of the generated tests MUST be from a critical category (security_boundary, business_rule, data_integrity, breaking_change) if such candidates exist in the GENERATE set.
|
|
483
497
|
- **Parallel generation (IMPORTANT for speed)**: Generate **independent tests in parallel** whenever possible. Tests targeting different endpoints with different output files can be generated concurrently in the same tool call batch. Specifically:
|
|
484
498
|
- Call \`skyramp_batch_scenario_test_generation\` for ALL integration scenarios AND \`skyramp_contract_test_generation\` for ALL contract tests **in the same tool call batch**.
|
|
@@ -565,7 +579,6 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
565
579
|
- **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
|
|
566
580
|
- **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
|
|
567
581
|
If \`browser_navigate\` fails (app not running / connection refused), apply skip condition (a) above: move the intended test to \`additionalRecommendations\` with the failure reason AND record the outage in \`issuesFound\`.
|
|
568
|
-
Record at most 2-3 UI traces per run to stay within tool call budget. Quality over quantity: 1 great test is better than 3 mediocre ones — do not pad to reach the count.
|
|
569
582
|
**Strategic assertions** — key checkpoints only, 3 to 5 per test:
|
|
570
583
|
- **After the main action completes**: verify the outcome is visible (new item appears, form saves, confirmation shows)
|
|
571
584
|
- **State transitions**: verify counts, totals, or status fields update correctly
|
|
@@ -631,6 +644,8 @@ If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated t
|
|
|
631
644
|
1. Read the error output to diagnose the root cause (4xx on prereq step, assertion mismatch, floating-point precision, 500 from app bug, timeout, etc.).
|
|
632
645
|
2. **Expected failure check (no retry):** If the failure is an assertion error or HTTP error that matches the issue identified in the code analysis (e.g. the test was generated specifically to document a broken endpoint, a UI rendering bug, or a missing validation), then this is the **intended outcome** — the test is correctly catching the real bug. Report it immediately as \`status: "Fail"\` and move on. Do NOT retry.
|
|
633
646
|
|
|
647
|
+
**A \`requirement_conflict\` test failing is likewise the intended outcome.** It asserts what the PR description requires and the code does not do; a red result is the finding, so report \`status: "Fail"\` with the conflict named in \`issuesFound\` and move on. Do NOT retry it, do NOT relax the assertion toward the implemented behavior, and do NOT delete the test — either of those hides the disagreement the run exists to surface.
|
|
648
|
+
|
|
634
649
|
This path also covers an assertion failure that application behavior outside this PR's diff explains — for example child records that survive the deletion of their parent, state inherited when an ID is recycled or reused, or a value that ignores a status the test set. Before you keep such a test red, confirm the cause in the source: read the handler, model, or query that should have done the work, and find the specific operation that is missing or wrong. If you find it, report the test as \`status: "Fail"\` and add an \`issuesFound\` entry for it. Do NOT retry.
|
|
635
650
|
|
|
636
651
|
If you cannot point at the missing or wrong line in application code, the app is not at fault — shared or seeded data, parallel test workers, or setup the test itself never did explain the collision. Treat that as an infrastructure failure: fix it and retry once as in step 3.
|
|
@@ -683,7 +698,7 @@ If you skipped here due to non-application changes (per Task 1), submit with emp
|
|
|
683
698
|
|
|
684
699
|
**If you generated zero new tests because the PR has no testable behavioral surface:**
|
|
685
700
|
This applies when the diff contains ONLY changes with no observable API or UI behavior change. Examples:
|
|
686
|
-
-
|
|
701
|
+
- Docs/comments only: JSDoc updates, comment-only changes
|
|
687
702
|
- Dependency-only: version bumps with no API surface change
|
|
688
703
|
- Dead code / unintegrated utility or component: a new helper function, utility, or UI component added to the codebase but not imported, mounted, or rendered anywhere — use this classification only after confirming the new symbol does not appear as an import or render call in any other source file; do NOT classify as dead code based solely on the diff. For UI components specifically: an unintegrated component has no DOM node in the running app and cannot be browser-tested regardless of how complex its logic is
|
|
689
704
|
- Config-only: linter rules, build config, environment variable additions with no runtime behavior change
|
|
@@ -694,6 +709,8 @@ In these cases:
|
|
|
694
709
|
- \`businessCaseAnalysis\` must be a one-sentence summary of what the PR actually does (do NOT leave it blank)
|
|
695
710
|
- \`additionalRecommendations\` must be \`[]\` — do NOT recommend tests for a no-surface PR
|
|
696
711
|
|
|
712
|
+
**This zero-test path does NOT apply when the requirement check found a \`requirement_conflict\`.** A requirement the PR description states and the diff did not implement is testable surface: report the conflict in \`issuesFound\` at severity \`high\` or above, and carry either the failing \`requirement_conflict\` test in \`newTestsCreated\` or its VERIFY-style entry in \`additionalRecommendations\`. Abstaining there hands the author back their own code as though it were the requirement.
|
|
713
|
+
|
|
697
714
|
${task3CountRule}
|
|
698
715
|
|
|
699
716
|
${reportLanguageBlock}Call \`skyramp_submit_report\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking, and the report is written beside it. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
@@ -703,7 +720,7 @@ ${hasRelatedRepos
|
|
|
703
720
|
- **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
|
|
704
721
|
`
|
|
705
722
|
: ""}
|
|
706
|
-
- **additionalRecommendations**: AT MOST ${maxRecommendations -
|
|
723
|
+
- **additionalRecommendations**: AT MOST ${maxRecommendations} minus the number of budget-counting tests in \`newTestsCreated\` — an unused generate slot becomes a recommendation slot. The one optional discovered-scenario test does not count against the budget, so it does not reduce this allowance either.
|
|
707
724
|
- For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
|
|
708
725
|
|
|
709
726
|
${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false, modularizeViaGenerationResult: UI_UTILS_REUSE })}`;
|
|
@@ -27,16 +27,50 @@ function floorBalancedPick(items, count) {
|
|
|
27
27
|
typesOrder.push(t);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
-
|
|
30
|
+
// Two passes, because "protected" is broader than "is a caught bug".
|
|
31
|
+
//
|
|
32
|
+
// A `bug_caught` candidate may take a type-floor slot: a caught bug is the
|
|
33
|
+
// strongest thing a run produces, and it outranks type spread. Every OTHER
|
|
34
|
+
// protected candidate — anything else CRITICAL, and attack-surface
|
|
35
|
+
// security_boundary items — still yields to the floor, which is what stops
|
|
36
|
+
// the SKYR-3879 skew where three CRITICAL integration candidates took all
|
|
37
|
+
// three slots and no contract test was generated.
|
|
38
|
+
//
|
|
39
|
+
// The second pass replaces `maxProtected = count - (typesOrder.length - 1)`,
|
|
40
|
+
// a static cap computed from the whole pool. It reserved a slot for every
|
|
41
|
+
// other type whether or not the picks already covered it, so at the default
|
|
42
|
+
// budget of 3 with three types present it evaluated to 1. Counting the types
|
|
43
|
+
// still uncovered keeps the same floor and drops the arithmetic that made a
|
|
44
|
+
// second finding unreachable.
|
|
45
|
+
// The same bound the prompt states (`maxGen - 1`, and the whole budget only
|
|
46
|
+
// at a budget of 1). Without it the first pass took every caught bug up to
|
|
47
|
+
// `count`, which put the SKYR-3879 skew back: at the default budget of 3, two
|
|
48
|
+
// same-type bugs left no slot for the type floor, and a measured 9-20% of
|
|
49
|
+
// backend-only pools lost a test type. It also evicted attack-surface
|
|
50
|
+
// security_boundary items the prompt in this same branch says to preserve
|
|
51
|
+
// while any non-bug slot remains.
|
|
52
|
+
const bugSlots = count <= 1 ? count : count - 1;
|
|
31
53
|
const selected = [];
|
|
54
|
+
const deferred = [];
|
|
32
55
|
const pool = [];
|
|
33
|
-
let
|
|
56
|
+
let bugsTaken = 0;
|
|
34
57
|
for (const it of items) {
|
|
58
|
+
if (it.scenario?.category === "bug_caught" && bugsTaken < bugSlots && selected.length < count) {
|
|
59
|
+
selected.push(it);
|
|
60
|
+
bugsTaken++;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
deferred.push(it);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const takenTypes = new Set(selected.map(typeOf));
|
|
67
|
+
for (const it of deferred) {
|
|
68
|
+
const stillUncovered = typesOrder.filter((t) => !takenTypes.has(t) && t !== typeOf(it)).length;
|
|
35
69
|
if (isProtectedCandidate(it.priority, it.scenario) &&
|
|
36
|
-
|
|
37
|
-
selected.length
|
|
70
|
+
selected.length < count &&
|
|
71
|
+
count - (selected.length + 1) >= stillUncovered) {
|
|
38
72
|
selected.push(it);
|
|
39
|
-
|
|
73
|
+
takenTypes.add(typeOf(it));
|
|
40
74
|
}
|
|
41
75
|
else {
|
|
42
76
|
pool.push(it);
|
|
@@ -1,13 +1,26 @@
|
|
|
1
1
|
import { Candidate, BudgetContext, SelectionResult } from "../../types/Recommendation.js";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* What each PR class GUARANTEES in GENERATE (SKYR-4275). These are floors, not
|
|
4
|
+
* quotas: the slots above them are open to either kind, decided by `pick`.
|
|
5
|
+
* An open slot goes to whichever candidate `pick` chooses on rank, so a backend
|
|
6
|
+
* candidate takes one when it outranks the UI candidates, not merely because it
|
|
7
|
+
* exists. Measured against main on random mixed pools at `maxGenerate` 5 with no
|
|
8
|
+
* caught bugs: 35% get fewer backend tests than main did, average 3.53 to 3.24.
|
|
9
|
+
*
|
|
10
|
+
* Backend-only floors nothing on purpose: of 86 backend-only diffs in the eval
|
|
11
|
+
* corpus, ZERO generated a UI test, so a floor there would guarantee what
|
|
12
|
+
* already happens every time.
|
|
13
|
+
*
|
|
14
|
+
* At one slot there is no room for a floor plus an open slot, so each class
|
|
15
|
+
* keeps that slot for its own kind.
|
|
16
|
+
*
|
|
17
|
+
* The floors always sum to at most `maxGenerate`, which is what lets
|
|
18
|
+
* `applyFloors` satisfy both without one starving the other.
|
|
7
19
|
*/
|
|
8
|
-
export declare function
|
|
9
|
-
|
|
10
|
-
|
|
20
|
+
export declare function generateFloors(ctx: BudgetContext): {
|
|
21
|
+
ui: number;
|
|
22
|
+
backend: number;
|
|
23
|
+
};
|
|
11
24
|
/**
|
|
12
25
|
* Shared budgeting pipeline. All Budgeters run the same external-dedup,
|
|
13
26
|
* attack-surface prioritization, and ADDITIONAL set-difference; they differ
|
|
@@ -16,4 +29,4 @@ export declare function reservedUISlots(ctx: BudgetContext): number;
|
|
|
16
29
|
* With `pick = roundRobinByType` this reproduces the pre-refactor selection in
|
|
17
30
|
* diffExecutionPlan.ts exactly.
|
|
18
31
|
*/
|
|
19
|
-
export declare function runBudget(ranked: Candidate[],
|
|
32
|
+
export declare function runBudget(ranked: Candidate[], rawCtx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;
|