@skyramp/mcp 0.3.7 → 0.3.9-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +1 -0
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +22 -1
- package/build/prompts/test-maintenance/actionsInstructions.d.ts +4 -0
- package/build/prompts/test-maintenance/actionsInstructions.js +14 -2
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +4 -1
- package/build/prompts/test-recommendation/diffExecutionPlan.js +41 -9
- package/build/prompts/test-recommendation/fullRepoCatalog.js +3 -2
- package/build/prompts/test-recommendation/recommendationSections.js +3 -3
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +2 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +58 -76
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +14 -1
- package/build/prompts/testbot/testbot-prompts.js +35 -15
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +39 -5
- package/build/recommendation/budgeters/shared.d.ts +21 -8
- package/build/recommendation/budgeters/shared.js +134 -56
- package/build/recommendation/planRanker.d.ts +5 -3
- package/build/recommendation/planRanker.js +1 -1
- package/build/services/containerEnv.d.ts +1 -1
- package/build/services/containerEnv.js +12 -0
- package/build/tools/executeSkyrampTestTool.d.ts +80 -0
- package/build/tools/executeSkyrampTestTool.js +246 -19
- package/build/tools/submitReportTool.d.ts +5 -5
- package/build/tools/submitReportTool.js +11 -2
- package/build/tools/test-management/actionsTool.js +71 -3
- package/build/tools/test-management/analyzeChangesTool.d.ts +10 -0
- package/build/tools/test-management/analyzeChangesTool.js +73 -0
- package/build/tools/test-management/registerTestPlanTool.d.ts +3 -3
- package/build/tools/test-management/registerTestPlanTool.js +29 -8
- package/build/types/RepositoryAnalysis.d.ts +10 -10
- package/build/types/TestAnalysis.d.ts +12 -0
- package/build/types/TestExecution.d.ts +4 -0
- package/build/types/TestRecommendation.d.ts +11 -1
- package/build/types/TestRecommendation.js +34 -0
- package/build/utils/AnalysisStateManager.d.ts +7 -0
- package/build/utils/assertion-verify/ui-lints.d.ts +0 -5
- package/build/utils/assertion-verify/ui-lints.js +32 -0
- package/build/utils/frontendSelectors.d.ts +33 -0
- package/build/utils/frontendSelectors.js +196 -5
- package/build/utils/pathSignatures.d.ts +4 -1
- package/build/utils/pathSignatures.js +14 -2
- package/build/utils/planOnlyMode.d.ts +33 -0
- package/build/utils/planOnlyMode.js +40 -0
- package/build/utils/rebaselineSnapshots.d.ts +24 -0
- package/build/utils/rebaselineSnapshots.js +65 -0
- package/build/utils/removedUiElements.d.ts +34 -0
- package/build/utils/removedUiElements.js +153 -0
- package/build/utils/reportVerification.d.ts +7 -6
- package/build/utils/reportVerification.js +68 -4
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +1 -1
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/assertHiddenTool.js +56 -0
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +2 -1
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +4 -1
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +160 -1
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderSignalProcessor.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder.js +5 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.-Id052Lr.js → index.B7KbSQcC.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderSignalProcessor.ts +7 -0
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder.ts +6 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +3 -3
|
@@ -163,6 +163,8 @@ Output should be concise and immediately actionable.`
|
|
|
163
163
|
|
|
164
164
|
For UI recommendations you *do* emit, ground the \`reasoning\` field in elements you have actually observed via \`browser_blueprint\` calls earlier in this session. If a recommendation's target element is not in any blueprint you have observed, either rephrase the recommendation around an element that IS observed, or describe the test target in higher-level terms. Do not invent element names from the PR description, source diff, or component name. If you have not captured any blueprints yet (e.g. backend-only PR, or pre-flight skipped), UI recommendations fall back to source-grounded prose drawn from the diff alone — that is a legitimate outcome, not a reason to invent.
|
|
165
165
|
|
|
166
|
+
**That rule governs element *identification*, not what you may assert about an identified element.** It does NOT forbid a description-derived **negative** assertion. When the PR description says an element should be gone and a captured page still renders it, the name is blueprint-grounded — you read it off the live page, not off the description — so a failing "is not present" assertion on that page is the correct output, and the requirement conflict is what it documents. Ground \`targetElements\` against the captured page where the element DOES render (the same grounding the negative-test guidance elsewhere in this prompt requires), and cite the test as a \`requirement_conflict\`. What stays forbidden is naming an element **no** blueprint contains because the description mentioned it.
|
|
167
|
+
|
|
166
168
|
Write UI recommendation \`reasoning\` fields in **natural prose** that names elements as a human would describe them (e.g. "the Notifications heading", "the disabled Mark all as read button"). **Do NOT mention "blueprint", "captured blueprint", "DOM analyzer", or any other internal MCP terminology in the reasoning text.** The reader of the report is a developer reviewing test recommendations on a PR; they don't know what a blueprint is and shouldn't have to. Phrases like "the captured blueprint shows X" or "visible from the captured blueprint" leak builder internals — instead just describe what the test verifies in plain product terms ("the disabled mark-all-read button in the empty state"). Likewise do NOT use internal-identifier syntax like \`role=button, accessibleName='X', logicalName=...\`.
|
|
167
169
|
</ui_recommendation_authoring_rules>
|
|
168
170
|
`;
|
|
@@ -264,10 +266,21 @@ Static hints available: ${routeDiscovery.staticHints.length}
|
|
|
264
266
|
${routeDiscovery.diffFilePath ? `Diff file: ${routeDiscovery.diffFilePath}` : ""}
|
|
265
267
|
`.trim()
|
|
266
268
|
: "";
|
|
269
|
+
// ── Source Priority ──
|
|
270
|
+
// The ladder ranks sources that describe what the system DOES. The PR
|
|
271
|
+
// title/description states what the change is SUPPOSED to do, so it is not a
|
|
272
|
+
// rung — resolving a description-vs-code disagreement by rung produced tests
|
|
273
|
+
// that asserted the code and buried the contradiction in prose (SKYR-4291).
|
|
274
|
+
// Only the ladder-placement rule lives here; the Execution Plan's Code Review
|
|
275
|
+
// step is the canonical home for how a conflict is handled, so this document
|
|
276
|
+
// states it once. Diff-scoped: repo mode has no PR to compare against.
|
|
277
|
+
const requirementConflictRules = isDiffScope ? `
|
|
278
|
+
That ladder ranks sources that describe what the system **does**. The **PR title and description** — plus any requirements file they reference — state what this change is **supposed to do**. They are the *intent* side; the ladder is the *implementation* side. Never settle a disagreement between the two by picking the higher rung: a requirement the implemented behavior contradicts is a \`requirement_conflict\`, NOT a case of "the description loses". The Code Review step of the Execution Plan below specifies how to detect, check, and handle one — follow it there.
|
|
279
|
+
` : "";
|
|
267
280
|
const sourcePriority = `
|
|
268
281
|
## Source Priority
|
|
269
282
|
When information conflicts, prefer: **Traces** (actual behavior) > **Source code read by the LLM** (implemented behavior) > **OpenAPI spec/docs** (documented behavior) > **Static parser hints** (best-effort, may be incomplete or framework-blind).
|
|
270
|
-
`;
|
|
283
|
+
${requirementConflictRules}`;
|
|
271
284
|
// Compact fingerprint of tests already covering endpoints in this repo (Skyramp + external).
|
|
272
285
|
// Re-derived fresh each run from test files on disk — no separate persistence needed.
|
|
273
286
|
const testLocations = analysis.existingTests?.testLocations ?? {};
|
|
@@ -3,6 +3,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
3
3
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
4
4
|
import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
|
|
5
5
|
import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
6
|
+
import { setPlanOnlyMode } from "../../utils/planOnlyMode.js";
|
|
6
7
|
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
7
8
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
8
9
|
import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, } from "../../utils/featureFlags.js";
|
|
@@ -119,6 +120,13 @@ export function getTestbotPrompt(opts) {
|
|
|
119
120
|
// en/argless render disarms a language captured earlier in a long-lived
|
|
120
121
|
// server process instead of falsely rejecting an English report.
|
|
121
122
|
setReportLanguage(language && language !== "en" ? language : undefined);
|
|
123
|
+
// SKYR-4250: same capture point, same last-render-wins rule, for the
|
|
124
|
+
// plan-only lane. skyramp_submit_report's SKYR-3883 guard
|
|
125
|
+
// (findUnchangedFileClaims) verifies newTestsCreated against the working
|
|
126
|
+
// tree; a plan-only run writes no files, so without this every declaration
|
|
127
|
+
// is unchanged-file-flagged and the agent is forced to demote it to
|
|
128
|
+
// additionalRecommendations.
|
|
129
|
+
setPlanOnlyMode(planOnly);
|
|
122
130
|
let reportLanguageBlock = "";
|
|
123
131
|
if (language && language !== "en") {
|
|
124
132
|
const reportLanguageName = new Intl.DisplayNames(["en"], { type: "language" }).of(language) ??
|
|
@@ -136,7 +144,7 @@ export function getTestbotPrompt(opts) {
|
|
|
136
144
|
maintenanceBeforeExecStep = ` d. Plan-only run: skip the pre-edit baseline execution — the application is not running. Record every maintenance verdict from static drift analysis alone; execution statuses simply remain unrecorded.`;
|
|
137
145
|
}
|
|
138
146
|
else {
|
|
139
|
-
maintenanceBeforeExecStep = ` d. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every UPDATE/REGENERATE/DELETE test. Exclude tests marked \`[external]\` — those are baselined in step 2(a) via \`skyramp_run_existing_tests\`. Run them sequentially, not in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail.`;
|
|
147
|
+
maintenanceBeforeExecStep = ` d. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every UPDATE/REGENERATE/DELETE test. Exclude tests marked \`[external]\` — those are baselined in step 2(a) via \`skyramp_run_existing_tests\`. Run them sequentially, not in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail. Never pass \`rebaselineSnapshots\` here: a \`Screenshot comparison failed\` on this run is the evidence that a visual baseline is stale, and the refresh belongs to the final execution.`;
|
|
140
148
|
}
|
|
141
149
|
// For follow-up requests: emit the @skyramp-testbot header + guardrails + retrieve-recommendations step.
|
|
142
150
|
// For first-run prompts: emit the full Task 1 analysis + maintenance section.
|
|
@@ -222,7 +230,10 @@ ${maintenanceBeforeExecStep}
|
|
|
222
230
|
- Incorrect arithmetic in business logic (discount calculations, price aggregation)
|
|
223
231
|
Log each finding in \`issuesFound\` with a \`severity\` (critical/high/medium/low). These bugs should inform your test design in Task 2.
|
|
224
232
|
|
|
225
|
-
|
|
233
|
+
**Requirement check — does the code do what the PR says it does?** Read the \`<TITLE>\` and \`<DESCRIPTION>\` above as a statement of *intended* behavior and check the diff against it, requirement by requirement. The description is not outranked by the code: when the two disagree, that is a \`requirement_conflict\`, not a case of "the description loses". Detection, the requirements-file rules, the four false-red pre-checks and the promotion ordering are specified in the Code Review step of the Execution Plan \`skyramp_analyze_changes\` returned — follow them there rather than improvising; read any requirements file the description names from \`${repositoryPath}\`.
|
|
234
|
+
- **On a mismatch, do both.** (a) Add an \`issuesFound\` entry at severity \`high\` or \`critical\` — never \`medium\` or below — quoting the requirement and naming the file and line that contradicts it. (b) Carry it into Task 2 as a \`requirement_conflict\` candidate: a test asserting **what the description requires**, which therefore FAILS on the current code. Where no such test is recordable, record the Execution Plan's VERIFY-style entry in \`additionalRecommendations\` instead — never resolve the disagreement by asserting what the code does.
|
|
235
|
+
|
|
236
|
+
4. **Blueprint Citation Invariant** (UI test recommendations only). Every named UI element in your recommendation must correspond to an element actually captured in one of the blueprints you captured from the UI Blueprint Capture section of \`skyramp_analyze_changes\` — with one exception: a removal guard's target is an element the PR removed, which no blueprint can contain; it is grounded in \`uiContext.removedElements\` instead (see the negative-test note below). For **every** UI recommendation — both \`newTestsCreated[]\` entries (tests you generated) AND \`additionalRecommendations[]\` entries (deferred tests) with \`testType: "ui"\` — populate **four** fields: \`description\`, \`reasoning\`, \`targetElements\`, and \`pageContext\`. \`description\` and \`reasoning\` carry different roles — fill BOTH (see Field 4 below). The structured fields (\`targetElements\`, \`pageContext\`) are required on UI recs in BOTH arrays; non-UI recs (contract / integration / e2e / batch-scenario) MUST omit them.
|
|
226
237
|
|
|
227
238
|
**Field 1 — \`targetElements\`** (the elements the test targets — array, length 1+):
|
|
228
239
|
\`\`\`json
|
|
@@ -336,7 +347,7 @@ ${maintenanceBeforeExecStep}
|
|
|
336
347
|
"reasoning": "Would assert the **Edit Order** button (\`edit-order-btn\`) is not rendered on cancelled orders; enforces the conditional-render guard added in OrderDetail.tsx:118."
|
|
337
348
|
}
|
|
338
349
|
\`\`\`
|
|
339
|
-
(For negative tests, ground \`targetElements\` against a captured page where the element DOES render — so the verifier can confirm the citation is real.)
|
|
350
|
+
(For negative tests, ground \`targetElements\` against a captured page where the element DOES render — so the verifier can confirm the citation is real. **Exception — removal guards:** an element listed in \`uiContext.removedElements\` renders on no page any more, so no blueprint can contain it. For its \`verify-removed-ui-<testId>\` recommendation, cite the element from that list (\`testId\` for a \`data-testid\`; otherwise name the \`attribute="value"\` pair in \`reasoning\` and leave \`testId\` null), take \`role\` and \`accessibleName\` from the deleted JSX in the diff, set \`pageContext.url\` to the page the list names, and prefix \`reasoning\` with \`[removed-element]\`.)
|
|
340
351
|
|
|
341
352
|
*Page-level / lifecycle test (no single dominant element), recommendatory voice:*
|
|
342
353
|
\`\`\`json
|
|
@@ -381,13 +392,13 @@ ${maintenanceBeforeExecStep}
|
|
|
381
392
|
|
|
382
393
|
**Self-check before submitting (per UI rec, in both \`newTestsCreated\` and \`additionalRecommendations\`):**
|
|
383
394
|
- **Both \`description\` AND \`reasoning\` are populated.** \`description\` is a 2-4 sentence walkthrough; \`reasoning\` is one sentence about what's verified + why. They must not duplicate each other.
|
|
384
|
-
- Every entry in \`targetElements\` must appear verbatim in a captured blueprint element (matched on \`accessibleName\` + \`role\`).
|
|
395
|
+
- Every entry in \`targetElements\` must appear verbatim in a captured blueprint element (matched on \`accessibleName\` + \`role\`) — except a removal guard's entry, whose \`testId\` (or, for a non-\`data-testid\` attribute, the \`attribute="value"\` named in \`reasoning\`) must instead appear in \`uiContext.removedElements\` and whose \`reasoning\` carries the \`[removed-element]\` prefix.
|
|
385
396
|
- Every element name appearing in \`reasoning\` must match an entry's \`accessibleName\` exactly. Every backticked identifier in \`reasoning\` must come from a \`testId\` or \`stableId\` of an entry.
|
|
386
397
|
- **Voice matches the array:** \`newTestsCreated\` uses descriptive voice ("verifies", "asserts", "documents"). \`additionalRecommendations\` uses recommendatory voice ("would verify", "should assert", "would cover").
|
|
387
|
-
- \`pageContext.url\` must equal the URL of the BlueprintCapture you lifted \`targetElements\` from.
|
|
398
|
+
- \`pageContext.url\` must equal the URL of the BlueprintCapture you lifted \`targetElements\` from (for a removal guard: the page \`uiContext.removedElements\` names for that id).
|
|
388
399
|
- Never mix two elements' fields in one \`targetElements\` entry — each entry is one captured element.
|
|
389
400
|
- No internal jargon in \`reasoning\` or \`description\`: no \`logicalName\`, \`fingerprint\`, \`pageHash\`, \`xpath\`.
|
|
390
|
-
- If any check fails, re-capture and verify, or drop that recommendation.
|
|
401
|
+
- If any check fails, re-capture and verify, or drop that recommendation (a removal guard passes these checks through the \`removedElements\` grounding above, never through invention).
|
|
391
402
|
|
|
392
403
|
**Non-UI entries (contract / integration / e2e / batch-scenario) are unaffected.** Use their pre-existing reasoning formats. Do NOT add \`targetElements\` or \`pageContext\` to non-UI entries — the schema rejects them.
|
|
393
404
|
|
|
@@ -431,7 +442,7 @@ ${maintenanceBeforeExecStep}
|
|
|
431
442
|
**Self-check before calling \`skyramp_submit_report\`:** count your final GENERATE list, then confirm \`newTestsCreated\` has exactly that many entries — one per GENERATE item. A GENERATE item always goes in \`newTestsCreated\` (even though no file was generated in this run — the declaration IS the deliverable); \`additionalRecommendations\` holds ONLY candidates that are NOT in the GENERATE list. Reporting a GENERATE item as a recommendation instead of a declaration is the single most common plan-only reporting error — recheck the placement before submitting.`;
|
|
432
443
|
}
|
|
433
444
|
else {
|
|
434
|
-
task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if
|
|
445
|
+
task3CountRule = `Otherwise (your Budget Plan is non-zero): in \`newTestsCreated\`, you must have exactly as many budget-counting new tests as your committed Budget Plan's generate count (at most ${maxGenerate}). Only new files (ADD) created for the planned GENERATE items count toward this target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the budget. If you have fewer budget-counting new tests than your generate count, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer if every remaining candidate failed after retry — and, on a PR with backend changes, the fallback single-contract test also failed. A frontend-only PR has no backend fallback: its exhausted candidates go to \`additionalRecommendations\` with the failure reason.`;
|
|
435
446
|
}
|
|
436
447
|
// Task 2 branches wholesale in plan-only eval runs (SKYR-3879 plan-only
|
|
437
448
|
// lane): the standard task mandates generation and execution, which a
|
|
@@ -443,9 +454,9 @@ ${maintenanceBeforeExecStep}
|
|
|
443
454
|
|
|
444
455
|
This is a plan-only evaluation run: the application under test is NOT running, and this run evaluates test SELECTION only. Nothing is generated or executed in this task.
|
|
445
456
|
|
|
446
|
-
- Draft your complete candidate list exactly as the Execution Plan directs — every
|
|
457
|
+
- Draft your complete candidate list exactly as the Execution Plan directs — every test you would generate OR recommend for this PR, grounded in the analysis output and the diff. Favor tests that would FAIL if the changed logic were buggy, not just tests that exercise the new surface.
|
|
447
458
|
- If a tool named \`skyramp_register_test_plan\` is available, call it with the full candidate union — include a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for every candidate that probes changed logic — and treat its returned GENERATE list as your final selection. If that tool is not available, commit to your Budget Plan's GENERATE selection (at most ${maxGenerate}).
|
|
448
|
-
-
|
|
459
|
+
- Include UI and E2E candidates on the same footing as the API types. The app is not running, so you cannot capture a blueprint — plan them ungrounded rather than drop them. For every UI entry, set \`targetElements: null\`, omit \`pageContext\`, and prefix BOTH \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Describe the page or feature the test would exercise; do not name an element you have not seen. This lane is the one case where you fall back without attempting a capture first, so do NOT log the fallback in \`issuesFound\` — capture was never applicable here, and nothing failed.
|
|
449
460
|
- Take no other actions in this task: no test generation tools, no browser traces or blueprint captures, no test files written, no test executions. Proceed directly to ${taskRef(TASK_SUBMIT)}.`;
|
|
450
461
|
}
|
|
451
462
|
else {
|
|
@@ -453,8 +464,8 @@ This is a plan-only evaluation run: the application under test is NOT running, a
|
|
|
453
464
|
|
|
454
465
|
${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
|
|
455
466
|
|
|
456
|
-
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
457
|
-
- **Do not fabricate tests outside the GENERATE list
|
|
467
|
+
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget. **Every \`requirement_conflict\` candidate from Task 1's requirement check belongs in this registration** — register it with category \`requirement_conflict\` so it ranks in the top tier and takes the first promoted slot rather than losing one to a bug-catching candidate; omitting it from the candidate list is how a stated requirement silently loses its test.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
468
|
+
- **Do not fabricate tests outside the GENERATE list returned by \`skyramp_register_test_plan\`.** A change to an EXISTING covered endpoint or component is maintenance only where an existing test asserts the changed value: handle that in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test. A changed value, option, or behavior that no existing test asserts is not maintenance — it is a candidate you register (GENERATE or ADDITIONAL), however small the change. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
|
|
458
469
|
- Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
|
|
459
470
|
- Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
|
|
460
471
|
- If the existing test source is \`[external]\`, skip the resource entirely — the external test already provides coverage. Do NOT UPDATE, REGENERATE, or DELETE external tests.
|
|
@@ -479,6 +490,9 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
479
490
|
- **Always generate a test for critical bugs, even if it will fail.** When a GENERATE-tagged item targets a page or endpoint with a known bug, do NOT skip it because you expect the test to fail — a failing test that documents a bug is more valuable than a text-only description. This applies within the existing GENERATE budget; do not add extra tests beyond the plan.
|
|
480
491
|
- For UI rendering bugs: navigate to the broken page and add a \`browser_assert\` that verifies the page rendered its expected content (e.g. assert the page heading is visible). The assertion will fail on the broken page, which is the correct outcome — it documents the bug as a failing test.
|
|
481
492
|
- The assertion MUST target the broken page itself, not a different page that works. If \`/orders/{id}/edit\` crashes, assert on \`/orders/{id}/edit\` (e.g. "Edit Order" heading visible), NOT on \`/orders\`.
|
|
493
|
+
- **The same rule covers a \`requirement_conflict\` from Task 1's requirement check — and it is promoted first.** Generate the test that asserts what the PR description requires, not what the code implements. It fails today; that is the deliverable, and it turns green when the code is fixed to match the description. Do NOT re-aim it at the implemented behavior to get a passing run, and do NOT drop it because a bug-catching test already occupies a slot — requirement conflicts take the first slots of the promotion bound, so the bug-catching tests yield to it, not the other way round.
|
|
494
|
+
- For a "should no longer be present" requirement: navigate to the page the description names, confirm from the capture that the element still renders, then assert its **absence**. The element name comes from the live capture, so the assertion is grounded; the failure is the conflict. Assert on the page the requirement is about — not a sibling page where the element is genuinely gone.
|
|
495
|
+
- For a "should now do X" requirement the code does not do: assert X. Same rule — the stated requirement is the expectation.
|
|
482
496
|
- **Critical categories first**: At least 1 of the generated tests MUST be from a critical category (security_boundary, business_rule, data_integrity, breaking_change) if such candidates exist in the GENERATE set.
|
|
483
497
|
- **Parallel generation (IMPORTANT for speed)**: Generate **independent tests in parallel** whenever possible. Tests targeting different endpoints with different output files can be generated concurrently in the same tool call batch. Specifically:
|
|
484
498
|
- Call \`skyramp_batch_scenario_test_generation\` for ALL integration scenarios AND \`skyramp_contract_test_generation\` for ALL contract tests **in the same tool call batch**.
|
|
@@ -562,10 +576,10 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
562
576
|
- \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
|
|
563
577
|
- \`skyramp_ui_test_generation\`: set \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript Playwright only — the default; leave \`codeReuse\` unset for other languages)
|
|
564
578
|
- **\`browser_assert\` — MANDATORY**: at least one per page navigated. Call multiple assertions in the same tool call batch when checking independent elements. If you navigate to 2 pages, assert on both. Each assertion should verify a business outcome (state change, computed value, error condition) — not just that an element is visible.
|
|
579
|
+
- **Removal guard — when \`uiContext.removedElements\` is non-empty**: the server has already resolved which elements this PR removed — each identified by a \`data-*\` attribute and value — and the page each rendered on (renames, moves and deleted files are excluded). Per listed PAGE: \`browser_navigate\` to it, wait for the app to render, then call \`browser_assert_hidden\` once per listed element using the target the list shows — \`testId: "<value>"\` when the attribute is \`data-testid\`, otherwise \`attribute: "<name>", value: "<value>"\` (e.g. \`attribute: "data-cy"\`); no \`ref\` — the element no longer exists, so no snapshot has one; the tool checks the live page and refuses to record on a blank or un-rendered page. Then at least one \`browser_assert\` on a RETAINED element of the same page verifying its text, value or state. If a UI trace for that page is already planned, record the guard assertions inside that trace; otherwise this is its own trace and spec, \`verify-removed-ui-<page-slug>\`. The generated test asserts \`toBeHidden()\` on \`getByTestId(...)\` or \`page.locator('[data-cy="..."]')\` accordingly. A removed element the server did not list still gets its test (the budget rule requires it) — ground that one in the diff and say so in \`reasoning\`.
|
|
565
580
|
- **\`browser_visual_snapshot\` — for visual/appearance checks**: when the instruction asks to take a screenshot, capture a baseline, or verify how a page/element/region *looks* (not its text or value), call \`browser_visual_snapshot\` — it records a \`toHaveScreenshot()\` assertion so the generated test pixel-compares against a baseline on every run. Do NOT use \`browser_take_screenshot\` for this: it captures a throwaway image that is dropped at export and never appears in the generated test (use it only to view the page yourself).
|
|
566
581
|
- **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
|
|
567
582
|
If \`browser_navigate\` fails (app not running / connection refused), apply skip condition (a) above: move the intended test to \`additionalRecommendations\` with the failure reason AND record the outage in \`issuesFound\`.
|
|
568
|
-
Record at most 2-3 UI traces per run to stay within tool call budget. Quality over quantity: 1 great test is better than 3 mediocre ones — do not pad to reach the count.
|
|
569
583
|
**Strategic assertions** — key checkpoints only, 3 to 5 per test:
|
|
570
584
|
- **After the main action completes**: verify the outcome is visible (new item appears, form saves, confirmation shows)
|
|
571
585
|
- **State transitions**: verify counts, totals, or status fields update correctly
|
|
@@ -631,11 +645,15 @@ If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated t
|
|
|
631
645
|
1. Read the error output to diagnose the root cause (4xx on prereq step, assertion mismatch, floating-point precision, 500 from app bug, timeout, etc.).
|
|
632
646
|
2. **Expected failure check (no retry):** If the failure is an assertion error or HTTP error that matches the issue identified in the code analysis (e.g. the test was generated specifically to document a broken endpoint, a UI rendering bug, or a missing validation), then this is the **intended outcome** — the test is correctly catching the real bug. Report it immediately as \`status: "Fail"\` and move on. Do NOT retry.
|
|
633
647
|
|
|
648
|
+
**A \`requirement_conflict\` test failing is likewise the intended outcome.** It asserts what the PR description requires and the code does not do; a red result is the finding, so report \`status: "Fail"\` with the conflict named in \`issuesFound\` and move on. Do NOT retry it, do NOT relax the assertion toward the implemented behavior, and do NOT delete the test — either of those hides the disagreement the run exists to surface.
|
|
649
|
+
|
|
634
650
|
This path also covers an assertion failure that application behavior outside this PR's diff explains — for example child records that survive the deletion of their parent, state inherited when an ID is recycled or reused, or a value that ignores a status the test set. Before you keep such a test red, confirm the cause in the source: read the handler, model, or query that should have done the work, and find the specific operation that is missing or wrong. If you find it, report the test as \`status: "Fail"\` and add an \`issuesFound\` entry for it. Do NOT retry.
|
|
635
651
|
|
|
636
652
|
If you cannot point at the missing or wrong line in application code, the app is not at fault — shared or seeded data, parallel test workers, or setup the test itself never did explain the collision. Treat that as an infrastructure failure: fix it and retry once as in step 3.
|
|
637
653
|
|
|
638
654
|
**If you did confirm the missing or wrong operation in the source, do NOT make the test pass.** Never add a reset, cleanup, or setup call for isolation. Never weaken the assertion — no \`==\` to \`>=\`, no exact value to a range. A failing test whose diagnosis names a pre-existing bug is the most valuable output of this run; a passing version of it reports nothing.
|
|
655
|
+
|
|
656
|
+
**Visual snapshots — maintained tests only, and never a third run.** A newly generated test writes its baseline on its first run and cannot be stale, so this applies to an existing spec whose committed baseline the PR made out of date. That mismatch is known from the \`phase: "before"\` run, and the refresh rides the final run: pass the \`rebaseline_snapshots\` list \`skyramp_actions\` returned as \`rebaselineSnapshots\` on that one call. It never costs an extra execution — if a screenshot mismatch on a maintained test first appears on the final run, report \`Fail\`; do not add a run. The execution result names which baselines were refreshed and which were not; the report row's description carries the refreshed names automatically, so \`afterDetails\` states only the outcome (e.g. \`1 passed in 6.1s\`). If no diff line explains a mismatch, keep the test \`Fail\` and add an \`issuesFound\` entry. Never raise \`maxDiffPixelRatio\`, never remove the \`toHaveScreenshot()\` call, never edit or delete a PNG by hand, and never pass \`rebaselineSnapshots\` for a mismatch the diff does not explain.
|
|
639
657
|
3. Apply a targeted fix and retry **once** only for **infrastructure failures** — that means exactly **2 total \`skyramp_execute_test\` calls per test file** for these cases. Examples of infrastructure failures worth fixing:
|
|
640
658
|
- Assertion mismatch from floating-point precision, or an expected value mis-transcribed from the observed response or computed with an arithmetic slip. If application behavior outside the diff explains the mismatch, it is not an infrastructure failure — use step 2 instead.
|
|
641
659
|
- Import error, syntax error, or missing dependency in the generated test file
|
|
@@ -658,7 +676,7 @@ Do not make any changes other than the code-reuse refactoring (step 3 and the ge
|
|
|
658
676
|
|
|
659
677
|
**Execution timing:**
|
|
660
678
|
- **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
|
|
661
|
-
- **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection. Exclude tests marked \`[external]\`.
|
|
679
|
+
- **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together. For a maintained spec whose \`skyramp_actions\` entry carried \`rebaseline_snapshots\`, pass that exact list as \`rebaselineSnapshots\` on this run and only this run. **Execute tests SEQUENTIALLY (one at a time)** — do NOT send multiple \`skyramp_execute_test\` calls in the same tool call batch, as concurrent execution overwhelms the stdio transport and causes MCP disconnection. Exclude tests marked \`[external]\`.
|
|
662
680
|
- Only report test results for files you actually ran.
|
|
663
681
|
**Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.`;
|
|
664
682
|
}
|
|
@@ -683,7 +701,7 @@ If you skipped here due to non-application changes (per Task 1), submit with emp
|
|
|
683
701
|
|
|
684
702
|
**If you generated zero new tests because the PR has no testable behavioral surface:**
|
|
685
703
|
This applies when the diff contains ONLY changes with no observable API or UI behavior change. Examples:
|
|
686
|
-
-
|
|
704
|
+
- Docs/comments only: JSDoc updates, comment-only changes
|
|
687
705
|
- Dependency-only: version bumps with no API surface change
|
|
688
706
|
- Dead code / unintegrated utility or component: a new helper function, utility, or UI component added to the codebase but not imported, mounted, or rendered anywhere — use this classification only after confirming the new symbol does not appear as an import or render call in any other source file; do NOT classify as dead code based solely on the diff. For UI components specifically: an unintegrated component has no DOM node in the running app and cannot be browser-tested regardless of how complex its logic is
|
|
689
707
|
- Config-only: linter rules, build config, environment variable additions with no runtime behavior change
|
|
@@ -694,6 +712,8 @@ In these cases:
|
|
|
694
712
|
- \`businessCaseAnalysis\` must be a one-sentence summary of what the PR actually does (do NOT leave it blank)
|
|
695
713
|
- \`additionalRecommendations\` must be \`[]\` — do NOT recommend tests for a no-surface PR
|
|
696
714
|
|
|
715
|
+
**This zero-test path does NOT apply when the requirement check found a \`requirement_conflict\`.** A requirement the PR description states and the diff did not implement is testable surface: report the conflict in \`issuesFound\` at severity \`high\` or above, and carry either the failing \`requirement_conflict\` test in \`newTestsCreated\` or its VERIFY-style entry in \`additionalRecommendations\`. Abstaining there hands the author back their own code as though it were the requirement.
|
|
716
|
+
|
|
697
717
|
${task3CountRule}
|
|
698
718
|
|
|
699
719
|
${reportLanguageBlock}Call \`skyramp_submit_report\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking, and the report is written beside it. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
@@ -703,7 +723,7 @@ ${hasRelatedRepos
|
|
|
703
723
|
- **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
|
|
704
724
|
`
|
|
705
725
|
: ""}
|
|
706
|
-
- **additionalRecommendations**: AT MOST ${maxRecommendations -
|
|
726
|
+
- **additionalRecommendations**: AT MOST ${maxRecommendations} minus the number of budget-counting tests in \`newTestsCreated\` — an unused generate slot becomes a recommendation slot. The one optional discovered-scenario test does not count against the budget, so it does not reduce this allowance either.
|
|
707
727
|
- For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
|
|
708
728
|
|
|
709
729
|
${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false, modularizeViaGenerationResult: UI_UTILS_REUSE })}`;
|
|
@@ -27,16 +27,50 @@ function floorBalancedPick(items, count) {
|
|
|
27
27
|
typesOrder.push(t);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
-
|
|
30
|
+
// Two passes, because "protected" is broader than "is a caught bug".
|
|
31
|
+
//
|
|
32
|
+
// A `bug_caught` candidate may take a type-floor slot: a caught bug is the
|
|
33
|
+
// strongest thing a run produces, and it outranks type spread. Every OTHER
|
|
34
|
+
// protected candidate — anything else CRITICAL, and attack-surface
|
|
35
|
+
// security_boundary items — still yields to the floor, which is what stops
|
|
36
|
+
// the SKYR-3879 skew where three CRITICAL integration candidates took all
|
|
37
|
+
// three slots and no contract test was generated.
|
|
38
|
+
//
|
|
39
|
+
// The second pass replaces `maxProtected = count - (typesOrder.length - 1)`,
|
|
40
|
+
// a static cap computed from the whole pool. It reserved a slot for every
|
|
41
|
+
// other type whether or not the picks already covered it, so at the default
|
|
42
|
+
// budget of 3 with three types present it evaluated to 1. Counting the types
|
|
43
|
+
// still uncovered keeps the same floor and drops the arithmetic that made a
|
|
44
|
+
// second finding unreachable.
|
|
45
|
+
// The same bound the prompt states (`maxGen - 1`, and the whole budget only
|
|
46
|
+
// at a budget of 1). Without it the first pass took every caught bug up to
|
|
47
|
+
// `count`, which put the SKYR-3879 skew back: at the default budget of 3, two
|
|
48
|
+
// same-type bugs left no slot for the type floor, and a measured 9-20% of
|
|
49
|
+
// backend-only pools lost a test type. It also evicted attack-surface
|
|
50
|
+
// security_boundary items the prompt in this same branch says to preserve
|
|
51
|
+
// while any non-bug slot remains.
|
|
52
|
+
const bugSlots = count <= 1 ? count : count - 1;
|
|
31
53
|
const selected = [];
|
|
54
|
+
const deferred = [];
|
|
32
55
|
const pool = [];
|
|
33
|
-
let
|
|
56
|
+
let bugsTaken = 0;
|
|
34
57
|
for (const it of items) {
|
|
58
|
+
if (it.scenario?.category === "bug_caught" && bugsTaken < bugSlots && selected.length < count) {
|
|
59
|
+
selected.push(it);
|
|
60
|
+
bugsTaken++;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
deferred.push(it);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const takenTypes = new Set(selected.map(typeOf));
|
|
67
|
+
for (const it of deferred) {
|
|
68
|
+
const stillUncovered = typesOrder.filter((t) => !takenTypes.has(t) && t !== typeOf(it)).length;
|
|
35
69
|
if (isProtectedCandidate(it.priority, it.scenario) &&
|
|
36
|
-
|
|
37
|
-
selected.length
|
|
70
|
+
selected.length < count &&
|
|
71
|
+
count - (selected.length + 1) >= stillUncovered) {
|
|
38
72
|
selected.push(it);
|
|
39
|
-
|
|
73
|
+
takenTypes.add(typeOf(it));
|
|
40
74
|
}
|
|
41
75
|
else {
|
|
42
76
|
pool.push(it);
|
|
@@ -1,13 +1,26 @@
|
|
|
1
1
|
import { Candidate, BudgetContext, SelectionResult } from "../../types/Recommendation.js";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* What each PR class GUARANTEES in GENERATE (SKYR-4275). These are floors, not
|
|
4
|
+
* quotas: the slots above them are open to either kind, decided by `pick`.
|
|
5
|
+
* An open slot goes to whichever candidate `pick` chooses on rank, so a backend
|
|
6
|
+
* candidate takes one when it outranks the UI candidates, not merely because it
|
|
7
|
+
* exists. Measured against main on random mixed pools at `maxGenerate` 5 with no
|
|
8
|
+
* caught bugs: 35% get fewer backend tests than main did, average 3.53 to 3.24.
|
|
9
|
+
*
|
|
10
|
+
* Backend-only floors nothing on purpose: of 86 backend-only diffs in the eval
|
|
11
|
+
* corpus, ZERO generated a UI test, so a floor there would guarantee what
|
|
12
|
+
* already happens every time.
|
|
13
|
+
*
|
|
14
|
+
* At one slot there is no room for a floor plus an open slot, so each class
|
|
15
|
+
* keeps that slot for its own kind.
|
|
16
|
+
*
|
|
17
|
+
* The floors always sum to at most `maxGenerate`, which is what lets
|
|
18
|
+
* `applyFloors` satisfy both without one starving the other.
|
|
7
19
|
*/
|
|
8
|
-
export declare function
|
|
9
|
-
|
|
10
|
-
|
|
20
|
+
export declare function generateFloors(ctx: BudgetContext): {
|
|
21
|
+
ui: number;
|
|
22
|
+
backend: number;
|
|
23
|
+
};
|
|
11
24
|
/**
|
|
12
25
|
* Shared budgeting pipeline. All Budgeters run the same external-dedup,
|
|
13
26
|
* attack-surface prioritization, and ADDITIONAL set-difference; they differ
|
|
@@ -16,4 +29,4 @@ export declare function reservedUISlots(ctx: BudgetContext): number;
|
|
|
16
29
|
* With `pick = roundRobinByType` this reproduces the pre-refactor selection in
|
|
17
30
|
* diffExecutionPlan.ts exactly.
|
|
18
31
|
*/
|
|
19
|
-
export declare function runBudget(ranked: Candidate[],
|
|
32
|
+
export declare function runBudget(ranked: Candidate[], rawCtx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;
|
|
@@ -1,28 +1,76 @@
|
|
|
1
1
|
import { TestType } from "../../types/TestTypes.js";
|
|
2
2
|
import { inferScenarioType, prioritizeAttackSurfaceBundles } from "../diversity.js";
|
|
3
3
|
import { externalDedupKeys, scenarioCoverageKeys, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
|
|
4
|
+
import { isFlawTargetingCategory } from "../../types/TestRecommendation.js";
|
|
4
5
|
import { logger } from "../../utils/logger.js";
|
|
5
6
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* What each PR class GUARANTEES in GENERATE (SKYR-4275). These are floors, not
|
|
8
|
+
* quotas: the slots above them are open to either kind, decided by `pick`.
|
|
9
|
+
* An open slot goes to whichever candidate `pick` chooses on rank, so a backend
|
|
10
|
+
* candidate takes one when it outranks the UI candidates, not merely because it
|
|
11
|
+
* exists. Measured against main on random mixed pools at `maxGenerate` 5 with no
|
|
12
|
+
* caught bugs: 35% get fewer backend tests than main did, average 3.53 to 3.24.
|
|
13
|
+
*
|
|
14
|
+
* Backend-only floors nothing on purpose: of 86 backend-only diffs in the eval
|
|
15
|
+
* corpus, ZERO generated a UI test, so a floor there would guarantee what
|
|
16
|
+
* already happens every time.
|
|
17
|
+
*
|
|
18
|
+
* At one slot there is no room for a floor plus an open slot, so each class
|
|
19
|
+
* keeps that slot for its own kind.
|
|
20
|
+
*
|
|
21
|
+
* The floors always sum to at most `maxGenerate`, which is what lets
|
|
22
|
+
* `applyFloors` satisfy both without one starving the other.
|
|
10
23
|
*/
|
|
11
|
-
export function
|
|
24
|
+
export function generateFloors(ctx) {
|
|
25
|
+
// Number.isFinite, not Math.max: `Math.max(0, NaN)` is NaN, which compares
|
|
26
|
+
// false against 0 and would fall through to the class branches below.
|
|
27
|
+
const max = Number.isFinite(ctx.maxGenerate) ? Math.max(0, Math.floor(ctx.maxGenerate)) : 0;
|
|
28
|
+
if (max === 0)
|
|
29
|
+
return { ui: 0, backend: 0 };
|
|
12
30
|
if (ctx.isUIOnlyPR)
|
|
13
|
-
return 0;
|
|
14
|
-
|
|
31
|
+
return { ui: max === 1 ? 1 : max - 1, backend: 0 };
|
|
32
|
+
if (ctx.hasFrontendChanges)
|
|
33
|
+
return max === 1 ? { ui: 1, backend: 0 } : { ui: 1, backend: 1 };
|
|
34
|
+
return { ui: 0, backend: 0 };
|
|
15
35
|
}
|
|
16
|
-
/**
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
36
|
+
/**
|
|
37
|
+
* Raise a chosen GENERATE set to meet the floors, by swapping rather than
|
|
38
|
+
* re-picking. `pick` runs ONCE over the whole pool, so its protected-first
|
|
39
|
+
* ordering, its one-slot-per-test-type coverage and attack-surface bundle
|
|
40
|
+
* adjacency are all settled before this runs. Picking each floor from its own
|
|
41
|
+
* slice instead would compute those guarantees per slice and collapse them —
|
|
42
|
+
* measured at 25% of mixed-PR inputs losing a test type, 8.3% losing contract
|
|
43
|
+
* coverage outright.
|
|
44
|
+
*
|
|
45
|
+
* A floor is raised only as far as the pool allows. Satisfying one floor cannot
|
|
46
|
+
* starve the other, because `generateFloors` never returns two that sum above
|
|
47
|
+
* the budget — `generateFloors.test.ts` pins that invariant.
|
|
48
|
+
*/
|
|
49
|
+
function applyFloors(generate, slotOrdered, floors, isUI) {
|
|
50
|
+
const isBackend = (item) => !isUI(item);
|
|
51
|
+
const result = [...generate];
|
|
52
|
+
for (const [want, wanted] of [
|
|
53
|
+
[floors.ui, isUI],
|
|
54
|
+
[floors.backend, isBackend],
|
|
55
|
+
]) {
|
|
56
|
+
const spare = slotOrdered.filter((item) => wanted(item) && !result.includes(item));
|
|
57
|
+
let shortfall = Math.min(want, result.filter(wanted).length + spare.length) - result.filter(wanted).length;
|
|
58
|
+
// Swap from the END: `pick` returns its slots in preference order, so the
|
|
59
|
+
// last one it chose is the one this PR class needs least.
|
|
60
|
+
for (let i = result.length - 1; i >= 0 && shortfall > 0; i--) {
|
|
61
|
+
if (wanted(result[i]))
|
|
62
|
+
continue;
|
|
63
|
+
result[i] = spare.shift();
|
|
64
|
+
shortfall--;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
21
68
|
}
|
|
22
69
|
/**
|
|
23
70
|
* Split candidates by whether an external test already covers them. Protected
|
|
24
|
-
* `bug_caught` / attack-surface scenarios always count
|
|
25
|
-
* require semantic flaw coverage the external test may not
|
|
71
|
+
* `bug_caught` / `requirement_conflict` / attack-surface scenarios always count
|
|
72
|
+
* as uncovered — they require semantic flaw coverage the external test may not
|
|
73
|
+
* provide.
|
|
26
74
|
*
|
|
27
75
|
* A candidate is covered only when the set holds EVERY one of its keys. One
|
|
28
76
|
* match is not coverage: a scenario exercising two changed endpoints would lose
|
|
@@ -44,8 +92,8 @@ function partitionByExternalCoverage(ranked, externalCoverage) {
|
|
|
44
92
|
uncovered.push(item);
|
|
45
93
|
continue;
|
|
46
94
|
}
|
|
47
|
-
if (item.scenario.category
|
|
48
|
-
logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
|
|
95
|
+
if (isFlawTargetingCategory(item.scenario.category) || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
96
|
+
logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/requirement-conflict/attack-surface scenario requires semantic flaw coverage`);
|
|
49
97
|
uncovered.push(item);
|
|
50
98
|
continue;
|
|
51
99
|
}
|
|
@@ -62,36 +110,27 @@ function partitionByExternalCoverage(ranked, externalCoverage) {
|
|
|
62
110
|
* With `pick = roundRobinByType` this reproduces the pre-refactor selection in
|
|
63
111
|
* diffExecutionPlan.ts exactly.
|
|
64
112
|
*/
|
|
65
|
-
export function runBudget(ranked,
|
|
113
|
+
export function runBudget(ranked, rawCtx, pick) {
|
|
114
|
+
// SKYR-4292: maxTotal is the budget; maxGenerate is how much of it to generate.
|
|
115
|
+
// Every producer clamps upstream (getTestbotPrompt, computeScoredCandidates,
|
|
116
|
+
// buildScopeAssessmentSection), but a state file is the tool boundary and can
|
|
117
|
+
// carry maxGenerate > maxTotal — the UI-only branch then reserved maxGenerate
|
|
118
|
+
// slots and planned more items than the total. Clamp once here so every
|
|
119
|
+
// budgeter agrees with the prompt. It must stay ABOVE generateFloors(ctx):
|
|
120
|
+
// the floors are computed from the clamped budget, and moving the clamp
|
|
121
|
+
// below them would let the floors sum past maxTotal again (review on #835).
|
|
122
|
+
const ctx = { ...rawCtx, maxGenerate: Math.min(rawCtx.maxGenerate, rawCtx.maxTotal) };
|
|
66
123
|
const dropped = [];
|
|
67
|
-
const
|
|
68
|
-
const uiSlots = reservedUISlots(ctx);
|
|
124
|
+
const floors = generateFloors(ctx);
|
|
69
125
|
const { uncovered, covered } = partitionByExternalCoverage(ranked, ctx.externalCoverage);
|
|
70
126
|
const slotOrdered = prioritizeAttackSurfaceBundles(uncovered);
|
|
71
|
-
|
|
72
|
-
//
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
const backendGenerate = pick(backendSource, Math.min(backend, backendSource.length));
|
|
79
|
-
// Fill the reserved UI slots the `pick` step leaves empty: the backend count
|
|
80
|
-
// is 0 on a UI-only PR (and maxGenerate-1 on a mixed PR), so UI candidates are
|
|
81
|
-
// never chosen above even though reservedUISlots budgets for them. Promote the
|
|
82
|
-
// top-ranked UI candidates not already chosen, up to the reserved count, so
|
|
83
|
-
// GENERATE is complete instead of an empty list plus a dropped count.
|
|
84
|
-
const chosen = new Set(backendGenerate);
|
|
85
|
-
const uiGenerate = [];
|
|
86
|
-
for (const item of slotOrdered) {
|
|
87
|
-
if (uiGenerate.length >= uiSlots)
|
|
88
|
-
break;
|
|
89
|
-
if (chosen.has(item) || inferScenarioType(item.scenario) !== TestType.UI)
|
|
90
|
-
continue;
|
|
91
|
-
uiGenerate.push(item);
|
|
92
|
-
chosen.add(item);
|
|
93
|
-
}
|
|
94
|
-
const generate = [...backendGenerate, ...uiGenerate];
|
|
127
|
+
const isUICandidate = (item) => inferScenarioType(item.scenario) === TestType.UI;
|
|
128
|
+
// ONE pick over the whole pool — the shape the backend pick always had — so
|
|
129
|
+
// `pick` decides protected-first order, per-type coverage and bundle adjacency
|
|
130
|
+
// exactly once. `applyFloors` then corrects only the UI/backend mix.
|
|
131
|
+
const budget = Number.isFinite(ctx.maxGenerate) ? Math.max(0, Math.floor(ctx.maxGenerate)) : 0;
|
|
132
|
+
const picked = pick(slotOrdered, Math.min(budget, slotOrdered.length));
|
|
133
|
+
const generate = applyFloors(picked, slotOrdered, floors, isUICandidate);
|
|
95
134
|
// SKYR-4024. A covered candidate is one an existing test already reaches, so
|
|
96
135
|
// generating a second test for it is wrong — maintenance updates the existing
|
|
97
136
|
// one. But dropping it outright removed it from the RECOMMENDATIONS too, and an
|
|
@@ -100,24 +139,50 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
100
139
|
// one UI candidate fills the reserved slot while every covered backend
|
|
101
140
|
// candidate would otherwise vanish.
|
|
102
141
|
//
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
// test file
|
|
107
|
-
//
|
|
108
|
-
//
|
|
109
|
-
|
|
142
|
+
// Three conditions keep the reserve shut where recommending nothing is right.
|
|
143
|
+
// A UI-only PR is left alone. A zero budget is an explicit abstention, so it
|
|
144
|
+
// must not be talked out of recommending nothing. And the PR must not touch a
|
|
145
|
+
// test file of its own: coverage is read from the working tree, and a test
|
|
146
|
+
// file without the Skyramp marker counts as external, so a PR that adds its
|
|
147
|
+
// own tests would have that coverage used as grounds to recommend duplicates
|
|
148
|
+
// of it.
|
|
149
|
+
//
|
|
150
|
+
// One pick over the whole pool leaves no "backend half" to count, so the
|
|
151
|
+
// emptiness test reproduces what that half used to mean per class. On a mixed
|
|
152
|
+
// PR the UI slot must not read as backend coverage, or a covered backend
|
|
153
|
+
// candidate vanishes behind it. On a backend-only PR the pick always drew from
|
|
154
|
+
// the whole pool, so a non-empty GENERATE closed the reserve even when every
|
|
155
|
+
// item was UI — keep that, or an all-UI pool opens a reserve it never opened.
|
|
156
|
+
const backendChosen = generate.filter((item) => !isUICandidate(item));
|
|
157
|
+
const backendHalfEmpty = ctx.hasFrontendChanges
|
|
158
|
+
? backendChosen.length === 0
|
|
159
|
+
: generate.length === 0;
|
|
160
|
+
// One per-class behaviour DID change here, deliberately. On a mixed PR at a
|
|
161
|
+
// budget of 1 the old gate also required `backendGenerateCount > 0`, which is
|
|
162
|
+
// 0 for that class, so a covered backend candidate was dropped outright. The
|
|
163
|
+
// reserve exists because dropping removes a candidate from the
|
|
164
|
+
// RECOMMENDATIONS too (SKYR-4024), and a budget of 1 is the case where that
|
|
165
|
+
// costs most — there is no second slot to find it later. It is now
|
|
166
|
+
// recommended instead of dropped. `reserveOpensForMixedPRAtBudgetOne` in
|
|
167
|
+
// shared.test.ts pins it, so a future change has to mean it.
|
|
168
|
+
const spendReserve = backendHalfEmpty &&
|
|
169
|
+
!ctx.isUIOnlyPR &&
|
|
170
|
+
budget > 0 &&
|
|
171
|
+
covered.length > 0 &&
|
|
172
|
+
!ctx.diffChangesTestFiles;
|
|
110
173
|
if (spendReserve) {
|
|
111
174
|
logger.info(`External dedup: GENERATE empty after dedup — recommending ${covered.length} held-back candidate(s) as ADDITIONAL rather than approving an empty plan`);
|
|
112
175
|
}
|
|
113
176
|
else if (covered.length > 0) {
|
|
114
177
|
// SKYR-4214. The reserve stayed shut, so these really are dropped — say so,
|
|
115
178
|
// with the reason, instead of leaving the agent an unexplained short list.
|
|
116
|
-
const reason =
|
|
117
|
-
? `the backend half of GENERATE is filled (${
|
|
118
|
-
:
|
|
119
|
-
? "
|
|
120
|
-
:
|
|
179
|
+
const reason = !backendHalfEmpty
|
|
180
|
+
? `the backend half of GENERATE is filled (${backendChosen.length || generate.length})`
|
|
181
|
+
: ctx.isUIOnlyPR
|
|
182
|
+
? "a UI-only PR keeps its slots for UI"
|
|
183
|
+
: budget === 0
|
|
184
|
+
? "the budget is zero"
|
|
185
|
+
: "the PR changes test files of its own";
|
|
121
186
|
logger.info(`External dedup: dropping ${covered.length} covered candidate(s) — reserve shut because ${reason}`);
|
|
122
187
|
for (const { item, keys } of covered) {
|
|
123
188
|
dropped.push({
|
|
@@ -142,6 +207,7 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
142
207
|
const additionalSource = spendReserve
|
|
143
208
|
? [...slotOrdered, ...prioritizeAttackSurfaceBundles(covered.map((c) => c.item))]
|
|
144
209
|
: slotOrdered;
|
|
210
|
+
const chosen = new Set(generate);
|
|
145
211
|
const notChosen = additionalSource.filter((it) => !chosen.has(it));
|
|
146
212
|
const remainingBudget = Math.max(0, ctx.maxTotal - generate.length);
|
|
147
213
|
const generatedCoverage = new Set(generate.flatMap((item) => scenarioCoverageKeys(item.scenario)));
|
|
@@ -149,6 +215,18 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
149
215
|
const keys = scenarioCoverageKeys(item.scenario);
|
|
150
216
|
if (keys.length === 0 || !keys.every((key) => generatedCoverage.has(key)))
|
|
151
217
|
return true;
|
|
218
|
+
// Same exemption the EXTERNAL dedup above already makes, for the same
|
|
219
|
+
// reason: a flaw-targeting scenario (a caught bug, or a requirement the code
|
|
220
|
+
// contradicts) or an attack-surface boundary asserts a specific defect, and
|
|
221
|
+
// sharing an endpoint with a GENERATE item is not evidence that the defect is
|
|
222
|
+
// covered. Without it a PR with two defects in one handler had its second one
|
|
223
|
+
// deleted here — the first took a GENERATE slot, and the key is method +
|
|
224
|
+
// resource + test type + interaction + status, which two tests on one
|
|
225
|
+
// endpoint share.
|
|
226
|
+
if (isFlawTargetingCategory(item.scenario.category) || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
227
|
+
logger.info(`GENERATE dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/requirement-conflict/attack-surface scenario requires semantic flaw coverage`);
|
|
228
|
+
return true;
|
|
229
|
+
}
|
|
152
230
|
dropped.push({
|
|
153
231
|
candidateId: item.candidateId,
|
|
154
232
|
reason: `covered by GENERATE (${keys.join(", ")})`,
|
|
@@ -164,5 +242,5 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
164
242
|
}
|
|
165
243
|
// Budgeting itself produces no demotions; the register-plan selection stage
|
|
166
244
|
// (planRanker.selectPlan) fills this channel from discriminator verification.
|
|
167
|
-
return { generate, additional, reservedUISlots:
|
|
245
|
+
return { generate, additional, reservedUISlots: floors.ui, demotions: [], dropped };
|
|
168
246
|
}
|