@skyramp/mcp 0.3.4 → 0.3.6-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +92 -30
- package/build/playwright/traceRecordingPrompt.d.ts +6 -0
- package/build/playwright/traceRecordingPrompt.js +6 -2
- package/build/prompts/code-reuse.d.ts +1 -2
- package/build/prompts/code-reuse.js +182 -77
- package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
- package/build/prompts/modularization/integration-test-modularization.js +83 -41
- package/build/prompts/modularization/render.d.ts +18 -0
- package/build/prompts/modularization/render.js +12 -0
- package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
- package/build/prompts/modularization/ui-test-modularization.js +89 -47
- package/build/prompts/pom-aware-code-reuse.js +3 -1
- package/build/prompts/shared-helper-policy.d.ts +57 -0
- package/build/prompts/shared-helper-policy.js +135 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
- package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
- package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
- package/build/prompts/test-recommendation/recommendationShared.js +90 -16
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +88 -33
- package/build/recommendation/budgeters/shared.js +105 -27
- package/build/recommendation/discriminators.js +13 -2
- package/build/recommendation/planRanker.d.ts +6 -6
- package/build/recommendation/planRanker.js +6 -61
- package/build/services/AnalyticsService.d.ts +7 -0
- package/build/services/AnalyticsService.js +7 -1
- package/build/services/ModularizationService.js +1 -3
- package/build/services/TestDiscoveryService.d.ts +0 -2
- package/build/services/TestDiscoveryService.js +2 -37
- package/build/services/TestGenerationService.d.ts +16 -0
- package/build/services/TestGenerationService.js +86 -10
- package/build/services/containerEnv.js +13 -12
- package/build/tools/code-refactor/codeReuseTool.js +279 -93
- package/build/tools/code-refactor/enhance-state.d.ts +49 -0
- package/build/tools/code-refactor/enhance-state.js +109 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
- package/build/tools/code-refactor/modularizationTool.js +9 -2
- package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
- package/build/tools/code-refactor/reuse-outcome.js +14 -4
- package/build/tools/code-refactor/reuse-state.d.ts +127 -5
- package/build/tools/code-refactor/reuse-state.js +628 -16
- package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
- package/build/tools/code-refactor/utils-verify-gates.js +100 -0
- package/build/tools/code-refactor/verify-gates.d.ts +2 -1
- package/build/tools/code-refactor/verify-gates.js +90 -25
- package/build/tools/executeSkyrampTestTool.d.ts +19 -0
- package/build/tools/executeSkyrampTestTool.js +158 -8
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
- package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
- package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
- package/build/tools/generate-tests/generateUIRestTool.js +22 -0
- package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
- package/build/tools/generate-tests/scenarioLint.js +127 -19
- package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
- package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
- package/build/tools/submitReportTool.d.ts +38 -38
- package/build/tools/submitReportTool.js +487 -104
- package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
- package/build/tools/test-management/analyzeChangesTool.js +75 -12
- package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
- package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
- package/build/tools/test-management/registerTestPlanTool.js +149 -23
- package/build/types/Recommendation.d.ts +34 -5
- package/build/types/RepositoryAnalysis.d.ts +133 -114
- package/build/types/RepositoryAnalysis.js +1 -1
- package/build/types/ReuseOutcome.d.ts +102 -6
- package/build/types/ReuseOutcome.js +16 -2
- package/build/types/TestRecommendation.js +21 -3
- package/build/types/TestTypes.js +14 -8
- package/build/types/TestbotReport.d.ts +25 -3
- package/build/types/index.d.ts +2 -2
- package/build/types/index.js +1 -1
- package/build/utils/AnalysisStateManager.d.ts +69 -1
- package/build/utils/AnalysisStateManager.js +69 -5
- package/build/utils/branchDiff.d.ts +10 -0
- package/build/utils/branchDiff.js +28 -0
- package/build/utils/changedRoutes.d.ts +29 -0
- package/build/utils/changedRoutes.js +87 -0
- package/build/utils/featureFlags.d.ts +21 -0
- package/build/utils/featureFlags.js +23 -0
- package/build/utils/frontendIntegration.js +34 -4
- package/build/utils/importerHop.d.ts +2 -8
- package/build/utils/importerHop.js +15 -53
- package/build/utils/pathMatching.d.ts +38 -0
- package/build/utils/pathMatching.js +71 -0
- package/build/utils/pathSignatures.d.ts +22 -0
- package/build/utils/pathSignatures.js +57 -0
- package/build/utils/planMatchKeys.d.ts +16 -3
- package/build/utils/planMatchKeys.js +26 -10
- package/build/utils/pluralization.d.ts +10 -0
- package/build/utils/pluralization.js +18 -0
- package/build/utils/pom-catalog-parse.d.ts +52 -0
- package/build/utils/pom-catalog-parse.js +141 -0
- package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
- package/build/utils/pom-scope/selector-extractor.js +34 -8
- package/build/utils/pom-verify/verify.d.ts +6 -5
- package/build/utils/pom-verify/verify.js +8 -6
- package/build/utils/reportLanguage.d.ts +43 -0
- package/build/utils/reportLanguage.js +125 -0
- package/build/utils/reportVerification.d.ts +74 -4
- package/build/utils/reportVerification.js +259 -3
- package/build/utils/reuseRouting.d.ts +3 -0
- package/build/utils/reuseRouting.js +50 -0
- package/build/utils/routeParsers.d.ts +2 -0
- package/build/utils/routeParsers.js +65 -8
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +57 -45
- package/build/utils/subjectEndpoints.d.ts +19 -0
- package/build/utils/subjectEndpoints.js +98 -0
- package/build/utils/testFileClassification.d.ts +11 -0
- package/build/utils/testFileClassification.js +47 -0
- package/build/utils/uiPageEnumerator.d.ts +45 -19
- package/build/utils/uiPageEnumerator.js +95 -51
- package/build/utils/utils-verify/allow.d.ts +16 -0
- package/build/utils/utils-verify/allow.js +68 -0
- package/build/utils/utils-verify/call-sites.d.ts +34 -0
- package/build/utils/utils-verify/call-sites.js +154 -0
- package/build/utils/utils-verify/index.d.ts +7 -0
- package/build/utils/utils-verify/index.js +7 -0
- package/build/utils/utils-verify/language-spec.d.ts +91 -0
- package/build/utils/utils-verify/language-spec.js +210 -0
- package/build/utils/utils-verify/locate.d.ts +39 -0
- package/build/utils/utils-verify/locate.js +199 -0
- package/build/utils/utils-verify/parse.d.ts +34 -0
- package/build/utils/utils-verify/parse.js +177 -0
- package/build/utils/utils-verify/stage.d.ts +24 -0
- package/build/utils/utils-verify/stage.js +107 -0
- package/build/utils/utils-verify/verify.d.ts +63 -0
- package/build/utils/utils-verify/verify.js +168 -0
- package/build/utils/utils.d.ts +3 -1
- package/build/utils/utils.js +3 -1
- package/build/workspace/workspace.d.ts +32 -32
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
|
@@ -2,11 +2,12 @@ import { z } from "zod";
|
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
4
4
|
import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
|
|
5
|
-
import {
|
|
5
|
+
import { setReportLanguage } from "../../utils/reportLanguage.js";
|
|
6
|
+
import { TASK_ANALYZE_MAINTAIN, TASK_GENERATE, TASK_SUBMIT, taskRef, } from "../test-recommendation/recommendationShared.js";
|
|
6
7
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
7
|
-
import { isContractConsumerModeEnabled, isPomReuseEnabled } from "../../utils/featureFlags.js";
|
|
8
|
+
import { isContractConsumerModeEnabled, isPomReuseEnabled, isUtilsReuseEnabled, } from "../../utils/featureFlags.js";
|
|
8
9
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
9
|
-
import { buildServiceContext, readWorkspaceServices
|
|
10
|
+
import { buildServiceContext, readWorkspaceServices } from "../prompt-utils.js";
|
|
10
11
|
// Cached at module-load — flags are process-wide and cannot change per call.
|
|
11
12
|
const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
|
|
12
13
|
const SERVICE_REFS = resolveServiceDetailsRef();
|
|
@@ -19,13 +20,46 @@ const CONTRACT_MODE_GUIDANCE = CONSUMER_MODE_ENABLED
|
|
|
19
20
|
Both modes (\`providerMode: true, consumerMode: true\`): For diff that contains BOTH provider signals (such as new/modified endpoint handlers, route changes this service owns) AND consumer signals (outbound HTTP client calls to another service, no new endpoint handlers).`
|
|
20
21
|
: ` Always add \`providerMode: true\` — the tool generates provider-side contract tests only.`;
|
|
21
22
|
const POM_REUSE_ENABLED = isPomReuseEnabled();
|
|
23
|
+
// Utils reuse (SkyrampUtils consolidation for integration tests, and for UI tests
|
|
24
|
+
// while the POM path is off) is customer-gated,
|
|
25
|
+
// default OFF — like the POM flag, its value is baked into the prompt at module load,
|
|
26
|
+
// so the MCP server must be restarted after flipping it.
|
|
27
|
+
const UTILS_REUSE_ENABLED = isUtilsReuseEnabled();
|
|
28
|
+
// The UI flow seeds a shared utils file (modularize-first) only with utils reuse
|
|
29
|
+
// on and the POM path off — the same condition isModularizeFirstTarget applies
|
|
30
|
+
// server-side. Every UI-chain statement in this prompt keys on this constant so
|
|
31
|
+
// no line can ban the modularization call the generation result instructs.
|
|
32
|
+
const UI_UTILS_REUSE = UTILS_REUSE_ENABLED && !POM_REUSE_ENABLED;
|
|
33
|
+
// Step 4 of the post-generation list. On the modularize-first UI flow the
|
|
34
|
+
// enhance call is the first sub-step of the generation result's chain — but
|
|
35
|
+
// that chain exists only for a TS/JS UI generation with enhanceAssertions on,
|
|
36
|
+
// which this prompt cannot see per call. So the step keeps the call and lets
|
|
37
|
+
// the agent skip it only when it already made it for that file.
|
|
38
|
+
const UI_ENHANCE_STEP = UI_UTILS_REUSE
|
|
39
|
+
? `Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`, and apply every instruction returned to that file — UNLESS you already called it for this file as the first sub-step of step 3's chain, in which case skip it (never enhance the same file twice; never leave a generated UI test un-enhanced). The guidance below on which assertions to add applies to whichever call enhances the file.`
|
|
40
|
+
: `Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.`;
|
|
22
41
|
// Post-generation code-reuse step for UI tests. The POM catalog, the `verify: true`
|
|
23
42
|
// loop and the `.raw.bak` restore only exist on the POM-aware path — when that path
|
|
24
43
|
// is flagged off, `skyramp_reuse_code` returns the SkyrampUtils workflow instead, so
|
|
25
44
|
// the agent must not be sent looking for artifacts nothing produces.
|
|
26
45
|
const UI_CODE_REUSE_STEP = POM_REUSE_ENABLED
|
|
27
46
|
? `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Two outcomes: (1) a response starting "No reusable POM layer detected" — this is a normal outcome, continue immediately (do NOT retry); (2) a refactoring workflow — follow it to completion INCLUDING its verification loop (\`skyramp_reuse_code\` with \`verify: true\`), finish only when it reports PASSED. If a reused test later fails execution and the failure points at a substituted POM call, restore the saved \`<testFile>.raw.bak\` over the test file and re-run — do NOT hand-edit the customer's POM methods (this re-run counts toward the 2-attempt execution cap — prefer this restore over the generic timeout fix-up when the failing locator came from a POM substitution). **Cleanup before reporting:** \`.raw.bak\` files are internal scratch — after ALL test executions are complete (pass or fail) and before calling \`skyramp_submit_report\`, delete every \`*.raw.bak\` you created so they are not committed to the customer-facing branch. The \`skyramp-pom-catalog.md\` is NOT scratch — leave it in place (later runs reuse it).`
|
|
28
|
-
:
|
|
47
|
+
: UI_UTILS_REUSE
|
|
48
|
+
? `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result carries three CRITICAL NEXT STEPS — \`skyramp_enhance_assertions\` (on the freshly generated file, while its selectors are still inline), then \`skyramp_modularization\`, then \`skyramp_reuse_code\` — with the exact arguments for each. Do all three, in that order and to completion, for EVERY generated UI test file — the enhance sub-step is that file's step 4 below (do not call \`skyramp_enhance_assertions\` a second time for the same file, and do not skip it for later files). Sibling tests offering nothing to reuse is a normal outcome, but the reuse steps still move this test's own helpers into the shared utils file and import them back, so the test file IS expected to change. Then continue (do NOT retry).`
|
|
49
|
+
: `3. **[MANDATORY] After \`skyramp_ui_test_generation\` with \`codeReuse: true\`**: the generation result directs you to call \`skyramp_reuse_code\` — do this BEFORE \`skyramp_enhance_assertions\`. Follow the returned steps exactly. If it finds no existing helper functions to reuse, that is a normal outcome — leave the test file unchanged and continue immediately (do NOT retry).`;
|
|
50
|
+
// Generation-call clause for the integration pipeline: only ask for codeReuse when
|
|
51
|
+
// utils reuse is enabled — with the flag off, integration generation behaves exactly
|
|
52
|
+
// as it did before the feature existed (no file-rewriting post-steps).
|
|
53
|
+
const INTEGRATION_CODE_REUSE_GEN_CLAUSE = UTILS_REUSE_ENABLED
|
|
54
|
+
? `, setting \`modularizeCode: false\` and \`codeReuse: true\` (TypeScript/JavaScript/Python — leave \`codeReuse\` unset for Java: cross-file helpers do not compile in the test executor)`
|
|
55
|
+
: ``;
|
|
56
|
+
// The post-generation modularize → reuse steps are NOT listed here: the
|
|
57
|
+
// generation tool's own result emits them (see TestGenerationService) when called
|
|
58
|
+
// with codeReuse: true on a modularize-first flow (routing: isModularizeFirstTarget).
|
|
59
|
+
// The UI clauses below keep `modularizeCode: false` deliberately: the
|
|
60
|
+
// modularization happens via that hand-off, not the flag, so the instructions
|
|
61
|
+
// exist only on runs that actually generated that test type. This prompt's job
|
|
62
|
+
// is only the generation-call clauses that ask for codeReuse.
|
|
29
63
|
/**
|
|
30
64
|
* Parse the JSON-encoded `relatedRepositories` argument passed via the testbot
|
|
31
65
|
* prompt/resource. Returns undefined for missing/blank input or malformed JSON so the
|
|
@@ -77,9 +111,18 @@ export function getTestbotPrompt(opts) {
|
|
|
77
111
|
// SKYR-4023: user-facing report language. English display names improve
|
|
78
112
|
// instruction-following vs bare codes; Intl.DisplayNames (Node >=18) names any
|
|
79
113
|
// code correctly, falling back to the raw code only if it can't be resolved.
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
114
|
+
// SKYR-4185: rendering the prompt is the single capture point for the
|
|
115
|
+
// report language — the prompt registration AND the testbot resource both
|
|
116
|
+
// come through here, so skyramp_submit_report's language guardrail sees it
|
|
117
|
+
// regardless of which surface served the prompt (no dual wiring to forget).
|
|
118
|
+
// Set on EVERY render, not just non-English ones: last render wins, so an
|
|
119
|
+
// en/argless render disarms a language captured earlier in a long-lived
|
|
120
|
+
// server process instead of falsely rejecting an English report.
|
|
121
|
+
setReportLanguage(language && language !== "en" ? language : undefined);
|
|
122
|
+
let reportLanguageBlock = "";
|
|
123
|
+
if (language && language !== "en") {
|
|
124
|
+
const reportLanguageName = new Intl.DisplayNames(["en"], { type: "language" }).of(language) ??
|
|
125
|
+
language;
|
|
83
126
|
reportLanguageBlock = `**Report language: ${reportLanguageName}.** Write ALL user-facing free-text report fields in ${reportLanguageName}: \`businessCaseAnalysis\`, every \`description\` and \`reasoning\`, \`testResults[].details\`, \`issuesFound[].description\`, \`nextSteps[]\`, test-maintenance \`beforeDetails\`/\`afterDetails\`, and \`commitMessage\`. Do NOT translate: code identifiers, endpoint paths, file names, test IDs, \`scenarioName\`, enum values (\`Pass\`/\`Fail\`/\`Skipped\`, severity values, \`testType\`), or anything inside backticks.
|
|
84
127
|
|
|
85
128
|
`;
|
|
@@ -134,7 +177,8 @@ Use those recommendations as your baseline. Only add or remove tests that the us
|
|
|
134
177
|
|
|
135
178
|
**If \`skyramp_analyze_changes\` returns an error:** retry once only if the error is transient (timeout, network blip, temporary unavailability) — do NOT retry for permanent errors (invalid repository path, missing required parameter, authentication failure). If it fails again, call \`skyramp_submit_report\` with a minimal valid payload: leave all test arrays empty and add the error to \`issuesFound\`. Refer to the \`skyramp_submit_report\` schema for required fields. Do NOT attempt Task 2 without a valid stateFile.
|
|
136
179
|
**If all changed files are non-application** (CI/CD, docs, lock files, config) → skip to Task 3 (Submit Report) with empty arrays. Put the one-paragraph summary in \`businessCaseAnalysis\` (always populated; that's where end-state narration belongs); leave \`issuesFound\` empty — a non-application diff is not an issue. Example narration for a Testbot onboarding PR (\`.github/workflows/skyramp-testbot.yml\` and/or files under \`.skyramp/\`): "This PR adds Skyramp Testbot GitHub Actions workflow configuration to enable automated test generation on every pull request. It also adds System Under Test (SUT) setup files under \`.skyramp/sut/\` required for the testbot workflow, to bring up services for testing. It contains no application code changes and has no testable behavioral surface."
|
|
137
|
-
${hasRelatedRepos
|
|
180
|
+
${hasRelatedRepos
|
|
181
|
+
? `
|
|
138
182
|
**MULTI-REPO CONTEXT (MANDATORY).** This run includes ${relatedRepositories.length} related ${relatedRepositories.length === 1 ? "repository" : "repositories"} listed in the \`<related_repositories>\` block below, each with an explicit \`repository\` (\`owner/repo\`), \`path\`, and \`base_branch\`. Use the \`repository\` value verbatim — do NOT infer it from git remotes or paths. You MUST analyze EACH related repository — exactly one \`skyramp_analyze_changes\` call per listed repo (${relatedRepositories.length} ${relatedRepositories.length === 1 ? "call" : "calls"}), in addition to the primary call in step 2.
|
|
139
183
|
|
|
140
184
|
**Run the primary call (step 2) FIRST, then the related repos in listed order — not in parallel.** All calls in this run automatically share ONE run-scoped state file — you do NOT pass a state-file path; setting \`repository\` is enough. The primary writes its root section; each related repo's call upserts its own section into that same file. Concurrent calls would race on the shared file, so they must be sequential. For each related repo:
|
|
@@ -153,7 +197,8 @@ ${hasRelatedRepos ? `
|
|
|
153
197
|
Within a type, higher score wins regardless of which repo it came from. Everything not selected becomes an ADDITIONAL recommendation. This guarantees a frontend-only primary repo cannot starve a related backend repo's contract/integration tests of GENERATE slots (and vice versa). When you generate a test for a related repo's endpoint:
|
|
154
198
|
- **Execute it only if that repo's service is already running and reachable.** The workflow's setup may have started multiple services; before generating an API test for a related repo, confirm its \`base_url\` (from that repo's workspace/Execution Plan) responds. If the service is unreachable, still GENERATE the test but mark its \`testResults\` status as \`Skipped\` with details "service not running in this run" — do NOT count an unreachable service as a failure.
|
|
155
199
|
- **Write the test file into that service's own \`testDirectory\`** — the one declared for the service in the unified workspace.yml (the related repo's services were registered there in step 1(a), each with its \`repository\`). The \`testDirectory\` is interpreted relative to the **single delivery root** (the configured test repo if set, otherwise the primary repo), so all generated tests are delivered together by the existing single-target delivery. Do NOT invent a per-source-repo subdirectory, and do NOT write into the related repo's own checkout — it is read-only context. (If two repos happen to declare the same \`testDirectory\`, their files coexist there; the \`repository\` field on each report item — below — is what attributes ownership, not the path.)
|
|
156
|
-
- **Set the \`repository\` field** (\`owner/repo\`) on every such \`newTestsCreated\` / \`testResults\` item so the report attributes it to the originating repo (see Report Guidelines).`
|
|
200
|
+
- **Set the \`repository\` field** (\`owner/repo\`) on every such \`newTestsCreated\` / \`testResults\` item so the report attributes it to the originating repo (see Report Guidelines).`
|
|
201
|
+
: ""}
|
|
157
202
|
|
|
158
203
|
2. **Maintain existing tests:**
|
|
159
204
|
|
|
@@ -169,7 +214,7 @@ ${maintenanceBeforeExecStep}
|
|
|
169
214
|
|
|
170
215
|
f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) and record each file's result as its \`afterStatus\`. Editing an \`[external]\` file that step 2(a) confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
|
|
171
216
|
|
|
172
|
-
3. **Code review:**
|
|
217
|
+
3. **Code review:** Find the logic bugs in the code that this change touches. Read the implementation of each changed endpoint: the route handler, and the functions that it calls to read or write data. For a changed screen, read the component and the functions that it calls. Read these files even when the diff does not contain them — a defect often sits in the code that the change depends on. Report each finding in \`issuesFound\` with a severity, and say which file and line holds it. Common patterns to flag:
|
|
173
218
|
- Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)
|
|
174
219
|
- Incomplete CRUD: create without cleanup, update that adds new records without removing old ones
|
|
175
220
|
- Missing input validation on new endpoints
|
|
@@ -186,8 +231,6 @@ ${maintenanceBeforeExecStep}
|
|
|
186
231
|
"role": "button",
|
|
187
232
|
"accessibleName": "Save changes",
|
|
188
233
|
"testId": "save-changes-btn",
|
|
189
|
-
"stableId": null,
|
|
190
|
-
"contextText": null,
|
|
191
234
|
"mutability": "mutable",
|
|
192
235
|
"widgetType": "native"
|
|
193
236
|
}
|
|
@@ -195,10 +238,10 @@ ${maintenanceBeforeExecStep}
|
|
|
195
238
|
\`\`\`
|
|
196
239
|
- **One element when the test has a single dominant target** (a click, a type, a single visibility check). Most tests fall here — use a length-1 array.
|
|
197
240
|
- **Multiple elements when the test verifies several elements together** — render-state tests (heading + input + button on a form), workflow tests (click button A, assert state appears in element B), or form-fill tests (multiple inputs + submit button). Each element is its own array entry.
|
|
198
|
-
- Each element's fields
|
|
241
|
+
- Each element's fields come from a captured blueprint element. Copy the values that are present, and never invent one. \`role\` and \`accessibleName\` are always in the capture; the capture omits \`testId\`/\`stableId\`/\`contextText\` when the element has no such value, so omit them too.
|
|
199
242
|
- \`mutability\` — copy from \`blueprint.element.mutability\`. \`'mutable'\` = behavioral-test target; \`'immutable'\` = smoke target.
|
|
200
243
|
- \`widgetType\` — copy from \`blueprint.element.widgetType\`. \`'custom'\` = JavaScript-composite control (Radix, MUI, etc.) requiring click-to-open interaction; \`'native'\` = standard HTML element.
|
|
201
|
-
- \`contextText\` — only for elements inside repeating sections (table rows, list items). Lift from \`repeatingElement.items[].contextText\`.
|
|
244
|
+
- \`contextText\` — only for elements inside repeating sections (table rows, list items). Lift from \`repeatingElement.items[].contextText\`. Omit it otherwise.
|
|
202
245
|
|
|
203
246
|
**Field 2 — \`pageContext\`** (where the test runs):
|
|
204
247
|
\`\`\`json
|
|
@@ -350,19 +393,21 @@ ${maintenanceBeforeExecStep}
|
|
|
350
393
|
|
|
351
394
|
**No upstream captures available?** Set \`targetElements\` to \`null\`, omit \`pageContext\`, and prefix \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Use page/feature-level prose; don't cite specific element names without grounding. Apply the marker per entry, not per PR — affected recs only. Log capture failures in \`issuesFound\` (one info-severity entry per failure mode, naming counts). Don't pre-emptively fall back without attempting capture first. Non-UI work is unaffected.
|
|
352
395
|
`;
|
|
353
|
-
const serviceContext = services?.length ? buildServiceContext(services) :
|
|
396
|
+
const serviceContext = services?.length ? buildServiceContext(services) : "";
|
|
354
397
|
// The <ui-credentials> tags are framing for the agent's prompt context —
|
|
355
398
|
// not real XML — so credentials pass through verbatim and the agent can
|
|
356
399
|
// type them directly into login fields. Reject the only string that would
|
|
357
400
|
// break the framing: a credential containing the closing tag itself.
|
|
358
401
|
const trimmedCredentials = uiCredentials?.trim();
|
|
359
|
-
if (trimmedCredentials && trimmedCredentials.includes(
|
|
402
|
+
if (trimmedCredentials && trimmedCredentials.includes("</ui-credentials>")) {
|
|
360
403
|
throw new Error("uiCredentials must not contain '</ui-credentials>'");
|
|
361
404
|
}
|
|
362
405
|
const uiCredentialsBlock = trimmedCredentials
|
|
363
406
|
? `<ui-credentials>\n${trimmedCredentials}\n</ui-credentials>`
|
|
364
|
-
:
|
|
365
|
-
const testsRepoDirBlock = testsRepoDir
|
|
407
|
+
: "";
|
|
408
|
+
const testsRepoDirBlock = testsRepoDir
|
|
409
|
+
? `<TESTS REPO DIR>${testsRepoDir}</TESTS REPO DIR>\n`
|
|
410
|
+
: "";
|
|
366
411
|
// Multi-repo context block. Each entry's `repositoryPath` is checked out at its
|
|
367
412
|
// FEATURE ref; `baseBranch` is that repo's default branch (the diff base), so
|
|
368
413
|
// skyramp_analyze_changes computes a real default…feature diff for the related repo.
|
|
@@ -373,7 +418,7 @@ ${maintenanceBeforeExecStep}
|
|
|
373
418
|
.map((r) => ` <repository repository="${r.repo}" path="${r.repositoryPath}" base_branch="${r.baseBranch || "auto-detect"}" />`)
|
|
374
419
|
.join("\n") +
|
|
375
420
|
`\n</related_repositories>\n`
|
|
376
|
-
:
|
|
421
|
+
: "";
|
|
377
422
|
const testDirInstruction = testsRepoDir
|
|
378
423
|
? `the \`<output_dir>\` from the \`<services>\` block, rooted under the test repository at \`${testsRepoDir}\` (i.e. \`${testsRepoDir}/<output_dir>\`). Write ALL test output files to paths under \`${testsRepoDir}\`, not under \`${repositoryPath}\`. Do NOT write any test files to the app repository.`
|
|
379
424
|
: `${SERVICE_REFS.testDirRef}. Do NOT create a new \`tests/\` directory at the repo root — use that path. If no \`testDirectory\` is configured, default to the language-conventional location (e.g. \`src/test/java/...\` for Java, \`tests/\` for Python).`;
|
|
@@ -408,7 +453,7 @@ This is a plan-only evaluation run: the application under test is NOT running, a
|
|
|
408
453
|
|
|
409
454
|
${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
|
|
410
455
|
|
|
411
|
-
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list. (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
456
|
+
- **MANDATORY — use the plan returned by \`skyramp_register_test_plan\` as-is**: Before generating anything, call \`skyramp_register_test_plan\` (\`stateFile\` required) with your complete candidate list — every test you would generate OR recommend, including the Execution Plan's own pre-ranked GENERATE/ADDITIONAL items and any candidate you drafted yourself, with a \`discriminator\` claim \`{kind, changedCodeAnchor}\` for candidates probing changed logic. Its returned GENERATE list — not the Execution Plan's raw pre-ranked GENERATE section — governs ADD actions from this point on. You MUST generate exactly those scenarios in the exact order listed, keeping each item's \`scenarioName\` exactly as registered — the generation tools match on it and reject renamed or substituted scenarios. If parameter grounding uncovers a distinct bug-catching scenario not already registered, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Register the pooled, type-distributed selection instead of any single repo's GENERATE list — call \`skyramp_register_test_plan\` ONCE for the whole run, with candidates from EVERY repo pooled into one list, including at least one UI candidate when any repo changed frontend files (the tool rejects a pooled registration without one). (In single-repo runs, register the GENERATE list exactly as-is.)` : ""}
|
|
412
457
|
- **Do not fabricate tests outside the GENERATE list provided by \`skyramp_analyze_changes\`.** Changes that only modify, delete, or add fields to an EXISTING covered endpoint or component are maintenance: handle them in ${taskRef(TASK_ANALYZE_MAINTAIN)} by UPDATE/DELETE of the existing test, never by creating a new spec. If the GENERATE list is empty, create zero new tests and proceed to ${taskRef(TASK_SUBMIT)}.
|
|
413
458
|
- Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
|
|
414
459
|
- Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
|
|
@@ -460,7 +505,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
460
505
|
4. Only pass \`authHeader: ""\` if you can confirm the endpoint is truly unauthenticated.
|
|
461
506
|
|
|
462
507
|
**How to generate each type (for ADD):**
|
|
463
|
-
- **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file.
|
|
508
|
+
- **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file${INTEGRATION_CODE_REUSE_GEN_CLAUSE}.
|
|
464
509
|
**Use the pre-built scenario JSON from the Execution Plan** — pass the steps array directly. Do NOT read source code models to construct request bodies if the plan already provides them.
|
|
465
510
|
Scenario JSON and test files go in ${testDirInstruction}
|
|
466
511
|
**Pipeline for speed**: Call ALL \`skyramp_batch_scenario_test_generation\` calls in one batch. When they return, call ALL \`skyramp_integration_test_generation\` calls in the next batch. Do NOT serialize per-scenario (batch→integration→batch→integration) — batch ALL scenarios first, then generate ALL integration tests.
|
|
@@ -478,7 +523,7 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
478
523
|
- Legacy format: \`username:password\` — the first \`:\` splits username from password.
|
|
479
524
|
These are format hints, not a strict grammar — apply judgment on ambiguous input (e.g. a \`=\` or \`;\` inside a value of the key=value form: split on the \`;\` that precedes a plausible login-field key).
|
|
480
525
|
|
|
481
|
-
**Credential selection**:
|
|
526
|
+
**Credential selection**: When several credentials are provided, reason carefully about which one each test case needs BEFORE logging in. Multiple credentials exist so tests can exercise the app as different identities — an app with authorization levels behaves differently per account, and a test only has value when it runs as the identity it is about: an admin workflow needs the admin account, a permission-boundary test needs the restricted one, a plain user flow needs an ordinary user. Read what each credential says about itself — a labeling field (\`role\`, or whatever the customer named it: \`accessLevel\`, \`permissionLevel\`, a team/tenant name, …), the username itself, any extra fields — and match that against the test case's intent. When nothing about a test case calls for a specific identity, use the first credential. If the identity a test case needs is not among the credentials, use the closest match and add a note to \`issuesFound\` naming the identity that was missing. NEVER mix fields across credential lines — type the username, password, and every extra field from the SAME line. The exact values you type identify which credential the generated test will read from the environment at replay time, so a mixed or altered value breaks that binding.
|
|
482
527
|
|
|
483
528
|
Type all values verbatim. Before navigating to ANY feature URL:
|
|
484
529
|
1. \`browser_navigate\` to the login URL (e.g. \`{baseUrl}/login\`, \`/user/login\`, \`/signin\` — infer from the app's base URL and framework)
|
|
@@ -567,7 +612,7 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
567
612
|
|
|
568
613
|
**The Blueprint Citation Invariant applies during recording too.** Every assertion you emit cites element names — those names must come from blueprint captures, not invention. For N user-intent-level actions, the reference target is N+1 \`browser_blueprint\` calls (the first returns full, the rest return deltas). Traces that follow the pattern produce assertions grounded in observable state changes; traces that skip captures fall back to author-inferred assertions and risk citing names that don't exist in the rendered DOM.
|
|
569
614
|
|
|
570
|
-
The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), then
|
|
615
|
+
The rest of the UI workflow stays the same: trace plan, browser auth, navigation, export (\`skyramp_export_zip\`), generation (\`skyramp_ui_test_generation\`), then the post-calls the generation result lists (${UI_UTILS_REUSE ? "`skyramp_enhance_assertions`, `skyramp_modularization`, `skyramp_reuse_code`, in that order" : "`skyramp_reuse_code` (when `codeReuse: true`) and `skyramp_enhance_assertions`"}). Capture-act-capture adds blueprint captures alongside the existing steps; it doesn't replace anything.
|
|
571
616
|
- **E2E**: Only if BOTH a backend trace \`.json\` AND a Playwright \`.zip\` already exist in the repo. Without both, move to \`additionalRecommendations\`.
|
|
572
617
|
- Skip smoke tests entirely.
|
|
573
618
|
|
|
@@ -585,8 +630,14 @@ If a test **generation** tool call fails:
|
|
|
585
630
|
If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated test:
|
|
586
631
|
1. Read the error output to diagnose the root cause (4xx on prereq step, assertion mismatch, floating-point precision, 500 from app bug, timeout, etc.).
|
|
587
632
|
2. **Expected failure check (no retry):** If the failure is an assertion error or HTTP error that matches the issue identified in the code analysis (e.g. the test was generated specifically to document a broken endpoint, a UI rendering bug, or a missing validation), then this is the **intended outcome** — the test is correctly catching the real bug. Report it immediately as \`status: "Fail"\` and move on. Do NOT retry.
|
|
633
|
+
|
|
634
|
+
This path also covers an assertion failure that application behavior outside this PR's diff explains — for example child records that survive the deletion of their parent, state inherited when an ID is recycled or reused, or a value that ignores a status the test set. Before you keep such a test red, confirm the cause in the source: read the handler, model, or query that should have done the work, and find the specific operation that is missing or wrong. If you find it, report the test as \`status: "Fail"\` and add an \`issuesFound\` entry for it. Do NOT retry.
|
|
635
|
+
|
|
636
|
+
If you cannot point at the missing or wrong line in application code, the app is not at fault — shared or seeded data, parallel test workers, or setup the test itself never did explain the collision. Treat that as an infrastructure failure: fix it and retry once as in step 3.
|
|
637
|
+
|
|
638
|
+
**If you did confirm the missing or wrong operation in the source, do NOT make the test pass.** Never add a reset, cleanup, or setup call for isolation. Never weaken the assertion — no \`==\` to \`>=\`, no exact value to a range. A failing test whose diagnosis names a pre-existing bug is the most valuable output of this run; a passing version of it reports nothing.
|
|
588
639
|
3. Apply a targeted fix and retry **once** only for **infrastructure failures** — that means exactly **2 total \`skyramp_execute_test\` calls per test file** for these cases. Examples of infrastructure failures worth fixing:
|
|
589
|
-
- Assertion mismatch
|
|
640
|
+
- Assertion mismatch from floating-point precision, or an expected value mis-transcribed from the observed response or computed with an arithmetic slip. If application behavior outside the diff explains the mismatch, it is not an infrastructure failure — use step 2 instead.
|
|
590
641
|
- Import error, syntax error, or missing dependency in the generated test file
|
|
591
642
|
- Connection refused or timeout unrelated to the app under test
|
|
592
643
|
4. If it still fails after the retry, report it as \`status: "Fail"\` with the error details and move on — do NOT edit and re-run a third time. A failing test that documents a real bug is a valid outcome.
|
|
@@ -597,13 +648,13 @@ If a generated UI test fails with a timeout waiting for an element after navigat
|
|
|
597
648
|
2. Add \`await page.locator('[data-testid="some-element"]').waitFor({ state: 'visible', timeout: 10000 });\` for the specific element the test needs.
|
|
598
649
|
Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than once — if the test still fails after this fix, report it as "Fail".
|
|
599
650
|
|
|
600
|
-
**After generation, you MUST do exactly these steps — nothing more, nothing less
|
|
651
|
+
**After generation, you MUST do exactly these steps — nothing more, nothing less** (generation results may add their own CRITICAL NEXT STEPS — e.g. modularize-then-reuse for integration or UI tests generated with \`codeReuse: true\` — follow those too, in the order the result states):
|
|
601
652
|
1. **[MANDATORY] After \`skyramp_integration_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated integration test file, \`testType: "integration"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
|
|
602
653
|
2. **[MANDATORY] After \`skyramp_contract_test_generation\` with \`providerMode\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
|
|
603
654
|
${UI_CODE_REUSE_STEP}
|
|
604
|
-
4. **[MANDATORY] After \`skyramp_ui_test_generation\`**:
|
|
605
|
-
5. **Wait**: Do NOT proceed to test execution until steps 1–4 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
|
|
606
|
-
Do not make any changes other than the code-reuse refactoring (step 3) and the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
|
|
655
|
+
4. **[MANDATORY] After \`skyramp_ui_test_generation\`**: ${UI_ENHANCE_STEP} The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
|
|
656
|
+
5. **Wait**: Do NOT proceed to test execution until steps 1–4 (plus any generation-result CRITICAL NEXT STEPS) are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
|
|
657
|
+
Do not make any changes other than the code-reuse refactoring (step 3 and the generation-result reuse steps) and the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
|
|
607
658
|
|
|
608
659
|
**Execution timing:**
|
|
609
660
|
- **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
|
|
@@ -611,11 +662,13 @@ Do not make any changes other than the code-reuse refactoring (step 3) and the a
|
|
|
611
662
|
- Only report test results for files you actually ran.
|
|
612
663
|
**Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.`;
|
|
613
664
|
}
|
|
614
|
-
const primaryRepoBlock = primaryRepo
|
|
665
|
+
const primaryRepoBlock = primaryRepo
|
|
666
|
+
? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n`
|
|
667
|
+
: "";
|
|
615
668
|
return `<TITLE>${prTitle}</TITLE>
|
|
616
669
|
<DESCRIPTION>${prDescription}</DESCRIPTION>
|
|
617
670
|
${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
|
|
618
|
-
${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext +
|
|
671
|
+
${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + "\n" : ""}${uiCredentialsBlock ? uiCredentialsBlock + "\n" : ""}## Goal
|
|
619
672
|
|
|
620
673
|
Every test this run delivers must be a usable functional test — one that exercises the running application through its real API or UI surface and that the user can keep running in CI. Optimize for catching real production bugs: business-rule and computed-field errors, data-integrity violations, security-boundary bypasses, broken user journeys. A test that would FAIL if the application's logic were wrong beats several that merely exercise new surface — prefer fewer, higher-signal tests over padded coverage. The tasks below define which tests are in scope for this run; use the Skyramp MCP server tools for all of them.
|
|
621
674
|
|
|
@@ -645,13 +698,15 @@ ${task3CountRule}
|
|
|
645
698
|
|
|
646
699
|
${reportLanguageBlock}Call \`skyramp_submit_report\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking, and the report is written beside it. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
647
700
|
|
|
648
|
-
${hasRelatedRepos
|
|
701
|
+
${hasRelatedRepos
|
|
702
|
+
? `
|
|
649
703
|
- **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
|
|
650
|
-
`
|
|
704
|
+
`
|
|
705
|
+
: ""}
|
|
651
706
|
- **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
|
|
652
707
|
- For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
|
|
653
708
|
|
|
654
|
-
${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false })}`;
|
|
709
|
+
${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false, modularizeViaGenerationResult: UI_UTILS_REUSE })}`;
|
|
655
710
|
// Neither path reaches the agent any more: SKYR-4147 made the report derive from the
|
|
656
711
|
// state file's directory, and that directory comes from the environment via
|
|
657
712
|
// runArtifactDir(). Keep it that way — a path the model retypes out of this prose is a
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { TestType } from "../../types/TestTypes.js";
|
|
2
2
|
import { inferScenarioType, prioritizeAttackSurfaceBundles } from "../diversity.js";
|
|
3
|
-
import {
|
|
3
|
+
import { externalDedupKeys, scenarioCoverageKeys, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
|
|
4
4
|
import { logger } from "../../utils/logger.js";
|
|
5
5
|
/**
|
|
6
6
|
* Backend GENERATE slot count:
|
|
@@ -20,25 +20,39 @@ export function reservedUISlots(ctx) {
|
|
|
20
20
|
return ctx.hasFrontendChanges && ctx.maxGenerate > 0 ? 1 : 0;
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
23
|
+
* Split candidates by whether an external test already covers them. Protected
|
|
24
|
+
* `bug_caught` / attack-surface scenarios always count as uncovered — they
|
|
25
|
+
* require semantic flaw coverage the external test may not provide.
|
|
26
|
+
*
|
|
27
|
+
* A candidate is covered only when the set holds EVERY one of its keys. One
|
|
28
|
+
* match is not coverage: a scenario exercising two changed endpoints would lose
|
|
29
|
+
* the endpoint the external test does not reach (SKYR-4214). An empty key list
|
|
30
|
+
* carries no information, so it never removes anything.
|
|
31
|
+
*
|
|
32
|
+
* The covered group is held back, not discarded. `runBudget` moves it into
|
|
33
|
+
* ADDITIONAL when GENERATE's backend half would otherwise be empty (SKYR-4024),
|
|
34
|
+
* and records it as dropped, with a reason, when it does not.
|
|
26
35
|
*/
|
|
27
|
-
function
|
|
36
|
+
function partitionByExternalCoverage(ranked, externalCoverage) {
|
|
28
37
|
if (externalCoverage.size === 0)
|
|
29
|
-
return ranked;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
logger.info(`External dedup: skipping "${item.scenario.scenarioName}" (${key}) — covered by external test`);
|
|
38
|
-
return false;
|
|
38
|
+
return { uncovered: ranked, covered: [] };
|
|
39
|
+
const uncovered = [];
|
|
40
|
+
const covered = [];
|
|
41
|
+
for (const item of ranked) {
|
|
42
|
+
const keys = externalDedupKeys(item.scenario);
|
|
43
|
+
if (keys.length === 0 || !keys.every((key) => externalCoverage.has(key))) {
|
|
44
|
+
uncovered.push(item);
|
|
45
|
+
continue;
|
|
39
46
|
}
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
|
|
48
|
+
logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${keys.join(", ")}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
|
|
49
|
+
uncovered.push(item);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
logger.info(`External dedup: "${item.scenario.scenarioName}" (${keys.join(", ")}) covered by an external test — eligible for the reserve`);
|
|
53
|
+
covered.push({ item, keys });
|
|
54
|
+
}
|
|
55
|
+
return { uncovered, covered };
|
|
42
56
|
}
|
|
43
57
|
/**
|
|
44
58
|
* Shared budgeting pipeline. All Budgeters run the same external-dedup,
|
|
@@ -49,10 +63,11 @@ function applyExternalDedup(ranked, externalCoverage) {
|
|
|
49
63
|
* diffExecutionPlan.ts exactly.
|
|
50
64
|
*/
|
|
51
65
|
export function runBudget(ranked, ctx, pick) {
|
|
66
|
+
const dropped = [];
|
|
52
67
|
const backend = backendGenerateCount(ctx);
|
|
53
68
|
const uiSlots = reservedUISlots(ctx);
|
|
54
|
-
const
|
|
55
|
-
const slotOrdered = prioritizeAttackSurfaceBundles(
|
|
69
|
+
const { uncovered, covered } = partitionByExternalCoverage(ranked, ctx.externalCoverage);
|
|
70
|
+
const slotOrdered = prioritizeAttackSurfaceBundles(uncovered);
|
|
56
71
|
// When UI slots are reserved, keep UI candidates out of the backend pick so the
|
|
57
72
|
// reserved slot can always be filled below. Otherwise a high-ranked (e.g.
|
|
58
73
|
// CRITICAL) UI candidate could be consumed by the backend pick on a mixed PR,
|
|
@@ -77,14 +92,77 @@ export function runBudget(ranked, ctx, pick) {
|
|
|
77
92
|
chosen.add(item);
|
|
78
93
|
}
|
|
79
94
|
const generate = [...backendGenerate, ...uiGenerate];
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
95
|
+
// SKYR-4024. A covered candidate is one an existing test already reaches, so
|
|
96
|
+
// generating a second test for it is wrong — maintenance updates the existing
|
|
97
|
+
// one. But dropping it outright removed it from the RECOMMENDATIONS too, and an
|
|
98
|
+
// empty GENERATE is authoritative downstream. The reserve surfaces it as
|
|
99
|
+
// ADDITIONAL instead. The emptiness test reads the backend half: on a mixed PR
|
|
100
|
+
// one UI candidate fills the reserved slot while every covered backend
|
|
101
|
+
// candidate would otherwise vanish.
|
|
102
|
+
//
|
|
103
|
+
// Two conditions keep the reserve shut where recommending nothing is right.
|
|
104
|
+
// Backend slots must exist, so a UI-only PR is left alone. And the PR must not
|
|
105
|
+
// touch a test file of its own: coverage is read from the working tree, and a
|
|
106
|
+
// test file without the Skyramp marker counts as external, so a PR that adds
|
|
107
|
+
// its own tests would have that coverage used as grounds to recommend
|
|
108
|
+
// duplicates of it.
|
|
109
|
+
const spendReserve = backendGenerate.length === 0 && backend > 0 && covered.length > 0 && !ctx.diffChangesTestFiles;
|
|
110
|
+
if (spendReserve) {
|
|
111
|
+
logger.info(`External dedup: GENERATE empty after dedup — recommending ${covered.length} held-back candidate(s) as ADDITIONAL rather than approving an empty plan`);
|
|
112
|
+
}
|
|
113
|
+
else if (covered.length > 0) {
|
|
114
|
+
// SKYR-4214. The reserve stayed shut, so these really are dropped — say so,
|
|
115
|
+
// with the reason, instead of leaving the agent an unexplained short list.
|
|
116
|
+
const reason = backendGenerate.length > 0
|
|
117
|
+
? `the backend half of GENERATE is filled (${backendGenerate.length})`
|
|
118
|
+
: backend === 0
|
|
119
|
+
? "no backend slots (UI-only PR)"
|
|
120
|
+
: "the PR changes test files of its own";
|
|
121
|
+
logger.info(`External dedup: dropping ${covered.length} covered candidate(s) — reserve shut because ${reason}`);
|
|
122
|
+
for (const { item, keys } of covered) {
|
|
123
|
+
dropped.push({
|
|
124
|
+
candidateId: item.candidateId,
|
|
125
|
+
reason: `covered by an external test (${keys.join(", ")})`,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
// ADDITIONAL = everything not chosen for GENERATE, in rank order, minus any
|
|
130
|
+
// whose subject endpoints are ALL already covered by GENERATE, then capped at
|
|
131
|
+
// the remaining budget. All, not any: an item that also covers an endpoint no
|
|
132
|
+
// GENERATE item reaches is not a duplicate. An empty key list never removes
|
|
133
|
+
// an item.
|
|
134
|
+
//
|
|
135
|
+
// The duplicate filter runs BEFORE the cap. In the other order a duplicate
|
|
136
|
+
// took a slot and then vacated it, and nothing moved up: eval run
|
|
137
|
+
// 32543498147, fixture cc15-org-reviewer-role, filled 19 of a maxTotal of 20
|
|
138
|
+
// while 16 candidates carried the reason "beyond the maxTotal budget of 20".
|
|
139
|
+
// A candidate that is both a duplicate and past the cap now reports the
|
|
140
|
+
// duplicate — the stronger reason, and the only one that does not depend on
|
|
141
|
+
// where the cap happened to fall.
|
|
142
|
+
const additionalSource = spendReserve
|
|
143
|
+
? [...slotOrdered, ...prioritizeAttackSurfaceBundles(covered.map((c) => c.item))]
|
|
144
|
+
: slotOrdered;
|
|
145
|
+
const notChosen = additionalSource.filter((it) => !chosen.has(it));
|
|
146
|
+
const remainingBudget = Math.max(0, ctx.maxTotal - generate.length);
|
|
147
|
+
const generatedCoverage = new Set(generate.flatMap((item) => scenarioCoverageKeys(item.scenario)));
|
|
148
|
+
const notDuplicate = notChosen.filter((item) => {
|
|
149
|
+
const keys = scenarioCoverageKeys(item.scenario);
|
|
150
|
+
if (keys.length === 0 || !keys.every((key) => generatedCoverage.has(key)))
|
|
151
|
+
return true;
|
|
152
|
+
dropped.push({
|
|
153
|
+
candidateId: item.candidateId,
|
|
154
|
+
reason: `covered by GENERATE (${keys.join(", ")})`,
|
|
155
|
+
});
|
|
156
|
+
return false;
|
|
157
|
+
});
|
|
158
|
+
const additional = notDuplicate.slice(0, remainingBudget);
|
|
159
|
+
for (const item of notDuplicate.slice(remainingBudget)) {
|
|
160
|
+
dropped.push({
|
|
161
|
+
candidateId: item.candidateId,
|
|
162
|
+
reason: `beyond the maxTotal budget of ${ctx.maxTotal}`,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
87
165
|
// Budgeting itself produces no demotions; the register-plan selection stage
|
|
88
166
|
// (planRanker.selectPlan) fills this channel from discriminator verification.
|
|
89
|
-
return { generate, additional, reservedUISlots: uiSlots, demotions: [] };
|
|
167
|
+
return { generate, additional, reservedUISlots: uiSlots, demotions: [], dropped };
|
|
90
168
|
}
|
|
@@ -281,9 +281,20 @@ function isNegativeAssertStep(step) {
|
|
|
281
281
|
if (step?.expectedStatusCode === 404)
|
|
282
282
|
return true;
|
|
283
283
|
const assertions = [...toStringArray(step?.expectedResponseFields), ...toStringArray(step?.bodyMustInclude)];
|
|
284
|
-
return assertsEmptyOrZero(assertions);
|
|
284
|
+
return assertsEmptyOrZero(assertions, step?.responseBody);
|
|
285
285
|
}
|
|
286
|
-
|
|
286
|
+
/**
|
|
287
|
+
* True when the step asserts an empty or zero result, reading the assertion
|
|
288
|
+
* strings AND `responseBody`. An EMPTY ARRAY body is itself the "no rows came
|
|
289
|
+
* back" assertion, and can be the only empty-result signal a step carries.
|
|
290
|
+
*
|
|
291
|
+
* `{}` deliberately does NOT count: an unspecified body is filled in as `{}`
|
|
292
|
+
* (ScenarioGenerationService), so it cannot be told apart from a body the agent
|
|
293
|
+
* never wrote. `isAssertStep` above already reads `{}` as asserting nothing.
|
|
294
|
+
*/
|
|
295
|
+
function assertsEmptyOrZero(strings, responseBody) {
|
|
296
|
+
if (Array.isArray(responseBody) && responseBody.length === 0)
|
|
297
|
+
return true;
|
|
287
298
|
return strings.some((raw) => {
|
|
288
299
|
const s = raw.toLowerCase();
|
|
289
300
|
if (/\bcount\b/.test(s) && /\b0\b/.test(s))
|
|
@@ -4,9 +4,9 @@ import { ScenarioCategory } from "../types/TestRecommendation.js";
|
|
|
4
4
|
export interface RankOptions {
|
|
5
5
|
/**
|
|
6
6
|
* Categories that take the top carve-out tier ahead of everything else.
|
|
7
|
-
* Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (
|
|
8
|
-
*
|
|
9
|
-
* the agent's priority tag as a ranking input.
|
|
7
|
+
* Defaults to the `CATEGORY_PRIORITY === "CRITICAL"` categories (bug_caught only —
|
|
8
|
+
* new_endpoint is MEDIUM, not carved out). Exposed so phase 2 can tune the carve-out
|
|
9
|
+
* WITHOUT reintroducing the agent's priority tag as a ranking input.
|
|
10
10
|
*/
|
|
11
11
|
carveOutCategories?: ScenarioCategory[];
|
|
12
12
|
/**
|
|
@@ -32,9 +32,9 @@ export interface SelectPlanContext extends BudgetContext {
|
|
|
32
32
|
* independent of input order (the final tiebreak is the stable `candidateId`).
|
|
33
33
|
*
|
|
34
34
|
* Ordering (highest first):
|
|
35
|
-
* 1. Carve-out —
|
|
36
|
-
*
|
|
37
|
-
* `
|
|
35
|
+
* 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
|
|
36
|
+
* protected-first convention of `roundRobinByType` /
|
|
37
|
+
* `prioritizeAttackSurfaceBundles`.
|
|
38
38
|
* 2. Verified discriminators — candidates whose declared discriminator survived
|
|
39
39
|
* `validateDiscriminator` (marked via `verifiedDiscriminator`) float ahead of
|
|
40
40
|
* unverified peers in the same tier.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CATEGORY_PRIORITY, PriorityTier } from "../types/TestRecommendation.js";
|
|
2
2
|
import { diversityBalancedBudgeter } from "./budgeters/diversityBalancedBudgeter.js";
|
|
3
|
-
import {
|
|
3
|
+
import { collectChangedRouteLines, findStepOnChangedRoute } from "../utils/changedRoutes.js";
|
|
4
4
|
const PRIORITY_RANK = {
|
|
5
5
|
CRITICAL: 0,
|
|
6
6
|
HIGH: 1,
|
|
@@ -14,9 +14,9 @@ const DEFAULT_CARVE_OUT_CATEGORIES = Object.keys(CATEGORY_PRIORITY).filter((cate
|
|
|
14
14
|
* independent of input order (the final tiebreak is the stable `candidateId`).
|
|
15
15
|
*
|
|
16
16
|
* Ordering (highest first):
|
|
17
|
-
* 1. Carve-out —
|
|
18
|
-
*
|
|
19
|
-
* `
|
|
17
|
+
* 1. Carve-out — CRITICAL-category scenarios (bug_caught), preserving the
|
|
18
|
+
* protected-first convention of `roundRobinByType` /
|
|
19
|
+
* `prioritizeAttackSurfaceBundles`.
|
|
20
20
|
* 2. Verified discriminators — candidates whose declared discriminator survived
|
|
21
21
|
* `validateDiscriminator` (marked via `verifiedDiscriminator`) float ahead of
|
|
22
22
|
* unverified peers in the same tier.
|
|
@@ -77,62 +77,7 @@ function categoryRank(candidate) {
|
|
|
77
77
|
const tier = CATEGORY_PRIORITY[candidate.scenario?.category] ?? PriorityTier.LOW;
|
|
78
78
|
return PRIORITY_RANK[tier];
|
|
79
79
|
}
|
|
80
|
-
/**
|
|
81
|
-
* Extract method+path from every changed (`+`/`-`, non-header) line of a raw
|
|
82
|
-
* unified diff. Reuses `parseRouteLine`, which already strips the leading
|
|
83
|
-
* `+`/`-` marker and matches the same route-decorator patterns the endpoint
|
|
84
|
-
* scanner does.
|
|
85
|
-
*/
|
|
86
|
-
function collectChangedRouteLines(diffText) {
|
|
87
|
-
const routes = [];
|
|
88
|
-
let currentFile = "";
|
|
89
|
-
for (const line of diffText.split("\n")) {
|
|
90
|
-
// Track the current file from the unified-diff header so parseRouteLine
|
|
91
|
-
// gets the real path — its UI-component guard (UI_COMPONENT_EXT) depends
|
|
92
|
-
// on it, or a route-shaped line inside a .tsx/.jsx file (e.g. a client
|
|
93
|
-
// router registration) gets misparsed as a changed backend route.
|
|
94
|
-
if (line.startsWith("+++ ")) {
|
|
95
|
-
const spec = line.slice(4).trim().split("\t")[0];
|
|
96
|
-
currentFile = spec === "/dev/null" ? "" : normalizeDiffPath(spec);
|
|
97
|
-
continue;
|
|
98
|
-
}
|
|
99
|
-
if (line.startsWith("--- ") || line.startsWith("diff --git") || line.startsWith("index "))
|
|
100
|
-
continue;
|
|
101
|
-
if (!(line.startsWith("+") || line.startsWith("-")))
|
|
102
|
-
continue;
|
|
103
|
-
const parsed = parseRouteLine(line, currentFile);
|
|
104
|
-
if (parsed)
|
|
105
|
-
routes.push({ method: parsed.method, path: parsed.path });
|
|
106
|
-
}
|
|
107
|
-
return routes;
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Whether any step of `scenario` targets a method+path on the changed hunk.
|
|
111
|
-
* Diff-extracted paths are local to the file's own router declaration (e.g.
|
|
112
|
-
* "/suggestions"); scenario step paths are fully mounted (e.g.
|
|
113
|
-
* "/api/recipes/suggestions") once drafted from a scanned/recovered endpoint.
|
|
114
|
-
* A local path matches when the step path ends with it, so the cross-file
|
|
115
|
-
* mount prefix difference (see recoverRemovedEndpointsFromBase, SKYR-4026)
|
|
116
|
-
* doesn't prevent the match.
|
|
117
|
-
*/
|
|
80
|
+
/** Whether any step of `scenario` targets a method+path on the changed hunk. */
|
|
118
81
|
function scenarioOnChangedHunk(scenario, changedRoutes) {
|
|
119
|
-
|
|
120
|
-
return false;
|
|
121
|
-
for (const step of scenario.steps ?? []) {
|
|
122
|
-
const stepMethod = (step.method ?? "").toUpperCase();
|
|
123
|
-
const stepPath = (step.path ?? "").replace(/\/+$/, "");
|
|
124
|
-
for (const route of changedRoutes) {
|
|
125
|
-
if (route.method.toUpperCase() !== stepMethod)
|
|
126
|
-
continue;
|
|
127
|
-
const routePath = route.path.replace(/\/+$/, "");
|
|
128
|
-
if (routePath === "") {
|
|
129
|
-
if (stepPath === "" || stepPath === "/")
|
|
130
|
-
return true;
|
|
131
|
-
continue;
|
|
132
|
-
}
|
|
133
|
-
if (stepPath === routePath || stepPath.endsWith(routePath))
|
|
134
|
-
return true;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
return false;
|
|
82
|
+
return !!findStepOnChangedRoute(scenario.steps, changedRoutes);
|
|
138
83
|
}
|