@skyramp/mcp 0.2.6 → 0.2.8-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/commandLibrary.d.ts +12 -0
- package/build/commands/recommendTestsAndExecuteCommand.d.ts +2 -0
- package/build/commands/testThisEndpointCommand.d.ts +2 -0
- package/build/index.d.ts +2 -0
- package/build/index.js +6 -4
- package/build/playwright/PlaywrightTraceService.d.ts +32 -0
- package/build/playwright/index.d.ts +3 -0
- package/build/playwright/registerPlaywrightTools.d.ts +8 -0
- package/build/playwright/registerPlaywrightTools.js +12 -11
- package/build/playwright/traceRecordingPrompt.d.ts +13 -0
- package/build/playwright/traceRecordingPrompt.js +3 -11
- package/build/prompts/code-reuse.d.ts +1 -0
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.d.ts +2 -0
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.d.ts +2 -0
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +15 -0
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.d.ts +2 -0
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -5
- package/build/prompts/fix-error-prompt.d.ts +1 -0
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.d.ts +1 -0
- package/build/prompts/modularization/integration-test-modularization.d.ts +1 -0
- package/build/prompts/modularization/ui-test-modularization.d.ts +1 -0
- package/build/prompts/personas.d.ts +21 -0
- package/build/prompts/pom-aware-code-reuse.d.ts +1 -0
- package/build/prompts/startTraceCollectionPrompts.d.ts +2 -0
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.d.ts +2 -0
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +114 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.d.ts +2 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +292 -0
- package/build/prompts/sut-setup/shared.d.ts +39 -0
- package/build/prompts/sut-setup/shared.js +132 -0
- package/build/prompts/test-maintenance/actionsInstructions.d.ts +35 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +4 -0
- package/build/prompts/test-maintenance/driftAnalysisSections.d.ts +13 -0
- package/build/prompts/test-maintenance/driftAnalysisSections.js +9 -3
- package/build/prompts/test-recommendation/analysisOutputPrompt.d.ts +84 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
- package/build/prompts/test-recommendation/analysisOutputPrompt.test.d.ts +1 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.test.js +1 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +17 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +83 -18
- package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +7 -0
- package/build/prompts/test-recommendation/mergeEnrichedScenarios.test.d.ts +1 -0
- package/build/prompts/test-recommendation/mergeEnrichedScenarios.test.js +1 -0
- package/build/prompts/test-recommendation/promptPlan.d.ts +203 -0
- package/build/prompts/test-recommendation/promptPlan.test.d.ts +1 -0
- package/build/prompts/test-recommendation/recommendationSections.d.ts +35 -0
- package/build/prompts/test-recommendation/recommendationShared.d.ts +31 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +7 -0
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +55 -0
- package/build/prompts/test-recommendation/scopeAssessment.js +2 -2
- package/build/prompts/test-recommendation/scopeAssessment.test.d.ts +1 -0
- package/build/prompts/test-recommendation/scopeAssessment.test.js +1 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +6 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +4 -4
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.d.ts +1 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +97 -99
- package/build/prompts/testbot/testbot-prompts.d.ts +26 -0
- package/build/prompts/testbot/testbot-prompts.js +301 -46
- package/build/prompts/testbot/testbot-prompts.test.d.ts +1 -0
- package/build/prompts/testbot/testbot-prompts.test.js +109 -117
- package/build/resources/analysisResources.d.ts +17 -0
- package/build/resources/progressResource.d.ts +2 -0
- package/build/resources/sutSetupResource.d.ts +2 -0
- package/build/resources/sutSetupResource.js +45 -0
- package/build/resources/testbotResource.d.ts +2 -0
- package/build/resources/testbotResource.js +2 -2
- package/build/services/AnalyticsService.d.ts +13 -0
- package/build/services/AnalyticsService.test.d.ts +1 -0
- package/build/services/AnalyticsService.test.js +86 -0
- package/build/services/ModularizationService.d.ts +11 -0
- package/build/services/ScenarioGenerationService.d.ts +37 -0
- package/build/services/ScenarioGenerationService.integration.test.d.ts +1 -0
- package/build/services/ScenarioGenerationService.integration.test.js +4 -0
- package/build/services/ScenarioGenerationService.test.d.ts +1 -0
- package/build/services/TestDiscoveryService.d.ts +128 -0
- package/build/services/TestDiscoveryService.js +22 -12
- package/build/services/TestDiscoveryService.test.d.ts +1 -0
- package/build/services/TestDiscoveryService.test.js +4 -3
- package/build/services/TestExecutionService.d.ts +55 -0
- package/build/services/TestExecutionService.js +158 -26
- package/build/services/TestExecutionService.test.d.ts +1 -0
- package/build/services/TestExecutionService.test.js +339 -25
- package/build/services/TestGenerationService.d.ts +56 -0
- package/build/services/TestGenerationService.test.d.ts +1 -0
- package/build/services/TestGenerationService.test.js +10 -8
- package/build/services/containerEnv.d.ts +14 -0
- package/build/tool-phase-coverage.test.d.ts +1 -0
- package/build/tool-phase-coverage.test.js +2 -0
- package/build/tool-phases.d.ts +38 -0
- package/build/tools/auth/loginTool.d.ts +2 -0
- package/build/tools/auth/logoutTool.d.ts +2 -0
- package/build/tools/code-refactor/codeReuseTool.d.ts +2 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.d.ts +2 -0
- package/build/tools/code-refactor/modularizationTool.d.ts +2 -0
- package/build/tools/executeSkyrampTestTool.d.ts +2 -0
- package/build/tools/executeSkyrampTestTool.js +26 -7
- package/build/tools/fixErrorTool.d.ts +2 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +45 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +1 -1
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.d.ts +1 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +11 -9
- package/build/tools/generate-tests/generateContractRestTool.d.ts +31 -0
- package/build/tools/generate-tests/generateE2ERestTool.d.ts +13 -0
- package/build/tools/generate-tests/generateFuzzRestTool.d.ts +10 -0
- package/build/tools/generate-tests/generateIntegrationRestTool.d.ts +16 -0
- package/build/tools/generate-tests/generateLoadRestTool.d.ts +18 -0
- package/build/tools/generate-tests/generateLoadRestTool.test.d.ts +1 -0
- package/build/tools/generate-tests/generateMockRestTool.d.ts +64 -0
- package/build/tools/generate-tests/generateSmokeRestTool.d.ts +8 -0
- package/build/tools/generate-tests/generateUIRestTool.d.ts +14 -0
- package/build/tools/generate-tests/generateUIRestTool.js +1 -1
- package/build/tools/generate-tests/loadTestSchema.d.ts +39 -0
- package/build/tools/one-click/oneClickTool.d.ts +11 -0
- package/build/tools/submitReportTool.d.ts +394 -0
- package/build/tools/submitReportTool.js +250 -41
- package/build/tools/submitReportTool.test.d.ts +1 -0
- package/build/tools/submitReportTool.test.js +511 -55
- package/build/tools/test-management/actionsTool.d.ts +10 -0
- package/build/tools/test-management/actionsTool.js +21 -7
- package/build/tools/test-management/analyzeChangesTool.d.ts +27 -0
- package/build/tools/test-management/analyzeChangesTool.js +150 -48
- package/build/tools/test-management/analyzeChangesTool.test.d.ts +1 -0
- package/build/tools/test-management/analyzeChangesTool.test.js +95 -78
- package/build/tools/test-management/analyzeTestHealthTool.d.ts +2 -0
- package/build/tools/test-management/analyzeTestHealthTool.js +16 -9
- package/build/tools/test-management/index.d.ts +3 -0
- package/build/tools/test-management/index.js +0 -1
- package/build/tools/trace/resolveSaveStoragePath.d.ts +7 -0
- package/build/tools/trace/resolveSaveStoragePath.test.d.ts +1 -0
- package/build/tools/trace/resolveSessionPaths.d.ts +40 -0
- package/build/tools/trace/resolveSessionPaths.test.d.ts +1 -0
- package/build/tools/trace/sessionState.d.ts +2 -0
- package/build/tools/trace/sessionState.test.d.ts +1 -0
- package/build/tools/trace/startTraceCollectionTool.d.ts +2 -0
- package/build/tools/trace/stopTraceCollectionTool.d.ts +2 -0
- package/build/tools/workspace/initScanWorkspaceTool.d.ts +4 -0
- package/build/tools/workspace/initializeWorkspaceTool.d.ts +2 -0
- package/build/tools/workspace/initializeWorkspaceTool.js +39 -22
- package/build/tools/workspace/initializeWorkspaceTool.test.d.ts +1 -0
- package/build/tools/workspace/initializeWorkspaceTool.test.js +134 -0
- package/build/tools/workspace/serviceUpsert.d.ts +12 -0
- package/build/tools/workspace/serviceUpsert.js +23 -0
- package/build/tools/workspace/serviceUpsert.test.d.ts +1 -0
- package/build/tools/workspace/serviceUpsert.test.js +50 -0
- package/build/types/OneClickCommands.d.ts +68 -0
- package/build/types/RepositoryAnalysis.d.ts +3004 -0
- package/build/types/TestAnalysis.d.ts +97 -0
- package/build/types/TestExecution.d.ts +63 -0
- package/build/types/TestExecution.js +11 -1
- package/build/types/TestRecommendation.d.ts +12 -0
- package/build/types/TestTypes.d.ts +234 -0
- package/build/types/TestbotReport.d.ts +68 -0
- package/build/types/TestbotReport.js +1 -0
- package/build/types/index.d.ts +4 -0
- package/build/types/index.js +3 -0
- package/build/utils/AnalysisStateManager.d.ts +242 -0
- package/build/utils/AnalysisStateManager.js +144 -3
- package/build/utils/AnalysisStateManager.test.d.ts +1 -0
- package/build/utils/AnalysisStateManager.test.js +98 -0
- package/build/utils/analyze-openapi.d.ts +1 -0
- package/build/utils/branchDiff.d.ts +18 -0
- package/build/utils/dartRouteExtractor.d.ts +45 -0
- package/build/utils/dartRouteExtractor.test.d.ts +1 -0
- package/build/utils/docker.d.ts +19 -0
- package/build/utils/docker.test.d.ts +1 -0
- package/build/utils/docker.test.js +10 -9
- package/build/utils/featureFlags.d.ts +37 -0
- package/build/utils/featureFlags.test.d.ts +1 -0
- package/build/utils/gitStaging.d.ts +24 -0
- package/build/utils/gitStaging.test.d.ts +1 -0
- package/build/utils/gitStaging.test.js +13 -7
- package/build/utils/httpDefaults.d.ts +10 -0
- package/build/utils/httpDefaults.test.d.ts +1 -0
- package/build/utils/initAgent.d.ts +36 -0
- package/build/utils/language-helper.d.ts +6 -0
- package/build/utils/logger.d.ts +11 -0
- package/build/utils/normalizeParams.d.ts +14 -0
- package/build/utils/normalizeSkyrampImports.d.ts +13 -0
- package/build/utils/pr-comment-parser.d.ts +31 -0
- package/build/utils/pr-comment-parser.test.d.ts +1 -0
- package/build/utils/pr-comment-parser.test.js +6 -5
- package/build/utils/projectMetadata.d.ts +15 -0
- package/build/utils/projectMetadata.test.d.ts +1 -0
- package/build/utils/proxy-terminal.d.ts +2 -0
- package/build/utils/repoScanner.d.ts +27 -0
- package/build/utils/repoScanner.test.d.ts +1 -0
- package/build/utils/routeParsers.d.ts +53 -0
- package/build/utils/routeParsers.js +31 -38
- package/build/utils/routeParsers.test.d.ts +1 -0
- package/build/utils/scenarioDrafting.d.ts +71 -0
- package/build/utils/scenarioDrafting.js +23 -1
- package/build/utils/scenarioDrafting.test.d.ts +1 -0
- package/build/utils/skyrampMdContent.d.ts +5 -0
- package/build/utils/sourceRouteExtractor.d.ts +48 -0
- package/build/utils/sourceRouteExtractor.test.d.ts +1 -0
- package/build/utils/telemetry.d.ts +32 -0
- package/build/utils/telemetry.test.d.ts +1 -0
- package/build/utils/trace-parser.d.ts +32 -0
- package/build/utils/trace-parser.test.d.ts +6 -0
- package/build/utils/uiPageEnumerator.d.ts +146 -0
- package/build/utils/uiPageEnumerator.js +58 -14
- package/build/utils/uiPageEnumerator.test.d.ts +1 -0
- package/build/utils/uiPageEnumerator.test.js +223 -43
- package/build/utils/utils.d.ts +45 -0
- package/build/utils/utils.js +10 -0
- package/build/utils/utils.test.d.ts +1 -0
- package/build/utils/versions.d.ts +3 -0
- package/build/utils/versions.js +1 -1
- package/build/utils/workspaceAuth.d.ts +127 -0
- package/build/utils/workspaceAuth.test.d.ts +1 -0
- package/build/utils/workspaceAuth.test.js +25 -25
- package/build/workspace/index.d.ts +1 -0
- package/build/workspace/workspace.d.ts +390 -0
- package/build/workspace/workspace.js +7 -7
- package/build/workspace/workspace.test.d.ts +1 -0
- package/build/workspace/workspace.test.js +6 -6
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +31 -13
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +88 -15
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -3
- package/node_modules/playwright/node_modules/playwright-core/ThirdPartyNotices.txt +23 -126
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderApp.js +79 -19
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/skyramp/aliasRemapper.js +84 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/skyramp/replayEngine.js +15 -4
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-CPZKmjFR.js → codeMirrorModule-D0BjbCb7.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-lvTRGFx-.js +193 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-B0JOjboO.js → codeMirrorModule-Cqp3cwEJ.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-1anWeyDf.js → defaultSettingsView-CYf9adZh.js} +79 -79
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Cc5029a3.js → index.B_7ywgmr.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.Wo5yvvVh.js → uiMode.u4_8VnCV.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderApp.ts +129 -43
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/skyramp/aliasRemapper.ts +119 -0
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/skyramp/replayEngine.ts +33 -10
- package/node_modules/playwright/package.json +1 -1
- package/package.json +16 -4
- package/build/tools/test-management/uiAnalyzeChangesTool.js +0 -155
- package/build/tools/test-management/uiAnalyzeChangesTool.test.js +0 -147
- package/node_modules/playwright/node_modules/playwright-core/bundles/mcp/node_modules/.bin/node-which +0 -52
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/is-docker +0 -5
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/mime +0 -46
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/yaml +0 -11
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-CsgPR2wy.js +0 -193
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.CP3Y8Zsb.js +0 -2
|
@@ -17,12 +17,53 @@ const CONTRACT_MODE_GUIDANCE = CONSUMER_MODE_ENABLED
|
|
|
17
17
|
For client-facing APIs consumed by frontend: add \`consumerMode: true\`.
|
|
18
18
|
Both modes (\`providerMode: true, consumerMode: true\`): For diff that contains BOTH provider signals (such as new/modified endpoint handlers, route changes this service owns) AND consumer signals (outbound HTTP client calls to another service, no new endpoint handlers).`
|
|
19
19
|
: ` Always add \`providerMode: true\` — the tool generates provider-side contract tests only.`;
|
|
20
|
+
/**
|
|
21
|
+
* Parse the JSON-encoded `relatedRepositories` argument passed via the testbot
|
|
22
|
+
* prompt/resource. Returns undefined for missing/blank input or malformed JSON so the
|
|
23
|
+
* caller cleanly falls back to single-repo behavior.
|
|
24
|
+
*/
|
|
25
|
+
export function parseRelatedRepositories(raw) {
|
|
26
|
+
if (!raw)
|
|
27
|
+
return undefined;
|
|
28
|
+
try {
|
|
29
|
+
const parsed = JSON.parse(raw);
|
|
30
|
+
if (!Array.isArray(parsed) || parsed.length === 0)
|
|
31
|
+
return undefined;
|
|
32
|
+
return parsed
|
|
33
|
+
.filter((r) => !!r &&
|
|
34
|
+
typeof r.repositoryPath === "string" &&
|
|
35
|
+
r.repositoryPath.length > 0 &&
|
|
36
|
+
typeof r.repo === "string" &&
|
|
37
|
+
r.repo.length > 0)
|
|
38
|
+
.map((r) => ({
|
|
39
|
+
repositoryPath: r.repositoryPath,
|
|
40
|
+
baseBranch: typeof r.baseBranch === "string" ? r.baseBranch : undefined,
|
|
41
|
+
repo: r.repo,
|
|
42
|
+
}));
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
20
48
|
export function getTestbotPrompt(prTitle, prDescription, summaryOutputFile, repositoryPath, baseBranch, maxRecommendations = MAX_RECOMMENDATIONS, maxGenerate = MAX_TESTS_TO_GENERATE, _maxCritical = MAX_CRITICAL_TESTS, // Reserved — accepted for API compat but not yet wired into prompt
|
|
21
|
-
prNumber, userPrompt, services,
|
|
49
|
+
prNumber, userPrompt, services, uiCredentials, testsRepoDir, relatedRepositories, primaryRepo) {
|
|
22
50
|
maxGenerate = Math.min(Math.max(maxGenerate, 0), maxRecommendations);
|
|
23
51
|
// TODO(SKYR-3636 follow-up): migrate Task 1 + Task 2 step bodies to PromptPlan
|
|
24
52
|
// (src/prompts/test-recommendation/promptPlan.ts) so step numbers don't have
|
|
25
53
|
// to be hand-maintained when steps are added or reordered.
|
|
54
|
+
// Multi-repo mode: when related repos are present, the agent analyzes each one
|
|
55
|
+
// (primary + related) and correlates cross-repo changes. The configured checkout
|
|
56
|
+
// HEAD of each related repo is its feature ref; `baseBranch` is that repo's
|
|
57
|
+
// default branch, so skyramp_analyze_changes computes a real default…feature
|
|
58
|
+
// diff. Every repo carries an explicit `repo` (owner/repo) resolved action-side,
|
|
59
|
+
// so the agent never infers it. GENERATE candidates are pooled across the primary
|
|
60
|
+
// + related repos into one shared ranked budget; each generated test executes
|
|
61
|
+
// against its own repo's service when reachable. Two distinct roots: the unified
|
|
62
|
+
// .skyramp/workspace.yml ALWAYS lives in the primary repo (where the workflow is
|
|
63
|
+
// installed), while test FILES are delivered into each service's declared
|
|
64
|
+
// testDirectory, relative to the delivery root (the configured test repo if set,
|
|
65
|
+
// otherwise the primary repo).
|
|
66
|
+
const hasRelatedRepos = !!relatedRepositories?.length;
|
|
26
67
|
// For follow-up requests: emit the @skyramp-testbot header + guardrails + retrieve-recommendations step.
|
|
27
68
|
// For first-run prompts: emit the full Task 1 analysis + maintenance section.
|
|
28
69
|
const task1Section = userPrompt
|
|
@@ -44,7 +85,7 @@ Verify the prompt inside <USER_PROMPT> is related to adding or removing tests fr
|
|
|
44
85
|
- If the prompt matches one or more tests in the Additional Recommendations → proceed to Task 1 (Skip Analysis).
|
|
45
86
|
|
|
46
87
|
### Task 1: Retrieve Previous Recommendations
|
|
47
|
-
Call \`skyramp_analyze_changes\` with \`repositoryPath\`: "${repositoryPath}", \`scope\`: "branch_diff"${baseBranch ? `, \`baseBranch\`: "${baseBranch}"` : ""}${prNumber ? `, \`prNumber\`: ${prNumber}` : ""}${
|
|
88
|
+
Call \`skyramp_analyze_changes\` with \`repositoryPath\`: "${repositoryPath}", \`scope\`: "branch_diff"${baseBranch ? `, \`baseBranch\`: "${baseBranch}"` : ""}${prNumber ? `, \`prNumber\`: ${prNumber}` : ""}${testsRepoDir ? `, \`testsRepoDir\`: "${testsRepoDir}"` : ""}.
|
|
48
89
|
This will fetch the previous TestBot report from the PR comments and return deduplicated recommendations.
|
|
49
90
|
Use those recommendations as your baseline. Only add or remove tests that the user requested AND that appear in the Additional Recommendations. Then proceed straight to Task 2: Generate New Tests.
|
|
50
91
|
`
|
|
@@ -53,27 +94,47 @@ Use those recommendations as your baseline. Only add or remove tests that the us
|
|
|
53
94
|
|
|
54
95
|
## Task 1: Analyze & Maintain
|
|
55
96
|
|
|
56
|
-
1.
|
|
97
|
+
1. Call \`skyramp_analyze_changes\` with \`repositoryPath\`: "${repositoryPath}", \`scope\`: "branch_diff", \`topN\`: ${maxRecommendations}, \`maxGenerate\`: ${maxGenerate}${baseBranch ? `, \`baseBranch\`: "${baseBranch}"` : ""}${prNumber ? `, \`prNumber\`: ${prNumber}` : ""}${testsRepoDir ? `, \`testsRepoDir\`: "${testsRepoDir}"` : ""}.
|
|
57
98
|
|
|
58
|
-
**
|
|
99
|
+
**Follow all instructions returned by \`skyramp_analyze_changes\`** — its response opens with a **UI Blueprint Capture** section that tells you which candidate URLs to capture (and how) before you write any UI recommendation \`reasoning\`.${uiCredentials ? " If the section lists candidate URLs, log in once via the credentials in your <ui-credentials> context before navigating to them." : ""}
|
|
59
100
|
|
|
60
|
-
**Otherwise:** for each candidate URL in the response${uiCredentials ? " (after logging in via the credentials provided)" : ""}, \`browser_navigate\` to the URL, then \`browser_blueprint\` to capture. The captures stay in your tool-result history — they're the element vocabulary you'll use when writing UI recommendation \`reasoning\` fields in \`skyramp_analyze_changes\`. You do NOT need to thread them back into a tool call.
|
|
61
101
|
|
|
62
|
-
|
|
102
|
+
**Feature context check:** After capturing a blueprint per the UI Blueprint Capture instructions, cross-check whether it actually shows the changed feature. Look at the diff — what component, field, or UI element was changed? If that component only renders in a specific context (a particular collection type, a non-empty list, a specific user role, a form that only appears after a prior action), and the blueprint you captured doesn't contain it, navigate to the right context before using this blueprint for recommendations. A blueprint that doesn't show the changed feature produces recommendations that test the wrong thing — for example, recommending tests against a system collection when the PR changed behavior on custom collections. If you can't determine the right context from the diff and workspace config, note it in \`issuesFound\` and fall back to source-grounded recommendations for that URL.
|
|
63
103
|
|
|
64
|
-
2. Call \`skyramp_analyze_changes\` with \`repositoryPath\`: "${repositoryPath}", \`scope\`: "branch_diff", \`topN\`: ${maxRecommendations}, \`maxGenerate\`: ${maxGenerate}${baseBranch ? `, \`baseBranch\`: "${baseBranch}"` : ""}${prNumber ? `, \`prNumber\`: ${prNumber}` : ""}${stateOutputFile ? `, \`stateOutputFile\`: "${stateOutputFile}"` : ""}${testsRepoDir ? `, \`testsRepoDir\`: "${testsRepoDir}"` : ""}. Use the blueprints already in your context (from \`skyramp_ui_analyze_changes\`) to ground UI recommendation reasoning.
|
|
65
104
|
**If \`skyramp_analyze_changes\` returns an error:** retry once only if the error is transient (timeout, network blip, temporary unavailability) — do NOT retry for permanent errors (invalid repository path, missing required parameter, authentication failure). If it fails again, call \`skyramp_submit_report\` with a minimal valid payload: leave all test arrays empty and add the error to \`issuesFound\`. Refer to the \`skyramp_submit_report\` schema for required fields. Do NOT attempt Task 2 without a valid stateFile.
|
|
66
|
-
**If all changed files are non-application** (CI/CD, docs, lock files, config
|
|
67
|
-
|
|
68
|
-
|
|
105
|
+
**If all changed files are non-application** (CI/CD, docs, lock files, config) → skip to Task 3 (Submit Report) with empty arrays and a single \`issuesFound\` entry explaining why (same format as the zero-test path below).
|
|
106
|
+
${hasRelatedRepos ? `
|
|
107
|
+
**MULTI-REPO CONTEXT (MANDATORY).** This run includes ${relatedRepositories.length} related ${relatedRepositories.length === 1 ? "repository" : "repositories"} listed in the \`<related_repositories>\` block below, each with an explicit \`repository\` (\`owner/repo\`), \`path\`, and \`base_branch\`. Use the \`repository\` value verbatim — do NOT infer it from git remotes or paths. You MUST analyze EACH related repository — exactly one \`skyramp_analyze_changes\` call per listed repo (${relatedRepositories.length} ${relatedRepositories.length === 1 ? "call" : "calls"}), in addition to the primary call in step 2.
|
|
108
|
+
|
|
109
|
+
**Run the primary call (step 2) FIRST, then the related repos in listed order — not in parallel.** All calls in this run automatically share ONE run-scoped state file — you do NOT pass a state-file path; setting \`repository\` is enough. The primary writes its root section; each related repo's call upserts its own section into that same file. Concurrent calls would race on the shared file, so they must be sequential. For each related repo:
|
|
110
|
+
- **(a)** Register its services into the **single unified workspace** — do NOT write a \`.skyramp/workspace.yml\` into the related repo (it is read-only context, and a per-repo file would need its own PR). The unified workspace always lives in the **primary repo** (where the testbot workflow is installed): \`${repositoryPath}\`${testsRepoDir ? ` — NOT in the configured test repo, even though generated test files are delivered there` : ""}. Steps: call \`skyramp_init_scan\` with \`workspacePath\`: the related repo's \`path\` to DISCOVER its services, then call \`skyramp_init_workspace\` with \`workspacePath\`: "${repositoryPath}" (the PRIMARY repo, NOT the related repo's path${testsRepoDir ? " and NOT the test repo" : ""}), \`merge\`: true, the \`scanToken\`, and the discovered services — each with its \`repository\` field set to that repo's \`repository\` value from the block. This upserts the related repo's services into the one unified \`.skyramp/workspace.yml\` (services are keyed by the composite of \`repository\` + \`serviceName\`).
|
|
111
|
+
- **(b)** Then call \`skyramp_analyze_changes\` with that repo's \`path\` (as \`repositoryPath\`), \`scope\`: "branch_diff", and \`repository\`: that repo's \`repository\` value. Passing \`repository\` stores this analysis as a section within the one run-scoped state file (the file is reused automatically across calls — do NOT pass any state-file path). Pass \`baseBranch\` only when the repo's \`base_branch\` attribute names a concrete branch; when \`base_branch\` is \`auto-detect\`, OMIT \`baseBranch\` and let the tool detect the repo's default branch.
|
|
112
|
+
- **(c)** If analysis of a related repo fails (e.g. workspace init or diff error), record a one-line \`issuesFound\` note and CONTINUE — a missing related repo must not block the primary analysis.
|
|
113
|
+
Keep ALL returned diffs and recommendations from every repo in your working context. **Correlate** cross-repo changes when forming recommendations and the business case — e.g. an API contract changed in one repo and consumed by another, a request/response shape that must stay compatible across the boundary.
|
|
114
|
+
|
|
115
|
+
**One state file, per-repo sections.** There is exactly ONE state file for the whole run (created automatically and reused across calls). Each \`skyramp_analyze_changes\` call writes its repo's section into it (keyed by \`repository\`). When you maintain/act on any repo's tests — \`skyramp_analyze_test_health\`, \`skyramp_actions\`, \`skyramp_execute_test\` — pass that repo's \`repository\` (\`owner/repo\`) as the \`repository\` argument alongside the \`stateFile\` path returned by \`skyramp_analyze_changes\`, so the tool reads/writes that repo's section. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; pass it for the primary repo's tests just as you do for related repos.
|
|
116
|
+
|
|
117
|
+
**Cross-repo test generation (Task 2 spans all repos).** Pool the GENERATE **and** ADDITIONAL recommendations from the primary AND every related repo into ONE candidate set — treat each repo's per-repo GENERATE list as **candidates, not a fixed selection**. (This OVERRIDES the "use the pre-ranked GENERATE list as-is" mandate in Task 2 below, which applies only to single-repo runs.) To fill the shared budget of ${maxGenerate} tests, distribute slots EVENLY across test types with spillover:
|
|
118
|
+
1. **Bucket** the pooled candidates by test type (UI/E2E, contract, integration), including ONLY the types that actually have candidates.
|
|
119
|
+
2. **Protected items first:** a promoted \`bug_caught\` (CRITICAL) test or an attack-surface \`security_boundary\` test always takes a GENERATE slot before round-robin fills the rest.
|
|
120
|
+
3. **Round-robin** one slot per type in priority order — the highest-scored candidate of the highest-ranked type first — taking one per type per round.
|
|
121
|
+
4. **Spillover:** when a type's candidates run out, its freed slots go to the next-highest-scored candidate of any remaining type.
|
|
122
|
+
Within a type, higher score wins regardless of which repo it came from. Everything not selected becomes an ADDITIONAL recommendation. This guarantees a frontend-only primary repo cannot starve a related backend repo's contract/integration tests of GENERATE slots (and vice versa). When you generate a test for a related repo's endpoint:
|
|
123
|
+
- **Execute it only if that repo's service is already running and reachable.** The workflow's setup may have started multiple services; before generating an API test for a related repo, confirm its \`base_url\` (from that repo's workspace/Execution Plan) responds. If the service is unreachable, still GENERATE the test but mark its \`testResults\` status as \`Skipped\` with details "service not running in this run" — do NOT count an unreachable service as a failure.
|
|
124
|
+
- **Write the test file into that service's own \`testDirectory\`** — the one declared for the service in the unified workspace.yml (the related repo's services were registered there in step (a), each with its \`repository\`). The \`testDirectory\` is interpreted relative to the **single delivery root** (the configured test repo if set, otherwise the primary repo), so all generated tests are delivered together by the existing single-target delivery. Do NOT invent a per-source-repo subdirectory, and do NOT write into the related repo's own checkout — it is read-only context. (If two repos happen to declare the same \`testDirectory\`, their files coexist there; the \`repository\` field on each report item — below — is what attributes ownership, not the path.)
|
|
125
|
+
- **Set the \`repository\` field** (\`owner/repo\`) on every such \`newTestsCreated\` / \`testResults\` item so the report attributes it to the originating repo (see Report Guidelines).` : ""}
|
|
126
|
+
|
|
127
|
+
2. **Maintain existing tests:**
|
|
69
128
|
|
|
70
129
|
a. Call \`skyramp_analyze_test_health\` with \`stateFile\` (from \`skyramp_analyze_changes\` output). **Do NOT read application source files** (routes, models, controllers) — all change information you need is in the \`skyramp_analyze_changes\` output and the diff.
|
|
71
130
|
|
|
72
131
|
b. Write \`updateInstructions\` for each UPDATE or REGENERATE test before calling \`skyramp_actions\` — articulating the change first prevents file content from overriding diff-based reasoning.
|
|
73
132
|
|
|
74
|
-
c. Call \`
|
|
133
|
+
c. Call \`skyramp_execute_test\` with \`phase: "before"\` and \`stateFile\` for every test whose action is UPDATE or REGENERATE. Run them in parallel. This captures the pre-edit baseline — do not skip even if you expect the test to fail.
|
|
75
134
|
|
|
76
|
-
|
|
135
|
+
d. Call \`skyramp_actions\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) and apply the edits it returns.
|
|
136
|
+
|
|
137
|
+
3. **Code review:** From the \`skyramp_analyze_changes\` output and the existing test files you read for maintenance, note any logic bugs. Do NOT read additional source files just for code review — use what is already available from the analysis and test file reads. Common patterns to flag:
|
|
77
138
|
- Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)
|
|
78
139
|
- Incomplete CRUD: create without cleanup, update that adds new records without removing old ones
|
|
79
140
|
- Missing input validation on new endpoints
|
|
@@ -81,15 +142,178 @@ Use those recommendations as your baseline. Only add or remove tests that the us
|
|
|
81
142
|
- Incorrect arithmetic in business logic (discount calculations, price aggregation)
|
|
82
143
|
Log each finding in \`issuesFound\` with a \`severity\` (critical/high/medium/low). These bugs should inform your test design in Task 2.
|
|
83
144
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
145
|
+
4. **Blueprint Citation Invariant** (UI test recommendations only). Every named UI element in your recommendation must correspond to an element actually captured in one of the blueprints you captured from the UI Blueprint Capture section of \`skyramp_analyze_changes\`. For **every** UI recommendation — both \`newTestsCreated[]\` entries (tests you generated) AND \`additionalRecommendations[]\` entries (deferred tests) with \`testType: "ui"\` — populate **four** fields: \`description\`, \`reasoning\`, \`targetElements\`, and \`pageContext\`. \`description\` and \`reasoning\` carry different roles — fill BOTH (see Field 4 below). The structured fields (\`targetElements\`, \`pageContext\`) are required on UI recs in BOTH arrays; non-UI recs (contract / integration / e2e / batch-scenario) MUST omit them.
|
|
146
|
+
|
|
147
|
+
**Field 1 — \`targetElements\`** (the elements the test targets — array, length 1+):
|
|
148
|
+
\`\`\`json
|
|
149
|
+
[
|
|
150
|
+
{
|
|
151
|
+
"role": "button",
|
|
152
|
+
"accessibleName": "Save changes",
|
|
153
|
+
"testId": "save-changes-btn",
|
|
154
|
+
"stableId": null,
|
|
155
|
+
"contextText": null,
|
|
156
|
+
"mutability": "mutable",
|
|
157
|
+
"widgetType": "native"
|
|
158
|
+
}
|
|
159
|
+
]
|
|
160
|
+
\`\`\`
|
|
161
|
+
- **One element when the test has a single dominant target** (a click, a type, a single visibility check). Most tests fall here — use a length-1 array.
|
|
162
|
+
- **Multiple elements when the test verifies several elements together** — render-state tests (heading + input + button on a form), workflow tests (click button A, assert state appears in element B), or form-fill tests (multiple inputs + submit button). Each element is its own array entry.
|
|
163
|
+
- Each element's fields are lifted verbatim from a captured blueprint element. \`role\`, \`accessibleName\` are required. \`testId\`/\`stableId\`/\`contextText\` are \`null\` when not present in the capture.
|
|
164
|
+
- \`mutability\` — copy from \`blueprint.element.mutability\`. \`'mutable'\` = behavioral-test target; \`'immutable'\` = smoke target.
|
|
165
|
+
- \`widgetType\` — copy from \`blueprint.element.widgetType\`. \`'custom'\` = JavaScript-composite control (Radix, MUI, etc.) requiring click-to-open interaction; \`'native'\` = standard HTML element.
|
|
166
|
+
- \`contextText\` — only for elements inside repeating sections (table rows, list items). Lift from \`repeatingElement.items[].contextText\`. \`null\` otherwise.
|
|
167
|
+
|
|
168
|
+
**Field 2 — \`pageContext\`** (where the test runs):
|
|
169
|
+
\`\`\`json
|
|
170
|
+
{ "url": "http://localhost:5173/orders/1", "pageHash": "10:434266447" }
|
|
171
|
+
\`\`\`
|
|
172
|
+
Lift \`url\` and \`pageHash\` from the BlueprintCapture object that fed \`targetElements\`. All entries in \`targetElements\` should be from the same page.
|
|
173
|
+
|
|
174
|
+
**Field 3 — \`reasoning\`** (one sentence — what is verified and why):
|
|
175
|
+
|
|
176
|
+
A **single sentence** (~25-50 words) that names the targeted element(s) and states what the test verifies and why it matters. There is no fixed template — write the sentence in whatever way reads naturally for the test. The constraints below are the only requirements.
|
|
177
|
+
|
|
178
|
+
- **Reference each element from \`targetElements\` by its \`accessibleName\`**. Bold each name. When the element has a \`testId\`, render it in inline code (e.g. \`save-changes-btn\`); fall back to \`stableId\` if no \`testId\`. Omit the parenthetical identifier when neither is captured. For elements inside repeating sections, include \`contextText\` parenthetically (e.g. \`(customer "Acme", total "$199.99")\`).
|
|
179
|
+
- **State what the test verifies or asserts** about those elements — the behavior, render state, business rule, or production risk.
|
|
180
|
+
- **Optionally add a "; this validates …" or "; covers …" clause** naming a formula, business rule, or source-line reference (e.g. "; this validates the \`subtotal * (1 - discountPercent/100)\` formula", "; covers the boundary clamp at EditOrderForm.tsx:42"). Skip the clause for smoke checks where there's no concrete code reference.
|
|
181
|
+
- **Voice matches which array the rec is in.** \`newTestsCreated\` uses descriptive voice ("verifies", "asserts", "documents"). \`additionalRecommendations\` uses recommendatory voice ("would verify", "should assert", "would cover"). Pick a sentence opening that matches; you do not need to start with a fixed phrase.
|
|
182
|
+
- **Multi-element recs** combine elements in one sentence using natural connectives ("and", "after clicking", "alongside") — not enumeration.
|
|
183
|
+
- Do NOT include \`logicalName\`, \`fingerprint\`, \`pageHash\`, \`xpath\`, or other internal identifiers — those leak builder internals into a user-facing report. (\`pageHash\` lives in \`pageContext\` for the verifier; never put it in prose.)
|
|
184
|
+
|
|
185
|
+
**Field 4 — \`description\`** (multi-sentence test walkthrough — REQUIRED for UI recs):
|
|
186
|
+
|
|
187
|
+
The \`description\` field complements \`reasoning\`. While \`reasoning\` is one sentence about what the test verifies and why, \`description\` is **2-4 sentences of free-form prose** describing the test as a walkthrough — the page navigated to, the actions taken, the assertions made, the data values used. Reads like documentation for whoever implements the test.
|
|
188
|
+
|
|
189
|
+
- For \`newTestsCreated\`: describe what the test does step by step. Example: "Navigates to /orders/1, opens the Edit Order form, enters discount=20%, submits, and asserts the real-time total updates to $799.99 before the PUT call."
|
|
190
|
+
- For \`additionalRecommendations\`: describe what the recommended test would do. Example: "Would navigate to a cancelled order's detail page and assert the Edit Order button is not rendered, enforcing the conditional guard added in OrderDetail.tsx."
|
|
191
|
+
- **\`description\` is NOT a duplicate of \`reasoning\`.** Keep them complementary: \`description\` = what the test does (walkthrough), \`reasoning\` = what's verified + why (one sentence).
|
|
192
|
+
- Same jargon-exclusion rules apply (no \`logicalName\` / \`pageHash\` / etc.).
|
|
193
|
+
- Same fallback rules apply: when no blueprint is available, prefix \`description\` and \`reasoning\` with \`[no-blueprint-data]\` and use page/feature-level prose.
|
|
194
|
+
|
|
195
|
+
**Examples (showing the full bundle — copy the structure, not the exact wording):**
|
|
196
|
+
|
|
197
|
+
*Single-element generated test (descriptive voice, with rationale clause):*
|
|
198
|
+
\`\`\`json
|
|
199
|
+
{
|
|
200
|
+
"testType": "ui",
|
|
201
|
+
"description": "Navigates to /orders/1, opens the Edit Order form, enters discount=150, submits, and asserts the validation error 'Discount must be ≤ 100' appears and the form does not close. Documents the boundary-clamp guard in EditOrderForm.tsx.",
|
|
202
|
+
"targetElements": [
|
|
203
|
+
{
|
|
204
|
+
"role": "button", "accessibleName": "Save changes",
|
|
205
|
+
"testId": "save-changes-btn", "stableId": null,
|
|
206
|
+
"contextText": null, "mutability": "mutable", "widgetType": "native"
|
|
207
|
+
}
|
|
208
|
+
],
|
|
209
|
+
"pageContext": {
|
|
210
|
+
"url": "http://localhost:5173/orders/1",
|
|
211
|
+
"pageHash": "10:434266447"
|
|
212
|
+
},
|
|
213
|
+
"reasoning": "Verifies the **Save changes** button (\`save-changes-btn\`) shows a validation error and prevents form close when discount > 100; this validates the \`Math.min(100, Math.max(0, value))\` clamp at EditOrderForm.tsx:42."
|
|
214
|
+
}
|
|
215
|
+
\`\`\`
|
|
216
|
+
|
|
217
|
+
*Multi-element generated test (render-state, descriptive voice):*
|
|
218
|
+
\`\`\`json
|
|
219
|
+
{
|
|
220
|
+
"testType": "ui",
|
|
221
|
+
"description": "Navigates to /orders/1, clicks Edit Order, and asserts both the 'Edit Order' heading and the Discount (%) input render with discount pre-filled to 0. Smoke check before any user interaction.",
|
|
222
|
+
"targetElements": [
|
|
223
|
+
{
|
|
224
|
+
"role": "heading", "accessibleName": "Edit Order",
|
|
225
|
+
"testId": "edit-order-heading", "stableId": null,
|
|
226
|
+
"contextText": null, "mutability": "immutable", "widgetType": "native"
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
"role": "spinbutton", "accessibleName": "Discount (%)",
|
|
230
|
+
"testId": "edit-order-input-discount", "stableId": null,
|
|
231
|
+
"contextText": null, "mutability": "mutable", "widgetType": "native"
|
|
232
|
+
}
|
|
233
|
+
],
|
|
234
|
+
"pageContext": {
|
|
235
|
+
"url": "http://localhost:5173/orders/1",
|
|
236
|
+
"pageHash": "13:-684516288"
|
|
237
|
+
},
|
|
238
|
+
"reasoning": "Asserts the **Edit Order** heading (\`edit-order-heading\`) and **Discount (%)** input (\`edit-order-input-discount\`) render with discount pre-filled to 0 on form mount; covers the form-mount default-value contract."
|
|
239
|
+
}
|
|
240
|
+
\`\`\`
|
|
241
|
+
|
|
242
|
+
*Negative test (asserts element is NOT rendered), recommendatory voice:*
|
|
243
|
+
\`\`\`json
|
|
244
|
+
{
|
|
245
|
+
"testType": "ui",
|
|
246
|
+
"description": "Would navigate to a cancelled order's detail page and assert the Edit Order button is not rendered. Enforces the conditional-render guard added in OrderDetail.tsx.",
|
|
247
|
+
"targetElements": [
|
|
248
|
+
{
|
|
249
|
+
"role": "button", "accessibleName": "Edit Order",
|
|
250
|
+
"testId": "edit-order-btn", "stableId": null,
|
|
251
|
+
"contextText": null, "mutability": "mutable", "widgetType": "native"
|
|
252
|
+
}
|
|
253
|
+
],
|
|
254
|
+
"pageContext": {
|
|
255
|
+
"url": "http://localhost:5173/orders/cancelled-99",
|
|
256
|
+
"pageHash": "9:712044820"
|
|
257
|
+
},
|
|
258
|
+
"reasoning": "Would assert the **Edit Order** button (\`edit-order-btn\`) is not rendered on cancelled orders; enforces the conditional-render guard added in OrderDetail.tsx:118."
|
|
259
|
+
}
|
|
260
|
+
\`\`\`
|
|
261
|
+
(For negative tests, ground \`targetElements\` against a captured page where the element DOES render — so the verifier can confirm the citation is real.)
|
|
262
|
+
|
|
263
|
+
*Page-level / lifecycle test (no single dominant element), recommendatory voice:*
|
|
264
|
+
\`\`\`json
|
|
265
|
+
{
|
|
266
|
+
"testType": "ui",
|
|
267
|
+
"description": "Would walk through create → edit → cancel of an order, asserting the Order details heading reflects each state transition (Pending → Editing → Cancelled). Covers the OrderStatus state machine added in this PR.",
|
|
268
|
+
"targetElements": [
|
|
269
|
+
{
|
|
270
|
+
"role": "heading", "accessibleName": "Order details",
|
|
271
|
+
"testId": null, "stableId": null, "contextText": null,
|
|
272
|
+
"mutability": "immutable", "widgetType": "native"
|
|
273
|
+
}
|
|
274
|
+
],
|
|
275
|
+
"pageContext": {
|
|
276
|
+
"url": "http://localhost:5173/orders/1",
|
|
277
|
+
"pageHash": "11:200115447"
|
|
278
|
+
},
|
|
279
|
+
"reasoning": "Would walk the create-edit-cancel order lifecycle and assert the **Order details** heading reflects each state transition; covers the OrderStatus state machine added in this PR."
|
|
280
|
+
}
|
|
281
|
+
\`\`\`
|
|
282
|
+
(When there's no single focus, anchor on the page heading or the most stable on-page element.)
|
|
283
|
+
|
|
284
|
+
*Single-element additional recommendation (recommendatory voice):*
|
|
285
|
+
\`\`\`json
|
|
286
|
+
{
|
|
287
|
+
"testType": "ui",
|
|
288
|
+
"description": "Would navigate to /admin/notifications when the inbox is empty and assert the Notifications heading renders alongside the empty-state message. Smoke check that the page handles the zero-rows case.",
|
|
289
|
+
"targetElements": [
|
|
290
|
+
{
|
|
291
|
+
"role": "heading", "accessibleName": "Notifications",
|
|
292
|
+
"testId": null, "stableId": null, "contextText": null,
|
|
293
|
+
"mutability": "immutable", "widgetType": "native"
|
|
294
|
+
}
|
|
295
|
+
],
|
|
296
|
+
"pageContext": {
|
|
297
|
+
"url": "http://localhost:5173/admin/notifications",
|
|
298
|
+
"pageHash": "8:992103445"
|
|
299
|
+
},
|
|
300
|
+
"reasoning": "Would assert the **Notifications** heading renders alongside the empty-state message when the inbox has zero rows; covers the empty-state render path added in NotificationsPage.tsx:31."
|
|
301
|
+
}
|
|
302
|
+
\`\`\`
|
|
303
|
+
|
|
304
|
+
**Self-check before submitting (per UI rec, in both \`newTestsCreated\` and \`additionalRecommendations\`):**
|
|
305
|
+
- **Both \`description\` AND \`reasoning\` are populated.** \`description\` is a 2-4 sentence walkthrough; \`reasoning\` is one sentence about what's verified + why. They must not duplicate each other.
|
|
306
|
+
- Every entry in \`targetElements\` must appear verbatim in a captured blueprint element (matched on \`accessibleName\` + \`role\`).
|
|
307
|
+
- Every element name appearing in \`reasoning\` must match an entry's \`accessibleName\` exactly. Every backticked identifier in \`reasoning\` must come from a \`testId\` or \`stableId\` of an entry.
|
|
308
|
+
- **Voice matches the array:** \`newTestsCreated\` uses descriptive voice ("verifies", "asserts", "documents"). \`additionalRecommendations\` uses recommendatory voice ("would verify", "should assert", "would cover").
|
|
309
|
+
- \`pageContext.url\` must equal the URL of the BlueprintCapture you lifted \`targetElements\` from.
|
|
310
|
+
- Never mix two elements' fields in one \`targetElements\` entry — each entry is one captured element.
|
|
311
|
+
- No internal jargon in \`reasoning\` or \`description\`: no \`logicalName\`, \`fingerprint\`, \`pageHash\`, \`xpath\`.
|
|
312
|
+
- If any check fails, re-capture and verify, or drop that recommendation.
|
|
313
|
+
|
|
314
|
+
**Non-UI entries (contract / integration / e2e / batch-scenario) are unaffected.** Use their pre-existing reasoning formats. Do NOT add \`targetElements\` or \`pageContext\` to non-UI entries — the schema rejects them.
|
|
315
|
+
|
|
316
|
+
**No upstream captures available?** Set \`targetElements\` to \`null\`, omit \`pageContext\`, and prefix \`description\` and \`reasoning\` with \`[no-blueprint-data]\`. Use page/feature-level prose; don't cite specific element names without grounding. Apply the marker per entry, not per PR — affected recs only. Log capture failures in \`issuesFound\` (one info-severity entry per failure mode, naming counts). Don't pre-emptively fall back without attempting capture first. Non-UI work is unaffected.
|
|
93
317
|
|
|
94
318
|
---`;
|
|
95
319
|
const serviceContext = services?.length ? buildServiceContext(services) : '';
|
|
@@ -105,13 +329,25 @@ Use those recommendations as your baseline. Only add or remove tests that the us
|
|
|
105
329
|
? `<ui-credentials>\n${trimmedCredentials}\n</ui-credentials>`
|
|
106
330
|
: '';
|
|
107
331
|
const testsRepoDirBlock = testsRepoDir ? `<TESTS REPO DIR>${testsRepoDir}</TESTS REPO DIR>\n` : '';
|
|
332
|
+
// Multi-repo context block. Each entry's `repositoryPath` is checked out at its
|
|
333
|
+
// FEATURE ref; `baseBranch` is that repo's default branch (the diff base), so
|
|
334
|
+
// skyramp_analyze_changes computes a real default…feature diff for the related repo.
|
|
335
|
+
// `repository` is the owner/repo resolved action-side (no agent inference).
|
|
336
|
+
const relatedReposBlock = hasRelatedRepos
|
|
337
|
+
? `<related_repositories>\n` +
|
|
338
|
+
relatedRepositories
|
|
339
|
+
.map((r) => ` <repository repository="${r.repo}" path="${r.repositoryPath}" base_branch="${r.baseBranch || "auto-detect"}" />`)
|
|
340
|
+
.join("\n") +
|
|
341
|
+
`\n</related_repositories>\n`
|
|
342
|
+
: '';
|
|
108
343
|
const testDirInstruction = testsRepoDir
|
|
109
344
|
? `the \`<output_dir>\` from the \`<services>\` block, rooted under the test repository at \`${testsRepoDir}\` (i.e. \`${testsRepoDir}/<output_dir>\`). Write ALL test output files to paths under \`${testsRepoDir}\`, not under \`${repositoryPath}\`. Do NOT write any test files to the app repository.`
|
|
110
345
|
: `${SERVICE_REFS.testDirRef}. Do NOT create a new \`tests/\` directory at the repo root — use that path. If no \`testDirectory\` is configured, default to the language-conventional location (e.g. \`src/test/java/...\` for Java, \`tests/\` for Python).`;
|
|
346
|
+
const primaryRepoBlock = primaryRepo ? `<REPOSITORY>${primaryRepo}</REPOSITORY>\n` : '';
|
|
111
347
|
return `<TITLE>${prTitle}</TITLE>
|
|
112
348
|
<DESCRIPTION>${prDescription}</DESCRIPTION>
|
|
113
|
-
<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
|
|
114
|
-
${testsRepoDirBlock}${serviceContext ? serviceContext + '\n' : ''}${uiCredentialsBlock ? uiCredentialsBlock + '\n' : ''}Use the Skyramp MCP server tools for all tasks below.
|
|
349
|
+
${primaryRepoBlock}<REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
|
|
350
|
+
${relatedReposBlock}${testsRepoDirBlock}${serviceContext ? serviceContext + '\n' : ''}${uiCredentialsBlock ? uiCredentialsBlock + '\n' : ''}Use the Skyramp MCP server tools for all tasks below.
|
|
115
351
|
|
|
116
352
|
${task1Section}
|
|
117
353
|
|
|
@@ -119,10 +355,10 @@ ${task1Section}
|
|
|
119
355
|
|
|
120
356
|
${userPrompt ? "Generate only the tests that the user requested from the Additional Recommendations. The rules below still apply." : "Drift-based maintenance (Task 1) is complete. This step only processes the GENERATE list. Exception: if a GENERATE item targets a resource with an existing `[skyramp]` contract test, UPDATE that test file (see covered-resource handling below) — a new test case added to an existing file counts toward the budget and is reported in `newTestsCreated`."}
|
|
121
357
|
|
|
122
|
-
- **MANDATORY — use the pre-ranked GENERATE list as-is**: The Execution Plan's GENERATE section governs ADD actions. You MUST generate exactly those scenarios in the exact order listed. Do NOT substitute, rename, or replace a GENERATE item. If parameter grounding uncovers a distinct bug-catching scenario not already in the GENERATE or ADDITIONAL list, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.
|
|
358
|
+
- **MANDATORY — use the pre-ranked GENERATE list as-is**: The Execution Plan's GENERATE section governs ADD actions. You MUST generate exactly those scenarios in the exact order listed. Do NOT substitute, rename, or replace a GENERATE item. If parameter grounding uncovers a distinct bug-catching scenario not already in the GENERATE or ADDITIONAL list, generate it after all planned GENERATE items are complete and report it in \`newTestsCreated\` — this is an additional test driven by source-code analysis and does not count against the GENERATE budget.${hasRelatedRepos ? `\n - **Multi-repo exception:** this run has related repositories, so the per-repo GENERATE lists are NOT final — they are candidates re-selected by the cross-repo round-robin described in Task 1's "Cross-repo test generation". Follow that pooled, type-distributed selection instead of any single repo's GENERATE list. (In single-repo runs the GENERATE list IS final — generate it exactly as-is.)` : ""}
|
|
123
359
|
- Scenario JSON files are always new files — always generate them for new methods. Every generated scenario JSON must have a corresponding new integration test generated from it via \`skyramp_integration_test_generation\`.
|
|
124
360
|
- Covered-resource handling (aligns with Execution Plan Step 0): When a GENERATE item targets a resource that already has an existing test file covering the same endpoint:
|
|
125
|
-
- If the existing test source is \`[external]\`, skip the resource entirely — the external test already provides coverage. Do NOT
|
|
361
|
+
- If the existing test source is \`[external]\`, skip the resource entirely — the external test already provides coverage. Do NOT UPDATE, REGENERATE, or DELETE external tests.
|
|
126
362
|
- If the existing test is tagged \`[skyramp]\`, apply type-specific rules:
|
|
127
363
|
- Contract tests: UPDATE the existing Skyramp test file (add the new method's test cases). A new test case is a new test even if the file already exists — report in \`newTestsCreated\` and count toward the budget.
|
|
128
364
|
- Integration/scenario tests: Always generate as a new file via the scenario pipeline (\`skyramp_batch_scenario_test_generation\` → \`skyramp_integration_test_generation\`), even if an existing integration test covers the same resource. A new multi-step scenario (e.g. create → PATCH → verify recalculation) is a distinct test file. Report in \`newTestsCreated\` and count toward the budget.
|
|
@@ -131,7 +367,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
131
367
|
- Example: If enrichment reveals that sending \`discount_value\` without \`discount_type\` silently orphans the value (a concrete bug), complete all planned GENERATE items first, then generate this discovered scenario as an extra test and report it in \`newTestsCreated\`.
|
|
132
368
|
- Total generated: Follow the "Budget: N generate" line in the Execution Plan. Process every GENERATE-tagged item in order. Backfill from ADDITIONAL candidates (highest-ranked first) until \`newTestsCreated\` reaches ${maxGenerate} or all candidates are exhausted.
|
|
133
369
|
- **UI test priority**: If the PR scope assessment shows any UI/E2E budget OR \`uiContext.changedFrontendFiles\` is non-empty (the deterministic server signal — populated for all supported frontend file types including \`.tsx\`/\`.jsx\`/\`.vue\`/\`.svelte\`/\`.dart\`), you MUST attempt to generate at least one UI test. Use \`browser_navigate\` to the app's base URL — if the app responds, record a trace and generate the test.
|
|
134
|
-
**Flutter web apps:** Skyramp's Playwright tools automatically enable Flutter's accessibility semantics tree on every \`browser_navigate\` call — you do NOT need to manually click \`flt-semantics-placeholder\` or add any activation step to the trace. Do NOT log an \`issuesFound\` entry about Flutter canvas rendering or accessibility activation — this is handled transparently. **Do NOT skip test generation or abstain from recording based on what you see in the Flutter source code** (e.g. \`SemanticsBinding.ensureSemantics()\` commented out, \`IS_TESTING\` flag absent, or similar) — Skyramp enables accessibility from the browser side regardless of the app's Dart code. Proceed with \`browser_navigate\` and test recording as normal.
|
|
370
|
+
**Flutter web apps:** Skyramp's Playwright tools automatically enable Flutter's accessibility semantics tree on every \`browser_navigate\` call — you do NOT need to manually click \`flt-semantics-placeholder\` or add any activation step to the trace. Do NOT log an \`issuesFound\` entry about Flutter canvas rendering or accessibility activation — this is handled transparently. **Do NOT skip test generation or abstain from recording based on what you see in the Flutter source code** (e.g. \`SemanticsBinding.ensureSemantics()\` commented out, \`IS_TESTING\` flag absent, or similar) — Skyramp enables accessibility from the browser side regardless of the app's Dart code. Proceed with \`browser_navigate\` and test recording as normal. **Start at the app's root URL** (e.g. \`{baseUrl}/\`) — do NOT \`browser_navigate\` straight to a deep sub-route (e.g. \`/authors\`, \`/orders/13\`). Flutter \`go_router\` SPAs route from the root: deep-linking on a cold page load often fails to render the expected screen (the route's widgets never mount, so the trace captures the wrong page). Load the root, let the app's own routing/auth-redirect render, then reach target screens by interaction. **After the initial login, navigate using in-app controls only** (tab buttons, links, back buttons) — do NOT call \`browser_navigate\` to a different URL after login. Flutter web apps are SPAs: a \`browser_navigate\` to a new URL after login triggers a full page reload which clears the auth session, causing redundant re-login cycles in the generated test. Use button clicks to reach target screens instead.
|
|
135
371
|
**Skip only if one of these conditions is met:**
|
|
136
372
|
- **(a) App is unreachable** — \`browser_navigate\` fails or connection is refused.
|
|
137
373
|
- **(b) Unintegrated non-route component** — the changed file is a leaf component (not a framework route/entrypoint) that has no integration point in the running app. To confirm:
|
|
@@ -139,7 +375,6 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
139
375
|
2. If no production file imports, re-exports, or renders it, the component has no DOM node in the running app → unintegrated.
|
|
140
376
|
3. **Exception**: if the same PR also adds a route/page file (e.g. under Next.js \`pages/\` or \`app/\`) that imports the component, the route IS the integration point — test through it.
|
|
141
377
|
**Never** apply the unintegrated heuristic to framework route/entrypoint files themselves — those are always reachable by convention.
|
|
142
|
-
- **(c) Cosmetic-only frontend change** — the frontend files in the diff are purely cosmetic with no observable rendering or interaction change (e.g. a \`.css\`/\`.scss\` reformat: property reordering, comment/whitespace edits, \`0px\`→\`0\`). \`uiContext.changedFrontendFiles\` being non-empty does NOT override this — that signal only means a frontend file changed, not that behavior changed. Take the zero-test abstention path in Task 3; do NOT record a trace just to satisfy the mandate.
|
|
143
378
|
**Never** generate tests for unrelated pages as a substitute for an unintegrated component.
|
|
144
379
|
This rule takes priority over generating additional backend-only tests.
|
|
145
380
|
- **Always generate a test for critical bugs, even if it will fail.** When a GENERATE-tagged item targets a page or endpoint with a known bug, do NOT skip it because you expect the test to fail — a failing test that documents a bug is more valuable than a text-only description. This applies within the existing GENERATE budget; do not add extra tests beyond the plan.
|
|
@@ -226,6 +461,7 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
226
461
|
- \`skyramp_export_zip\` outputPath: \`${repositoryPath}/.skyramp/<test_name>_trace.zip\`
|
|
227
462
|
- \`skyramp_ui_test_generation\`: set \`modularizeCode: false\`
|
|
228
463
|
- **\`browser_assert\` — MANDATORY**: at least one per page navigated. Call multiple assertions in the same tool call batch when checking independent elements. If you navigate to 2 pages, assert on both. Each assertion should verify a business outcome (state change, computed value, error condition) — not just that an element is visible.
|
|
464
|
+
- **Wait for stable state before the second capture**: After performing an action that affects computed fields (filling a discount, submitting a form, adding an item), check the current page state before calling the second \`browser_blueprint\` (the capture after the action). If a computed field — total, price, count, derived text — still shows its initial empty or zero value (e.g. \`$0.00\`, \`0\`, \`Loading...\`, empty string), that means async data hasn't finished loading yet. Use \`browser_wait_for\` to wait up to 10 seconds for the field to update to a real value (for example, wait for the total to show a non-zero amount like \`$799.99\` instead of \`$0.00\`). Once the field shows a real value, THEN call the second \`browser_blueprint\` to capture stable state. If after 10 seconds the field still hasn't updated, skip the assertion on that field — don't capture and assert a value that hasn't loaded.
|
|
229
465
|
If \`browser_navigate\` fails (app not running / connection refused), move to \`additionalRecommendations\` with the failure reason.
|
|
230
466
|
Record at most 2-3 UI traces per run to stay within tool call budget. Quality over quantity: 1 great test is better than 3 mediocre ones — do not pad to reach the count.
|
|
231
467
|
**Strategic assertions** — key checkpoints only, 3 to 5 per test:
|
|
@@ -238,9 +474,9 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
238
474
|
|
|
239
475
|
**Capture-act-capture (applies only when recording a UI trace):**
|
|
240
476
|
|
|
241
|
-
**Skip this entire section if \`uiContext\` was absent or \`changedFrontendFiles\` was empty in the \`
|
|
477
|
+
**Skip this entire section if \`uiContext\` was absent or \`changedFrontendFiles\` was empty in the \`skyramp_analyze_changes\` response** (backend-only PR). The capture-act-capture pattern is for UI trace recording only — there's no UI trace to record on a backend-only PR. Continue to the non-UI test-type instructions below.
|
|
242
478
|
|
|
243
|
-
**Reminder — the UI test priority rule above still applies.** If the diff contains frontend/UI changes, you still MUST attempt to generate at least one UI test. Capture-act-capture is **how** you record that test, not **whether** you record one — do not substitute UI recommendations for actually recording a trace. UI recommendation reasoning was already grounded in the
|
|
479
|
+
**Reminder — the UI test priority rule above still applies.** If the diff contains frontend/UI changes, you still MUST attempt to generate at least one UI test. Capture-act-capture is **how** you record that test, not **whether** you record one — do not substitute UI recommendations for actually recording a trace. UI recommendation reasoning was already grounded in the blueprints you captured from the UI Blueprint Capture section of \`skyramp_analyze_changes\`; Task 2's capture-act-capture is for the trace's own assertions, not for retroactively rewriting recommendation reasoning.
|
|
244
480
|
|
|
245
481
|
This pattern produces delta-derived assertions from blueprint diffs. Diff-derived assertions catch state changes more reliably than author-inference — the diff tells you what actually changed on the page so the assertion is grounded in observable state, not in guessing what "success" looks like.
|
|
246
482
|
|
|
@@ -254,15 +490,23 @@ ${CONTRACT_MODE_GUIDANCE}
|
|
|
254
490
|
|
|
255
491
|
1. **Before** the action: \`browser_blueprint\`. Identify the semantic target by \`role\`, \`accessibleName\`, and \`stableId\`/\`testId\`.
|
|
256
492
|
|
|
257
|
-
2. If the target's \`widgetType\` is \`"custom"\` or \`"unknown"
|
|
493
|
+
2. If the target's \`widgetType\` is \`"custom"\` or \`"unknown"\`, it is a JavaScript-composite control (e.g. a Radix/MUI combobox, a date picker, a custom multi-select) that does not behave like a native HTML element. Do NOT call \`browser_fill\` or \`browser_select_option\` directly on it. Instead: click the trigger element first, use \`browser_wait_for\` to wait for the overlay/dropdown to appear, then interact with its contents. Use \`browser_snapshot\` to inspect what appeared before choosing what to click next.
|
|
258
494
|
|
|
259
495
|
3. Execute the action via \`browser_click\` / \`browser_type\` / \`browser_navigate\`. The \`ref\` comes from \`browser_snapshot\` as today.
|
|
260
496
|
|
|
261
497
|
4. **After** the action: \`browser_blueprint\` again. The response shape depends on whether the action navigated:
|
|
262
|
-
- **Same URL (modal/tab/in-place mutation):** \`{ isFullCapture: false, pageHash, previousPageHash, delta, possibleAssertions }\`. The \`delta\` field contains \`elementsAdded\`, \`elementsRemoved\`, \`textChanges\`, \`repeatingCountChanges\`. The \`possibleAssertions\` field is a mechanical translation of those entries into Playwright \`expect(...)\` candidates — see step 5.
|
|
263
|
-
- **Navigated to a new URL** (e.g. router transition, link click, programmatic \`browser_navigate\`): \`{ isFullCapture: true, pageHash, blueprint }\` — a fresh full capture of the new page. No \`possibleAssertions\` here (no delta to translate).
|
|
498
|
+
- **Same URL (modal/tab/in-place mutation):** \`{ isFullCapture: false, pageHash, previousPageHash, delta, possibleAssertions }\`. The \`delta\` field contains \`elementsAdded\`, \`elementsRemoved\`, \`textChanges\`, \`repeatingCountChanges\`. The \`possibleAssertions\` field is a mechanical translation of those entries into Playwright \`expect(...)\` candidates — see step 5. An empty delta (all arrays empty) is itself a meaningful signal: the action did not change observable DOM (e.g. a silent failure the test should catch).
|
|
499
|
+
- **Navigated to a new URL** (e.g. router transition, link click, programmatic \`browser_navigate\`): \`{ isFullCapture: true, pageHash, blueprint }\` — a fresh full capture of the new page. No \`possibleAssertions\` here (no delta to translate). Search the new blueprint for the elements your assertion will target.
|
|
264
500
|
|
|
265
|
-
5. **The
|
|
501
|
+
5. **The second \`browser_blueprint\` response (the capture after the action) includes a \`possibleAssertions[]\` array — mechanical translations of delta entries into ready-to-use Playwright \`expect(...)\` candidates.** Each entry has \`{ code, rationale, tier }\`. Use tier to drive your assertion choices:
|
|
502
|
+
|
|
503
|
+
- **HIGH** — always assert on these. HIGH candidates translate state-observable changes: text content that updated (\`toHaveText\`), URL transitions (\`toHaveURL\`), count changes (\`toHaveCount\`). If a HIGH-tier candidate exists after an action, it is the primary assertion for that action. (If you followed the stable-state rule above and waited for real data before capturing, HIGH candidates will already contain real values — not loading state.)
|
|
504
|
+
- **MEDIUM** — use as supplementary assertions. MEDIUM candidates translate structural changes: an element appeared (\`toBeVisible\`) or disappeared (\`not.toBeVisible\`) after the action. Useful alongside HIGH assertions but insufficient alone for state-changing actions.
|
|
505
|
+
- **LOW** — treat as context only. LOW candidates are incidental visibility checks on elements that happened to be on-page. Do not use LOW candidates as the sole assertion on a state-changing action.
|
|
506
|
+
|
|
507
|
+
**If no HIGH-tier candidate exists after a state-changing action** (form submit, button click that mutates data), write a targeted assertion yourself — look in the second blueprint capture for computed values, status fields, or derived text that changed. One well-targeted \`toHaveText\` or \`toHaveValue\` beats five \`toBeVisible\` checks.
|
|
508
|
+
|
|
509
|
+
**The pre-existing rule still applies:** at least one \`browser_assert\` per page navigated, verifying a business outcome — not just that an element is visible.
|
|
266
510
|
|
|
267
511
|
**The Blueprint Citation Invariant applies during recording too.** Every assertion you emit cites element names — those names must come from blueprint captures, not invention. For N user-intent-level actions, the reference target is N+1 \`browser_blueprint\` calls (the first returns full, the rest return deltas). Traces that follow the pattern produce assertions grounded in observable state changes; traces that skip captures fall back to author-inferred assertions and risk citing names that don't exist in the rendered DOM.
|
|
268
512
|
|
|
@@ -299,18 +543,14 @@ Do NOT use \`page.waitForTimeout()\` with fixed delays. Do NOT retry more than o
|
|
|
299
543
|
**After generation, you MUST do exactly these steps — nothing more, nothing less:**
|
|
300
544
|
1. **[MANDATORY] After \`skyramp_integration_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated integration test file, \`testType: "integration"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
|
|
301
545
|
2. **[MANDATORY] After \`skyramp_contract_test_generation\` with \`providerMode\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
|
|
302
|
-
3. **[MANDATORY] After \`skyramp_ui_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
|
|
546
|
+
3. **[MANDATORY] After \`skyramp_ui_test_generation\`**: Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated UI test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file. The HIGH-tier \`possibleAssertions\` from your second \`browser_blueprint\` captures (after each action) during trace recording are in your context — when the enhance instructions ask you to add assertions for state-changing actions, use those grounded candidates first (they contain exact computed values from the DOM delta, e.g. \`toHaveText('Total: $899.98')\`). Only fall back to deriving values from the test file or source code when no HIGH-tier candidate covers the action.
|
|
303
547
|
4. **Wait**: Do NOT proceed to test execution until steps 1–3 are complete and the verification checklist in the \`skyramp_enhance_assertions\` tool result has been validated for EVERY generated test file.
|
|
304
548
|
Do not make any changes other than the assertion enhancements described above. For example: do not modify auth headers, cookies, tokens, env vars, or imports that the generation tool already set correctly — those are correct by construction and changing them breaks auth or execution.
|
|
305
549
|
|
|
306
|
-
**
|
|
307
|
-
- **beforeStatus** (maintained tests only): execute each maintained test file **once at the start** (before any edits) to capture \`beforeStatus\`. This is the only execution allowed before edits.
|
|
308
|
-
- **Final execution**: Do NOT call \`skyramp_execute_test\` again until ALL maintenance edits AND ALL new test generation/enhancement are complete. Then execute every test file once — maintained files (for \`afterStatus\`) and new files together. Run independent files in parallel (same tool call batch).
|
|
550
|
+
**Final execution (mandatory):** Do NOT call \`skyramp_execute_test\` until ALL maintenance edits AND ALL new test generation/enhancement are complete.
|
|
309
551
|
- Only report test results for files you actually ran.
|
|
310
552
|
**Auth**: If \`skyramp_analyze_changes\` reports an auth token or \`SKYRAMP_TEST_TOKEN\` is set, pass it in **every** \`skyramp_execute_test\` call from the first attempt — do NOT wait for a 401/403 to discover auth is needed.
|
|
311
553
|
|
|
312
|
-
---
|
|
313
|
-
|
|
314
554
|
## Task 3: Submit Report
|
|
315
555
|
|
|
316
556
|
**Before calling \`skyramp_submit_report\` — mandatory count check:**
|
|
@@ -331,15 +571,26 @@ In these cases:
|
|
|
331
571
|
|
|
332
572
|
Otherwise: in \`newTestsCreated\`, you must have exactly ${maxGenerate} budget-counting new tests for the planned GENERATE items. Only new files (ADD) created for those planned GENERATE items count toward this ${maxGenerate} target — GENERATE items converted to UPDATE do not. You may also include at most one additional discovered-scenario file in \`newTestsCreated\` (the bug-catching test generated after all planned items); that extra test does **not** count against the ${maxGenerate} budget. If you have fewer than ${maxGenerate} budget-counting new tests, backfill from the remaining ADDITIONAL candidates before proceeding. Only proceed with fewer than ${maxGenerate} budget-counting new tests if all candidates failed after retry AND the fallback single-contract test also failed.
|
|
333
573
|
|
|
334
|
-
Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}". Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
335
|
-
|
|
574
|
+
Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}" and \`stateFile\` (from \`skyramp_analyze_changes\` output) — the stateFile is required for execution outcome tracking. Field names, types, and formats are defined in the tool's parameter schema — follow them exactly.
|
|
575
|
+
${hasRelatedRepos ? `
|
|
576
|
+
- **MULTI-REPO attribution**: Set the \`repository\` field (\`owner/repo\`) on EVERY \`newTestsCreated\`, \`testResults\`, \`issuesFound\`, and \`additionalRecommendations\` item — including items about the PRIMARY repo — so each finding is unambiguously attributed. The primary repo's \`repository\` is \`${primaryRepo || "<the primary repo's owner/repo>"}\`; items derived from a related repo's diff (from the \`<related_repositories>\` analysis) carry that repo's \`repository\` value. In \`businessCaseAnalysis\`, include a short per-repo subsection and call out any cross-repo correlations you found.
|
|
577
|
+
` : ""}
|
|
336
578
|
- **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
|
|
337
579
|
- For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
|
|
338
580
|
- For \`testType: "integration"\` or \`"e2e"\` entries: omit \`primaryEndpoint\` — use \`description\` to list the endpoints involved instead.
|
|
581
|
+
- **testMaintenance**: Use \`[]\` **only** if no existing Skyramp tests were found in the repository. If existing tests were found (any score), include one entry per test. Set \`action\` to the exact drift action you chose from the Action Decision Matrix (\`UPDATE\`, \`REGENERATE\`, \`DELETE\`, \`VERIFY\`, or \`IGNORE\`). For UPDATE/REGENERATE/DELETE tests that were modified and executed, populate \`beforeDetails\` and \`afterDetails\` from what you observed in \`skyramp_execute_test\` output — one line each, no newlines. For passing runs write the count and timing (e.g. \`"4 passed in 15.09s"\`). For failing runs write the test name and failure reason (e.g. \`"1 failed: test_foo - assert X == Y — shared-state pollution"\`). For VERIFY/IGNORE tests (not modified), omit \`beforeDetails\` and \`afterDetails\`. Do **not** add entries for tests that were not returned by the health analysis.
|
|
582
|
+
- **testResults**: For UI/E2E test entries, include the \`videoPath\` field if \`skyramp_execute_test\` returned one.
|
|
339
583
|
|
|
340
584
|
---
|
|
341
585
|
|
|
342
586
|
${getTraceRecordingPromptText({ outputDir: `${repositoryPath}/.skyramp`, modularize: false })}`;
|
|
587
|
+
// TODO: merge stateFile and summaryOutputFile into a single file (follow-up PR).
|
|
588
|
+
// Currently the stateFile path is derived server-side from RUNNER_TEMP (GitHub Actions only);
|
|
589
|
+
// other CI systems (Jenkins, GitLab, Buildkite) require the caller to set RUNNER_TEMP.
|
|
590
|
+
// Once merged, the path flows through summaryOutputFile — already testbot-controlled and
|
|
591
|
+
// CI-agnostic — eliminating the RUNNER_TEMP dependency and the LLM confusion that motivated
|
|
592
|
+
// removing stateOutputFile from the prompt schema. Remove the RUNNER_TEMP branch in
|
|
593
|
+
// AnalysisStateManager.ts when this is done.
|
|
343
594
|
}
|
|
344
595
|
function escapeXml(value) {
|
|
345
596
|
return value
|
|
@@ -414,10 +665,6 @@ export function registerTestbotPrompt(server) {
|
|
|
414
665
|
.string()
|
|
415
666
|
.optional()
|
|
416
667
|
.describe("Natural language prompt from the user (via @skyramp-testbot comment) to add or remove specific recommendations."),
|
|
417
|
-
stateOutputFile: z
|
|
418
|
-
.string()
|
|
419
|
-
.optional()
|
|
420
|
-
.describe("Absolute path where skyramp_analyze_changes should write its state file. When provided, the caller can locate the file without log parsing."),
|
|
421
668
|
uiCredentials: z
|
|
422
669
|
.string()
|
|
423
670
|
.optional()
|
|
@@ -430,10 +677,18 @@ export function registerTestbotPrompt(server) {
|
|
|
430
677
|
.string()
|
|
431
678
|
.optional()
|
|
432
679
|
.describe("Absolute path to a cloned test repository. When set, the agent writes generated test files there instead of the app repository (cross-repo test delivery)."),
|
|
680
|
+
relatedRepositories: z
|
|
681
|
+
.string()
|
|
682
|
+
.optional()
|
|
683
|
+
.describe("JSON-encoded array of {repo, repositoryPath, baseBranch?} for related repositories to analyze (multi-repo mode). `repo` is the owner/repo slug (resolved by the action; used verbatim, never inferred). Each is analyzed with its own skyramp_analyze_changes call; the agent correlates cross-repo changes and draws GENERATE candidates from all repos into one shared ranked budget. Generated test files are delivered into each service's declared testDirectory, relative to the single delivery root (the primary repo, or the configured test repo)."),
|
|
684
|
+
primaryRepo: z
|
|
685
|
+
.string()
|
|
686
|
+
.optional()
|
|
687
|
+
.describe("The primary repository's owner/repo slug (where the testbot workflow runs). Used verbatim as the `repository` attribution for the primary repo's analysis, tests, and report items in multi-repo runs."),
|
|
433
688
|
},
|
|
434
689
|
}, async (args) => {
|
|
435
690
|
const services = await readWorkspaceServices(args.repositoryPath);
|
|
436
|
-
let prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.maxCritical, args.prNumber, args.userPrompt, services.length ? services : undefined, args.
|
|
691
|
+
let prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.maxCritical, args.prNumber, args.userPrompt, services.length ? services : undefined, args.uiCredentials, args.testsRepoDir, parseRelatedRepositories(args.relatedRepositories), args.primaryRepo);
|
|
437
692
|
if (args.workspaceValidationFailed) {
|
|
438
693
|
prompt = buildWorkspaceRecoveryPrefix(args.repositoryPath) + prompt;
|
|
439
694
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|