@skyramp/mcp 0.3.8 → 0.4.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/build/commands/commandLibrary.d.ts +1 -1
  2. package/build/commands/commandLibrary.js +3 -3
  3. package/build/commands/recommendTestsAndExecuteCommand.d.ts +1 -1
  4. package/build/commands/recommendTestsAndExecuteCommand.js +35 -20
  5. package/build/commands/testThisEndpointCommand.js +35 -19
  6. package/build/index.js +9 -3
  7. package/build/playwright/blueprintDigest.d.ts +15 -0
  8. package/build/playwright/blueprintDigest.js +152 -0
  9. package/build/playwright/blueprintDigestStore.d.ts +31 -0
  10. package/build/playwright/blueprintDigestStore.js +117 -0
  11. package/build/playwright/registerPlaywrightTools.js +60 -12
  12. package/build/playwright/traceRecordingPrompt.js +8 -7
  13. package/build/prompts/enhance-assertions/sharedAssertionRules.js +9 -8
  14. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -2
  15. package/build/prompts/promptAssets.d.ts +20 -0
  16. package/build/prompts/promptAssets.js +55 -0
  17. package/build/prompts/sut-setup/modes/dockerComposePrompt.js +19 -5
  18. package/build/prompts/test-maintenance/actionsInstructions.d.ts +4 -0
  19. package/build/prompts/test-maintenance/actionsInstructions.js +14 -2
  20. package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +0 -10
  21. package/build/prompts/test-maintenance/drift-analysis-prompt.js +2 -11
  22. package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +8 -4
  23. package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +5 -22
  24. package/build/prompts/test-recommendation/diffExecutionPlan.js +37 -465
  25. package/build/prompts/test-recommendation/recommendationSections.d.ts +7 -17
  26. package/build/prompts/test-recommendation/recommendationSections.js +67 -309
  27. package/build/prompts/test-recommendation/recommendationShared.d.ts +19 -47
  28. package/build/prompts/test-recommendation/recommendationShared.js +49 -155
  29. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +0 -5
  30. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +10 -153
  31. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -29
  32. package/build/prompts/test-recommendation/test-recommendation-prompt.js +32 -457
  33. package/build/prompts/testbot/planDeclarations.d.ts +6 -0
  34. package/build/prompts/testbot/planDeclarations.js +9 -0
  35. package/build/prompts/testbot/testbot-prompts.d.ts +8 -0
  36. package/build/prompts/testbot/testbot-prompts.js +256 -381
  37. package/build/recommendation/answers.d.ts +35 -0
  38. package/build/recommendation/answers.js +96 -0
  39. package/build/recommendation/registerPlan.d.ts +49 -0
  40. package/build/recommendation/registerPlan.js +117 -0
  41. package/build/recommendation/runVerifiers.d.ts +10 -0
  42. package/build/recommendation/runVerifiers.js +49 -0
  43. package/build/recommendation/subjectStep.d.ts +42 -0
  44. package/build/recommendation/subjectStep.js +86 -0
  45. package/build/recommendation/types.d.ts +163 -0
  46. package/build/recommendation/types.js +20 -0
  47. package/build/recommendation/verifierContracts.d.ts +382 -0
  48. package/build/recommendation/verifierContracts.js +263 -0
  49. package/build/recommendation/verifiers/changedFile.d.ts +2 -0
  50. package/build/recommendation/verifiers/changedFile.js +82 -0
  51. package/build/recommendation/verifiers/citedPath.d.ts +12 -0
  52. package/build/recommendation/verifiers/citedPath.js +35 -0
  53. package/build/recommendation/verifiers/coverage.d.ts +7 -0
  54. package/build/recommendation/verifiers/coverage.js +617 -0
  55. package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +11 -0
  56. package/build/recommendation/verifiers/deliveredMatchesPlan.js +33 -0
  57. package/build/recommendation/verifiers/endpointGrounded.d.ts +17 -0
  58. package/build/recommendation/verifiers/endpointGrounded.js +128 -0
  59. package/build/recommendation/verifiers/existingCoverage.d.ts +6 -0
  60. package/build/recommendation/verifiers/existingCoverage.js +51 -0
  61. package/build/recommendation/verifiers/expectedOutcome.d.ts +31 -0
  62. package/build/recommendation/verifiers/expectedOutcome.js +105 -0
  63. package/build/recommendation/verifiers/removedElementGuarded.d.ts +2 -0
  64. package/build/recommendation/verifiers/removedElementGuarded.js +57 -0
  65. package/build/recommendation/verifiers/reportedCategory.d.ts +26 -0
  66. package/build/recommendation/verifiers/reportedCategory.js +84 -0
  67. package/build/recommendation/verifiers/screenRoute.d.ts +10 -0
  68. package/build/recommendation/verifiers/screenRoute.js +118 -0
  69. package/build/recommendation/verifiers/statedDifference.d.ts +6 -0
  70. package/build/recommendation/verifiers/statedDifference.js +140 -0
  71. package/build/recommendation/verifiers/uiElementGrounded.d.ts +7 -0
  72. package/build/recommendation/verifiers/uiElementGrounded.js +318 -0
  73. package/build/resources/analysisResources.js +1 -114
  74. package/build/resources/testbotResource.js +23 -13
  75. package/build/services/ModularizationService.js +2 -1
  76. package/build/services/TestDiscoveryService.d.ts +3 -72
  77. package/build/services/TestDiscoveryService.js +10 -303
  78. package/build/services/containerEnv.d.ts +1 -1
  79. package/build/services/containerEnv.js +12 -0
  80. package/build/skills/fixTestImportErrorsSkill.d.ts +13 -0
  81. package/build/skills/fixTestImportErrorsSkill.js +20 -0
  82. package/build/toolNames.d.ts +1 -0
  83. package/build/toolNames.js +1 -0
  84. package/build/tools/code-refactor/enhanceAssertionsTool.js +3 -3
  85. package/build/tools/code-refactor/modularizationTool.js +2 -1
  86. package/build/tools/executeSkyrampTestTool.d.ts +80 -0
  87. package/build/tools/executeSkyrampTestTool.js +246 -19
  88. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +6 -0
  89. package/build/tools/generate-tests/generateContractRestTool.js +3 -3
  90. package/build/tools/generate-tests/planGuard.d.ts +2 -2
  91. package/build/tools/generate-tests/planGuard.js +78 -18
  92. package/build/tools/one-click/oneClickTool.d.ts +0 -1
  93. package/build/tools/one-click/oneClickTool.js +0 -5
  94. package/build/tools/submitReportTool.d.ts +48 -42
  95. package/build/tools/submitReportTool.js +576 -193
  96. package/build/tools/test-management/actionsTool.js +72 -4
  97. package/build/tools/test-management/analyzeChangesTool.d.ts +144 -48
  98. package/build/tools/test-management/analyzeChangesTool.js +212 -1219
  99. package/build/tools/test-management/analyzeTestHealthTool.js +13 -24
  100. package/build/tools/test-management/index.d.ts +1 -0
  101. package/build/tools/test-management/index.js +1 -0
  102. package/build/tools/test-management/registerTestPlanTool.d.ts +795 -172
  103. package/build/tools/test-management/registerTestPlanTool.js +609 -542
  104. package/build/tools/test-management/resolveScreenTool.d.ts +75 -0
  105. package/build/tools/test-management/resolveScreenTool.js +289 -0
  106. package/build/types/BlueprintDigest.d.ts +34 -0
  107. package/build/types/BlueprintDigest.js +1 -0
  108. package/build/types/RepositoryAnalysis.d.ts +20 -1559
  109. package/build/types/RepositoryAnalysis.js +2 -58
  110. package/build/types/StepMethod.d.ts +40 -0
  111. package/build/types/StepMethod.js +77 -0
  112. package/build/types/TestAnalysis.d.ts +12 -0
  113. package/build/types/TestExecution.d.ts +4 -0
  114. package/build/types/TestRecommendation.d.ts +24 -24
  115. package/build/types/TestRecommendation.js +91 -89
  116. package/build/types/TestbotPromptOptions.d.ts +0 -4
  117. package/build/types/TestbotReport.d.ts +64 -2
  118. package/build/utils/AnalysisStateManager.d.ts +79 -113
  119. package/build/utils/AnalysisStateManager.js +147 -57
  120. package/build/utils/assertion-verify/api-shared-lints.js +1 -1
  121. package/build/utils/assertion-verify/metrics.js +85 -36
  122. package/build/utils/assertion-verify/ui-lints.d.ts +0 -5
  123. package/build/utils/assertion-verify/ui-lints.js +32 -0
  124. package/build/utils/branchDiff.d.ts +63 -31
  125. package/build/utils/branchDiff.js +242 -94
  126. package/build/utils/containedPath.d.ts +18 -0
  127. package/build/utils/containedPath.js +73 -0
  128. package/build/utils/dartRouteExtractor.d.ts +18 -34
  129. package/build/utils/dartRouteExtractor.js +101 -173
  130. package/build/utils/featureFlags.d.ts +12 -0
  131. package/build/utils/featureFlags.js +14 -0
  132. package/build/utils/frontendSelectors.d.ts +48 -27
  133. package/build/utils/frontendSelectors.js +241 -80
  134. package/build/utils/pathMatching.d.ts +2 -4
  135. package/build/utils/pathMatching.js +2 -4
  136. package/build/utils/planMatchKeys.d.ts +38 -47
  137. package/build/utils/planMatchKeys.js +143 -81
  138. package/build/utils/rebaselineSnapshots.d.ts +24 -0
  139. package/build/utils/rebaselineSnapshots.js +65 -0
  140. package/build/utils/removedUiElements.d.ts +22 -0
  141. package/build/utils/removedUiElements.js +106 -0
  142. package/build/utils/reportVerification.d.ts +2 -6
  143. package/build/utils/reportVerification.js +61 -2
  144. package/build/utils/screenRoutes.d.ts +66 -0
  145. package/build/utils/screenRoutes.js +727 -0
  146. package/build/utils/sourceRouteExtractor.js +320 -112
  147. package/build/utils/testFileClassification.d.ts +11 -2
  148. package/build/utils/testFileClassification.js +44 -2
  149. package/build/utils/testFixtures.d.ts +5 -0
  150. package/build/utils/testFixtures.js +13 -0
  151. package/build/utils/utils.d.ts +0 -1
  152. package/build/utils/utils.js +0 -11
  153. package/build/utils/versions.d.ts +3 -3
  154. package/build/utils/versions.js +1 -1
  155. package/build/workspace/workspace.d.ts +12 -12
  156. package/node_modules/playwright/lib/mcp/skyramp/assertHiddenTool.js +56 -0
  157. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +2 -1
  158. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +10 -0
  159. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +4 -1
  160. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +160 -1
  161. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -2
  162. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
  163. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderSignalProcessor.js +2 -0
  164. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder.js +5 -1
  165. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.-Id052Lr.js → index.B7KbSQcC.js} +1 -1
  166. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
  167. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  168. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  169. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderSignalProcessor.ts +7 -0
  170. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder.ts +6 -1
  171. package/node_modules/playwright/package.json +1 -1
  172. package/package.json +4 -3
  173. package/plugin/.claude-plugin/plugin.json +8 -0
  174. package/plugin/plugin.json +6 -0
  175. package/plugin/prompts/declaring-a-plan.md +20 -0
  176. package/plugin/prompts/generate-tests/context-fetching.md +4 -0
  177. package/plugin/prompts/generate-tests/execution-plan.md +63 -0
  178. package/plugin/prompts/generate-tests/generation.md +108 -0
  179. package/plugin/prompts/generate-tests/path-parameters.md +1 -0
  180. package/plugin/prompts/generate-tests/reasoning-protocol.md +17 -0
  181. package/plugin/prompts/generate-tests/tool-workflow-variants.md +61 -0
  182. package/plugin/prompts/generate-tests/tool-workflows.md +65 -0
  183. package/plugin/prompts/plan-tests.md +42 -0
  184. package/plugin/prompts/testbot-task1.md +82 -0
  185. package/plugin/skills/fix-test-import-errors/SKILL.md +98 -0
  186. package/build/prompts/test-recommendation/analysisOutputPrompt.d.ts +0 -84
  187. package/build/prompts/test-recommendation/analysisOutputPrompt.js +0 -369
  188. package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +0 -7
  189. package/build/prompts/test-recommendation/fullRepoCatalog.js +0 -283
  190. package/build/prompts/test-recommendation/scopeAssessment.d.ts +0 -81
  191. package/build/prompts/test-recommendation/scopeAssessment.js +0 -359
  192. package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +0 -7
  193. package/build/recommendation/budgeters/diversityBalancedBudgeter.js +0 -105
  194. package/build/recommendation/budgeters/fixedNBudgeter.d.ts +0 -7
  195. package/build/recommendation/budgeters/fixedNBudgeter.js +0 -11
  196. package/build/recommendation/budgeters/shared.d.ts +0 -32
  197. package/build/recommendation/budgeters/shared.js +0 -246
  198. package/build/recommendation/discriminators.d.ts +0 -37
  199. package/build/recommendation/discriminators.js +0 -379
  200. package/build/recommendation/diversity.d.ts +0 -47
  201. package/build/recommendation/diversity.js +0 -101
  202. package/build/recommendation/planRanker.d.ts +0 -65
  203. package/build/recommendation/planRanker.js +0 -83
  204. package/build/recommendation/testFixtures.d.ts +0 -25
  205. package/build/recommendation/testFixtures.js +0 -45
  206. package/build/types/FrontendIntegration.d.ts +0 -28
  207. package/build/types/FrontendIntegration.js +0 -22
  208. package/build/types/Recommendation.d.ts +0 -146
  209. package/build/types/Recommendation.js +0 -74
  210. package/build/utils/changedRoutes.d.ts +0 -29
  211. package/build/utils/changedRoutes.js +0 -87
  212. package/build/utils/frontendIntegration.d.ts +0 -9
  213. package/build/utils/frontendIntegration.js +0 -243
  214. package/build/utils/importerHop.d.ts +0 -135
  215. package/build/utils/importerHop.js +0 -489
  216. package/build/utils/pathAffinityClassification.d.ts +0 -49
  217. package/build/utils/pathAffinityClassification.js +0 -180
  218. package/build/utils/pythonMountPrefixes.d.ts +0 -25
  219. package/build/utils/pythonMountPrefixes.js +0 -347
  220. package/build/utils/repoScanner.d.ts +0 -34
  221. package/build/utils/repoScanner.js +0 -300
  222. package/build/utils/routeParsers.d.ts +0 -95
  223. package/build/utils/routeParsers.js +0 -951
  224. package/build/utils/scenarioDrafting.d.ts +0 -92
  225. package/build/utils/scenarioDrafting.js +0 -951
  226. package/build/utils/subjectEndpoints.d.ts +0 -19
  227. package/build/utils/subjectEndpoints.js +0 -98
  228. package/build/utils/uiPageEnumerator.d.ts +0 -172
  229. package/build/utils/uiPageEnumerator.js +0 -474
@@ -1,608 +1,675 @@
1
1
  import { z } from "zod";
2
- import { TOOL_BATCH_SCENARIO_TEST_GENERATION, TOOL_CONTRACT_TEST_GENERATION, TOOL_E2E_TEST_GENERATION, TOOL_INTEGRATION_TEST_GENERATION, TOOL_REGISTER_TEST_PLAN, TOOL_UI_TEST_GENERATION, } from "../../toolNames.js";
2
+ import * as fs from "fs";
3
3
  import * as path from "path";
4
- import * as crypto from "crypto";
4
+ import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
5
5
  import { logger } from "../../utils/logger.js";
6
- import { StateManager, } from "../../utils/AnalysisStateManager.js";
6
+ import { StateManager, resolveRunStatePath } from "../../utils/AnalysisStateManager.js";
7
7
  import { toolError } from "../../utils/utils.js";
8
- import { buildApprovedPlanItem } from "../../utils/planMatchKeys.js";
9
- import { SCENARIO_CATEGORIES, CATEGORY_PRIORITY, Novelty, PriorityTier } from "../../types/TestRecommendation.js";
10
- import { HttpMethod, TestType } from "../../types/TestTypes.js";
11
- import { CandidateSource, computeCandidateId, scenarioMergeKey, DiscriminatorKind } from "../../types/Recommendation.js";
12
- import { selectPlan } from "../../recommendation/planRanker.js";
13
- import { resolveSubjectEndpoints } from "../../utils/subjectEndpoints.js";
14
- import { collectChangedRouteLines } from "../../utils/changedRoutes.js";
15
- import { generateFloors } from "../../recommendation/budgeters/shared.js";
16
- import { inferScenarioType } from "../../recommendation/diversity.js";
17
- import { validateDiscriminator } from "../../recommendation/discriminators.js";
18
- import { isAttackSurfaceSecurityBoundary } from "../../prompts/test-recommendation/recommendationShared.js";
19
- import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
8
+ import { TestType } from "../../types/TestTypes.js";
9
+ import { changedFileNames, computeBranchDiff } from "../../utils/branchDiff.js";
10
+ import { LOCKFILE, NOT_SOURCE_DIR } from "../../recommendation/verifiers/endpointGrounded.js";
11
+ import { hasUITestTypeLabel, isHTTPStep } from "../../recommendation/subjectStep.js";
12
+ import { STEP_METHOD_DESCRIPTION, normalizeStepMethod, stepMethodSchema } from "../../types/StepMethod.js";
13
+ import { registerPlan } from "../../recommendation/registerPlan.js";
14
+ import { targetElementSchema } from "../submitReportTool.js";
15
+ import { capturedBlueprintDigests } from "../../playwright/blueprintDigestStore.js";
16
+ import { fileExistsWithinRoot } from "../../utils/containedPath.js";
17
+ import { createScreenRouteResolver } from "../../utils/screenRoutes.js";
18
+ import { categoryMenu, SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
20
19
  const TOOL_NAME = TOOL_REGISTER_TEST_PLAN;
21
20
  // ── Zod schema ──────────────────────────────────────────────────────────────
22
- /** Test types eligible for a registered test plan — a subset of TestType
23
- * (excludes smoke/fuzz/load/mock/unit/unknown, which this checkpoint doesn't
24
- * govern). */
25
- const PLAN_TEST_TYPES = new Set([
26
- TestType.CONTRACT,
27
- TestType.INTEGRATION,
28
- TestType.E2E,
29
- TestType.UI,
30
- ]);
31
- const chainingRefSchema = z.object({
32
- sourceStep: z.number().int(),
33
- sourceField: z.string(),
34
- sourceLocation: z.enum(["body", "header", "cookie"]),
35
- targetParam: z.string(),
36
- targetLocation: z.enum(["path", "body", "query", "header", "cookie"]),
37
- });
21
+ /** One sentence, shared by the step fields that carry a value and by the case
22
+ * that names one, so the two sides state the same convention. A plan that spells
23
+ * the same created value two different ways reads as a case no step sends. */
24
+ const CREATED_VALUE_PLACEHOLDER = "Write a value an earlier step creates as a placeholder in braces, and use the same name wherever else the test names it.";
38
25
  const registerStepSchema = z.object({
39
- order: z.number().int().min(1).describe("Order of this step within the scenario, starting at 1."),
40
- method: z.nativeEnum(HttpMethod).describe("HTTP method of this step's request."),
41
- path: z.string().describe("Endpoint path, e.g. '/api/v1/products/{product_id}'."),
42
- description: z.string().describe("What this step does."),
43
- interactionType: z
44
- .enum(["success", "error", "edge-case"])
45
- .describe("Whether this step exercises the happy path, an expected error, or an edge case."),
46
- requestBody: z
47
- .record(z.any())
48
- .optional()
49
- .describe("Request body fields with concrete values. Used to verify boundary_equality claims (two distinct fields with equal values)."),
26
+ order: z.number().int().positive().describe("This step's position in the scenario, from 1. `stepUnderTest` and `routes` name a step by this number."),
27
+ method: stepMethodSchema.describe(`${STEP_METHOD_DESCRIPTION} A \`ui\` or \`e2e\` step is a page interaction.`),
28
+ path: z
29
+ .string()
30
+ .refine((value) => value.trim().length > 0, { message: "path must not be blank" })
31
+ .describe("Request path as the running service serves it, mount prefix included — `/ghost/api/admin/site/stats`, not the `/site/stats` the router file declares. For a page interaction, its target instead (`checkout button`); for an `OPERATION` step, the operation's name."),
32
+ description: z.string().optional().describe("What this step does, in one line."),
33
+ interactionType: z.string().optional().describe("success, error, or another label for what this step exercises."),
34
+ requestBody: z.unknown().optional().describe(`Request payload for this step, when it sends one. ${CREATED_VALUE_PLACEHOLDER}`),
50
35
  queryParams: z
51
- .record(z.any())
52
- .optional()
53
- .describe("Query parameters with concrete values. Also scanned for boundary_equality and negative_match verification."),
54
- responseBody: z
55
- .union([z.record(z.any()), z.array(z.any())])
36
+ .unknown()
56
37
  .optional()
57
- .describe("Expected response body shape/values, when asserted. An array for a collection/list endpoint that returns a JSON array."),
58
- expectedStatusCode: z.number().int().describe("HTTP status code this step asserts (e.g. 201, 400, 404)."),
59
- expectedResponseFields: z
38
+ .describe(`Query-string parameters for this step, when it sends any. ${CREATED_VALUE_PLACEHOLDER}`),
39
+ omits: z
60
40
  .array(z.string())
41
+ .min(1)
42
+ .optional()
43
+ .describe("Parameters this step leaves out on purpose."),
44
+ expectedStatusCode: z.number().int().optional().describe("Status code this step expects."),
45
+ })
46
+ // A `navigate` step's path is the screen the runner opens, so it has to be a
47
+ // thing a browser can be pointed at. Runs have written "the login page" there,
48
+ // which no runner can open and no later check refused.
49
+ .refine((step) => {
50
+ if (normalizeStepMethod(step.method) !== "navigate")
51
+ return true;
52
+ const target = step.path.trim().toLowerCase();
53
+ return target.startsWith("/") || target.startsWith("http");
54
+ }, { message: "a navigate step's path is a URL path such as /login", path: ["path"] });
55
+ // The fields `declarations` accepts. Named on its own so one list of them can be
56
+ // derived from it — a hand-written second copy had already fallen behind.
57
+ export const declarationFieldsSchema = z.object({
58
+ changedFile: z
59
+ .string()
61
60
  .optional()
62
- .describe("Response fields (optionally with expected values, e.g. 'account_count=2') this step asserts. Used to verify multi_record_aggregate claims."),
63
- bodyMustInclude: z
61
+ .describe("The changed file this test is about, repository-relative and spelled as the diff spells it — `src/routes/orders.ts`, not `./src/routes/orders.ts`. Checked: the name is one this pull request changed. Not checked: which line, so name the changed line in `description`."),
62
+ changes: z
64
63
  .array(z.string())
65
64
  .optional()
66
- .describe("Fields that MUST appear in the request body for this step."),
67
- chainsFrom: z.union([chainingRefSchema, z.array(chainingRefSchema)]).optional(),
68
- });
69
- const discriminatorClaimSchema = z.object({
70
- kind: z.nativeEnum(DiscriminatorKind),
71
- changedCodeAnchor: z
65
+ .describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change."),
66
+ asserts: z
72
67
  .string()
73
- .min(1)
74
- .describe("Verbatim snippet copied EXACTLY from the diff that this test probes. Must occur character-for-character in the PR diff to verify — a paraphrase or reconstruction will fail verification and demote the claim."),
75
- });
76
- const registerCandidateSchema = z.object({
77
- scenarioName: z.string().min(1).describe("Name for this test/scenario. Reused as the match key against the generation tool's scenarioName — keep it identical when you later call the generation tool."),
78
- description: z.string().min(1),
79
- category: z.enum(SCENARIO_CATEGORIES).describe("Test category — drives ranking (never your own priority tag). `bug_caught` targets a `<bug_found>` flaw from code review. `requirement_conflict` asserts a requirement the PR title/description (or a requirements file it references) states and the implemented behavior contradicts — it fails on the current code by design. Both rank in the top tier but are carved out independently, so a requirement conflict never loses its slot to a code-review bug."),
80
- priority: z.enum(["high", "medium", "low"]).describe("Your own assessment — recorded, but ranking is derived from category, not this field."),
81
- testType: z.nativeEnum(TestType).refine((t) => PLAN_TEST_TYPES.has(t), {
82
- message: `testType must be one of ${[...PLAN_TEST_TYPES].join(", ")}`,
83
- }),
84
- steps: z.array(registerStepSchema).min(1),
85
- discriminator: discriminatorClaimSchema
68
+ .describe("What this test checks."),
69
+ stepUnderTest: z
70
+ .number()
71
+ .int()
72
+ .optional()
73
+ .describe("The `order` of the step this test is really about. Required whenever the planned test makes more than one call; one call is its own subject. In a create-then-verify pair the write is the subject, not the read-back. Two checks read it; nothing infers it."),
74
+ expected: z
75
+ .object({
76
+ outcome: z
77
+ .enum(["fail", "pass"])
78
+ .describe("Whether this test should fail or pass against the app as it stands."),
79
+ why: z
80
+ .string()
81
+ .describe("Why it has that outcome. A test that documents a defect asserts what the FIXED code returns — the rejection, the preserved value, the correct status — so it is red today and green after the fix, never today's defective response as correct."),
82
+ })
83
+ .strict()
84
+ .describe("What this test does against the app as it stands, and why."),
85
+ differsFrom: z
86
+ .array(z.object({
87
+ plannedTestId: z.string().describe("The other planned test this one is distinct from, named by its `scenarioName`."),
88
+ difference: z.string().describe("How the two differ."),
89
+ }).strict())
90
+ .optional()
91
+ .describe("For another planned test in THIS plan whose subject step calls the same method and path, what this one asserts that it does not."),
92
+ existingTests: z
93
+ .array(z.object({
94
+ file: z.string().describe("Repository-relative path of an existing test you read that exercises this change."),
95
+ differsBy: z.string().describe("What this planned test checks that the existing test does not."),
96
+ }).strict())
97
+ .optional()
98
+ .describe("The existing tests you read on this change. Naming none is an answer. Checked: each file named is in the checkout. Not checked: what the file contains."),
99
+ startState: z
100
+ .string()
101
+ .optional()
102
+ .describe("The state the record this test mutates is in when the mutation runs, in one sentence. A record with no rows yet is a state. The test puts the record in that state itself. Checked: present. Not checked: the sentence."),
103
+ routes: z
104
+ .array(z.object({
105
+ step: z.number().int().describe("The `order` of the step this file declares."),
106
+ file: z.string().describe("Repository-relative path of the file that declares what that step calls."),
107
+ }).strict())
86
108
  .optional()
87
- .describe("Declare when this candidate specifically probes changed logic (a structural bug-shape). Verified against `steps[]` and the PR diff — a failed claim demotes (with a reason) but never rejects the candidate."),
109
+ .describe("One entry per step that calls the app, naming the repository file that declares what it calls. Checked: the file is in the checkout. Not checked: whether the route appears in it — frameworks build paths from scopes and prefixes, so it often does not."),
110
+ screenEvidence: z
111
+ .object({
112
+ file: z.string().describe("Repository-relative path of the changed file that renders this screen."),
113
+ })
114
+ .strict()
115
+ .optional()
116
+ .describe("UI planned tests only: the changed file that renders the screen `elements.pageUrl` opens. Checked: it is one of the files this pull request changed. Not checked: whether it really renders that screen."),
117
+ // The report's own schemas, reused so the two spellings cannot drift.
118
+ elements: z
119
+ .object({
120
+ pageUrl: z
121
+ .string()
122
+ .optional()
123
+ .describe("The page this test opens, spelled either as a route path starting with `/` or as an absolute `http` URL. Required on a `ui` or `e2e` planned test whose steps make no HTTP call."),
124
+ items: z
125
+ .array(targetElementSchema)
126
+ .nullish()
127
+ .describe("The elements this test targets, each one copied out of a `browser_blueprint` captured during this run. `null` when no capture holds them. Checked: each entry appears in a capture this run took of `pageUrl`. Not checked: whether the test needs that element."),
128
+ })
129
+ .strict()
130
+ .nullish()
131
+ .describe("UI planned tests only: the page this test opens and the elements it targets on it."),
132
+ });
133
+ // STRICT, here and on the planned test around it: zod strips an unrecognised key,
134
+ // so a misplaced declaration vanished and the check then objected to its absence.
135
+ const declarationsSchema = declarationFieldsSchema
136
+ .strict()
137
+ .superRefine((declared, ctx) => {
138
+ // Refuse a malformed shape; a judgment call gets an objection instead. An entry
139
+ // naming no element cannot be looked up, so there is nothing to report on.
140
+ const items = declared.elements?.items;
141
+ if (!Array.isArray(items))
142
+ return;
143
+ items.forEach((element, index) => {
144
+ const identifies = [element.accessibleName, element.testId, element.stableId].some((value) => typeof value === "string" && value.trim().length > 0);
145
+ if (identifies)
146
+ return;
147
+ ctx.addIssue({
148
+ code: z.ZodIssueCode.custom,
149
+ path: ["elements", "items", index],
150
+ message: "every `elements.items` entry needs at least one of `accessibleName`, `testId` or `stableId` with a value in it — an entry with none names no element that can be looked up in a capture.",
151
+ });
152
+ });
88
153
  });
154
+ // A plan drafts only these four; `TestType`'s others are maintenance outcomes.
155
+ const PLAN_TEST_TYPES = [TestType.CONTRACT, TestType.INTEGRATION, TestType.E2E, TestType.UI];
156
+ /** Whether an `elements.pageUrl` names a screen `uiElementGrounded` can look up: a
157
+ * route path or an absolute http URL. Constrained here and never on the shared
158
+ * `pageContextSchema`, which the report fills from a capture carrying any url. */
159
+ function namesAScreen(url) {
160
+ if (url.startsWith("/"))
161
+ return true;
162
+ try {
163
+ return ["http:", "https:"].includes(new URL(url).protocol);
164
+ }
165
+ catch {
166
+ return false;
167
+ }
168
+ }
169
+ /** The fields that belong under `declarations`. Read by the hoist below only.
170
+ * Derived from the schema, not written out again: the hand-written copy was
171
+ * missing `stepUnderTest`, so a stray one stayed at planned test level and zod
172
+ * dropped it. */
173
+ const DECLARATION_FIELDS = Object.keys(declarationFieldsSchema.shape);
174
+ function isPlainObject(value) {
175
+ return typeof value === "object" && value !== null && !Array.isArray(value);
176
+ }
177
+ /** One value as text with object keys sorted, so the same declaration written
178
+ * twice by hand is not a conflict merely because its keys are ordered
179
+ * differently. List order still counts: two orders are two different lists. */
180
+ function canonicalJson(value) {
181
+ return JSON.stringify(value, (_key, entry) => isPlainObject(entry)
182
+ ? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]]))
183
+ : entry);
184
+ }
185
+ /** Move a declaration field written one level too high into `declarations`.
186
+ * MEASURED: about one planned test in five did this across 141 eval planned tests, and
187
+ * every stray planned test-level key was a declaration field. A field in BOTH places
188
+ * with different values stays put, so `.strict()` refuses it by name. */
189
+ export function hoistDeclarationFields(value) {
190
+ if (!isPlainObject(value))
191
+ return value;
192
+ // A non-object `declarations` is left as written: hoisting would discard it
193
+ // and the error would name a field the agent never wrote.
194
+ if (value.declarations !== undefined && !isPlainObject(value.declarations))
195
+ return value;
196
+ const plannedTest = { ...value };
197
+ const original = isPlainObject(plannedTest.declarations) ? plannedTest.declarations : undefined;
198
+ const declarations = { ...(original ?? {}) };
199
+ let hoisted = false;
200
+ for (const field of DECLARATION_FIELDS) {
201
+ if (!(field in plannedTest))
202
+ continue;
203
+ const atCandidateLevel = plannedTest[field];
204
+ if (declarations[field] === undefined) {
205
+ declarations[field] = atCandidateLevel;
206
+ delete plannedTest[field];
207
+ hoisted = true;
208
+ continue;
209
+ }
210
+ if (canonicalJson(declarations[field]) === canonicalJson(atCandidateLevel)) {
211
+ // The same value twice says the same thing; drop the copy.
212
+ delete plannedTest[field];
213
+ }
214
+ }
215
+ if (hoisted || original)
216
+ plannedTest.declarations = declarations;
217
+ return plannedTest;
218
+ }
219
+ const registerPlannedTestSchema = z.preprocess(hoistDeclarationFields, z.object({
220
+ scenarioName: z
221
+ .string()
222
+ .refine((value) => value.trim().length > 0, { message: "scenarioName must not be blank" })
223
+ .describe("Short name for this test, and its id: objections name it, `differsFrom` refers to it, and the report joins delivered tests on it. Keep it stable across registrations."),
224
+ repository: z
225
+ .string()
226
+ .optional()
227
+ .describe("On a multi-repo run, the `owner/repo` this test is for — the same value you passed to `skyramp_analyze_changes`."),
228
+ description: z.string().describe("What this test does, in one or two lines."),
229
+ category: z
230
+ .enum(SCENARIO_CATEGORIES)
231
+ .describe(`What kind of test this is. Write one of:\n${categoryMenu()}`),
232
+ testType: z
233
+ .enum(PLAN_TEST_TYPES)
234
+ .describe("contract, integration, e2e or ui."),
235
+ steps: z
236
+ .array(registerStepSchema)
237
+ .describe("The requests this test makes, in order. At least one, unless `testType` is `ui` or `e2e` and the test makes no HTTP call."),
238
+ declarations: declarationsSchema
239
+ .optional()
240
+ .describe("The changes that make this planned test checkable. Every change belongs under THIS key; one written beside `steps` is moved here for you and reported as `declarations.<field>`."),
241
+ })
242
+ .strict()
243
+ .superRefine((plannedTest, ctx) => {
244
+ // Three checks read the steps for the subject endpoint, so an empty list
245
+ // passed all three in silence.
246
+ const isUILabel = hasUITestTypeLabel(plannedTest.testType);
247
+ if (plannedTest.steps.length === 0 && !isUILabel) {
248
+ ctx.addIssue({
249
+ code: z.ZodIssueCode.custom,
250
+ path: ["steps"],
251
+ message: "steps must not be empty for a contract, integration or other non-UI planned test: the route, duplicate and existing-coverage checks all read the steps to find the endpoint this test is about. List the requests the test makes.",
252
+ });
253
+ }
254
+ // Ruling A: a ui or e2e planned test whose steps make no HTTP call opens a
255
+ // screen instead of a request, so it must name that screen. Empty steps
256
+ // stay legal for it — a planned test naming no request and no screen is not
257
+ // a plan the server can read either.
258
+ const hasHTTPStep = plannedTest.steps.some((step) => isHTTPStep(step));
259
+ const rawUrl = plannedTest.declarations?.elements?.pageUrl;
260
+ const url = typeof rawUrl === "string" ? rawUrl.trim() : "";
261
+ if (isUILabel && !hasHTTPStep && url.length === 0) {
262
+ ctx.addIssue({
263
+ code: z.ZodIssueCode.custom,
264
+ path: ["declarations", "elements", "pageUrl"],
265
+ message: "a ui or e2e planned test whose steps make no HTTP call must set `declarations.elements.pageUrl` to the screen it opens: a planned test naming no request and no screen is not a plan the server can read.",
266
+ });
267
+ }
268
+ // The other half of Ruling A. Naming a screen and naming SOMETHING are
269
+ // different obligations: a one-word url passed this rule, and the page
270
+ // match and the relatedness check both compare it as written, so it named
271
+ // no page either could look up and drew no objection from either.
272
+ if (isUILabel && url.length > 0 && !namesAScreen(url)) {
273
+ ctx.addIssue({
274
+ code: z.ZodIssueCode.custom,
275
+ path: ["declarations", "elements", "pageUrl"],
276
+ message: "`declarations.elements.pageUrl` is accepted in two spellings and no others: a route path starting with `/`, e.g. `/orders/1`, or an absolute `http` or `https` URL, e.g. `http://localhost:5173/orders/1`. Give the path when this run has no host to name.",
277
+ });
278
+ }
279
+ }));
89
280
  const registerTestPlanSchema = {
90
281
  stateFile: z
91
282
  .string()
92
283
  .refine((p) => path.isAbsolute(p), { message: "stateFile must be an absolute path" })
93
- .describe("Path to state file from skyramp_analyze_changes"),
94
- // No `repository` argument, deliberately (SKYR-4204 reopen). The approved
95
- // plan is run-wide — one pooled registration covers every repository in the
96
- // run — so unlike the per-repo tools (analyze_test_health, actions,
97
- // execute_test) there is no section to select: the tool reads the root and
98
- // derives run-wide facts (diff union, budget flags, scenario lookup) itself.
99
- // A legacy caller still passing `repository` has it stripped harmlessly.
100
- candidates: z
101
- .array(registerCandidateSchema)
284
+ .optional()
285
+ .describe("Optional. A Testbot run finds its own state file, so leave this out. Give an absolute path only to name a state file the run would not find by itself."),
286
+ // `.default([])`, where `plannedTests` is required: an empty list here cannot pass
287
+ // in silence the way an empty planned test list did. It raises `coverage:noChanges`,
288
+ // which is exactly the objection a plan that states no change should draw.
289
+ changes: z
290
+ .array(z
291
+ .object({
292
+ id: z
293
+ .string()
294
+ .refine((value) => value.trim().length > 0, { message: "a change id must not be blank" })
295
+ .describe("Short slug for this change. A planned test cites it by this id."),
296
+ text: z.string().describe("The change in one sentence, as the pull request must make it."),
297
+ source: z
298
+ .string()
299
+ .describe("Where you read it: `pr-description`, `spec:<path>` for a requirements file the description names, or `diff`."),
300
+ surfaces: z
301
+ .array(z.enum(["api", "page"]))
302
+ .min(1)
303
+ .describe("Where a user or caller meets this change: `api` for a request and response, `page` for something rendered, both when both."),
304
+ cases: z
305
+ .array(z
306
+ .object({
307
+ param: z.string().describe("The parameter this case sets."),
308
+ value: z
309
+ .union([z.string(), z.number(), z.boolean(), z.null()])
310
+ .optional()
311
+ .describe(`The value a test sends for that parameter. ${CREATED_VALUE_PLACEHOLDER}`),
312
+ absent: z.literal(true).optional().describe("The parameter is left out of the request."),
313
+ expect: z
314
+ .enum(["accept", "reject"])
315
+ .describe("Whether the route must accept this value or reject it."),
316
+ })
317
+ .strict()
318
+ .refine((entry) => (entry.value !== undefined) !== (entry.absent === true), {
319
+ message: "a case states either `value` or `absent`, never both and never neither",
320
+ }))
321
+ .min(1, { message: "a `cases` list with no entry states no input — leave it out instead" })
322
+ .optional()
323
+ .describe("The inputs that prove this change: for each parameter the change constrains, the values the route must accept and the values it must reject. A range gives the last accepted value and the first rejected one at each end; a set gives each allowed value and one outside it; a required field gives the field left out. A case names a query parameter, a body field, a path segment, or `method` for the HTTP verb; the steps of the test that cites this change say which route it is, and a rejection that depends on who calls belongs in the change text instead."),
324
+ })
325
+ .strict())
326
+ .superRefine((changes, ctx) => {
327
+ const first = new Map();
328
+ changes.forEach((change, index) => {
329
+ const key = String(change?.id ?? "").trim();
330
+ if (!key)
331
+ return;
332
+ const seen = first.get(key);
333
+ if (seen === undefined) {
334
+ first.set(key, index);
335
+ return;
336
+ }
337
+ ctx.addIssue({
338
+ code: z.ZodIssueCode.custom,
339
+ path: [index, "id"],
340
+ message: `change id "${change.id}" is already used by change ${seen + 1}. Every change needs its own id — planned tests cite it by that id.`,
341
+ });
342
+ });
343
+ })
344
+ .default([])
345
+ .describe("The changes the pull request must make, read out of the pull request title and description, a requirements file they name, or the diff itself. Coverage per file is the basic minimum; coverage per change is the target, and anything more is a bonus."),
346
+ // REQUIRED, not `.default([])`. The MCP SDK wraps this shape in a plain
347
+ // `z.object`, which STRIPS an undeclared top-level key, so with a default
348
+ // `planned test:` for `planned tests:` stored an EMPTY plan and counted the
349
+ // registration. A run that warrants no test sends `[]` deliberately.
350
+ plannedTests: z
351
+ .array(registerPlannedTestSchema)
352
+ // A duplicated `scenarioName` is REFUSED here rather than objected to later,
353
+ // because the name is the planned test's id: two entries carrying one id share
354
+ // every objection about them and one answer closes both. Compared on the
355
+ // TRIMMED name and otherwise exactly — case is NOT folded, because the report
356
+ // joins on the exact id, and folding here would make "Login" and "LOGIN" one
357
+ // name to the plan and two ids to the report.
358
+ .superRefine((plannedTests, ctx) => {
359
+ const seen = new Map();
360
+ plannedTests.forEach((plannedTest, index) => {
361
+ const key = String(plannedTest?.scenarioName ?? "").trim();
362
+ if (!key)
363
+ return;
364
+ const first = seen.get(key);
365
+ if (first === undefined) {
366
+ seen.set(key, index);
367
+ return;
368
+ }
369
+ ctx.addIssue({
370
+ code: z.ZodIssueCode.custom,
371
+ path: [index, "scenarioName"],
372
+ message: `scenarioName "${plannedTest.scenarioName}" is already used by planned test ${first + 1}. Every planned test needs its own name — the name is its id.`,
373
+ });
374
+ });
375
+ })
376
+ .describe("Your complete plan, unordered. Nothing ranks, dedups, reorders or trims it, and the server adds no planned test of its own."),
377
+ answers: z
378
+ .array(z.object({
379
+ objectionId: z.string().describe("The objection's id, exactly as this tool reported it. Answers are matched by id, never by position."),
380
+ answer: z.string().describe("Why the objection does not apply, or what you decided instead."),
381
+ blocker: z
382
+ .string()
383
+ .optional()
384
+ .describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change with no test closes only with this. A reason the change is not worth testing is not a blocker."),
385
+ }).strict())
102
386
  .default([])
103
- .describe("Your COMPLETE candidate list — every test you would generate OR recommend this run, including ones already suggested by skyramp_analyze_changes. Merged with any server pre-ranked candidates, re-ranked, and split into GENERATE vs ADDITIONAL. Call this again (with your full updated list) if you want to add or change candidates — each call replaces your previous submission."),
387
+ .describe("One answer per objection from your previous registration that you do not intend to fix. There is no registration limit."),
104
388
  };
105
- // ── Handler helpers ─────────────────────────────────────────────────────────
106
- function derivePriorityTier(scenario) {
107
- if (isAttackSurfaceSecurityBoundary(scenario))
108
- return PriorityTier.CRITICAL;
109
- return CATEGORY_PRIORITY[scenario.category] ?? PriorityTier.LOW;
110
- }
111
- function toScenarioStep(step) {
389
+ // ── Registration ──────────────────────────────────────────────────────────────
390
+ /** Map the schema-validated input onto a `PlanInput`. `subjectEndpoints` is
391
+ * deliberately not resolved: every verifier derives the subject from `steps`
392
+ * through one helper. The planned test id IS the scenario name, so an objection
393
+ * names the plan entry in the words the agent wrote for it. */
394
+ export function buildRegistration(params, registrationNumber) {
112
395
  return {
113
- order: step.order,
114
- method: step.method,
115
- path: step.path,
116
- description: step.description,
117
- interactionType: step.interactionType,
118
- requestBody: step.requestBody,
119
- queryParams: step.queryParams,
120
- responseBody: step.responseBody,
121
- expectedStatusCode: step.expectedStatusCode,
122
- expectedResponseFields: step.expectedResponseFields,
123
- bodyMustInclude: step.bodyMustInclude,
124
- chainsFrom: step.chainsFrom,
396
+ changes: params.changes ?? [],
397
+ answers: params.answers ?? [],
398
+ // The tool counts the run's registrations; whatever the caller sent is ignored.
399
+ registrationNumber,
400
+ plannedTests: (params.plannedTests ?? []).map((input) => {
401
+ const scenario = {
402
+ scenarioName: input.scenarioName,
403
+ description: input.description,
404
+ category: input.category,
405
+ steps: input.steps,
406
+ testType: input.testType,
407
+ };
408
+ const declared = input.declarations;
409
+ const { elements, ...rest } = declared ?? {};
410
+ return {
411
+ // TRIMMED: the report side joins on `plannedTestId.trim()`.
412
+ plannedTestId: input.scenarioName.trim(),
413
+ scenario,
414
+ ...(input.repository ? { repository: input.repository } : {}),
415
+ // Passed through, never rebuilt field by field: a field the schema accepts
416
+ // and this function forgot to copy was dead once. Absent stays absent, so
417
+ // a field the agent did not write reads back as missing, not as a default
418
+ // the checks would then judge. `elements: null` is the model's "not
419
+ // applicable" habit and is dropped; `elements.items: null` MEANS capture
420
+ // failed and is kept.
421
+ declarations: { ...rest, asserts: declared?.asserts ?? "", ...(elements ? { elements } : {}) },
422
+ };
423
+ }),
125
424
  };
126
425
  }
127
- function toDraftedScenario(input, changedRoutes) {
128
- const scenario = {
129
- scenarioName: input.scenarioName,
130
- description: input.description,
131
- category: input.category,
132
- priority: input.priority,
133
- steps: input.steps.map(toScenarioStep),
134
- chainingKeys: [],
135
- requiresAuth: false,
136
- estimatedComplexity: "moderate",
137
- testType: input.testType,
138
- };
139
- // Resolved here, at the one boundary that has both the steps and the diff, so
140
- // every later dedup stage reads one recorded value (SKYR-4214).
141
- return { ...scenario, subjectEndpoints: resolveSubjectEndpoints(scenario, { changedRoutes }) };
426
+ /** The primary repository plus every related one, in that order. */
427
+ function repoSections(state) {
428
+ if (!state)
429
+ return [];
430
+ return [
431
+ { repo: state.metadata?.repository ?? "the primary repository", repositoryPath: state.metadata?.repositoryPath, data: state },
432
+ ...Object.entries(state.relatedRepos ?? {}).map(([repo, section]) => ({
433
+ repo,
434
+ repositoryPath: section?.repositoryPath,
435
+ data: section?.data,
436
+ })),
437
+ ];
142
438
  }
143
- /** Build the agent-submitted candidates. Discriminator claims are verified
144
- * later, by {@link applyDiscriminatorClaims}, once the merge has settled which
145
- * scenario each name resolves to. Exported for the direct unit test in
146
- * registerTestPlanTool.test.ts — a candidate that does not survive selection
147
- * never reaches the persisted plan, so its subjectEndpoints fill is not
148
- * observable through the tool's output, and testing here avoids coupling the
149
- * fill to what selection happens to keep. */
150
- export function buildAgentCandidates(candidates, changedRoutes) {
151
- return candidates.map((input) => {
152
- const scenario = toDraftedScenario(input, changedRoutes);
153
- return {
154
- scenario,
155
- priority: derivePriorityTier(scenario),
156
- novelty: Novelty.NEW,
157
- source: CandidateSource.AGENT,
158
- candidateId: computeCandidateId(scenario),
159
- };
160
- });
439
+ /**
440
+ * Whether a cited repository file EXISTS inside the checkout. Reads no content,
441
+ * and never throws: a bad citation is an objection, not a crashed tool.
442
+ *
443
+ * CONFINEMENT IS THIS FUNCTION'S JOB. The sibling verifier's string check misses
444
+ * `file://` and `http://` URLs, a drive-relative `C:routes.rb` and a
445
+ * percent-encoded `..`, so the containment in `containedPath` is the only thing
446
+ * between an agent-written path and a read outside the checkout.
447
+ *
448
+ * CASE IS THE FILESYSTEM'S RULE: a case-only misspelling passes on a
449
+ * case-insensitive checkout and is not found on a case-sensitive one. CI is the
450
+ * case-sensitive one, so that is the answer a real run gets.
451
+ */
452
+ export function makeCitedFileCheck(state) {
453
+ const roots = [];
454
+ for (const section of repoSections(state)) {
455
+ // The literal "unknown" means unresolved; resolving against it would read
456
+ // the process working directory instead of a checkout.
457
+ if (!section.repositoryPath || section.repositoryPath === "unknown")
458
+ continue;
459
+ try {
460
+ const real = fs.realpathSync(section.repositoryPath);
461
+ if (!roots.includes(real))
462
+ roots.push(real);
463
+ }
464
+ catch {
465
+ // A repository path that no longer resolves holds no readable file.
466
+ }
467
+ }
468
+ return (relativePath) => {
469
+ if (typeof relativePath !== "string")
470
+ return false;
471
+ const cited = relativePath.trim();
472
+ if (!cited)
473
+ return false;
474
+ return roots.some((root) => fileExistsWithinRoot(root, cited));
475
+ };
161
476
  }
162
477
  /**
163
- * Verify each declared discriminator against the candidate that actually won
164
- * the merge, and mark it. Failed claims demote (never reject).
478
+ * The route patterns that render a changed frontend file, across every repository
479
+ * of the run.
165
480
  *
166
- * Runs AFTER the merge, not while building the agent's candidates: a server
167
- * pre-seeded candidate wins the name, so verifying the agent's copy first
168
- * attaches the result to a candidate that is about to be discarded — silently
169
- * stripping the verification. Verification is against the WINNER's
170
- * steps, so a claim that only held for the agent's rewrite is correctly refused
171
- * rather than carried across.
481
+ * Built like {@link makeCitedFileCheck}: one resolver per checkout, asked in turn,
482
+ * so a planned test does not have to say which repository its file lives in. The
483
+ * scan is lazy inside each resolver and reused across planned tests, so a plan with
484
+ * ten UI tests reads a repository once.
172
485
  *
173
- * The marker is cleared before the current claim is applied. Each call replaces
174
- * the agent's previous submission (see the `candidates` schema), and
175
- * recoverServerCandidates restores whatever the LAST plan recorded — so keeping
176
- * it would let a boost earned by an earlier registration outlive the claim that
177
- * earned it, on a submission that no longer makes that claim or whose new claim
178
- * fails.
486
+ * A repository that CAN speak for the file wins: `undefined` from a resolver means
487
+ * "not my file, or not a framework I read", so it is skipped, and the first
488
+ * definite answer is returned. Every resolver saying `undefined` is the answer the
489
+ * check stays silent on.
179
490
  */
180
- function applyDiscriminatorClaims(candidates, claimsByKey, diffText, reposWithoutDiff = []) {
181
- const demotions = [];
182
- const marked = candidates.map((candidate) => {
183
- const { verifiedDiscriminator: _stale, ...unmarked } = candidate;
184
- const claim = claimsByKey.get(scenarioMergeKey(candidate.scenario.scenarioName) || candidate.candidateId);
185
- if (!claim)
186
- return unmarked;
187
- const result = validateDiscriminator(candidate.scenario, claim, diffText, reposWithoutDiff);
188
- if (result.verified) {
189
- return { ...unmarked, verifiedDiscriminator: claim.kind };
491
+ export function makeScreenRouteResolver(state) {
492
+ const resolvers = repoSections(state)
493
+ .filter((section) => section.repositoryPath && section.repositoryPath !== "unknown")
494
+ .map((section) => createScreenRouteResolver(section.repositoryPath));
495
+ return (relativePath) => {
496
+ let unmounted;
497
+ for (const resolve of resolvers) {
498
+ const routes = resolve(relativePath);
499
+ if (routes === undefined)
500
+ continue;
501
+ // A non-empty answer is a fact about a real mount; an empty one is only the
502
+ // answer when no repository found a mount at all.
503
+ if (routes.length > 0)
504
+ return routes;
505
+ unmounted = routes;
190
506
  }
191
- demotions.push({
192
- candidateId: candidate.candidateId,
193
- reason: result.reason ?? `${claim.kind} discriminator claim could not be verified.`,
194
- });
195
- return unmarked;
196
- });
197
- return { candidates: marked, demotions };
507
+ return unmounted;
508
+ };
198
509
  }
199
- /** Recover full Candidates for pre-seeded ("server") plan items by matching
200
- * candidateId back to the analyzed scenarios (repositoryAnalysis.scenarios) —
201
- * ApprovedPlanItem is deliberately slim and doesn't carry steps[]. */
202
- function recoverServerCandidates(priorPlan, allScenarios) {
203
- if (!priorPlan)
204
- return [];
205
- const scenarioById = new Map();
206
- for (const scenario of allScenarios) {
207
- scenarioById.set(computeCandidateId(scenario), scenario);
510
+ /** Which files each repository's branch diff changed. NAMES ONLY, never persisted.
511
+ * Lockfiles and non-source directories are dropped, so a planned test cannot cite
512
+ * `package-lock.json` and nothing demands a test for one. */
513
+ async function changedFilesForVerification(section) {
514
+ const repositoryPath = section.repositoryPath;
515
+ if (!repositoryPath || repositoryPath === "unknown")
516
+ return undefined;
517
+ const baseBranch = section.data?.repositoryAnalysis?.diff?.baseBranch;
518
+ try {
519
+ const files = changedFileNames(await computeBranchDiff(repositoryPath, baseBranch, false, true)).filter((file) => !(LOCKFILE.test(file.path) || NOT_SOURCE_DIR.test(file.path)));
520
+ return files.length > 0 ? files : undefined;
208
521
  }
209
- const serverItems = [...priorPlan.generate, ...priorPlan.additional].filter((item) => item.source === CandidateSource.SERVER);
210
- const recovered = [];
211
- for (const item of serverItems) {
212
- const scenario = scenarioById.get(item.candidateId);
213
- if (!scenario) {
214
- logger.warning(`${TOOL_NAME}: pre-seeded server candidate ${item.candidateId} not found in repositoryAnalysis.scenarios — dropping`);
215
- continue;
216
- }
217
- recovered.push({
218
- scenario,
219
- priority: derivePriorityTier(scenario),
220
- novelty: Novelty.EXISTING,
221
- source: CandidateSource.SERVER,
222
- candidateId: item.candidateId,
223
- ...(item.verifiedDiscriminator ? { verifiedDiscriminator: item.verifiedDiscriminator } : {}),
522
+ catch (error) {
523
+ // A repository git cannot read yields no changed files, which
524
+ // `reposWithoutDiff` already reports to the agent — it must not fail the
525
+ // registration.
526
+ logger.warning("Verify-time changed-file listing failed; this repository has no diff for the checks", {
527
+ repositoryPath,
528
+ error: error instanceof Error ? error.message : String(error),
224
529
  });
530
+ return undefined;
225
531
  }
226
- return recovered;
227
- }
228
- /** Whether a repo section's persisted diff classification contains any new,
229
- * modified, or removed endpoint — the same signal `computeScoredCandidates`
230
- * derives `hasApiChanges` from at analyze time. */
231
- function sectionHasApiChanges(data) {
232
- const diff = data?.repositoryAnalysis?.diff;
233
- return ((diff?.newEndpoints?.length ?? 0) > 0 ||
234
- (diff?.modifiedEndpoints?.length ?? 0) > 0 ||
235
- (diff?.removedEndpoints?.length ?? 0) > 0);
236
532
  }
237
- function resolveBudgetContext(stateData, fullState) {
238
- const pbc = stateData.planBudgetContext;
239
- const base = {
240
- maxGenerate: pbc?.maxGenerate ?? MAX_TESTS_TO_GENERATE,
241
- maxTotal: pbc?.maxTotal ?? MAX_RECOMMENDATIONS,
242
- isUIOnlyPR: pbc?.isUIOnlyPR ?? false,
243
- hasFrontendChanges: pbc?.hasFrontendChanges ?? false,
244
- externalCoverage: new Set(pbc?.externalCoverageKeys ?? []),
245
- // `true` for the same reason `computeDiffChangesTestFiles` answers `true` with
246
- // no diff: an unknown must shut the reserve, not open it. State written by an
247
- // older MCP build carries no such field, and if that PR did add its own tests
248
- // the reserve would recommend duplicates of them — the failure this flag
249
- // exists to stop, reached by version skew alone. It is also the pre-reserve
250
- // behaviour: covered candidates were dropped.
251
- diffChangesTestFiles: pbc?.diffChangesTestFiles ?? true,
252
- };
253
- const relatedSections = Object.values(fullState?.relatedRepos ?? {}).map((section) => section.data);
254
- if (relatedSections.length === 0)
255
- return base;
256
- // SKYR-4204: isUIOnlyPR/hasFrontendChanges are per-SECTION facts — each
257
- // analyze call computes them from its own repo's diff. The approved plan is
258
- // one run-wide decision, so a frontend-only PRIMARY diff must not put the
259
- // budgeter on its all-UI branch (backendGenerateCount() = 0) while a related
260
- // repo carries endpoint changes; that pinned GENERATE to UI placeholders and
261
- // left every API candidate in ADDITIONAL regardless of maxGenerate. Derive
262
- // the flags across ALL sections. A related repo checked out on its default
263
- // branch classifies no endpoint changes, so it leaves a UI-only run UI-only.
264
- // Budget numbers come from the root (primary) section: the testbot prompt
265
- // passes topN/maxGenerate only to the primary analyze call, so related
266
- // sections carry defaults.
267
- const sections = [fullState ?? undefined, ...relatedSections];
268
- const hasFrontendChanges = sections.some((s) => s?.planBudgetContext?.hasFrontendChanges ?? false);
269
- const hasApiChanges = sections.some(sectionHasApiChanges);
270
- const rootPbc = fullState?.planBudgetContext;
271
- // SKYR-4211: external-test coverage is also a per-section fact — a backend
272
- // candidate already covered by an external test recorded in a RELATED repo's
273
- // section must dedup the same as one covered in the primary. Union the keys
274
- // across all sections, like the flags above.
275
- const externalCoverage = new Set(sections.flatMap((s) => s?.planBudgetContext?.externalCoverageKeys ?? []));
533
+ /** Every fact the verifiers check against, gathered once per registration. */
534
+ export async function buildVerifyContext(state) {
535
+ const sections = repoSections(state);
536
+ const listings = await Promise.all(sections.map((section) => changedFilesForVerification(section)));
537
+ // Per repository, so a planned test that names one is checked against it alone.
538
+ // A planned test that names none reads every entry, which is every planned test of a
539
+ // single-repository run.
276
540
  return {
277
- maxGenerate: rootPbc?.maxGenerate ?? base.maxGenerate,
278
- maxTotal: rootPbc?.maxTotal ?? base.maxTotal,
279
- isUIOnlyPR: hasFrontendChanges && !hasApiChanges,
280
- hasFrontendChanges,
281
- externalCoverage,
282
- // Unioned across sections like the flags above: a pull request that adds
283
- // its own tests in ANY repo of the run must shut the reserve, or that repo's
284
- // own coverage becomes grounds to recommend duplicates of it. A section with
285
- // no budget context is an unknown, and an unknown shuts the reserve — the
286
- // same direction `computeDiffChangesTestFiles` takes with no diff.
287
- diffChangesTestFiles: sections.some((s) => s?.planBudgetContext?.diffChangesTestFiles ?? true),
541
+ changedFiles: Object.fromEntries(sections.map((section, i) => [section.repo, listings[i] ?? []])),
542
+ reposWithoutDiff: sections.filter((_s, i) => !listings[i]?.length).map((s) => s.repo),
543
+ // From the process, not the state file: the browser tools and this one are
544
+ // registered on the same server, so captures and reader share a process.
545
+ uiCaptures: capturedBlueprintDigests(),
546
+ // From the analyze result on the state file, across every repository: a
547
+ // removal guard names an element that renders nowhere, so this list is the
548
+ // only thing that can tell an honest one from an invented element.
549
+ removedUiElements: sections.flatMap((section) => section.data?.uiContext?.removedElements ?? []),
550
+ citedFileExists: makeCitedFileCheck(state),
551
+ routesRenderingFile: makeScreenRouteResolver(state),
288
552
  };
289
553
  }
290
- function describeGenerationCall(item) {
291
- switch (item.testType) {
292
- case "contract":
293
- return item.scenarioName
294
- ? `call ${TOOL_CONTRACT_TEST_GENERATION} with scenarioName "${item.scenarioName}"`
295
- : `call ${TOOL_CONTRACT_TEST_GENERATION}`;
296
- case "integration":
297
- return `call ${TOOL_BATCH_SCENARIO_TEST_GENERATION}, then ${TOOL_INTEGRATION_TEST_GENERATION} with the returned scenarioFile`;
298
- case "e2e":
299
- return `call ${TOOL_E2E_TEST_GENERATION}`;
300
- case "ui":
301
- return `call ${TOOL_UI_TEST_GENERATION}`;
302
- default:
303
- return "use the generation tool matching this testType";
304
- }
305
- }
306
- function renderPlanItem(item, rank) {
307
- const discriminatorNote = item.verifiedDiscriminator ? `, verified discriminator: ${item.verifiedDiscriminator}` : "";
308
- return `${rank}. [${item.testType}] "${item.scenarioName}" (${item.category}${discriminatorNote}) — ${describeGenerationCall(item)}`;
309
- }
554
+ // ── Output ──────────────────────────────────────────────────────────────────
310
555
  /**
311
- * Whether this run generates UI/E2E tests is a deterministic property of the
312
- * approved plan, so it is decided here rather than left to the agent to resolve
313
- * from prompt prose. The prompt owns HOW to record a UI trace and the runtime
314
- * conditions the server cannot know (app unreachable, unintegrated component);
315
- * this directive owns WHETHER and HOW MANY.
556
+ * What the agent reads back. Every objection carries its id so it can be answered,
557
+ * and the text says the plan is stored either way — a objections list without that
558
+ * line reads as a rejection and invites a needless second registration.
316
559
  */
317
- function renderGenerationDirective(plan) {
318
- const uiCount = plan.generate.filter((item) => item.testType === TestType.UI || item.testType === TestType.E2E).length;
319
- const nonUICount = plan.generate.length - uiCount;
320
- const tests = (n) => `${n} test${n === 1 ? "" : "s"}`;
321
- if (plan.generate.length === 0) {
322
- return [
323
- "### Generation directive: GENERATE NOTHING",
324
- "Create zero new tests of any type this run and proceed to the report. An empty `newTestsCreated` is the CORRECT " +
325
- "result for this PR — there is no new observable surface to cover. Do not record a browser trace and do not invent " +
326
- "a spec to have something to report; report your maintenance work and the ADDITIONAL candidates as recommendations instead.",
327
- ];
328
- }
329
- if (uiCount === 0) {
330
- return [
331
- "### Generation directive: NO UI/E2E GENERATION",
332
- `This plan allocates no UI/E2E generation. Generate the ${tests(nonUICount)} listed above and create zero UI tests ` +
333
- "— do not record a browser trace and do not add a UI spec to fill `newTestsCreated`.",
334
- ];
335
- }
336
- return [
337
- `### Generation directive: UI/E2E GENERATION REQUIRED (${uiCount})`,
338
- `This plan allocates ${tests(uiCount)} of type UI/E2E. You MUST attempt to record and generate each one — do not ` +
339
- "downgrade them to recommendations while the app is reachable." +
340
- (nonUICount > 0 ? ` Generate the ${tests(nonUICount)} of other types as well.` : ""),
341
- ];
342
- }
343
- /** SKYR-4292: the server cannot count the diff's changed behaviors, so an
344
- * under-budget plan is flagged (not rejected) with what a missing candidate looks
345
- * like. Wipro PR #24 registered 3 candidates against a 20-slot UI-only budget and
346
- * the tool answered "AUTHORITATIVE and FINAL" with no pushback. */
347
- function renderBudgetCheck(plan, maxTotal) {
348
- const filled = plan.generate.length + plan.additional.length;
349
- if (filled >= maxTotal)
350
- return [];
351
- return [
352
- "",
353
- `### Budget check: ${filled} of ${maxTotal} slots filled`,
354
- "Every observable behavior this diff changes — each option value added, removed, renamed or reordered; each label, " +
355
- "element, route, state, validation rule, computed value or style value (spacing, size, color, radius, font weight, " +
356
- "class name) that renders differently; each removed element, asserted absent — needs a candidate above, however " +
357
- "small the change. A change is covered only when an existing test asserts the changed value, not when a test merely " +
358
- "reaches the component. If any changed behavior has no candidate, call skyramp_register_test_plan again with your " +
359
- "full updated list. If the diff truly changes fewer behaviors than the budget, this plan stands as-is.",
360
- ];
361
- }
362
- function renderPlanText(plan, dropped, maxTotal) {
560
+ export function renderPlanResult(result) {
363
561
  const lines = [];
364
- lines.push(`## Approved Test Plan (${plan.planId})`);
365
- lines.push("");
366
- lines.push("This plan is AUTHORITATIVE and FINAL. Generation tools reject scenarios that are not in the GENERATE list below. " +
367
- "If you need to add, drop, or change a candidate, call skyramp_register_test_plan again with your full updated candidate list.");
368
- lines.push("");
369
- lines.push(`### GENERATE (${plan.generate.length}) — generate exactly these, in this order`);
370
- if (plan.generate.length === 0) {
371
- lines.push(" (none — nothing to generate this run)");
372
- }
373
- else {
374
- plan.generate.forEach((item, i) => lines.push(renderPlanItem(item, i + 1)));
375
- }
376
- lines.push("");
377
- lines.push(`### ADDITIONAL (${plan.additional.length}) — report as recommendations; only generate if backfilling a failed GENERATE item`);
378
- if (plan.additional.length === 0) {
379
- lines.push(" (none)");
380
- }
381
- else {
382
- plan.additional.forEach((item, i) => lines.push(renderPlanItem(item, plan.generate.length + i + 1)));
562
+ lines.push(`Registration ${result.plan.registrationNumber} stored as this run's plan.`);
563
+ lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s).`);
564
+ // A convenience: `differsFrom` also accepts a `scenarioName`.
565
+ for (const plannedTest of result.plan.plannedTests) {
566
+ lines.push(`- ${plannedTest.plannedTestId} — ${plannedTest.scenario?.scenarioName ?? "(unnamed)"}`);
383
567
  }
384
- if (plan.demotions.length > 0) {
385
- // A demotion (failed discriminator claim) and a drop (removed by the
386
- // budgeter for an unrelated reason — coverage or budget) are independent:
387
- // a candidate can suffer both. Split so the report never claims a
388
- // dropped candidate is "still registered".
389
- const dropReasonByCandidateId = new Map(dropped.map((d) => [d.candidateId, d.reason]));
390
- const registeredDemotions = plan.demotions.filter((d) => !dropReasonByCandidateId.has(d.candidateId));
391
- const removedDemotions = plan.demotions.filter((d) => dropReasonByCandidateId.has(d.candidateId));
392
- if (registeredDemotions.length > 0) {
393
- lines.push("");
394
- lines.push(`### Demotions (${registeredDemotions.length}) — discriminator claims that did NOT verify (candidate still registered, claim dropped)`);
395
- for (const demotion of registeredDemotions) {
396
- lines.push(`- ${demotion.candidateId}: ${demotion.reason}`);
397
- }
398
- }
399
- if (removedDemotions.length > 0) {
400
- lines.push("");
401
- lines.push(`### Demoted AND removed (${removedDemotions.length}) — the claim failed to verify AND the candidate left the plan. It is NOT registered.`);
402
- for (const demotion of removedDemotions) {
403
- lines.push(`- ${demotion.candidateId}: claim failed — ${demotion.reason}; removed — ${dropReasonByCandidateId.get(demotion.candidateId)}`);
404
- }
405
- }
568
+ lines.push(result.summary);
569
+ // The agent sends the answers it wants read; these came off the stored plan, and
570
+ // the plan they answered has changed since.
571
+ if (result.carried) {
572
+ lines.push("");
573
+ lines.push(`Answers carried from registration ${result.carried.registrationNumber} — recheck each; revise it if the plan change made it wrong:`);
574
+ for (const { objectionId, answer } of result.carried.answers)
575
+ lines.push(`- ${objectionId} — ${answer}`);
406
576
  }
407
- // Every other removal — budget or coverage, with no failed claim attached.
408
- // This section is independent of `demotions`: the two channels are unrelated,
409
- // and the motivating case has drops and NO demotions. Nesting it inside the
410
- // demotion block returned empty GENERATE/ADDITIONAL lists with no candidate id
411
- // and no reason, which is the state item 5 exists to make visible. Measured on
412
- // eval run 32586610615, fixture repo15-P14: 10 drops, 0 demotions.
413
- const demotedIds = new Set(plan.demotions.map((d) => d.candidateId));
414
- const removedWithoutDemotion = dropped.filter((d) => !demotedIds.has(d.candidateId));
415
- if (removedWithoutDemotion.length > 0) {
577
+ if (result.objections.length > 0) {
416
578
  lines.push("");
417
- lines.push(`### Removed (${removedWithoutDemotion.length}) — candidates the selection stage dropped. They are NOT registered.`);
418
- for (const drop of removedWithoutDemotion) {
419
- lines.push(`- ${drop.candidateId}: ${drop.reason}`);
579
+ lines.push("Objections:");
580
+ for (const objection of result.objections) {
581
+ lines.push(`- ${objection.objectionId} — ${objection.message}`);
582
+ lines.push(` evidence: ${objection.evidence}`);
583
+ lines.push(` next: ${objection.suggestion}`);
420
584
  }
585
+ lines.push("");
586
+ lines.push("The plan is stored either way. Revise it and call this tool again — there is no registration limit — " +
587
+ "or answer an objection through `answers` using the id above. Anything left open is rendered in the report.");
421
588
  }
422
- lines.push("");
423
- lines.push(...renderGenerationDirective(plan));
424
- lines.push(...renderBudgetCheck(plan, maxTotal));
425
589
  return lines.join("\n");
426
590
  }
427
591
  // ── Tool registration ───────────────────────────────────────────────────────
592
+ /** One registration at a time per state file: the registration number is a
593
+ * read-modify-write with an await between the halves, so two overlapping calls
594
+ * returned one number for two plans. In-process serialisation is enough. */
595
+ const registrationsInFlight = new Map();
596
+ /** The one file two calls contend for, whatever they spelled. Keying on the raw
597
+ * string let `/a/b/state.json` and `/a/b//state.json` take different locks on one
598
+ * file. `realpathSync` throws before the file exists; the resolved path is the
599
+ * right key then. */
600
+ function lockKeyFor(stateFile) {
601
+ const resolved = path.resolve(stateFile);
602
+ try {
603
+ return fs.realpathSync(resolved);
604
+ }
605
+ catch {
606
+ return resolved;
607
+ }
608
+ }
609
+ function oneAtATime(stateFile, work) {
610
+ const key = lockKeyFor(stateFile);
611
+ const queued = (registrationsInFlight.get(key) ?? Promise.resolve()).then(work, work);
612
+ // Never rejects, so one failed registration cannot break the chain.
613
+ const settled = queued.catch(() => undefined);
614
+ registrationsInFlight.set(key, settled);
615
+ // Dropped when nothing is queued, so the map does not grow without bound.
616
+ void settled.then(() => {
617
+ if (registrationsInFlight.get(key) === settled)
618
+ registrationsInFlight.delete(key);
619
+ });
620
+ return queued;
621
+ }
428
622
  export function registerRegisterTestPlanTool(server) {
429
623
  server.registerTool(TOOL_NAME, {
430
624
  annotations: {
431
- destructiveHint: false,
625
+ // It REPLACES the stored plan and the registration count.
626
+ destructiveHint: true,
432
627
  readOnlyHint: false,
433
628
  idempotentHint: false,
434
629
  openWorldHint: false,
435
630
  },
436
- description: "REQUIRED before generation tools in testbot runs — the deterministic selection checkpoint that decides which candidate tests get generated. " +
437
- "Register your COMPLETE candidate list (every test you would generate OR recommend) BEFORE calling any generation tool. " +
438
- "Merges your candidates with any server pre-ranked candidates from skyramp_analyze_changes, verifies declared `discriminator` " +
439
- "claims against the PR diff, re-ranks the union, and returns the mandated GENERATE list plus ADDITIONAL recommendations and " +
440
- "demotion reasons for claims that failed verification. In an active testbot run, generation tools (skyramp_batch_scenario_test_generation, " +
441
- "skyramp_contract_test_generation, skyramp_integration_test_generation) reject any scenario not in the returned GENERATE list — " +
442
- "call this tool first, then generate exactly what it returns. Standalone/IDE MCP usage (no active testbot run) is unaffected.",
631
+ description: "Register your complete test plan for this run, with the changes that make each planned test checkable. " +
632
+ "The planning procedure in your prompt says how to decide and declare the plan. " +
633
+ "Every registration the schema accepts is stored, and no check ever refuses one. " +
634
+ "The schema refuses the whole call on ordinary field validation, and on three rules that read more than one field: a non-UI planned test with an empty `steps` list, a `ui` or `e2e` planned test whose steps make no HTTP call and names no `elements.pageUrl`, and an `elements.items` entry carrying no name and no identifier. " +
635
+ "An undeclared key is refused by name on a planned test and on `declarations`, so a declaration written one level too high is corrected rather than dropped in silence. " +
636
+ "Checks run against the stored plan and return objections. Fix one and register it again as often as you like, or answer it by id through `answers`. " +
637
+ "There is no fallback plan: if you never register, this run has no plan.",
443
638
  inputSchema: registerTestPlanSchema,
444
639
  }, async (params) => {
445
- let errorResult;
446
- try {
447
- logger.info("Registering run-wide test plan", {
448
- candidateCount: params.candidates?.length ?? 0,
449
- });
450
- const stateManager = StateManager.fromStatePath(params.stateFile);
451
- // The plan is run-wide, so register always reads the ROOT (primary)
452
- // section — there is no section selector (routing by one is what let a
453
- // related-repo registration run with that section's context —
454
- // SKYR-4204 reopen). Run-wide facts (diff union, budget flags,
455
- // scenario lookup) come from fullState below.
456
- const stateData = await stateManager.readData();
457
- if (!stateData) {
458
- errorResult = toolError(`State file is empty or invalid: ${params.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
459
- return errorResult;
460
- }
461
- // SKYR-4203: one register call carries candidates for EVERY repo of the
462
- // run, but `stateData` is a single repo's section — and the candidate
463
- // schema has no per-candidate repository, so a related repo's anchors
464
- // were unverifiable by construction. Verify anchors (and rank diff-hunk
465
- // proximity) against the union of ALL sections' diffs: an anchor
466
- // grounded in a related repo's change is as real as one in the
467
- // primary's. Sections with no diff (a related repo checked out on its
468
- // default branch) are collected so an unmatched anchor reads as
469
- // uncheckable rather than ungrounded.
470
- const fullState = await stateManager.readFullState();
471
- const diffSections = [
472
- { repo: fullState?.metadata?.repository ?? "the primary repository", diff: fullState?.diffText },
473
- ...Object.entries(fullState?.relatedRepos ?? {}).map(([repo, section]) => ({
474
- repo,
475
- diff: section.data?.diffText,
476
- })),
477
- ];
478
- const diffText = diffSections.map((s) => s.diff).filter(Boolean).join("\n");
479
- const reposWithoutDiff = diffSections.filter((s) => !s.diff).map((s) => s.repo);
480
- // Scenario lookup for server-candidate recovery spans ALL sections —
481
- // the primary's drafted scenarios must resolve even when `repository`
482
- // targets a related repo (SKYR-4204 reopen: the same single-section
483
- // shape as the diff union above).
484
- const allScenarios = [
485
- ...(fullState?.repositoryAnalysis?.scenarios ?? []),
486
- ...Object.values(fullState?.relatedRepos ?? {}).flatMap((section) => section.data?.repositoryAnalysis?.scenarios ?? []),
487
- ];
488
- const changedRoutes = collectChangedRouteLines(diffText);
489
- const agentCandidates = buildAgentCandidates(params.candidates ?? [], changedRoutes);
490
- // The approved plan is run-wide and persists at the ROOT (see the
491
- // persist step below), so prior-plan recovery reads the root first;
492
- // the section fallback covers state written by older builds.
493
- const serverCandidates = recoverServerCandidates(fullState?.approvedPlan ?? stateData.approvedPlan, allScenarios);
494
- // Merge by scenario-name identity, NOT the full content-hashed candidateId:
495
- // the server (analyze_changes) and the agent frequently draft their own
496
- // independent steps[] for "the same" scenario (identical scenarioName),
497
- // which hash to different candidateIds — deduping on the full id let
498
- // both survive as apparent duplicates (SKYR-4026). A pre-seeded server
499
- // candidate wins over an agent submission for the same scenario name —
500
- // source: "server" is preserved per the unified-plan design.
501
- //
502
- // Uses scenarioMergeKey (untruncated), not scenarioNameSlug — the
503
- // latter's 48-char cap is fine for a short candidateId prefix combined
504
- // with a content hash, but bare as a Map key it would silently collapse
505
- // two distinct long scenario names sharing a common prefix. A missing
506
- // scenarioName (schema requires one for agent candidates; only a
507
- // malformed server-recovered scenario could lack one) falls back to
508
- // the already-unique candidateId rather than a shared literal.
509
- const mergeKey = (c) => scenarioMergeKey(c.scenario.scenarioName) || c.candidateId;
510
- const merged = new Map();
511
- for (const candidate of agentCandidates)
512
- merged.set(mergeKey(candidate), candidate);
513
- for (const candidate of serverCandidates)
514
- merged.set(mergeKey(candidate), candidate);
515
- // The agent declares a claim per scenario NAME, so key the claims the
516
- // same way the merge does and verify against whichever copy survived.
517
- const claimsByKey = new Map();
518
- // Keyed through mergeKey, not scenarioMergeKey alone: a name that
519
- // slugifies to nothing (e.g. an all-non-ASCII name) would be stored
520
- // under "" while the lookup falls back to the candidateId, so the claim
521
- // would be neither verified nor demoted.
522
- (params.candidates ?? []).forEach((input, i) => {
523
- if (input.discriminator)
524
- claimsByKey.set(mergeKey(agentCandidates[i]), input.discriminator);
525
- });
526
- const { candidates: allCandidates, demotions } = applyDiscriminatorClaims([...merged.values()], claimsByKey, diffText, reposWithoutDiff);
527
- // An empty union would persist an authoritative plan with an empty
528
- // GENERATE list, which the generation gate then enforces — bricking
529
- // the run until re-registration. Reject instead so the agent can
530
- // self-correct in one step.
531
- if (allCandidates.length === 0) {
532
- errorResult = toolError("No candidates submitted and no server pre-ranked candidates exist for this run. " +
533
- "Submit your complete candidate list (every test you would generate or recommend) in `candidates` — an empty plan would block all generation.");
534
- return errorResult;
535
- }
536
- // Review (mcp#774): enforce ONE run-wide plan at the tool boundary. A
537
- // re-registration whose pool shares no scenario with the active plan
538
- // is the per-repo-partial signature (the demoshop rig's second,
539
- // backend-only call) — accepting it would silently REPLACE the run's
540
- // plan with a subset and drop every earlier agent candidate. Genuine
541
- // full-list updates overlap the active plan and replace it as before.
542
- // The overlap test spans GENERATE ∪ ADDITIONAL: with maxGenerate=0 the
543
- // active plan's GENERATE is empty while ADDITIONAL carries every
544
- // candidate, and a disjoint partial could otherwise still replace it
545
- // silently (Copilot review on #774).
546
- const priorItems = [
547
- ...(fullState?.approvedPlan?.generate ?? []),
548
- ...(fullState?.approvedPlan?.additional ?? []),
549
- ];
550
- if (priorItems.length > 0) {
551
- const poolKeys = new Set(allCandidates.map(mergeKey));
552
- const overlaps = priorItems.some((item) => poolKeys.has(scenarioMergeKey(item.scenarioName) || item.candidateId));
553
- if (!overlaps) {
554
- errorResult = toolError(`A run-wide plan is already active (${fullState.approvedPlan.planId}: ` +
555
- `${priorItems.map((i) => i.scenarioName).join(", ")}) and this submission shares none of its ` +
556
- "scenarios. skyramp_register_test_plan REPLACES the whole run's plan — resubmit ONE pooled candidate " +
557
- "list for the entire run (your previous candidates plus these changes), not a per-repo subset.");
558
- return errorResult;
559
- }
560
- }
561
- const budgetContext = resolveBudgetContext(stateData, fullState);
562
- // SKYR-4204 reopen: a reserved UI slot is only fillable from the
563
- // candidate pool. A pool with no UI candidate on a run with frontend
564
- // changes would silently plan zero UI tests (runs 32419025266,
565
- // 32431042133, 32431106864) — reject so the agent self-corrects in one
566
- // step, exactly like the empty-candidates rejection above.
567
- const uiFloor = generateFloors(budgetContext).ui;
568
- if (uiFloor > 0 && !allCandidates.some((c) => inferScenarioType(c.scenario) === TestType.UI)) {
569
- errorResult = toolError(`This run has frontend changes, so the plan MUST include UI tests — at least ${uiFloor} of the ` +
570
- "GENERATE slots are held for them, and more are open to UI candidates on rank. No submitted or " +
571
- "recovered candidate has testType 'ui'. Resubmit your full candidate list including UI candidates " +
572
- "for the changed frontend (see uiContext.changedFrontendFiles).");
573
- return errorResult;
574
- }
575
- const result = selectPlan(allCandidates, { ...budgetContext, demotions, diffText });
576
- const approvedPlan = {
577
- planId: crypto.randomUUID(),
578
- createdAt: new Date().toISOString(),
579
- generate: result.generate.map(buildApprovedPlanItem),
580
- additional: result.additional.map(buildApprovedPlanItem),
581
- demotions: result.demotions,
582
- dropped: result.dropped,
583
- };
584
- // The approved plan is the ONE run-wide authority (the prompt mandates
585
- // a single pooled registration on multi-repo runs), so it persists at
586
- // the state-file ROOT regardless of `repository`. Per-section plans
587
- // blinded every root-reading consumer — planGuard let generation run
588
- // unguarded, submitReportTool's reconciliation was inert, and the eval
589
- // extractor saw "no plan" (SKYR-4204 reopen). writeData preserves the
590
- // relatedRepos sections and the root metadata.
640
+ // The run's own state file, unless the agent named one. Outside a run there
641
+ // is no path to fall back to, and no analysis to check a plan against.
642
+ const stateFile = resolveRunStatePath(params.stateFile);
643
+ if (!stateFile) {
644
+ return toolError("This run has no state file. Call skyramp_analyze_changes first — it writes the state file this plan is checked against.");
645
+ }
646
+ return oneAtATime(stateFile, async () => {
591
647
  try {
592
- const { metadata: _meta, relatedRepos: _sections, ...rootData } = fullState ?? {};
593
- await stateManager.writeData({ ...rootData, approvedPlan }, { step: TOOL_NAME });
648
+ const stateManager = StateManager.fromStatePath(stateFile);
649
+ // Run-wide: one registration covers every repository, so the facts come
650
+ // from the whole state file and the plan persists at its root.
651
+ const fullState = await stateManager.readFullState();
652
+ if (!fullState) {
653
+ return toolError(`State file is empty or invalid: ${stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
654
+ }
655
+ const registrationNumber = (fullState.registrationCount ?? 0) + 1;
656
+ const registration = buildRegistration(params, registrationNumber);
657
+ // The previous registration's answers are already on the stored plan, which
658
+ // the write below replaces.
659
+ const result = registerPlan(registration, await buildVerifyContext(fullState), fullState.plan);
660
+ try {
661
+ const { metadata: _metadata, relatedRepos: _sections, ...rootData } = fullState;
662
+ await stateManager.writeData({ ...rootData, plan: result.plan, registrationCount: registrationNumber }, { step: TOOL_NAME });
663
+ }
664
+ catch (error) {
665
+ return toolError(`Failed to persist the test plan to the state file: ${error.message}. Retry this call.`);
666
+ }
667
+ logger.info(`${TOOL_NAME}: registration ${registrationNumber}, ${result.plan.plannedTests.length} planned test(s), ${result.objections.length} objection(s)`);
668
+ return { content: [{ type: "text", text: renderPlanResult(result) }] };
594
669
  }
595
670
  catch (error) {
596
- errorResult = toolError(`Failed to persist approved test plan to state file: ${error.message}. Retry this call.`);
597
- return errorResult;
671
+ return toolError(`${TOOL_NAME} failed: ${error?.message ?? String(error)}`);
598
672
  }
599
- return {
600
- content: [{ type: "text", text: renderPlanText(approvedPlan, result.dropped, budgetContext.maxTotal) }],
601
- };
602
- }
603
- catch (error) {
604
- errorResult = toolError(`skyramp_register_test_plan failed: ${error?.message ?? String(error)}`);
605
- return errorResult;
606
- }
673
+ });
607
674
  });
608
675
  }