@skyramp/mcp 0.3.1 → 0.3.2-rc.pom-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/build/index.js +2 -1
  2. package/build/prompts/code-reuse.d.ts +7 -1
  3. package/build/prompts/code-reuse.js +8 -5
  4. package/build/prompts/code-reuse.test.d.ts +1 -0
  5. package/build/prompts/code-reuse.test.js +62 -0
  6. package/build/prompts/pom-aware-code-reuse.d.ts +6 -1
  7. package/build/prompts/pom-aware-code-reuse.js +100 -53
  8. package/build/prompts/pom-aware-code-reuse.test.d.ts +1 -0
  9. package/build/prompts/pom-aware-code-reuse.test.js +11 -0
  10. package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +4 -2
  11. package/build/prompts/test-recommendation/diffExecutionPlan.js +11 -65
  12. package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +2 -2
  13. package/build/prompts/test-recommendation/recommendationSections.js +5 -2
  14. package/build/prompts/test-recommendation/scopeAssessment.js +1 -1
  15. package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +26 -1
  16. package/build/prompts/test-recommendation/test-recommendation-prompt.js +68 -56
  17. package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +48 -11
  18. package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
  19. package/build/prompts/testbot/testbot-prompts.js +67 -21
  20. package/build/prompts/testbot/testbot-prompts.test.js +44 -0
  21. package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +7 -0
  22. package/build/recommendation/budgeters/diversityBalancedBudgeter.js +71 -0
  23. package/build/recommendation/budgeters/diversityBalancedBudgeter.test.d.ts +1 -0
  24. package/build/recommendation/budgeters/diversityBalancedBudgeter.test.js +75 -0
  25. package/build/recommendation/budgeters/fixedNBudgeter.d.ts +7 -0
  26. package/build/recommendation/budgeters/fixedNBudgeter.js +11 -0
  27. package/build/recommendation/budgeters/fixedNBudgeter.test.d.ts +1 -0
  28. package/build/recommendation/budgeters/fixedNBudgeter.test.js +66 -0
  29. package/build/recommendation/budgeters/shared.d.ts +19 -0
  30. package/build/recommendation/budgeters/shared.js +66 -0
  31. package/build/recommendation/discriminators.d.ts +31 -0
  32. package/build/recommendation/discriminators.js +355 -0
  33. package/build/recommendation/discriminators.test.d.ts +1 -0
  34. package/build/recommendation/discriminators.test.js +324 -0
  35. package/build/recommendation/diversity.d.ts +47 -0
  36. package/build/recommendation/diversity.js +101 -0
  37. package/build/recommendation/diversity.test.d.ts +1 -0
  38. package/build/recommendation/diversity.test.js +77 -0
  39. package/build/recommendation/planRanker.d.ts +50 -0
  40. package/build/recommendation/planRanker.js +67 -0
  41. package/build/recommendation/planRanker.test.d.ts +1 -0
  42. package/build/recommendation/planRanker.test.js +110 -0
  43. package/build/recommendation/testFixtures.d.ts +25 -0
  44. package/build/recommendation/testFixtures.js +45 -0
  45. package/build/resources/testbotResource.js +4 -1
  46. package/build/services/ScenarioGenerationService.d.ts +5 -0
  47. package/build/services/ScenarioGenerationService.js +16 -1
  48. package/build/services/ScenarioGenerationService.test.js +44 -0
  49. package/build/services/TestExecutionService.d.ts +15 -1
  50. package/build/services/TestExecutionService.js +210 -55
  51. package/build/services/TestExecutionService.test.js +397 -0
  52. package/build/services/TestGenerationService.js +19 -1
  53. package/build/services/TestGenerationService.test.js +58 -0
  54. package/build/tool-phases.js +1 -0
  55. package/build/toolNames.d.ts +19 -0
  56. package/build/toolNames.js +19 -0
  57. package/build/tools/code-refactor/codeReuseTool.d.ts +7 -0
  58. package/build/tools/code-refactor/codeReuseTool.js +130 -4
  59. package/build/tools/code-refactor/codeReuseTool.test.d.ts +1 -0
  60. package/build/tools/code-refactor/codeReuseTool.test.js +290 -0
  61. package/build/tools/executeSkyrampTestTool.js +8 -2
  62. package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +6 -1
  63. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +110 -17
  64. package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +147 -0
  65. package/build/tools/generate-tests/generateContractRestTool.js +11 -1
  66. package/build/tools/generate-tests/generateIntegrationRestTool.js +24 -1
  67. package/build/tools/generate-tests/generateIntegrationRestTool.test.d.ts +1 -0
  68. package/build/tools/generate-tests/generateIntegrationRestTool.test.js +159 -0
  69. package/build/tools/generate-tests/planGuard.d.ts +13 -0
  70. package/build/tools/generate-tests/planGuard.js +78 -0
  71. package/build/tools/generate-tests/planGuard.test.d.ts +1 -0
  72. package/build/tools/generate-tests/planGuard.test.js +185 -0
  73. package/build/tools/generate-tests/scenarioFileIdentity.d.ts +10 -0
  74. package/build/tools/generate-tests/scenarioFileIdentity.js +46 -0
  75. package/build/tools/generate-tests/scenarioLint.d.ts +30 -0
  76. package/build/tools/generate-tests/scenarioLint.js +150 -0
  77. package/build/tools/generate-tests/scenarioLint.test.d.ts +1 -0
  78. package/build/tools/generate-tests/scenarioLint.test.js +100 -0
  79. package/build/tools/submitReportTool.js +78 -0
  80. package/build/tools/submitReportTool.test.js +255 -0
  81. package/build/tools/test-management/analyzeChangesTool.js +55 -2
  82. package/build/tools/test-management/analyzeChangesTool.test.js +12 -0
  83. package/build/tools/test-management/index.d.ts +1 -0
  84. package/build/tools/test-management/index.js +1 -0
  85. package/build/tools/test-management/registerTestPlanTool.d.ts +2 -0
  86. package/build/tools/test-management/registerTestPlanTool.js +329 -0
  87. package/build/tools/test-management/registerTestPlanTool.test.d.ts +1 -0
  88. package/build/tools/test-management/registerTestPlanTool.test.js +296 -0
  89. package/build/types/Recommendation.d.ts +97 -0
  90. package/build/types/Recommendation.js +48 -0
  91. package/build/types/RepositoryAnalysis.d.ts +14 -14
  92. package/build/types/TestExecution.d.ts +2 -0
  93. package/build/types/TestRecommendation.d.ts +12 -1
  94. package/build/types/TestRecommendation.js +26 -11
  95. package/build/types/TestTypes.js +1 -1
  96. package/build/utils/AnalysisStateManager.d.ts +47 -0
  97. package/build/utils/docker.test.js +1 -1
  98. package/build/utils/planMatchKeys.d.ts +61 -0
  99. package/build/utils/planMatchKeys.js +125 -0
  100. package/build/utils/pom-scope/import-expansion.d.ts +5 -0
  101. package/build/utils/pom-scope/import-expansion.js +32 -0
  102. package/build/utils/pom-scope/index.d.ts +39 -0
  103. package/build/utils/pom-scope/index.js +120 -0
  104. package/build/utils/pom-scope/index.test.d.ts +1 -0
  105. package/build/utils/pom-scope/index.test.js +239 -0
  106. package/build/utils/pom-scope/pom-files.d.ts +3 -0
  107. package/build/utils/pom-scope/pom-files.js +48 -0
  108. package/build/utils/pom-scope/pom-files.test.d.ts +1 -0
  109. package/build/utils/pom-scope/pom-files.test.js +29 -0
  110. package/build/utils/pom-scope/scoring.d.ts +20 -0
  111. package/build/utils/pom-scope/scoring.js +45 -0
  112. package/build/utils/pom-scope/scoring.test.d.ts +1 -0
  113. package/build/utils/pom-scope/scoring.test.js +39 -0
  114. package/build/utils/pom-scope/selector-extractor.d.ts +7 -0
  115. package/build/utils/pom-scope/selector-extractor.js +57 -0
  116. package/build/utils/pom-scope/selector-extractor.test.d.ts +1 -0
  117. package/build/utils/pom-scope/selector-extractor.test.js +67 -0
  118. package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.d.ts +5 -0
  119. package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.js +5 -0
  120. package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.d.ts +5 -0
  121. package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.js +9 -0
  122. package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.d.ts +4 -0
  123. package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.js +4 -0
  124. package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.d.ts +4 -0
  125. package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.js +4 -0
  126. package/build/utils/pom-verify/bindings.d.ts +19 -0
  127. package/build/utils/pom-verify/bindings.js +161 -0
  128. package/build/utils/pom-verify/bindings.test.d.ts +1 -0
  129. package/build/utils/pom-verify/bindings.test.js +164 -0
  130. package/build/utils/pom-verify/calls.d.ts +16 -0
  131. package/build/utils/pom-verify/calls.js +42 -0
  132. package/build/utils/pom-verify/calls.test.d.ts +1 -0
  133. package/build/utils/pom-verify/calls.test.js +61 -0
  134. package/build/utils/pom-verify/index.d.ts +4 -0
  135. package/build/utils/pom-verify/index.js +4 -0
  136. package/build/utils/pom-verify/resolve.d.ts +7 -0
  137. package/build/utils/pom-verify/resolve.js +27 -0
  138. package/build/utils/pom-verify/resolve.test.d.ts +1 -0
  139. package/build/utils/pom-verify/resolve.test.js +68 -0
  140. package/build/utils/pom-verify/strip.d.ts +9 -0
  141. package/build/utils/pom-verify/strip.js +89 -0
  142. package/build/utils/pom-verify/verify.d.ts +14 -0
  143. package/build/utils/pom-verify/verify.js +158 -0
  144. package/build/utils/pom-verify/verify.test.d.ts +1 -0
  145. package/build/utils/pom-verify/verify.test.js +325 -0
  146. package/build/utils/reportVerification.d.ts +61 -0
  147. package/build/utils/reportVerification.js +104 -0
  148. package/build/utils/reportVerification.test.d.ts +1 -0
  149. package/build/utils/reportVerification.test.js +185 -0
  150. package/build/utils/scenarioDrafting.js +5 -5
  151. package/build/utils/versions.d.ts +3 -3
  152. package/build/utils/versions.js +1 -1
  153. package/build/utils/workspaceAuth.d.ts +9 -1
  154. package/build/utils/workspaceAuth.js +25 -5
  155. package/build/utils/workspaceAuth.test.js +48 -0
  156. package/build/workspace/workspace.d.ts +20 -0
  157. package/build/workspace/workspace.js +4 -0
  158. package/build/workspace/workspace.test.js +10 -0
  159. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +77 -8
  160. package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
  161. package/node_modules/playwright/node_modules/playwright-core/lib/generated/pollingRecorderSource.js +1 -1
  162. package/node_modules/playwright/node_modules/playwright-core/lib/utils/isomorphic/volatileDate.js +101 -0
  163. package/node_modules/playwright/node_modules/playwright-core/lib/utils.js +2 -0
  164. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-aszq5EdG.js → codeMirrorModule-Bzd72-bG.js} +1 -1
  165. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-BxS7Jm4s.js → defaultSettingsView-DzxTioTK.js} +101 -101
  166. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.D4JTTy4R.js → index.BGc30U3S.js} +1 -1
  167. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
  168. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.DaRMQKOI.js → uiMode.IaDrb29A.js} +1 -1
  169. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
  170. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  171. package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
  172. package/node_modules/playwright/node_modules/playwright-core/src/generated/pollingRecorderSource.ts +1 -1
  173. package/node_modules/playwright/node_modules/playwright-core/src/utils/isomorphic/volatileDate.ts +131 -0
  174. package/node_modules/playwright/node_modules/playwright-core/src/utils.ts +1 -0
  175. package/node_modules/playwright/package.json +1 -1
  176. package/package.json +3 -3
  177. package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
@@ -9,6 +9,23 @@ import { DriftAction } from "../types/TestAnalysis.js";
9
9
  import { TestExecutionStatus } from "../types/TestExecution.js";
10
10
  import { StateManager } from "../utils/AnalysisStateManager.js";
11
11
  import { toolError, testFileMatches } from "../utils/utils.js";
12
+ import { matchesApprovedPlan } from "../utils/planMatchKeys.js";
13
+ import { isTestbotEnabled } from "../utils/featureFlags.js";
14
+ import { findUnbackedClaims, listChangedFiles } from "../utils/reportVerification.js";
15
+ // SKYR-3879 Path B: which testTypes the register-plan checkpoint gates. Mirrors
16
+ // the generation tools actually wired to planGuard (batch-scenario/integration,
17
+ // contract) — UI and E2E are on a separate blueprint-grounded pipeline and are
18
+ // NOT gated at generation time (see planGuard.ts wiring), so they are excluded
19
+ // here too rather than surprising the agent with a report-time-only gate.
20
+ const PLAN_GATED_TEST_TYPES = new Set([TestType.CONTRACT, TestType.INTEGRATION]);
21
+ /** Split an `endpoint` field ("GET /api/v1/products") into method + path. */
22
+ function parseEndpointField(endpoint) {
23
+ const trimmed = (endpoint ?? "").trim();
24
+ const spaceIdx = trimmed.indexOf(" ");
25
+ if (spaceIdx <= 0)
26
+ return { path: trimmed || undefined };
27
+ return { method: trimmed.slice(0, spaceIdx), path: trimmed.slice(spaceIdx + 1).trim() };
28
+ }
12
29
  // Drift actions that actually modify a test file. VERIFY and IGNORE are
13
30
  // no-ops (the test was assessed but left unchanged), so they must not count
14
31
  // toward "tests maintained" telemetry.
@@ -430,6 +447,33 @@ export function registerSubmitReportTool(server) {
430
447
  "Call skyramp_actions (with recommendations: [] if no existing tests needed action) before skyramp_submit_report.");
431
448
  return errorResult;
432
449
  }
450
+ // SKYR-3879 Path B: every gated newTestsCreated entry (contract/integration —
451
+ // the testTypes actually wired to planGuard) must be admissible under the
452
+ // approved plan from skyramp_register_test_plan — the same predicate the
453
+ // generation-tool gate uses (GENERATE, or ADDITIONAL backfill when GENERATE
454
+ // is non-empty), so a generation the gate permitted is never rejected at
455
+ // report time. Absent approvedPlan (register tool never ran / never
456
+ // pre-seeded anything) → no check, back-compat.
457
+ if (stateData.approvedPlan) {
458
+ const approvedPlan = stateData.approvedPlan;
459
+ const unapproved = dedupedNewTests.filter((t) => {
460
+ if (!PLAN_GATED_TEST_TYPES.has(t.testType))
461
+ return false;
462
+ const { method, path } = parseEndpointField(t.endpoint);
463
+ return !matchesApprovedPlan(approvedPlan, { testType: t.testType, method, path });
464
+ });
465
+ if (unapproved.length > 0) {
466
+ const approvedList = approvedPlan.generate.length > 0
467
+ ? approvedPlan.generate.map((item) => `[${item.testType}] ${item.scenarioName}`).join(", ")
468
+ : "(none)";
469
+ errorResult = toolError(`${unapproved.length} newTestsCreated entr${unapproved.length === 1 ? "y" : "ies"} not in the approved plan from ` +
470
+ `skyramp_register_test_plan (plan ${approvedPlan.planId}) — neither its GENERATE list nor its ADDITIONAL backfill pool: ` +
471
+ `${unapproved.map((t) => `${t.testId} (${t.testType} ${t.endpoint})`).join(", ")}. ` +
472
+ `Approved GENERATE items: ${approvedList}. Call skyramp_register_test_plan to register this test before reporting it, ` +
473
+ `or remove it from newTestsCreated.`);
474
+ return errorResult;
475
+ }
476
+ }
433
477
  // TestbotReport.testMaintenance has no repository field, so a related repo's
434
478
  // reconciled verdicts have nowhere to go in this report. Fail loud instead of
435
479
  // silently omitting real maintenance work skyramp_actions already did and persisted.
@@ -493,6 +537,40 @@ export function registerSubmitReportTool(server) {
493
537
  "Add a testMaintenanceDetails entry with the missing field(s), drafted from the execution output you already saw.");
494
538
  return errorResult;
495
539
  }
540
+ // SKYR-3883: in a testbot run, refuse to ship a report that claims file work
541
+ // the working tree doesn't reflect. The delivery step can only commit what the
542
+ // agent actually created/edited, so a report claiming otherwise erodes trust
543
+ // (a reviewer believes maintenance happened that never landed). Fail loudly so
544
+ // the agent makes the edit — or drops the claim — instead of hallucinating it.
545
+ // Only the working tree exists at report time (delivery commits later), so we
546
+ // verify against created/modified/untracked paths, not the eventual commit.
547
+ if (isTestbotEnabled()) {
548
+ const fullState = await stateManager.readFullState();
549
+ const repoRoot = fullState?.metadata?.repositoryPath;
550
+ if (repoRoot && repoRoot !== "unknown") {
551
+ try {
552
+ const changedFiles = await listChangedFiles(repoRoot);
553
+ const unbacked = findUnbackedClaims({
554
+ repoRoot,
555
+ changedFiles,
556
+ newTests: dedupedNewTests,
557
+ verdicts: stateData.maintenanceVerdicts ?? [],
558
+ primaryRepository: fullState?.metadata?.repository,
559
+ });
560
+ if (unbacked.length > 0) {
561
+ errorResult = toolError(`${unbacked.length} report claim(s) are not backed by any change in the working tree — ` +
562
+ `Testbot will not report file work that hasn't actually been made. Make the edit (or remove the claim), then resubmit:\n` +
563
+ unbacked.map((u) => ` - ${u}`).join("\n"));
564
+ return errorResult;
565
+ }
566
+ }
567
+ catch (err) {
568
+ // Never block a valid report because verification itself failed (path not a
569
+ // git repo, git unavailable) — degrade to a warning instead of false-rejecting.
570
+ logger.warning(`Skipping report/working-tree verification (could not enumerate changes): ${err.message}`);
571
+ }
572
+ }
573
+ }
496
574
  // Strip generation-artifact fields from newTestsCreated before writing.
497
575
  // scenarioFile, traceFile, frontendTrace are internal paths used during
498
576
  // generation — downstream scoring scripts don't expect them and fail if
@@ -6,6 +6,7 @@ import { TestExecutionStatus } from "../types/TestExecution.js";
6
6
  import * as fs from "fs/promises";
7
7
  import * as path from "path";
8
8
  import * as os from "os";
9
+ import { execFileSync } from "child_process";
9
10
  import { z } from "zod";
10
11
  jest.unstable_mockModule("../utils/logger.js", () => ({
11
12
  logger: { info: jest.fn(), error: jest.fn(), warning: jest.fn(), debug: jest.fn() },
@@ -16,6 +17,7 @@ jest.unstable_mockModule("../services/AnalyticsService.js", () => ({
16
17
  // @ts-ignore
17
18
  const { registerSubmitReportTool, additionalRecommendationSchema, newTestSchema, targetElementSchema, pageContextSchema } = await import("./submitReportTool.js");
18
19
  const { AnalyticsService } = await import("../services/AnalyticsService.js");
20
+ const { CandidateSource } = await import("../types/Recommendation.js");
19
21
  const pushMCPToolEventMock = AnalyticsService.pushMCPToolEvent;
20
22
  function captureToolHandler() {
21
23
  let handler;
@@ -1170,4 +1172,257 @@ describe("registerSubmitReportTool", () => {
1170
1172
  expect(result.content[0].text).not.toContain("beforeDetails");
1171
1173
  });
1172
1174
  });
1175
+ describe("approvedPlan cross-check (SKYR-3879 Path B)", () => {
1176
+ async function writeStateFileWithPlan(dir, approvedPlan) {
1177
+ const stateFile = path.join(dir, "analyze-changes-state.json");
1178
+ await fs.writeFile(stateFile, JSON.stringify({
1179
+ existingTests: [],
1180
+ maintenanceVerdicts: [],
1181
+ ...(approvedPlan ? { approvedPlan } : {}),
1182
+ metadata: sampleMetadata(dir),
1183
+ }), "utf-8");
1184
+ return stateFile;
1185
+ }
1186
+ const APPROVED_PLAN = {
1187
+ planId: "plan-42",
1188
+ createdAt: "2026-01-01T00:00:00.000Z",
1189
+ generate: [
1190
+ {
1191
+ candidateId: "products-search-abcd1234",
1192
+ scenarioName: "products_search",
1193
+ testType: "contract",
1194
+ category: "business_rule",
1195
+ source: CandidateSource.AGENT,
1196
+ matchKeys: ["name:products-search", "ep:contract:GET:/api/v1/products/search"],
1197
+ },
1198
+ ],
1199
+ additional: [],
1200
+ demotions: [],
1201
+ };
1202
+ function contractNewTest(overrides = {}) {
1203
+ return {
1204
+ testId: "contract-get-products-search",
1205
+ testType: TestType.CONTRACT,
1206
+ category: "business_rule",
1207
+ endpoint: "GET /api/v1/products/search",
1208
+ fileName: "products_search_contract_test.py",
1209
+ description: "Searches products and asserts the response schema.",
1210
+ reasoning: "Validates the contract for the new search endpoint.",
1211
+ ...overrides,
1212
+ };
1213
+ }
1214
+ it("passes when every gated newTestsCreated entry matches an approved GENERATE item", async () => {
1215
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1216
+ tmpDirs.push(tmpDir);
1217
+ const outputFile = path.join(tmpDir, "report.json");
1218
+ const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
1219
+ const result = await handler({
1220
+ summaryOutputFile: outputFile,
1221
+ stateFile,
1222
+ businessCaseAnalysis: "Adds product search.",
1223
+ newTestsCreated: [contractNewTest()],
1224
+ testResults: [],
1225
+ issuesFound: [],
1226
+ });
1227
+ expect(result.isError).toBeUndefined();
1228
+ });
1229
+ it("rejects an unapproved gated entry, naming the offending entry and the approved list", async () => {
1230
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1231
+ tmpDirs.push(tmpDir);
1232
+ const outputFile = path.join(tmpDir, "report.json");
1233
+ const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
1234
+ const result = await handler({
1235
+ summaryOutputFile: outputFile,
1236
+ stateFile,
1237
+ businessCaseAnalysis: "Adds an unregistered contract test.",
1238
+ newTestsCreated: [
1239
+ contractNewTest({
1240
+ testId: "contract-get-products-unregistered",
1241
+ endpoint: "GET /api/v1/products/unregistered",
1242
+ }),
1243
+ ],
1244
+ testResults: [],
1245
+ issuesFound: [],
1246
+ });
1247
+ expect(result.isError).toBe(true);
1248
+ expect(result.content[0].text).toContain("plan-42");
1249
+ expect(result.content[0].text).toContain("contract-get-products-unregistered");
1250
+ expect(result.content[0].text).toContain("products_search");
1251
+ });
1252
+ it("accepts a gated entry matching an approved ADDITIONAL item — backfill parity with planGuard", async () => {
1253
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1254
+ tmpDirs.push(tmpDir);
1255
+ const outputFile = path.join(tmpDir, "report.json");
1256
+ const stateFile = await writeStateFileWithPlan(tmpDir, {
1257
+ ...APPROVED_PLAN,
1258
+ additional: [
1259
+ {
1260
+ candidateId: "products-backfill-efgh5678",
1261
+ scenarioName: "products_backfill",
1262
+ testType: "contract",
1263
+ category: "business_rule",
1264
+ source: CandidateSource.AGENT,
1265
+ matchKeys: ["name:products-backfill", "ep:contract:GET:/api/v1/products/backfill"],
1266
+ },
1267
+ ],
1268
+ });
1269
+ const result = await handler({
1270
+ summaryOutputFile: outputFile,
1271
+ stateFile,
1272
+ businessCaseAnalysis: "GENERATE item failed; backfilled from ADDITIONAL as the plan instructs.",
1273
+ newTestsCreated: [
1274
+ contractNewTest({
1275
+ testId: "contract-get-products-backfill",
1276
+ endpoint: "GET /api/v1/products/backfill",
1277
+ }),
1278
+ ],
1279
+ testResults: [],
1280
+ issuesFound: [],
1281
+ });
1282
+ expect(result.isError).toBeUndefined();
1283
+ });
1284
+ it("rejects an ADDITIONAL-matching entry when the GENERATE list is empty — zero-budget plans have nothing to backfill", async () => {
1285
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1286
+ tmpDirs.push(tmpDir);
1287
+ const outputFile = path.join(tmpDir, "report.json");
1288
+ const stateFile = await writeStateFileWithPlan(tmpDir, {
1289
+ ...APPROVED_PLAN,
1290
+ generate: [],
1291
+ additional: APPROVED_PLAN.generate,
1292
+ });
1293
+ const result = await handler({
1294
+ summaryOutputFile: outputFile,
1295
+ stateFile,
1296
+ businessCaseAnalysis: "Generated a test on a zero-budget plan.",
1297
+ newTestsCreated: [contractNewTest()],
1298
+ testResults: [],
1299
+ issuesFound: [],
1300
+ });
1301
+ expect(result.isError).toBe(true);
1302
+ expect(result.content[0].text).toContain("plan-42");
1303
+ });
1304
+ it("does not check newTestsCreated when approvedPlan is absent (back-compat)", async () => {
1305
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1306
+ tmpDirs.push(tmpDir);
1307
+ const outputFile = path.join(tmpDir, "report.json");
1308
+ const stateFile = await writeStateFileWithPlan(tmpDir, undefined);
1309
+ const result = await handler({
1310
+ summaryOutputFile: outputFile,
1311
+ stateFile,
1312
+ businessCaseAnalysis: "No plan was ever registered for this run.",
1313
+ newTestsCreated: [contractNewTest({ endpoint: "GET /api/v1/products/anything" })],
1314
+ testResults: [],
1315
+ issuesFound: [],
1316
+ });
1317
+ expect(result.isError).toBeUndefined();
1318
+ });
1319
+ it("does not gate UI/E2E entries even when unmatched — those tools are outside planGuard's scope", async () => {
1320
+ const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
1321
+ tmpDirs.push(tmpDir);
1322
+ const outputFile = path.join(tmpDir, "report.json");
1323
+ const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
1324
+ const result = await handler({
1325
+ summaryOutputFile: outputFile,
1326
+ stateFile,
1327
+ businessCaseAnalysis: "Adds a UI test not present in the approved plan.",
1328
+ newTestsCreated: [
1329
+ {
1330
+ testId: "ui-checkout-flow",
1331
+ testType: TestType.UI,
1332
+ category: "workflow",
1333
+ endpoint: "N/A",
1334
+ fileName: "checkout_ui_test.spec.ts",
1335
+ description: "Walks through the checkout flow. [no-blueprint-data]",
1336
+ reasoning: "Covers the new checkout UI. [no-blueprint-data]",
1337
+ targetElements: null,
1338
+ },
1339
+ ],
1340
+ testResults: [],
1341
+ issuesFound: [],
1342
+ });
1343
+ expect(result.isError).toBeUndefined();
1344
+ });
1345
+ });
1346
+ });
1347
+ // SKYR-3883: end-to-end coverage of the working-tree verification gate. The unit
1348
+ // behavior lives in reportVerification.test.ts; these exercise the wiring through
1349
+ // the real tool handler (feature-flag gating, git enumeration, error surfacing).
1350
+ describe("submit_report working-tree verification (SKYR-3883)", () => {
1351
+ let handler;
1352
+ let tmpDirs = [];
1353
+ const prevFlag = process.env.SKYRAMP_FEATURE_TESTBOT;
1354
+ beforeAll(() => {
1355
+ handler = captureToolHandler();
1356
+ });
1357
+ beforeEach(() => {
1358
+ process.env.SKYRAMP_FEATURE_TESTBOT = "1";
1359
+ });
1360
+ afterEach(async () => {
1361
+ if (prevFlag === undefined)
1362
+ delete process.env.SKYRAMP_FEATURE_TESTBOT;
1363
+ else
1364
+ process.env.SKYRAMP_FEATURE_TESTBOT = prevFlag;
1365
+ for (const dir of tmpDirs)
1366
+ await fs.rm(dir, { recursive: true, force: true });
1367
+ tmpDirs = [];
1368
+ });
1369
+ async function initGitRepo() {
1370
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-wt-"));
1371
+ tmpDirs.push(dir);
1372
+ execFileSync("git", ["init", "-q"], { cwd: dir });
1373
+ execFileSync("git", ["config", "user.email", "t@t.dev"], { cwd: dir });
1374
+ execFileSync("git", ["config", "user.name", "t"], { cwd: dir });
1375
+ return dir;
1376
+ }
1377
+ it("rejects a report whose newTestsCreated file was never written", async () => {
1378
+ const dir = await initGitRepo();
1379
+ const outputFile = path.join(dir, "report.json");
1380
+ const stateFile = await writeStateFile(dir, []);
1381
+ // search_smoke_test.py is claimed but never created → unbacked.
1382
+ const result = await handler(sampleReportParams(outputFile, stateFile));
1383
+ expect(result.isError).toBe(true);
1384
+ expect(result.content[0].text).toContain("not backed");
1385
+ expect(result.content[0].text).toContain("search_smoke_test.py");
1386
+ });
1387
+ it("accepts the report once the claimed file exists in the working tree", async () => {
1388
+ const dir = await initGitRepo();
1389
+ const outputFile = path.join(dir, "report.json");
1390
+ const stateFile = await writeStateFile(dir, []);
1391
+ await fs.writeFile(path.join(dir, "search_smoke_test.py"), "def test_search(): pass\n");
1392
+ const result = await handler(sampleReportParams(outputFile, stateFile));
1393
+ expect(result.isError).toBeUndefined();
1394
+ });
1395
+ it("rejects an UPDATE maintenance verdict with no working-tree change, exempting DELETE", async () => {
1396
+ const dir = await initGitRepo();
1397
+ const outputFile = path.join(dir, "report.json");
1398
+ const stateFile = await writeReconciledStateFile(dir, [], [
1399
+ { testFilePath: path.join(dir, "dcim/tests/test_filtersets.py"), action: DriftAction.Update, rationale: "physical_address filter", testType: TestType.INTEGRATION, endpoint: "GET /api/dcim/sites/" },
1400
+ { testFilePath: path.join(dir, "dcim/tests/gone.py"), action: DriftAction.Delete, rationale: "obsolete", testType: TestType.INTEGRATION, endpoint: "GET /api/dcim/sites/" },
1401
+ ]);
1402
+ // Keep the newTestsCreated claim backed so the failure isolates to the verdict.
1403
+ await fs.writeFile(path.join(dir, "search_smoke_test.py"), "x\n");
1404
+ const result = await handler(sampleReportParams(outputFile, stateFile));
1405
+ expect(result.isError).toBe(true);
1406
+ expect(result.content[0].text).toContain("test_filtersets.py");
1407
+ // DELETE is applied at delivery, never a working-tree claim — must not be flagged.
1408
+ expect(result.content[0].text).not.toContain("gone.py");
1409
+ });
1410
+ it("skips verification entirely when the testbot feature flag is off", async () => {
1411
+ delete process.env.SKYRAMP_FEATURE_TESTBOT;
1412
+ const dir = await initGitRepo();
1413
+ const outputFile = path.join(dir, "report.json");
1414
+ const stateFile = await writeStateFile(dir, []);
1415
+ // Claimed file absent, but the gate is off → report goes through.
1416
+ const result = await handler(sampleReportParams(outputFile, stateFile));
1417
+ expect(result.isError).toBeUndefined();
1418
+ });
1419
+ it("degrades to a pass when repositoryPath is not a git repo (verification can't run)", async () => {
1420
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-nogit-"));
1421
+ tmpDirs.push(dir);
1422
+ const outputFile = path.join(dir, "report.json");
1423
+ const stateFile = await writeStateFile(dir, []); // repositoryPath=dir, but no git init
1424
+ const result = await handler(sampleReportParams(outputFile, stateFile));
1425
+ // git status throws → caught → warning → report not falsely rejected.
1426
+ expect(result.isError).toBeUndefined();
1427
+ });
1173
1428
  });
@@ -10,8 +10,12 @@ import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWo
10
10
  import { AnalyticsService } from "../../services/AnalyticsService.js";
11
11
  import { makeProgressReporter } from "../../utils/progress.js";
12
12
  import { StateManager, registerSession, storeSessionData, setTestsRepoDir, getActiveRunStatePath, setActiveRunStatePath, } from "../../utils/AnalysisStateManager.js";
13
- import { buildRecommendationPrompt } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
13
+ import { buildRecommendationPrompt, computeScoredCandidates } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
14
14
  import { hasFlutterSdkDep, isFrontendFile, isTestFile } from "../../prompts/test-recommendation/scopeAssessment.js";
15
+ import { buildExternalCoverageSet } from "../../prompts/test-recommendation/recommendationShared.js";
16
+ import { CandidateSource, computeCandidateId } from "../../types/Recommendation.js";
17
+ import { selectPlan } from "../../recommendation/planRanker.js";
18
+ import { buildApprovedPlanItem } from "../../utils/planMatchKeys.js";
15
19
  import { enumerateCandidateUiPages } from "../../utils/uiPageEnumerator.js";
16
20
  import { checkFrontendFileIntegration } from "../../utils/frontendIntegration.js";
17
21
  import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
@@ -1278,6 +1282,12 @@ export function registerAnalyzeChangesTool(server) {
1278
1282
  diffFilePath = path.join(os.tmpdir(), `skyramp-diff-${sessionId}.diff`);
1279
1283
  await fs.promises.writeFile(diffFilePath, diffData.diffContent, { encoding: "utf-8", mode: 0o600 });
1280
1284
  }
1285
+ // SKYR-3879 Path B: size-capped raw diff text, persisted (not just the
1286
+ // file path above) so skyramp_register_test_plan can verify a
1287
+ // discriminator's changedCodeAnchor occurs verbatim in the diff without
1288
+ // an extra file read of its own.
1289
+ const DIFF_TEXT_CAP = 200_000;
1290
+ const diffText = diffData?.diffContent ? diffData.diffContent.slice(0, DIFF_TEXT_CAP) : undefined;
1281
1291
  // Compute UI context from the diff's changed files using the shared
1282
1292
  // `isFrontendFile` classifier. Persisting this in the stateFile lets
1283
1293
  // skyramp_analyze_test_health and the recommendation prompt consume the
@@ -1326,11 +1336,54 @@ export function registerAnalyzeChangesTool(server) {
1326
1336
  (unintegratedFrontendFiles.length > 0
1327
1337
  ? `\n\n${buildUnintegratedInstructions(unintegratedFrontendFiles)}`
1328
1338
  : "");
1339
+ // ── SKYR-3879 Path B: budget context + pre-seeded approved plan ──
1340
+ // Computed BEFORE the state write (not after, alongside the
1341
+ // recommendation prompt) so both this pre-seed and buildRecommendationPrompt
1342
+ // derive their GENERATE/ADDITIONAL split from the exact same numbers —
1343
+ // computeScoredCandidates is the single source of truth for both.
1344
+ const topN = params.topN ?? MAX_RECOMMENDATIONS;
1345
+ const scoredResult = computeScoredCandidates(fullAnalysis, analysisScope, topN, params.maxGenerate);
1346
+ const externalCoverage = buildExternalCoverageSet(testLocationsByType);
1347
+ const planBudgetContext = {
1348
+ maxGenerate: scoredResult.maxGen,
1349
+ maxTotal: topN,
1350
+ isUIOnlyPR: scoredResult.isUIOnlyPR,
1351
+ hasFrontendChanges: scoredResult.hasFrontendChanges,
1352
+ externalCoverageKeys: [...externalCoverage],
1353
+ };
1354
+ let approvedPlan;
1355
+ if (scoredResult.scored.length > 0) {
1356
+ const budgetCtx = {
1357
+ maxGenerate: planBudgetContext.maxGenerate,
1358
+ maxTotal: planBudgetContext.maxTotal,
1359
+ isUIOnlyPR: planBudgetContext.isUIOnlyPR,
1360
+ hasFrontendChanges: planBudgetContext.hasFrontendChanges,
1361
+ externalCoverage,
1362
+ };
1363
+ const serverCandidates = scoredResult.scored.map(({ scenario, priority, novelty }) => ({
1364
+ scenario,
1365
+ priority,
1366
+ novelty,
1367
+ source: CandidateSource.SERVER,
1368
+ candidateId: computeCandidateId(scenario),
1369
+ }));
1370
+ const plan = selectPlan(serverCandidates, budgetCtx);
1371
+ approvedPlan = {
1372
+ planId: crypto.randomUUID(),
1373
+ createdAt: new Date().toISOString(),
1374
+ generate: plan.generate.map(buildApprovedPlanItem),
1375
+ additional: plan.additional.map(buildApprovedPlanItem),
1376
+ demotions: [],
1377
+ };
1378
+ }
1329
1379
  const unifiedState = {
1330
1380
  existingTests,
1331
1381
  newEndpoints: newEndpointsForDrafting,
1332
1382
  analysisScope,
1333
1383
  ...(uiContext ? { uiContext } : {}),
1384
+ ...(diffText ? { diffText } : {}),
1385
+ planBudgetContext,
1386
+ ...(approvedPlan ? { approvedPlan } : {}),
1334
1387
  repositoryAnalysis: {
1335
1388
  skeletonEndpoints,
1336
1389
  projectMeta,
@@ -1419,7 +1472,7 @@ export function registerAnalyzeChangesTool(server) {
1419
1472
  }
1420
1473
  await sendProgress(90, 100, "Generating ranked recommendations...");
1421
1474
  // ── Step 12: Generate ranked recommendations inline ──
1422
- const topN = params.topN ?? MAX_RECOMMENDATIONS;
1475
+ // topN computed earlier (SKYR-3879 Path B pre-seed, before the state write).
1423
1476
  // ── Step 13: Fetch PR comment history for deduplication ──
1424
1477
  let prContext;
1425
1478
  if (params.prNumber) {
@@ -9,6 +9,18 @@ jest.unstable_mockModule("../../services/AnalyticsService.js", () => ({
9
9
  }));
10
10
  jest.unstable_mockModule("../../prompts/test-recommendation/test-recommendation-prompt.js", () => ({
11
11
  buildRecommendationPrompt: jest.fn(),
12
+ // SKYR-3879 Path B pre-seed reads this before the state write — default to
13
+ // "nothing pre-ranked" so existing tests (which don't exercise the plan
14
+ // pre-seed) see byte-identical unifiedState/writeRepoData behavior.
15
+ computeScoredCandidates: jest.fn(() => ({
16
+ scored: [],
17
+ filteredChangedFiles: [],
18
+ isUIOnlyPR: false,
19
+ hasFrontendChanges: false,
20
+ hasApiChanges: false,
21
+ maxGen: 0,
22
+ seed: "",
23
+ })),
12
24
  }));
13
25
  jest.unstable_mockModule("../../prompts/test-recommendation/recommendationSections.js", () => ({
14
26
  MAX_RECOMMENDATIONS: 10,
@@ -1,3 +1,4 @@
1
1
  export { registerAnalyzeChangesTool } from "./analyzeChangesTool.js";
2
2
  export { registerAnalyzeTestHealthTool } from "./analyzeTestHealthTool.js";
3
3
  export { registerActionsTool } from "./actionsTool.js";
4
+ export { registerRegisterTestPlanTool } from "./registerTestPlanTool.js";
@@ -1,3 +1,4 @@
1
1
  export { registerAnalyzeChangesTool } from "./analyzeChangesTool.js";
2
2
  export { registerAnalyzeTestHealthTool } from "./analyzeTestHealthTool.js";
3
3
  export { registerActionsTool } from "./actionsTool.js";
4
+ export { registerRegisterTestPlanTool } from "./registerTestPlanTool.js";
@@ -0,0 +1,2 @@
1
+ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ export declare function registerRegisterTestPlanTool(server: McpServer): void;