@skyramp/mcp 0.4.3 → 0.4.4-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/build/index.js +239 -210
  2. package/build/playwright/PlaywrightTraceService.d.ts +1 -1
  3. package/build/playwright/blueprintDigestStore.d.ts +1 -1
  4. package/build/playwright/registerPlaywrightTools.d.ts +1 -1
  5. package/build/playwright/traceRecordingPrompt.d.ts +1 -1
  6. package/build/playwright/traceRecordingPrompt.js +4 -3
  7. package/build/prompts/local-dev/local-dev-prompts.d.ts +1 -1
  8. package/build/prompts/local-dev/local-dev-prompts.js +2 -2
  9. package/build/prompts/orderAssertionRule.d.ts +9 -0
  10. package/build/prompts/orderAssertionRule.js +9 -0
  11. package/build/prompts/startTraceCollectionPrompts.d.ts +1 -1
  12. package/build/prompts/startTraceCollectionPrompts.js +2 -3
  13. package/build/prompts/test-maintenance/driftAnalysisShared.js +2 -0
  14. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +1 -1
  15. package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +2 -2
  16. package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
  17. package/build/prompts/testbot/testbot-prompts.js +9 -9
  18. package/build/recommendation/answers.d.ts +3 -3
  19. package/build/recommendation/answers.js +3 -2
  20. package/build/recommendation/verifierContracts.d.ts +4 -4
  21. package/build/recommendation/verifierContracts.js +6 -5
  22. package/build/recommendation/verifiers/coverage.js +25 -25
  23. package/build/recommendation/verifiers/deliveredMatchesPlan.js +3 -3
  24. package/build/resources/analysisResources.d.ts +1 -1
  25. package/build/resources/analysisResources.js +1 -1
  26. package/build/resources/progressResource.d.ts +1 -1
  27. package/build/resources/sutSetupResource.d.ts +1 -1
  28. package/build/resources/sutSetupResource.js +1 -1
  29. package/build/resources/testbotResource.d.ts +1 -1
  30. package/build/resources/testbotResource.js +1 -1
  31. package/build/services/AnalyticsService.d.ts +1 -1
  32. package/build/services/FleetExecutionService.d.ts +130 -0
  33. package/build/services/FleetExecutionService.js +391 -0
  34. package/build/services/ModularizationService.d.ts +1 -1
  35. package/build/services/TestExecutionService.d.ts +7 -0
  36. package/build/services/TestExecutionService.js +21 -0
  37. package/build/services/TestGenerationService.d.ts +1 -1
  38. package/build/tool-phases.js +2 -0
  39. package/build/tools/auth/loginTool.d.ts +1 -1
  40. package/build/tools/auth/loginTool.js +2 -2
  41. package/build/tools/auth/logoutTool.d.ts +1 -1
  42. package/build/tools/auth/logoutTool.js +2 -2
  43. package/build/tools/code-refactor/alignAssertionsTool.d.ts +1 -1
  44. package/build/tools/code-refactor/alignAssertionsTool.js +1 -1
  45. package/build/tools/code-refactor/assertionSkillTools.d.ts +1 -1
  46. package/build/tools/code-refactor/assertionSkillTools.js +2 -2
  47. package/build/tools/code-refactor/codeReuseTool.d.ts +1 -1
  48. package/build/tools/code-refactor/codeReuseTool.js +2 -2
  49. package/build/tools/code-refactor/enhanceAssertionsTool.d.ts +1 -1
  50. package/build/tools/code-refactor/enhanceAssertionsTool.js +1 -1
  51. package/build/tools/code-refactor/modularizationTool.d.ts +1 -1
  52. package/build/tools/code-refactor/modularizationTool.js +2 -2
  53. package/build/tools/enrichTestWithMocksTool.d.ts +1 -1
  54. package/build/tools/enrichTestWithMocksTool.js +2 -2
  55. package/build/tools/executeFleetTool.d.ts +28 -0
  56. package/build/tools/executeFleetTool.js +107 -0
  57. package/build/tools/executeSkyrampTestTool.d.ts +8 -5
  58. package/build/tools/executeSkyrampTestTool.js +13 -5
  59. package/build/tools/fixErrorTool.d.ts +1 -1
  60. package/build/tools/fixErrorTool.js +1 -1
  61. package/build/tools/generate-tests/batchMockGenerationTool.d.ts +6 -57
  62. package/build/tools/generate-tests/batchMockGenerationTool.js +3 -3
  63. package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +9 -45
  64. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +29 -11
  65. package/build/tools/generate-tests/generateContractRestTool.d.ts +1 -2
  66. package/build/tools/generate-tests/generateContractRestTool.js +1 -1
  67. package/build/tools/generate-tests/generateE2ERestTool.d.ts +2 -2
  68. package/build/tools/generate-tests/generateE2ERestTool.js +10 -2
  69. package/build/tools/generate-tests/generateFuzzRestTool.d.ts +1 -1
  70. package/build/tools/generate-tests/generateFuzzRestTool.js +1 -1
  71. package/build/tools/generate-tests/generateIntegrationRestTool.d.ts +1 -2
  72. package/build/tools/generate-tests/generateIntegrationRestTool.js +1 -1
  73. package/build/tools/generate-tests/generateLoadRestTool.d.ts +1 -2
  74. package/build/tools/generate-tests/generateLoadRestTool.js +2 -1
  75. package/build/tools/generate-tests/generateMockRestTool.d.ts +1 -2
  76. package/build/tools/generate-tests/generateMockRestTool.js +3 -3
  77. package/build/tools/generate-tests/generateSmokeRestTool.d.ts +1 -1
  78. package/build/tools/generate-tests/generateSmokeRestTool.js +1 -1
  79. package/build/tools/generate-tests/generateUILoadRestTool.d.ts +64 -0
  80. package/build/tools/generate-tests/generateUILoadRestTool.js +512 -0
  81. package/build/tools/generate-tests/generateUIRestTool.d.ts +8 -7
  82. package/build/tools/generate-tests/generateUIRestTool.js +10 -2
  83. package/build/tools/generate-tests/loadTestSchema.d.ts +15 -15
  84. package/build/tools/generate-tests/trace-reuse-guard.d.ts +1 -1
  85. package/build/tools/generateEnrichedIntegrationTestTool.d.ts +1 -2
  86. package/build/tools/generateEnrichedIntegrationTestTool.js +1 -1
  87. package/build/tools/localDevWorkerComposeTool.d.ts +1 -1
  88. package/build/tools/localDevWorkerComposeTool.js +2 -2
  89. package/build/tools/one-click/oneClickTool.d.ts +1 -1
  90. package/build/tools/one-click/oneClickTool.js +1 -1
  91. package/build/tools/preflightMockCheckTool.d.ts +1 -1
  92. package/build/tools/preflightMockCheckTool.js +3 -3
  93. package/build/tools/queryProxyMocksTool.d.ts +1 -1
  94. package/build/tools/queryProxyMocksTool.js +3 -3
  95. package/build/tools/submitReportTool.d.ts +111 -414
  96. package/build/tools/submitReportTool.js +88 -38
  97. package/build/tools/test-management/actionsTool.d.ts +1 -1
  98. package/build/tools/test-management/actionsTool.js +7 -7
  99. package/build/tools/test-management/analyzeChangesTool.d.ts +22 -118
  100. package/build/tools/test-management/analyzeChangesTool.js +4 -4
  101. package/build/tools/test-management/analyzeTestHealthTool.d.ts +1 -1
  102. package/build/tools/test-management/analyzeTestHealthTool.js +4 -4
  103. package/build/tools/test-management/registerTestPlanTool.d.ts +107 -733
  104. package/build/tools/test-management/registerTestPlanTool.js +12 -12
  105. package/build/tools/test-management/resolveScreenTool.d.ts +1 -1
  106. package/build/tools/test-management/resolveScreenTool.js +6 -2
  107. package/build/tools/trace/startTraceCollectionTool.d.ts +1 -1
  108. package/build/tools/trace/startTraceCollectionTool.js +4 -4
  109. package/build/tools/trace/stopTraceCollectionTool.d.ts +1 -1
  110. package/build/tools/trace/stopTraceCollectionTool.js +2 -2
  111. package/build/tools/verifyTestDependenciesTool.d.ts +1 -1
  112. package/build/tools/verifyTestDependenciesTool.js +1 -1
  113. package/build/tools/workspace/initScanWorkspaceTool.d.ts +1 -1
  114. package/build/tools/workspace/initScanWorkspaceTool.js +1 -1
  115. package/build/tools/workspace/initializeWorkspaceTool.d.ts +1 -1
  116. package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
  117. package/build/types/RepositoryAnalysis.d.ts +164 -1170
  118. package/build/types/RepositoryAnalysis.js +12 -10
  119. package/build/types/StepMethod.d.ts +24 -1
  120. package/build/types/StepMethod.js +1 -1
  121. package/build/types/TestTypes.d.ts +41 -133
  122. package/build/types/TestTypes.js +25 -3
  123. package/build/types/TestbotReport.d.ts +4 -1
  124. package/build/utils/assertion-verify/metrics.js +12 -0
  125. package/build/utils/docker.d.ts +29 -7
  126. package/build/utils/docker.js +83 -27
  127. package/build/utils/findUp.d.ts +20 -0
  128. package/build/utils/findUp.js +59 -0
  129. package/build/utils/initAgent.d.ts +3 -3
  130. package/build/utils/initAgent.js +58 -26
  131. package/build/utils/language-helper.js +1 -1
  132. package/build/utils/progress.d.ts +3 -4
  133. package/build/utils/progress.js +4 -4
  134. package/build/utils/repoSdkVersion.d.ts +16 -0
  135. package/build/utils/repoSdkVersion.js +88 -0
  136. package/build/utils/testDependencyPolicy.d.ts +20 -0
  137. package/build/utils/testDependencyPolicy.js +64 -4
  138. package/build/utils/utils-verify/typecheck.js +3 -18
  139. package/build/utils/utils.d.ts +1 -1
  140. package/build/utils/versions.d.ts +3 -3
  141. package/build/utils/versions.js +1 -1
  142. package/build/workspace/workspace.d.ts +117 -538
  143. package/build/workspace/workspace.js +2 -2
  144. package/node_modules/playwright/lib/mcp/skyramp/assertOrder.js +66 -0
  145. package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +12 -3
  146. package/node_modules/playwright/lib/mcp/skyramp/common/liveReadError.js +40 -0
  147. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +21 -0
  148. package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -1
  149. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +96 -16
  150. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +10 -0
  151. package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
  152. package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderRunner.js +22 -0
  153. package/node_modules/playwright/node_modules/playwright-core/lib/utils/isomorphic/elementOrder.js +57 -0
  154. package/node_modules/playwright/node_modules/playwright-core/lib/utils.js +2 -0
  155. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.uiPTFncY.js → index.0MJW0uG7.js} +1 -1
  156. package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
  157. package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
  158. package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
  159. package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderRunner.ts +26 -1
  160. package/node_modules/playwright/node_modules/playwright-core/src/utils/isomorphic/elementOrder.ts +83 -0
  161. package/node_modules/playwright/node_modules/playwright-core/src/utils.ts +1 -0
  162. package/node_modules/playwright/package.json +1 -1
  163. package/package.json +9 -6
  164. package/plugin/prompts/plan-tests.md +9 -17
  165. package/plugin/skills/enhance-assertions/reference/shared-rules.md +1 -1
  166. package/plugin/skills/enhance-assertions/reference/ui.md +2 -2
  167. package/plugin/skills/fix-test-import-errors/SKILL.md +1 -1
  168. package/plugin/skills/repair-staged-files/SKILL.md +17 -0
@@ -5,10 +5,10 @@ import { constants as fsConstants } from "fs";
5
5
  import * as path from "path";
6
6
  import { AnalyticsService } from "../services/AnalyticsService.js";
7
7
  import { SCENARIO_CATEGORIES, categoryExpectsRed, } from "../types/TestRecommendation.js";
8
- import { TESTBOT_SUPPORTED_TEST_TYPES, TestType } from "../types/TestTypes.js";
8
+ import { ProgrammingLanguage, TESTBOT_SUPPORTED_TEST_TYPES, TestType, } from "../types/TestTypes.js";
9
9
  import { STEP_METHOD_DESCRIPTION, stepMethodSchema, } from "../types/StepMethod.js";
10
10
  import { DriftAction } from "../types/TestAnalysis.js";
11
- import { TestExecutionStatus } from "../types/TestExecution.js";
11
+ import { isResult, TestExecutionStatus } from "../types/TestExecution.js";
12
12
  import { IssueFoundCategory } from "../types/TestbotReport.js";
13
13
  import { StateManager, runArtifactDir, getTestsRepoDir, resolveOwnRunStatePath, } from "../utils/AnalysisStateManager.js";
14
14
  import { toolError, testFileMatches } from "../utils/utils.js";
@@ -79,6 +79,28 @@ function rowsLeavingCoverage(rows) {
79
79
  MAINTENANCE_COVERAGE_ACTIONS.has(row.action) &&
80
80
  row?.reportOnly !== true);
81
81
  }
82
+ /** The VERIFY rows this run owes one run: the test is read, not edited, so its
83
+ * only run is the one after the edits. */
84
+ function verifyRows(rows) {
85
+ return (rows ?? []).filter((row) => row?.action === DriftAction.Verify && row?.reportOnly !== true);
86
+ }
87
+ /** The rows that back a `maintains` claim: an edit this run made, or a VERIFY
88
+ * whose test this server ran after the edits to a result. A VERIFY that never
89
+ * ran, or whose run left only the Unknown reservation, backs nothing. In the
90
+ * plan-only lane nothing runs, so a VERIFY backs the claim without a run. */
91
+ function rowsBackingMaintains(rows, executions, planOnly) {
92
+ return [
93
+ ...rowsLeavingCoverage(rows),
94
+ ...verifyRows(rows).filter((row) => {
95
+ if (planOnly)
96
+ return true;
97
+ if (typeof row.testFilePath !== "string")
98
+ return false;
99
+ const run = executions?.[canonicalTestPath(row.testFilePath)];
100
+ return (run?.phase === "after" && isResult(run.status));
101
+ }),
102
+ ];
103
+ }
82
104
  /** Objection ids for an untested declared change, `coverage:change:<id>`. The
83
105
  * change table reads the agent's answer back off them. */
84
106
  const CHANGE_OBJECTION_PREFIX = "coverage:change:";
@@ -98,7 +120,7 @@ const repositoryField = z
98
120
  // supplies anyway. SKYR-4156 is what happens when the agent owns that field instead.
99
121
  const testResultSchema = z.object({
100
122
  testType: z
101
- .nativeEnum(TestType)
123
+ .enum(TestType)
102
124
  .describe("Type of test. Do not include priority or other metadata in this field."),
103
125
  endpoint: z
104
126
  .string()
@@ -213,7 +235,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
213
235
  "Required on EVERY entry, UI included. That is deliberately wider than the approved-plan generation gate, which stays contract and integration only: " +
214
236
  "this id is the exact key the report is joined back to the plan on, and an entry without one falls back to guessing from the test name and endpoint."),
215
237
  testType: z
216
- .nativeEnum(TestType)
238
+ .enum(TestType)
217
239
  .describe("Type of test created. Do not include priority or other metadata in this field."),
218
240
  category: z
219
241
  .enum(SCENARIO_CATEGORIES)
@@ -260,7 +282,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
260
282
  // optional above and required here so the message says what to do.
261
283
  if ((rec.plannedTestId ?? "").trim().length === 0) {
262
284
  ctx.addIssue({
263
- code: z.ZodIssueCode.custom,
285
+ code: "custom",
264
286
  path: ["plannedTestId"],
265
287
  message: "plannedTestId is required on every newTestsCreated entry. Use the plannedTestId of the plan entry this test was approved under, from skyramp_register_test_plan.",
266
288
  });
@@ -270,7 +292,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
270
292
  for (const field of ["targetElements", "pageContext"]) {
271
293
  if (rec[field] !== undefined) {
272
294
  ctx.addIssue({
273
- code: z.ZodIssueCode.custom,
295
+ code: "custom",
274
296
  path: [field],
275
297
  message: `${field} is only valid for testType: 'ui'`,
276
298
  });
@@ -281,7 +303,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
281
303
  // description is required for UI tests (walkthrough of steps/assertions).
282
304
  if (!rec.description) {
283
305
  ctx.addIssue({
284
- code: z.ZodIssueCode.custom,
306
+ code: "custom",
285
307
  path: ["description"],
286
308
  message: "description is required for testType: 'ui'.",
287
309
  });
@@ -289,7 +311,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
289
311
  // targetElements must be present (array or null — not omitted entirely).
290
312
  if (rec.targetElements === undefined) {
291
313
  ctx.addIssue({
292
- code: z.ZodIssueCode.custom,
314
+ code: "custom",
293
315
  path: ["targetElements"],
294
316
  message: "targetElements is required for testType: 'ui'. Use null when blueprint capture failed; the null is the signal and needs no marker in description or reasoning.",
295
317
  });
@@ -300,7 +322,7 @@ export const newTestSchema = z.preprocess(stripNullGroundingFields, z
300
322
  if (Array.isArray(rec.targetElements) &&
301
323
  rec.pageContext === undefined) {
302
324
  ctx.addIssue({
303
- code: z.ZodIssueCode.custom,
325
+ code: "custom",
304
326
  path: ["pageContext"],
305
327
  message: "pageContext is required for testType: 'ui' when targetElements is an array.",
306
328
  });
@@ -331,7 +353,7 @@ const issueFoundSchema = z
331
353
  "report an issue here at 'critical' or 'high'. A stated requirement the code does not meet is not a " +
332
354
  "medium. The check raises an objection, never a rejection."),
333
355
  category: z
334
- .nativeEnum(IssueFoundCategory)
356
+ .enum(IssueFoundCategory)
335
357
  .describe("Issue classification. bug = a product/code defect, e.g. found by a test or in the diff. " +
336
358
  "lint = a linter or formatter objection (eslint, flake8, prettier). " +
337
359
  "type = a type-check failure (tsc, mypy). " +
@@ -384,7 +406,7 @@ const issueFoundSchema = z
384
406
  if (issue.sourceFile !== undefined)
385
407
  return;
386
408
  ctx.addIssue({
387
- code: z.ZodIssueCode.custom,
409
+ code: "custom",
388
410
  path: ["sourceFile"],
389
411
  message: `sourceFile is required when category is 'bug' — a bug entry must name the code it is about. ` +
390
412
  `Open the file, then set sourceFile (path relative to the repository root). Also set sourceSymbol (the function or identifier inside it) when the file has one, and sourceLine (the line of the offending statement) when you have it — both are optional. ` +
@@ -407,11 +429,11 @@ const scenarioStepSchema = z.object({
407
429
  .optional()
408
430
  .describe("Expected HTTP status code, e.g. 200, 201, 404"),
409
431
  requestBody: z
410
- .record(z.any())
432
+ .record(z.string(), z.any())
411
433
  .optional()
412
434
  .describe("Example request body with realistic field values"),
413
435
  responseBody: z
414
- .union([z.record(z.any()), z.array(z.any())])
436
+ .union([z.record(z.string(), z.any()), z.array(z.any())])
415
437
  .optional()
416
438
  .describe("Key response fields to verify, e.g. { id: 'number', name: 'string', in_stock: 'boolean?' }. An array for a collection/list endpoint that returns a JSON array."),
417
439
  });
@@ -421,7 +443,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
421
443
  .string()
422
444
  .describe("Human-readable kebab-case identifier, e.g. 'integration-products-orders-workflow' or 'e2e-checkout-flow'. Format: '<testType>-<scenario-slug>'. Must be unique within the report."),
423
445
  testType: z
424
- .nativeEnum(TestType)
446
+ .enum(TestType)
425
447
  .describe("Type of test. Do not include priority or other metadata in this field."),
426
448
  category: z
427
449
  .enum(SCENARIO_CATEGORIES)
@@ -475,7 +497,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
475
497
  .superRefine((rec, ctx) => {
476
498
  if (rec.testType === TestType.CONTRACT && !rec.primaryEndpoint) {
477
499
  ctx.addIssue({
478
- code: z.ZodIssueCode.custom,
500
+ code: "custom",
479
501
  path: ["primaryEndpoint"],
480
502
  message: "primaryEndpoint is required for contract tests — provide the HTTP verb and path, e.g. 'PATCH /api/v1/orders/{order_id}'.",
481
503
  });
@@ -485,7 +507,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
485
507
  for (const field of ["targetElements", "pageContext"]) {
486
508
  if (rec[field] !== undefined) {
487
509
  ctx.addIssue({
488
- code: z.ZodIssueCode.custom,
510
+ code: "custom",
489
511
  path: [field],
490
512
  message: `${field} is only valid for testType: 'ui'`,
491
513
  });
@@ -495,7 +517,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
495
517
  else {
496
518
  if (rec.targetElements === undefined) {
497
519
  ctx.addIssue({
498
- code: z.ZodIssueCode.custom,
520
+ code: "custom",
499
521
  path: ["targetElements"],
500
522
  message: "targetElements is required for testType: 'ui'. Use null when no capture covers the screen.",
501
523
  });
@@ -503,7 +525,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
503
525
  if (Array.isArray(rec.targetElements) &&
504
526
  rec.pageContext === undefined) {
505
527
  ctx.addIssue({
506
- code: z.ZodIssueCode.custom,
528
+ code: "custom",
507
529
  path: ["pageContext"],
508
530
  message: "pageContext is required for testType: 'ui' when targetElements is an array.",
509
531
  });
@@ -513,7 +535,7 @@ export const additionalRecommendationSchema = z.preprocess(stripNullGroundingFie
513
535
  }
514
536
  }));
515
537
  const testMaintenanceSchema = z.object({
516
- testType: z.nativeEnum(TestType).describe("Type of test."),
538
+ testType: z.enum(TestType).describe("Type of test."),
517
539
  endpoint: z
518
540
  .string()
519
541
  .describe("For contract and integration tests, the HTTP verb and path, e.g. 'GET /api/v1/products'. For ui and e2e tests, the page path the test opens, e.g. '/notifications'."),
@@ -524,7 +546,7 @@ const testMaintenanceSchema = z.object({
524
546
  })
525
547
  .describe("Absolute path of the test file that was maintained, e.g. '/repo/tests/products_smoke_test.py' — the same path you passed to skyramp_execute_test's testFile param. Consumers should basename this for display."),
526
548
  action: z
527
- .nativeEnum(DriftAction)
549
+ .enum(DriftAction)
528
550
  .describe("The drift action assigned to this test during maintenance triage."),
529
551
  description: z.string().describe("What was changed and why"),
530
552
  beforeDetails: z
@@ -552,8 +574,8 @@ const testMaintenanceSchema = z.object({
552
574
  // "probably the primary" cannot tell a single-repo row from a mis-stamped one.
553
575
  repository: z.string(),
554
576
  // Server-populated from stateFile execution records — never supplied by the LLM.
555
- beforeStatus: z.nativeEnum(TestExecutionStatus),
556
- afterStatus: z.nativeEnum(TestExecutionStatus),
577
+ beforeStatus: z.enum(TestExecutionStatus),
578
+ afterStatus: z.enum(TestExecutionStatus),
557
579
  });
558
580
  // LLM input contract: status fields are server-populated from stateFile, and
559
581
  // `defects` is attached from the per-file detail the agent drafts, not from the
@@ -1217,6 +1239,9 @@ async function pathExists(file) {
1217
1239
  * missing timestamp on either side falls back to "a run is a run", which is the
1218
1240
  * behaviour before this and never invents an objection. */
1219
1241
  function runOfWhatWasWritten(file) {
1242
+ if (file.how === "verified")
1243
+ return (file.recordedPhase === "after" &&
1244
+ file.recordedStatus !== TestExecutionStatus.Unknown);
1220
1245
  if (file.how === "updated")
1221
1246
  return file.recordedPhase === "after";
1222
1247
  if (file.recordedPhase === undefined)
@@ -1229,7 +1254,8 @@ function runOfWhatWasWritten(file) {
1229
1254
  return true;
1230
1255
  return ran >= wrote;
1231
1256
  }
1232
- /** Every test file this run wrote has an execution this server performed.
1257
+ /** Every test file this run wrote, and every VERIFY file it read, has an
1258
+ * execution this server performed.
1233
1259
  *
1234
1260
  * Informational, like every check here: it surfaces the gap for a person to read
1235
1261
  * and the agent can answer it. The agent is not blocked — a hard ban on running a
@@ -1257,9 +1283,11 @@ function checkEveryWrittenTestRan(written) {
1257
1283
  // drove itself is not visible here.
1258
1284
  message: file.how === "generated"
1259
1285
  ? "This run generated this test file, and this server recorded no run of it after it was written."
1260
- : file.recordedPhase === "before"
1261
- ? "This run edited this test file, and the only run of it this server recorded is from before the edit."
1262
- : "This run edited this test file, and this server recorded no run of it.",
1286
+ : file.how === "verified"
1287
+ ? "This run marked this test file VERIFY, and this server recorded no run of it after the edits."
1288
+ : file.recordedPhase === "before"
1289
+ ? "This run edited this test file, and the only run of it this server recorded is from before the edit."
1290
+ : "This run edited this test file, and this server recorded no run of it.",
1263
1291
  evidence: file.recordedPhase === "before"
1264
1292
  ? `${file.file} — the only recorded skyramp_execute_test run of this file is the pre-edit baseline`
1265
1293
  : file.how === "generated" && file.recordedPhase !== undefined
@@ -1331,7 +1359,7 @@ export function registerSubmitReportTool(server) {
1331
1359
  "This is the ONLY way to submit the report — do NOT write the report to a file manually. " +
1332
1360
  "If a check objects to what you reported, the objection names an id: fix what you can, and answer the rest through `answers` on a second call. " +
1333
1361
  "On an ACCEPTED call this tool also edits the files this run changed: a comment addressed to a Skyramp gate — `reuse-verify: allow …` or `// kept inline: …` — is removed from each of them, because it speaks to a gate and not to the customer who owns the file. Nothing else in those files is touched. A REFUSED call leaves every source file exactly as it found it; it still records its own run state, such as the refusal count and the objections it raised.",
1334
- inputSchema: {
1362
+ inputSchema: z.object({
1335
1363
  businessCaseAnalysis: z
1336
1364
  .string()
1337
1365
  .min(1, "businessCaseAnalysis is required — describe what the PR does, even for no-test runs (config-only, non-application, Testbot onboarding). It is never optional and must not be empty.")
@@ -1395,7 +1423,7 @@ export function registerSubmitReportTool(server) {
1395
1423
  "another run's execution results, and the call is refused. " +
1396
1424
  "The report is written beside it, and it holds the execution results, the " +
1397
1425
  "page captures and the videos this report needs."),
1398
- },
1426
+ }),
1399
1427
  _meta: {
1400
1428
  keywords: ["report", "summary", "testbot", "submit"],
1401
1429
  },
@@ -2195,9 +2223,10 @@ export function registerSubmitReportTool(server) {
2195
2223
  // Every file this run wrote, from the server's own records — one canonical
2196
2224
  // absolute path each, which is also the key an execution is recorded under.
2197
2225
  // Generated files come from the generators; edited files from the maintenance
2198
- // triage, narrowed by `rowsLeavingCoverage` to UPDATE and REGENERATE. The rest
2199
- // cannot be run: DELETE removed the file, VERIFY and IGNORE never edited one,
2200
- // and a `reportOnly` row only advised a change this run did not make.
2226
+ // triage, narrowed by `rowsLeavingCoverage` to UPDATE and REGENERATE. A VERIFY
2227
+ // file was not written but owes the same after-edit run. The rest owe none:
2228
+ // DELETE removed the file, IGNORE never touched one, and a `reportOnly` row
2229
+ // only advised a change this run did not make.
2201
2230
  //
2202
2231
  // A verdict names the SPEC, never `pomFile`: a POM-backed row edits the page
2203
2232
  // object, but the execution is recorded against the spec.
@@ -2232,6 +2261,23 @@ export function registerSubmitReportTool(server) {
2232
2261
  filesWritten.set(key, "updated");
2233
2262
  }
2234
2263
  }
2264
+ // A VERIFY file is not written, but owes the same after-edit run
2265
+ // (SKYR-4523). Primary only: the server turns every edit to a related
2266
+ // repository's test into a VERIFY the agent never chose. A language
2267
+ // skyramp_execute_test does not accept can never record a run.
2268
+ const runnableLanguages = Object.values(ProgrammingLanguage);
2269
+ for (const verdict of verifyRows(stateData.maintenanceVerdicts)) {
2270
+ const raw = verdict.testFilePath?.trim();
2271
+ if (!raw)
2272
+ continue;
2273
+ const key = canonicalTestPath(raw);
2274
+ const row = stateData.existingTests?.find((t) => canonicalTestPath(t.testFile) === key);
2275
+ // An uncatalogued file has no row for its result to land on.
2276
+ if (!row || !runnableLanguages.includes(row.language))
2277
+ continue;
2278
+ if (!filesWritten.has(key) && !deletedPaths.has(key))
2279
+ filesWritten.set(key, "verified");
2280
+ }
2235
2281
  // The LATEST run of each file, because a re-run replaces the record. An edited
2236
2282
  // file needs a run of the edit, so its pre-edit baseline does not answer for
2237
2283
  // it — `phase` is what tells the two apart. A generated file has no baseline,
@@ -2240,6 +2286,7 @@ export function registerSubmitReportTool(server) {
2240
2286
  file,
2241
2287
  {
2242
2288
  phase: String(record?.phase ?? ""),
2289
+ status: String(record?.status ?? ""),
2243
2290
  executedAt: String(record?.executedAt ?? ""),
2244
2291
  },
2245
2292
  ])));
@@ -2256,6 +2303,7 @@ export function registerSubmitReportTool(server) {
2256
2303
  file,
2257
2304
  how,
2258
2305
  recordedPhase: run?.phase,
2306
+ recordedStatus: run?.status,
2259
2307
  executedAt: run?.executedAt,
2260
2308
  generatedAt: generatedAt.get(file),
2261
2309
  });
@@ -2265,6 +2313,7 @@ export function registerSubmitReportTool(server) {
2265
2313
  const executionCoverage = {
2266
2314
  generated: written.filter((f) => f.how === "generated").length,
2267
2315
  updated: written.filter((f) => f.how === "updated").length,
2316
+ verified: written.filter((f) => f.how === "verified").length,
2268
2317
  executed: written.filter(runOfWhatWasWritten).length,
2269
2318
  };
2270
2319
  // A skipped test is not an execution. Refuse before writing a report so the
@@ -2423,10 +2472,11 @@ export function registerSubmitReportTool(server) {
2423
2472
  }, dedupedNewTests, params.testResults, params.issuesFound ?? [], isPlanOnlyMode(),
2424
2473
  // The triage's own verdicts, not this tool's input, so the join is against
2425
2474
  // what the run is recorded as having edited — and only the rows that left a
2426
- // test standing and were actually applied, so a no-op assessment, a deletion
2427
- // or a report-only recommendation cannot corroborate. The verdicts rather
2428
- // than the published rows because `reportOnly` lives here.
2429
- rowsLeavingCoverage(stateData.maintenanceVerdicts));
2475
+ // test standing and were actually applied, or a VERIFY whose test ran after
2476
+ // the edits — so an unrun assessment, a deletion or a report-only
2477
+ // recommendation cannot corroborate. The verdicts rather than the
2478
+ // published rows because `reportOnly` lives here.
2479
+ rowsBackingMaintains(stateData.maintenanceVerdicts, stateData.testExecutions, isPlanOnlyMode()));
2430
2480
  // Read defensively, like the crash-contained checks above: the plan comes
2431
2481
  // off disk, so its declared array type holds only on the validated path.
2432
2482
  const stillOpen = Array.isArray(stateData.plan?.openObjections)
@@ -2593,10 +2643,10 @@ export function registerSubmitReportTool(server) {
2593
2643
  //
2594
2644
  // `testFileMatches` rather than a basename, so a spec under one directory
2595
2645
  // cannot lend its coverage to a same-named spec under another; and only the
2596
- // rows that left a test standing, so DELETE and the no-op actions cannot
2597
- // corroborate a claim that an existing test covers the change.
2646
+ // rows `rowsBackingMaintains` keeps, so DELETE, IGNORE and an unrun VERIFY
2647
+ // cannot corroborate a claim that an existing test covers the change.
2598
2648
  const editedPaths = [];
2599
- for (const row of rowsLeavingCoverage(stateData.maintenanceVerdicts)) {
2649
+ for (const row of rowsBackingMaintains(stateData.maintenanceVerdicts, stateData.testExecutions, isPlanOnlyMode())) {
2600
2650
  for (const full of [
2601
2651
  row?.testFilePath,
2602
2652
  row?.pomFile,
@@ -1,4 +1,4 @@
1
- import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
1
+ import { McpServer } from "@modelcontextprotocol/server";
2
2
  /**
3
3
  * Compute a suggested new filename when an endpoint is renamed.
4
4
  */
@@ -69,19 +69,19 @@ const recommendationSchema = z.object({
69
69
  })
70
70
  .describe("Absolute path to the page object (POM) file to edit, when the spec delegates selectors to a page object. When set, skyramp_actions edits this file instead of testFilePath. testFilePath (the spec) is still used for catalog validation and the maintenance report."),
71
71
  action: z
72
- .nativeEnum(DriftAction)
72
+ .enum(DriftAction)
73
73
  .refine((a) => a !== DriftAction.Ignore, {
74
74
  message: "Omit IGNORE entries entirely — a discovered test you don't recommend is automatically treated as needing no action.",
75
75
  })
76
76
  .describe("The maintenance action for this test (UPDATE, REGENERATE, DELETE, or VERIFY). Do not send IGNORE — omit no-action tests from `recommendations` entirely; anything you leave out is recorded as needing no action."),
77
77
  testType: z
78
- .nativeEnum(TestType)
78
+ .enum(TestType)
79
79
  .describe("Type of this test — carried through to the maintenance report."),
80
80
  endpoint: z
81
81
  .string()
82
82
  .describe("HTTP verb and path this test covers, e.g. 'GET /api/v1/products'."),
83
83
  priority: z
84
- .nativeEnum(RecommendationPriority)
84
+ .enum(RecommendationPriority)
85
85
  .optional()
86
86
  .describe("Update priority"),
87
87
  rationale: z
@@ -90,7 +90,7 @@ const recommendationSchema = z.object({
90
90
  .min(1)
91
91
  .describe("1-2 sentence explanation of why this action is needed — becomes the maintenance report's description, so it must not be blank."),
92
92
  estimatedWork: z
93
- .nativeEnum(EstimatedWork)
93
+ .enum(EstimatedWork)
94
94
  .optional()
95
95
  .describe("Estimated effort to apply the update"),
96
96
  updateInstructions: z
@@ -169,15 +169,15 @@ export function registerActionsTool(server) {
169
169
  description: `Execute test maintenance actions — final step of the unified Test Health Analysis Flow. Call this once per run even when no tests need action (pass \`recommendations: []\`) — it is where the maintenance verdicts are persisted before the run moves on. Supply only the tests that need an action; any discovered test you omit from \`recommendations\` is treated as needing no action and recorded automatically, so you never enumerate the no-action tests yourself.`,
170
170
  // SKYR-4194: the payload is returned via structuredContent, which requires a
171
171
  // declared outputSchema — see dualChannelResult.
172
- outputSchema: {
172
+ outputSchema: z.object({
173
173
  summary: z
174
174
  .string()
175
175
  .describe("Human-readable report of the maintenance actions to take."),
176
176
  instructions: z
177
177
  .string()
178
178
  .describe("JSON instructions for carrying out those actions — file lists, per-file edit context, and the update/regenerate/delete strategies. Read and follow this field."),
179
- },
180
- inputSchema: actionsSchema,
179
+ }),
180
+ inputSchema: z.object(actionsSchema),
181
181
  }, async (args) => {
182
182
  let errorResult;
183
183
  try {
@@ -1,7 +1,6 @@
1
- import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
1
+ import { McpServer, CallToolResult } from "@modelcontextprotocol/server";
2
2
  import { z } from "zod";
3
3
  import { AuthFinding } from "../../utils/workspaceAuth.js";
4
- import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
5
4
  import { TraceFile } from "../../types/RepositoryAnalysis.js";
6
5
  import { ChangedFileName } from "../../utils/branchDiff.js";
7
6
  import { RemovedUiElement } from "../../utils/removedUiElements.js";
@@ -12,11 +11,14 @@ export declare function isGraphQLFile(filePath: string, repositoryPath: string):
12
11
  export declare const analyzeChangesInputSchema: {
13
12
  repositoryPath: z.ZodString;
14
13
  repository: z.ZodString;
15
- scope: z.ZodOptional<z.ZodDefault<z.ZodEnum<["full_repo", "branch_diff"]>>>;
14
+ scope: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
15
+ full_repo: "full_repo";
16
+ branch_diff: "branch_diff";
17
+ }>>>;
16
18
  baseBranch: z.ZodOptional<z.ZodString>;
17
19
  testDirectory: z.ZodOptional<z.ZodString>;
18
20
  prNumber: z.ZodOptional<z.ZodNumber>;
19
- testsRepoDir: z.ZodOptional<z.ZodEffects<z.ZodString, string, string>>;
21
+ testsRepoDir: z.ZodOptional<z.ZodString>;
20
22
  includeUncommitted: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
21
23
  };
22
24
  /**
@@ -86,7 +88,7 @@ export declare const analyzeChangesOutputSchema: {
86
88
  repositoryPath: z.ZodString;
87
89
  repositoryName: z.ZodString;
88
90
  repository: z.ZodString;
89
- relatedRepositories: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
91
+ relatedRepositories: z.ZodOptional<z.ZodArray<z.ZodString>>;
90
92
  analysisScope: z.ZodString;
91
93
  planOnly: z.ZodBoolean;
92
94
  currentBranch: z.ZodOptional<z.ZodString>;
@@ -94,26 +96,12 @@ export declare const analyzeChangesOutputSchema: {
94
96
  changedFiles: z.ZodArray<z.ZodObject<{
95
97
  path: z.ZodString;
96
98
  deleted: z.ZodBoolean;
97
- }, "strip", z.ZodTypeAny, {
98
- path: string;
99
- deleted: boolean;
100
- }, {
101
- path: string;
102
- deleted: boolean;
103
- }>, "many">;
99
+ }, z.core.$strip>>;
104
100
  existingTestFiles: z.ZodOptional<z.ZodObject<{
105
- skyramp: z.ZodArray<z.ZodString, "many">;
106
- external: z.ZodArray<z.ZodString, "many">;
107
- relevantExternal: z.ZodArray<z.ZodString, "many">;
108
- }, "strip", z.ZodTypeAny, {
109
- skyramp: string[];
110
- external: string[];
111
- relevantExternal: string[];
112
- }, {
113
- skyramp: string[];
114
- external: string[];
115
- relevantExternal: string[];
116
- }>>;
101
+ skyramp: z.ZodArray<z.ZodString>;
102
+ external: z.ZodArray<z.ZodString>;
103
+ relevantExternal: z.ZodArray<z.ZodString>;
104
+ }, z.core.$strip>>;
117
105
  workspace: z.ZodOptional<z.ZodObject<{
118
106
  baseUrl: z.ZodString;
119
107
  authMethod: z.ZodString;
@@ -124,113 +112,29 @@ export declare const analyzeChangesOutputSchema: {
124
112
  kind: z.ZodLiteral<"workspace-disagreement">;
125
113
  message: z.ZodString;
126
114
  serviceName: z.ZodString;
127
- field: z.ZodEnum<["authType", "authHeader", "authScheme"]>;
115
+ field: z.ZodEnum<{
116
+ authHeader: "authHeader";
117
+ authScheme: "authScheme";
118
+ authType: "authType";
119
+ }>;
128
120
  primary: z.ZodString;
129
121
  analysed: z.ZodString;
130
122
  primaryWorkspaceFile: z.ZodString;
131
123
  analysedWorkspaceFile: z.ZodString;
132
- }, "strip", z.ZodTypeAny, {
133
- message: string;
134
- serviceName: string;
135
- kind: "workspace-disagreement";
136
- field: "authHeader" | "authScheme" | "authType";
137
- primaryWorkspaceFile: string;
138
- analysedWorkspaceFile: string;
139
- primary: string;
140
- analysed: string;
141
- }, {
142
- message: string;
143
- serviceName: string;
144
- kind: "workspace-disagreement";
145
- field: "authHeader" | "authScheme" | "authType";
146
- primaryWorkspaceFile: string;
147
- analysedWorkspaceFile: string;
148
- primary: string;
149
- analysed: string;
150
- }>, "many">>;
151
- }, "strip", z.ZodTypeAny, {
152
- baseUrl: string;
153
- authMethod: string;
154
- openApiSpecLoaded: boolean;
155
- authHeader?: string | undefined;
156
- openApiSpecPath?: string | undefined;
157
- authFindings?: {
158
- message: string;
159
- serviceName: string;
160
- kind: "workspace-disagreement";
161
- field: "authHeader" | "authScheme" | "authType";
162
- primaryWorkspaceFile: string;
163
- analysedWorkspaceFile: string;
164
- primary: string;
165
- analysed: string;
166
- }[] | undefined;
167
- }, {
168
- baseUrl: string;
169
- authMethod: string;
170
- openApiSpecLoaded: boolean;
171
- authHeader?: string | undefined;
172
- openApiSpecPath?: string | undefined;
173
- authFindings?: {
174
- message: string;
175
- serviceName: string;
176
- kind: "workspace-disagreement";
177
- field: "authHeader" | "authScheme" | "authType";
178
- primaryWorkspaceFile: string;
179
- analysedWorkspaceFile: string;
180
- primary: string;
181
- analysed: string;
182
- }[] | undefined;
183
- }>>;
124
+ }, z.core.$strip>>>;
125
+ }, z.core.$strip>>;
184
126
  uiContext: z.ZodOptional<z.ZodObject<{
185
127
  removedElements: z.ZodOptional<z.ZodArray<z.ZodObject<{
186
128
  attribute: z.ZodString;
187
129
  value: z.ZodString;
188
130
  file: z.ZodString;
189
- }, "strip", z.ZodTypeAny, {
190
- value: string;
191
- file: string;
192
- attribute: string;
193
- }, {
194
- value: string;
195
- file: string;
196
- attribute: string;
197
- }>, "many">>;
131
+ }, z.core.$strip>>>;
198
132
  retiredElements: z.ZodOptional<z.ZodArray<z.ZodObject<{
199
133
  attribute: z.ZodString;
200
134
  value: z.ZodString;
201
135
  file: z.ZodString;
202
- }, "strip", z.ZodTypeAny, {
203
- value: string;
204
- file: string;
205
- attribute: string;
206
- }, {
207
- value: string;
208
- file: string;
209
- attribute: string;
210
- }>, "many">>;
211
- }, "strip", z.ZodTypeAny, {
212
- removedElements?: {
213
- value: string;
214
- file: string;
215
- attribute: string;
216
- }[] | undefined;
217
- retiredElements?: {
218
- value: string;
219
- file: string;
220
- attribute: string;
221
- }[] | undefined;
222
- }, {
223
- removedElements?: {
224
- value: string;
225
- file: string;
226
- attribute: string;
227
- }[] | undefined;
228
- retiredElements?: {
229
- value: string;
230
- file: string;
231
- attribute: string;
232
- }[] | undefined;
233
- }>>;
136
+ }, z.core.$strip>>>;
137
+ }, z.core.$strip>>;
234
138
  nextStep: z.ZodOptional<z.ZodString>;
235
139
  };
236
140
  /**
@@ -187,11 +187,11 @@ export function registerAnalyzeChangesTool(server) {
187
187
  description: `Read the repository and the current change, and discover the existing tests — first step of the unified Test Health Analysis Flow. Returns DATA only, no instructions: a stateFile path, the branch and its base, the changed files with their status, the existing test files, the workspace base URL and auth, the OpenAPI spec path, the repository's seed and fixture data files, and the identifying attributes the change removed from surviving files. It proposes no tests and it tells you nothing about the diff — read the diff and decide what the change warrants. Pass stateFile to skyramp_register_test_plan (required before you write any test), skyramp_analyze_test_health and skyramp_actions.`,
188
188
  // SKYR-4182: the payload is returned via structuredContent, which requires a
189
189
  // declared outputSchema.
190
- outputSchema: analyzeChangesOutputSchema,
191
- inputSchema: analyzeChangesInputSchema,
192
- }, async (params, extra) => {
190
+ outputSchema: z.object(analyzeChangesOutputSchema),
191
+ inputSchema: z.object(analyzeChangesInputSchema),
192
+ }, async (params, ctx) => {
193
193
  let errorResult;
194
- const sendProgress = makeProgressReporter(extra);
194
+ const sendProgress = makeProgressReporter(ctx);
195
195
  try {
196
196
  const analysisScope = params.scope === "branch_diff" || !params.scope
197
197
  ? AnalysisScope.CurrentBranchDiff
@@ -1,2 +1,2 @@
1
- import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
1
+ import { McpServer } from "@modelcontextprotocol/server";
2
2
  export declare function registerAnalyzeTestHealthTool(server: McpServer): void;
@@ -44,7 +44,7 @@ export function registerAnalyzeTestHealthTool(server) {
44
44
  openWorldHint: false,
45
45
  },
46
46
  description: `Assess existing tests for drift against the current branch diff and generate maintenance instructions (UPDATE, REGENERATE, DELETE, VERIFY, or IGNORE) for each test. Covers both API tests and UI tests (Playwright, Cypress, RTL, component tests). Requires a stateFile from skyramp_analyze_changes.`,
47
- inputSchema: {
47
+ inputSchema: z.object({
48
48
  stateFile: z
49
49
  .string()
50
50
  .describe("Path to the analyze-changes-state.json file returned by skyramp_analyze_changes (the stateFile field in its output). Do NOT pass testbot-result.txt here."),
@@ -57,12 +57,12 @@ export function registerAnalyzeTestHealthTool(server) {
57
57
  .trim()
58
58
  .min(1)
59
59
  .describe("The owner/repo whose analysis section to assess (e.g. 'letsramp/api-insight'). Set it on every call — the primary's owner/repo for the primary repo, or a related repo's owner/repo for that repo's section within the run-scoped stateFile."),
60
- },
61
- outputSchema: {
60
+ }),
61
+ outputSchema: z.object({
62
62
  prompt: z
63
63
  .string()
64
64
  .describe("LLM drift analysis prompt and assessment instructions."),
65
- },
65
+ }),
66
66
  }, async (args) => {
67
67
  let errorResult;
68
68
  try {