@skyramp/mcp 0.4.0 → 0.4.1-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
  2. package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
  3. package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
  4. package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
  5. package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
  6. package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
  7. package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
  8. package/build/prompts/testbot/testbot-prompts.js +19 -12
  9. package/build/recommendation/answers.d.ts +11 -7
  10. package/build/recommendation/answers.js +14 -10
  11. package/build/recommendation/pullRequestText.d.ts +18 -0
  12. package/build/recommendation/pullRequestText.js +31 -0
  13. package/build/recommendation/registerPlan.d.ts +9 -1
  14. package/build/recommendation/registerPlan.js +8 -1
  15. package/build/recommendation/runVerifiers.js +6 -0
  16. package/build/recommendation/types.d.ts +58 -0
  17. package/build/recommendation/verifierContracts.d.ts +108 -13
  18. package/build/recommendation/verifierContracts.js +147 -29
  19. package/build/recommendation/verifiers/coverage.d.ts +10 -0
  20. package/build/recommendation/verifiers/coverage.js +144 -22
  21. package/build/recommendation/verifiers/defects.d.ts +9 -0
  22. package/build/recommendation/verifiers/defects.js +117 -0
  23. package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
  24. package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
  25. package/build/recommendation/verifiers/existingCoverage.js +53 -0
  26. package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
  27. package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
  28. package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
  29. package/build/recommendation/verifiers/issueTraceability.js +197 -0
  30. package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
  31. package/build/recommendation/verifiers/requirementSourced.js +168 -0
  32. package/build/services/TestGenerationService.js +3 -1
  33. package/build/tools/code-refactor/codeReuseTool.js +1 -1
  34. package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
  35. package/build/tools/code-refactor/reuse-state.d.ts +85 -7
  36. package/build/tools/code-refactor/reuse-state.js +239 -34
  37. package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
  38. package/build/tools/code-refactor/utils-verify-gates.js +103 -11
  39. package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
  40. package/build/tools/submitReportTool.js +276 -39
  41. package/build/tools/test-management/actionsTool.js +5 -0
  42. package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
  43. package/build/tools/test-management/analyzeChangesTool.js +55 -2
  44. package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
  45. package/build/tools/test-management/registerTestPlanTool.js +105 -16
  46. package/build/types/ReuseOutcome.d.ts +73 -7
  47. package/build/types/TestAnalysis.d.ts +6 -0
  48. package/build/types/TestbotReport.d.ts +22 -4
  49. package/build/utils/AnalysisStateManager.d.ts +7 -1
  50. package/build/utils/AnalysisStateManager.js +5 -1
  51. package/build/utils/assertion-verify/api-shared-lints.js +118 -11
  52. package/build/utils/assertion-verify/format.js +2 -2
  53. package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
  54. package/build/utils/assertion-verify/helper-imports.js +119 -27
  55. package/build/utils/assertion-verify/lint-types.d.ts +31 -2
  56. package/build/utils/assertion-verify/lint-types.js +66 -0
  57. package/build/utils/assertion-verify/metrics.d.ts +13 -0
  58. package/build/utils/assertion-verify/metrics.js +16 -0
  59. package/build/utils/assertion-verify/verify.d.ts +11 -6
  60. package/build/utils/assertion-verify/verify.js +56 -15
  61. package/build/utils/canonicalJson.d.ts +11 -0
  62. package/build/utils/canonicalJson.js +17 -0
  63. package/build/utils/utils-verify/action-key.d.ts +27 -0
  64. package/build/utils/utils-verify/action-key.js +292 -0
  65. package/build/utils/utils-verify/allow.d.ts +8 -1
  66. package/build/utils/utils-verify/allow.js +14 -1
  67. package/build/utils/utils-verify/call-sites.d.ts +76 -8
  68. package/build/utils/utils-verify/call-sites.js +256 -70
  69. package/build/utils/utils-verify/language-spec.d.ts +3 -2
  70. package/build/utils/utils-verify/parse.d.ts +22 -3
  71. package/build/utils/utils-verify/parse.js +123 -52
  72. package/build/utils/utils-verify/verify.d.ts +33 -3
  73. package/build/utils/utils-verify/verify.js +126 -12
  74. package/build/utils/workspaceAuth.d.ts +59 -19
  75. package/build/utils/workspaceAuth.js +228 -31
  76. package/package.json +1 -1
  77. package/plugin/prompts/generate-tests/execution-plan.md +3 -3
  78. package/plugin/prompts/generate-tests/generation.md +1 -0
  79. package/plugin/prompts/plan-tests.md +41 -15
  80. package/plugin/prompts/testbot-task1.md +3 -9
  81. package/build/prompts/testbot/planDeclarations.d.ts +0 -6
  82. package/build/prompts/testbot/planDeclarations.js +0 -9
  83. package/plugin/prompts/declaring-a-plan.md +0 -20
@@ -4,7 +4,7 @@ import * as fs from "fs";
4
4
  import * as path from "path";
5
5
  import yaml from "js-yaml";
6
6
  import { logger } from "../../utils/logger.js";
7
- import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWorkspaceConfigRaw } from "../../utils/workspaceAuth.js";
7
+ import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWorkspaceConfigRaw, resolveWorkspaceApiService, findWorkspaceAuthDisagreement } from "../../utils/workspaceAuth.js";
8
8
  import { AnalyticsService } from "../../services/AnalyticsService.js";
9
9
  import { makeProgressReporter } from "../../utils/progress.js";
10
10
  import { StateManager, registerSession, storeSessionData, rememberTestsRepoDir, getActiveRunStatePath, setActiveRunStatePath, } from "../../utils/AnalysisStateManager.js";
@@ -101,6 +101,20 @@ export const analyzeChangesInputSchema = {
101
101
  .default(false)
102
102
  .describe("When true, diffs the base ref against the working tree (captures uncommitted and unstaged changes). Use for local-dev workflows. Defaults to false (CI mode — committed changes only)."),
103
103
  };
104
+ /** The shape an auth finding takes. Declared here beside the other output shapes;
105
+ * the assignment below fails to compile if it drifts from `AuthFinding`. */
106
+ const authFindingShape = z.object({
107
+ kind: z.literal("workspace-disagreement"),
108
+ message: z.string(),
109
+ serviceName: z.string(),
110
+ field: z.enum(["authType", "authHeader", "authScheme"]),
111
+ primary: z.string(),
112
+ analysed: z.string(),
113
+ primaryWorkspaceFile: z.string(),
114
+ analysedWorkspaceFile: z.string(),
115
+ });
116
+ const _authFindingShapeMatchesType = {};
117
+ void _authFindingShapeMatchesType;
104
118
  /** Both halves of `uiContext` carry the same record, so the shape is declared once.
105
119
  * It mirrors `RemovedUiElement`; the assignment below fails to compile if either drifts. */
106
120
  const uiIdentifierShape = z.object({ attribute: z.string(), value: z.string(), file: z.string() });
@@ -135,6 +149,7 @@ export const analyzeChangesOutputSchema = {
135
149
  authHeader: z.string().optional(),
136
150
  openApiSpecPath: z.string().optional(),
137
151
  openApiSpecLoaded: z.boolean(),
152
+ authFindings: z.array(authFindingShape).optional(),
138
153
  })
139
154
  .optional(),
140
155
  uiContext: z
@@ -285,9 +300,12 @@ export function registerAnalyzeChangesTool(server) {
285
300
  let wsAuthMethod = "none";
286
301
  let wsAuthType;
287
302
  let wsAuthScheme;
303
+ // What the agent has to decide about this repository's auth. Empty on a
304
+ // workspace whose config and code agree, which is every clean run.
305
+ const authFindings = [];
288
306
  const rawConfig = await readWorkspaceConfigRaw(params.repositoryPath);
289
307
  if (rawConfig) {
290
- const svc = rawConfig.services?.[0];
308
+ const svc = resolveWorkspaceApiService(rawConfig);
291
309
  if (svc?.api?.baseUrl)
292
310
  wsBaseUrl = svc.api.baseUrl;
293
311
  if (svc?.api?.authHeader !== undefined)
@@ -626,6 +644,40 @@ export function registerAnalyzeChangesTool(server) {
626
644
  if (params.repository)
627
645
  setActiveRunStatePath(stateFile);
628
646
  registerSession(sessionId, stateFile);
647
+ // A related repo reads its OWN .skyramp/workspace.yml, so it can disagree
648
+ // with the primary's unified entry for the same service and nothing would
649
+ // say so.
650
+ //
651
+ // Runs AFTER the state write, because both checkout paths come off the run's
652
+ // state file and this call's own path is not in it until then. Write order is
653
+ // not fixed: given a declared `primaryRepository`, a related repo may write
654
+ // first and the root is then created empty (AnalysisStateManager.writeRepoData).
655
+ // So whichever call arrives second holds both sides and is the one that
656
+ // reports — the primary against the related repos already recorded, a related
657
+ // repo against the primary.
658
+ if (rawConfig && params.repository) {
659
+ try {
660
+ const checkouts = await stateManager.listRepoCheckouts();
661
+ const analysingPrimary = checkouts.some((checkout) => checkout.primary && checkout.root === params.repositoryPath);
662
+ const others = checkouts.filter((checkout) => checkout.primary !== analysingPrimary &&
663
+ checkout.root !== params.repositoryPath);
664
+ for (const other of others) {
665
+ const otherConfig = await readWorkspaceConfigRaw(other.root);
666
+ const disagreement = analysingPrimary
667
+ ? findWorkspaceAuthDisagreement(rawConfig, otherConfig, params.repositoryPath, other.root, other.repository)
668
+ : findWorkspaceAuthDisagreement(otherConfig, rawConfig, other.root, params.repositoryPath, params.repository);
669
+ if (disagreement) {
670
+ authFindings.push(disagreement);
671
+ break;
672
+ }
673
+ }
674
+ }
675
+ catch (error) {
676
+ logger.debug("Could not compare this repository's workspace against the run's others", {
677
+ error: error instanceof Error ? error.message : String(error),
678
+ });
679
+ }
680
+ }
629
681
  try {
630
682
  await server.server.sendResourceListChanged();
631
683
  }
@@ -672,6 +724,7 @@ export function registerAnalyzeChangesTool(server) {
672
724
  ...(wsAuthHeader ? { authHeader: wsAuthHeader } : {}),
673
725
  ...(wsSchemaPath ? { openApiSpecPath: wsSchemaPath } : {}),
674
726
  openApiSpecLoaded: specFetchSucceeded,
727
+ ...(authFindings.length ? { authFindings } : {}),
675
728
  },
676
729
  // Removed half only. No prompt describes `retiredElements`, and beside the guard
677
730
  // list it reads as more guards to record; the plan checks read the state file.
@@ -3,13 +3,14 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
3
3
  import { StateManager, UnifiedAnalysisState } from "../../utils/AnalysisStateManager.js";
4
4
  import { TestType } from "../../types/TestTypes.js";
5
5
  import { PlanResult } from "../../recommendation/registerPlan.js";
6
- import { ObjectionAnswer, PlanChange, PlanInput, VerifyContext } from "../../recommendation/types.js";
6
+ import { ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlanMaintenance, VerifyContext } from "../../recommendation/types.js";
7
7
  /** The whole run's state file: the primary repository at the root, every related
8
8
  * repository under `relatedRepos`. */
9
9
  type RunState = Awaited<ReturnType<StateManager<UnifiedAnalysisState>["readFullState"]>>;
10
10
  export declare const declarationFieldsSchema: z.ZodObject<{
11
11
  changedFile: z.ZodOptional<z.ZodString>;
12
12
  changes: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
13
+ defects: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
13
14
  asserts: z.ZodString;
14
15
  stepUnderTest: z.ZodOptional<z.ZodNumber>;
15
16
  expected: z.ZodObject<{
@@ -121,11 +122,12 @@ export declare const declarationFieldsSchema: z.ZodObject<{
121
122
  differsBy: string;
122
123
  }[] | undefined;
123
124
  changedFile?: string | undefined;
124
- changes?: string[] | undefined;
125
+ defects?: string[] | undefined;
125
126
  routes?: {
126
127
  file: string;
127
128
  step: number;
128
129
  }[] | undefined;
130
+ changes?: string[] | undefined;
129
131
  elements?: {
130
132
  items?: {
131
133
  role: string;
@@ -158,11 +160,12 @@ export declare const declarationFieldsSchema: z.ZodObject<{
158
160
  differsBy: string;
159
161
  }[] | undefined;
160
162
  changedFile?: string | undefined;
161
- changes?: string[] | undefined;
163
+ defects?: string[] | undefined;
162
164
  routes?: {
163
165
  file: string;
164
166
  step: number;
165
167
  }[] | undefined;
168
+ changes?: string[] | undefined;
166
169
  elements?: {
167
170
  items?: {
168
171
  role: string;
@@ -250,6 +253,7 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
250
253
  declarations: z.ZodOptional<z.ZodEffects<z.ZodObject<{
251
254
  changedFile: z.ZodOptional<z.ZodString>;
252
255
  changes: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
256
+ defects: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
253
257
  asserts: z.ZodString;
254
258
  stepUnderTest: z.ZodOptional<z.ZodNumber>;
255
259
  expected: z.ZodObject<{
@@ -361,11 +365,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
361
365
  differsBy: string;
362
366
  }[] | undefined;
363
367
  changedFile?: string | undefined;
364
- changes?: string[] | undefined;
368
+ defects?: string[] | undefined;
365
369
  routes?: {
366
370
  file: string;
367
371
  step: number;
368
372
  }[] | undefined;
373
+ changes?: string[] | undefined;
369
374
  elements?: {
370
375
  items?: {
371
376
  role: string;
@@ -398,11 +403,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
398
403
  differsBy: string;
399
404
  }[] | undefined;
400
405
  changedFile?: string | undefined;
401
- changes?: string[] | undefined;
406
+ defects?: string[] | undefined;
402
407
  routes?: {
403
408
  file: string;
404
409
  step: number;
405
410
  }[] | undefined;
411
+ changes?: string[] | undefined;
406
412
  elements?: {
407
413
  items?: {
408
414
  role: string;
@@ -435,11 +441,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
435
441
  differsBy: string;
436
442
  }[] | undefined;
437
443
  changedFile?: string | undefined;
438
- changes?: string[] | undefined;
444
+ defects?: string[] | undefined;
439
445
  routes?: {
440
446
  file: string;
441
447
  step: number;
442
448
  }[] | undefined;
449
+ changes?: string[] | undefined;
443
450
  elements?: {
444
451
  items?: {
445
452
  role: string;
@@ -472,11 +479,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
472
479
  differsBy: string;
473
480
  }[] | undefined;
474
481
  changedFile?: string | undefined;
475
- changes?: string[] | undefined;
482
+ defects?: string[] | undefined;
476
483
  routes?: {
477
484
  file: string;
478
485
  step: number;
479
486
  }[] | undefined;
487
+ changes?: string[] | undefined;
480
488
  elements?: {
481
489
  items?: {
482
490
  role: string;
@@ -502,8 +510,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
502
510
  }, "strict", z.ZodTypeAny, {
503
511
  description: string;
504
512
  testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
505
- scenarioName: string;
506
513
  category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
514
+ scenarioName: string;
507
515
  steps: {
508
516
  path: string;
509
517
  method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
@@ -527,11 +535,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
527
535
  differsBy: string;
528
536
  }[] | undefined;
529
537
  changedFile?: string | undefined;
530
- changes?: string[] | undefined;
538
+ defects?: string[] | undefined;
531
539
  routes?: {
532
540
  file: string;
533
541
  step: number;
534
542
  }[] | undefined;
543
+ changes?: string[] | undefined;
535
544
  elements?: {
536
545
  items?: {
537
546
  role: string;
@@ -557,8 +566,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
557
566
  }, {
558
567
  description: string;
559
568
  testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
560
- scenarioName: string;
561
569
  category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
570
+ scenarioName: string;
562
571
  steps: {
563
572
  path: string;
564
573
  order: number;
@@ -582,11 +591,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
582
591
  differsBy: string;
583
592
  }[] | undefined;
584
593
  changedFile?: string | undefined;
585
- changes?: string[] | undefined;
594
+ defects?: string[] | undefined;
586
595
  routes?: {
587
596
  file: string;
588
597
  step: number;
589
598
  }[] | undefined;
599
+ changes?: string[] | undefined;
590
600
  elements?: {
591
601
  items?: {
592
602
  role: string;
@@ -612,8 +622,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
612
622
  }>, {
613
623
  description: string;
614
624
  testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
615
- scenarioName: string;
616
625
  category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
626
+ scenarioName: string;
617
627
  steps: {
618
628
  path: string;
619
629
  method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
@@ -637,11 +647,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
637
647
  differsBy: string;
638
648
  }[] | undefined;
639
649
  changedFile?: string | undefined;
640
- changes?: string[] | undefined;
650
+ defects?: string[] | undefined;
641
651
  routes?: {
642
652
  file: string;
643
653
  step: number;
644
654
  }[] | undefined;
655
+ changes?: string[] | undefined;
645
656
  elements?: {
646
657
  items?: {
647
658
  role: string;
@@ -667,8 +678,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
667
678
  }, {
668
679
  description: string;
669
680
  testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
670
- scenarioName: string;
671
681
  category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
682
+ scenarioName: string;
672
683
  steps: {
673
684
  path: string;
674
685
  order: number;
@@ -692,11 +703,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
692
703
  differsBy: string;
693
704
  }[] | undefined;
694
705
  changedFile?: string | undefined;
695
- changes?: string[] | undefined;
706
+ defects?: string[] | undefined;
696
707
  routes?: {
697
708
  file: string;
698
709
  step: number;
699
710
  }[] | undefined;
711
+ changes?: string[] | undefined;
700
712
  elements?: {
701
713
  items?: {
702
714
  role: string;
@@ -722,8 +734,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
722
734
  }>, {
723
735
  description: string;
724
736
  testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
725
- scenarioName: string;
726
737
  category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
738
+ scenarioName: string;
727
739
  steps: {
728
740
  path: string;
729
741
  method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
@@ -747,11 +759,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
747
759
  differsBy: string;
748
760
  }[] | undefined;
749
761
  changedFile?: string | undefined;
750
- changes?: string[] | undefined;
762
+ defects?: string[] | undefined;
751
763
  routes?: {
752
764
  file: string;
753
765
  step: number;
754
766
  }[] | undefined;
767
+ changes?: string[] | undefined;
755
768
  elements?: {
756
769
  items?: {
757
770
  role: string;
@@ -779,7 +792,9 @@ type RegisterPlannedTestInput = z.infer<typeof registerPlannedTestSchema>;
779
792
  export interface RegisterTestPlanParams {
780
793
  stateFile?: string;
781
794
  changes?: PlanChange[];
795
+ defects?: PlanDefect[];
782
796
  plannedTests?: RegisterPlannedTestInput[];
797
+ maintains?: PlanMaintenance[];
783
798
  answers?: ObjectionAnswer[];
784
799
  }
785
800
  /** Map the schema-validated input onto a `PlanInput`. `subjectEndpoints` is
@@ -3,6 +3,7 @@ import * as fs from "fs";
3
3
  import * as path from "path";
4
4
  import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
5
5
  import { logger } from "../../utils/logger.js";
6
+ import { canonicalJson, isPlainObject } from "../../utils/canonicalJson.js";
6
7
  import { StateManager, resolveRunStatePath } from "../../utils/AnalysisStateManager.js";
7
8
  import { toolError } from "../../utils/utils.js";
8
9
  import { TestType } from "../../types/TestTypes.js";
@@ -13,6 +14,7 @@ import { STEP_METHOD_DESCRIPTION, normalizeStepMethod, stepMethodSchema } from "
13
14
  import { registerPlan } from "../../recommendation/registerPlan.js";
14
15
  import { targetElementSchema } from "../submitReportTool.js";
15
16
  import { capturedBlueprintDigests } from "../../playwright/blueprintDigestStore.js";
17
+ import { pullRequestText } from "../../recommendation/pullRequestText.js";
16
18
  import { fileExistsWithinRoot } from "../../utils/containedPath.js";
17
19
  import { createScreenRouteResolver } from "../../utils/screenRoutes.js";
18
20
  import { categoryMenu, SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
@@ -62,7 +64,11 @@ export const declarationFieldsSchema = z.object({
62
64
  changes: z
63
65
  .array(z.string())
64
66
  .optional()
65
- .describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change."),
67
+ .describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change. CHECKED on a `requirement_conflict` test: at least one cited change must have been read from the pull request or a requirements file, never only from the diff."),
68
+ defects: z
69
+ .array(z.string())
70
+ .optional()
71
+ .describe("The ids of the defects that this test proves. Take the ids from your `defects` list. The server checks that each declared defect has a planned test that cites it and expects to fail, or a blocker naming what stopped the test. The server does not check that the test reaches the defect. CHECKED on a `requirement_conflict` test: it must cite at least one defect."),
66
72
  asserts: z
67
73
  .string()
68
74
  .describe("What this test checks."),
@@ -75,7 +81,7 @@ export const declarationFieldsSchema = z.object({
75
81
  .object({
76
82
  outcome: z
77
83
  .enum(["fail", "pass"])
78
- .describe("Whether this test should fail or pass against the app as it stands."),
84
+ .describe("Whether this test should fail or pass against the app as it stands. Every case a `fail` test sends carries `expectedValue` or `derived`, never both."),
79
85
  why: z
80
86
  .string()
81
87
  .describe("Why it has that outcome. A test that documents a defect asserts what the FIXED code returns — the rejection, the preserved value, the correct status — so it is red today and green after the fix, never today's defective response as correct."),
@@ -171,17 +177,6 @@ function namesAScreen(url) {
171
177
  * missing `stepUnderTest`, so a stray one stayed at planned test level and zod
172
178
  * dropped it. */
173
179
  const DECLARATION_FIELDS = Object.keys(declarationFieldsSchema.shape);
174
- function isPlainObject(value) {
175
- return typeof value === "object" && value !== null && !Array.isArray(value);
176
- }
177
- /** One value as text with object keys sorted, so the same declaration written
178
- * twice by hand is not a conflict merely because its keys are ordered
179
- * differently. List order still counts: two orders are two different lists. */
180
- function canonicalJson(value) {
181
- return JSON.stringify(value, (_key, entry) => isPlainObject(entry)
182
- ? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]]))
183
- : entry);
184
- }
185
180
  /** Move a declaration field written one level too high into `declarations`.
186
181
  * MEASURED: about one planned test in five did this across 141 eval planned tests, and
187
182
  * every stray planned test-level key was a declaration field. A field in BOTH places
@@ -294,9 +289,16 @@ const registerTestPlanSchema = {
294
289
  .refine((value) => value.trim().length > 0, { message: "a change id must not be blank" })
295
290
  .describe("Short slug for this change. A planned test cites it by this id."),
296
291
  text: z.string().describe("The change in one sentence, as the pull request must make it."),
292
+ quote: z
293
+ .string()
294
+ .optional()
295
+ .describe("The requirement in the words the pull request uses, when `source` is `pr-title` or `pr-description`. CHECKED on a change a `requirement_conflict` test cites: the quote must appear in the title or description."),
297
296
  source: z
298
297
  .string()
299
- .describe("Where you read it: `pr-description`, `spec:<path>` for a requirements file the description names, or `diff`."),
298
+ .refine((value) => /^(pr-title|pr-description|diff|spec:\S.*)$/i.test(value.trim()), {
299
+ message: "source must be `pr-title`, `pr-description`, `diff`, or `spec:<path>`",
300
+ })
301
+ .describe("Where you read it: `pr-title`, `pr-description`, `spec:<path>` for a requirements file the description names, or `diff`."),
300
302
  surfaces: z
301
303
  .array(z.enum(["api", "page"]))
302
304
  .min(1)
@@ -313,10 +315,31 @@ const registerTestPlanSchema = {
313
315
  expect: z
314
316
  .enum(["accept", "reject"])
315
317
  .describe("Whether the route must accept this value or reject it."),
318
+ expectedValue: z
319
+ .union([z.string(), z.number(), z.boolean(), z.null()])
320
+ .optional()
321
+ .describe("What the response must carry when this case is sent — the value itself, not a description of it. Declare it here so the test asserts what the change is supposed to do; a value decided while the test is being written can only be the value the running app returned, which is the one value a defect cannot contradict. Leave it out when no source states the value, and the test will assert the shape instead. `expectedValue` and `derived` are alternatives: a case declares one of them, never both. A test that expects to fail declares one of them on every case it sends."),
322
+ derived: z
323
+ .string()
324
+ .optional()
325
+ .describe("The rule the value follows from, in the source's words, when the source states a rule and not the value: `10 percent of the subtotal`. A test computes the value from what it sent. Use this instead of `expectedValue` whenever the pull request states the rule but never writes the number."),
326
+ expectedFrom: z
327
+ .string()
328
+ .refine((value) => /^(pr-title|pr-description|code|spec:\S.*|convention:\S.*)$/i.test(value.trim()), {
329
+ message: "expectedFrom must be `pr-title`, `pr-description`, `code`, `spec:<path>`, or `convention:<file:line>`",
330
+ })
331
+ .optional()
332
+ .describe("Where you read `expectedValue` or `derived`: `pr-title`, `pr-description`, `spec:<path>` for a requirements file the description names, `convention:<file:line>` for a rule the application already follows in code this pull request did not change, or `code` for the code under test. `code` draws an objection — it is the one source that agrees with a defect — so name it only when nothing else states the value. A value read from the pull request has to be WRITTEN there — in the quoted sentence or elsewhere in the title or description. If the pull request states the rule and never writes the number, leave `expectedValue` out and put the rule in `derived`. A case that declares `expectedValue` or `derived` and names no source here draws an objection."),
316
333
  })
317
334
  .strict()
318
335
  .refine((entry) => (entry.value !== undefined) !== (entry.absent === true), {
319
336
  message: "a case states either `value` or `absent`, never both and never neither",
337
+ })
338
+ // The description called them alternatives and the schema took both,
339
+ // which left generation told to pin a number AND to compute it.
340
+ .refine((entry) => entry.expectedValue === undefined || String(entry.derived ?? "").trim().length === 0, {
341
+ message: "a case states either `expectedValue` or `derived`, never both: `derived` is the case saying no source writes the value",
342
+ path: ["derived"],
320
343
  }))
321
344
  .min(1, { message: "a `cases` list with no entry states no input — leave it out instead" })
322
345
  .optional()
@@ -343,6 +366,43 @@ const registerTestPlanSchema = {
343
366
  })
344
367
  .default([])
345
368
  .describe("The changes the pull request must make, read out of the pull request title and description, a requirements file they name, or the diff itself. Coverage per file is the basic minimum; coverage per change is the target, and anything more is a bonus."),
369
+ // `.default([])`, like `changes`: an empty list draws `defects:none`, which is
370
+ // exactly the objection a plan that reports no review should draw.
371
+ defects: z
372
+ .array(z
373
+ .object({
374
+ id: z
375
+ .string()
376
+ .refine((value) => value.trim().length > 0, { message: "a defect id must not be blank" })
377
+ .describe("Short id for this defect, such as `D1`. A planned test cites it by this id, and the report's `issuesFound` entry names it as `defectId`."),
378
+ file: z.string().describe("Repository-relative path of the file that holds the defect, spelled as the diff spells it."),
379
+ line: z.number().int().positive().optional().describe("1-based line of the wrong statement. Advisory; nothing checks it."),
380
+ description: z.string().describe("What is wrong, in one line."),
381
+ severity: z
382
+ .enum(["critical", "high", "medium", "low"])
383
+ .describe("critical = feature broken or data corrupted; high = wrong behaviour; medium = minor functional gap; low = cosmetic."),
384
+ })
385
+ .strict())
386
+ .superRefine((defects, ctx) => {
387
+ const first = new Map();
388
+ defects.forEach((defect, index) => {
389
+ const key = String(defect?.id ?? "").trim();
390
+ if (!key)
391
+ return;
392
+ const seen = first.get(key);
393
+ if (seen === undefined) {
394
+ first.set(key, index);
395
+ return;
396
+ }
397
+ ctx.addIssue({
398
+ code: z.ZodIssueCode.custom,
399
+ path: [index, "id"],
400
+ message: `defect id "${defect.id}" is already used by defect ${seen + 1}. Every defect needs its own id — planned tests cite it by that id.`,
401
+ });
402
+ });
403
+ })
404
+ .default([])
405
+ .describe("The defects your code review found in the code that serves each change, whether or not the diff contains the line. Every defect gets a planned test that cites it in `defects` and expects to fail, or a blocker naming what stopped the test. A pre-existing defect in a file this pull request touches belongs on this list. An empty list says the review found none; the check asks you to confirm that in one line. A requirement conflict is a defect: list it here, and the `requirement_conflict` test cites it."),
346
406
  // REQUIRED, not `.default([])`. The MCP SDK wraps this shape in a plain
347
407
  // `z.object`, which STRIPS an undeclared top-level key, so with a default
348
408
  // `planned test:` for `planned tests:` stored an EMPTY plan and counted the
@@ -374,6 +434,20 @@ const registerTestPlanSchema = {
374
434
  });
375
435
  })
376
436
  .describe("Your complete plan, unordered. Nothing ranks, dedups, reorders or trims it, and the server adds no planned test of its own."),
437
+ maintains: z
438
+ .array(z
439
+ .object({
440
+ file: z
441
+ .string()
442
+ .describe("Repository-relative path of the existing test this run edits. It has to resolve in the checkout: a change is credited as covered only when it does, and `existingCoverage` objects when it does not."),
443
+ differsBy: z.string().describe("What your edit changes in that test, in one sentence."),
444
+ changes: z
445
+ .array(z.string())
446
+ .describe("The ids of the declared changes this maintenance covers, spelled as your own `changes` list gives them."),
447
+ })
448
+ .strict())
449
+ .optional()
450
+ .describe("Existing tests this run edits instead of planning new ones. Coverage counts the changes these cite, so a change already covered by a spec you are updating needs no planned test — and a duplicate spec beside the one you edited is the mistake. Declare maintenance here even when the plan holds no planned tests at all: a maintenance-only plan is a complete plan."),
377
451
  answers: z
378
452
  .array(z.object({
379
453
  objectionId: z.string().describe("The objection's id, exactly as this tool reported it. Answers are matched by id, never by position."),
@@ -381,7 +455,7 @@ const registerTestPlanSchema = {
381
455
  blocker: z
382
456
  .string()
383
457
  .optional()
384
- .describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change with no test closes only with this. A reason the change is not worth testing is not a blocker."),
458
+ .describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change or a defect with no test closes only with this. A reason the change is not worth testing is not a blocker."),
385
459
  }).strict())
386
460
  .default([])
387
461
  .describe("One answer per objection from your previous registration that you do not intend to fix. There is no registration limit."),
@@ -394,7 +468,9 @@ const registerTestPlanSchema = {
394
468
  export function buildRegistration(params, registrationNumber) {
395
469
  return {
396
470
  changes: params.changes ?? [],
471
+ defects: params.defects ?? [],
397
472
  answers: params.answers ?? [],
473
+ ...(params.maintains?.length ? { maintains: params.maintains } : {}),
398
474
  // The tool counts the run's registrations; whatever the caller sent is ignored.
399
475
  registrationNumber,
400
476
  plannedTests: (params.plannedTests ?? []).map((input) => {
@@ -530,6 +606,16 @@ async function changedFilesForVerification(section) {
530
606
  return undefined;
531
607
  }
532
608
  }
609
+ /** The pull request the prompt was rendered with. Blank on both fields only when
610
+ * no entry point recorded one, which is not a state the testbot produces: it
611
+ * fetches its prompt from this server. */
612
+ function recordedPullRequest() {
613
+ const recorded = pullRequestText();
614
+ if (recorded)
615
+ return recorded;
616
+ logger.warning("No pull request text was recorded before the plan was registered; the checks that read it will see an empty pull request.");
617
+ return { title: "", description: "" };
618
+ }
533
619
  /** Every fact the verifiers check against, gathered once per registration. */
534
620
  export async function buildVerifyContext(state) {
535
621
  const sections = repoSections(state);
@@ -543,6 +629,9 @@ export async function buildVerifyContext(state) {
543
629
  // From the process, not the state file: the browser tools and this one are
544
630
  // registered on the same server, so captures and reader share a process.
545
631
  uiCaptures: capturedBlueprintDigests(),
632
+ // From the process for the same reason: this server rendered the prompt, so it
633
+ // already holds the title and description the run was given.
634
+ pullRequest: recordedPullRequest(),
546
635
  // From the analyze result on the state file, across every repository: a
547
636
  // removal guard names an element that renders nowhere, so this list is the
548
637
  // only thing that can tell an honest one from an invented element.
@@ -561,7 +650,7 @@ export async function buildVerifyContext(state) {
561
650
  export function renderPlanResult(result) {
562
651
  const lines = [];
563
652
  lines.push(`Registration ${result.plan.registrationNumber} stored as this run's plan.`);
564
- lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s).`);
653
+ lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s), ${(result.plan.defects ?? []).length} defect(s).`);
565
654
  // A convenience: `differsFrom` also accepts a `scenarioName`.
566
655
  for (const plannedTest of result.plan.plannedTests) {
567
656
  lines.push(`- ${plannedTest.plannedTestId} — ${plannedTest.scenario?.scenarioName ?? "(unnamed)"}`);