@skyramp/mcp 0.4.0 → 0.4.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/enhance-assertions/contractProviderAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/integrationAssertionsPrompt.js +2 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +57 -22
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +17 -9
- package/build/prompts/test-recommendation/diffExecutionPlan.js +0 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -12
- package/build/recommendation/answers.d.ts +11 -7
- package/build/recommendation/answers.js +14 -10
- package/build/recommendation/pullRequestText.d.ts +18 -0
- package/build/recommendation/pullRequestText.js +31 -0
- package/build/recommendation/registerPlan.d.ts +9 -1
- package/build/recommendation/registerPlan.js +8 -1
- package/build/recommendation/runVerifiers.js +6 -0
- package/build/recommendation/types.d.ts +58 -0
- package/build/recommendation/verifierContracts.d.ts +108 -13
- package/build/recommendation/verifierContracts.js +147 -29
- package/build/recommendation/verifiers/coverage.d.ts +10 -0
- package/build/recommendation/verifiers/coverage.js +144 -22
- package/build/recommendation/verifiers/defects.d.ts +9 -0
- package/build/recommendation/verifiers/defects.js +117 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +22 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +43 -0
- package/build/recommendation/verifiers/existingCoverage.js +53 -0
- package/build/recommendation/verifiers/expectedValueSourced.d.ts +14 -0
- package/build/recommendation/verifiers/expectedValueSourced.js +246 -0
- package/build/recommendation/verifiers/issueTraceability.d.ts +52 -0
- package/build/recommendation/verifiers/issueTraceability.js +197 -0
- package/build/recommendation/verifiers/requirementSourced.d.ts +2 -0
- package/build/recommendation/verifiers/requirementSourced.js +168 -0
- package/build/services/TestGenerationService.js +3 -1
- package/build/tools/code-refactor/codeReuseTool.js +1 -1
- package/build/tools/code-refactor/reuse-outcome.d.ts +1 -1
- package/build/tools/code-refactor/reuse-state.d.ts +85 -7
- package/build/tools/code-refactor/reuse-state.js +239 -34
- package/build/tools/code-refactor/utils-verify-gates.d.ts +5 -0
- package/build/tools/code-refactor/utils-verify-gates.js +103 -11
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -1
- package/build/tools/submitReportTool.js +276 -39
- package/build/tools/test-management/actionsTool.js +5 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +53 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/registerTestPlanTool.d.ts +32 -17
- package/build/tools/test-management/registerTestPlanTool.js +105 -16
- package/build/types/ReuseOutcome.d.ts +73 -7
- package/build/types/TestAnalysis.d.ts +6 -0
- package/build/types/TestbotReport.d.ts +22 -4
- package/build/utils/AnalysisStateManager.d.ts +7 -1
- package/build/utils/AnalysisStateManager.js +5 -1
- package/build/utils/assertion-verify/api-shared-lints.js +118 -11
- package/build/utils/assertion-verify/format.js +2 -2
- package/build/utils/assertion-verify/helper-imports.d.ts +7 -0
- package/build/utils/assertion-verify/helper-imports.js +119 -27
- package/build/utils/assertion-verify/lint-types.d.ts +31 -2
- package/build/utils/assertion-verify/lint-types.js +66 -0
- package/build/utils/assertion-verify/metrics.d.ts +13 -0
- package/build/utils/assertion-verify/metrics.js +16 -0
- package/build/utils/assertion-verify/verify.d.ts +11 -6
- package/build/utils/assertion-verify/verify.js +56 -15
- package/build/utils/canonicalJson.d.ts +11 -0
- package/build/utils/canonicalJson.js +17 -0
- package/build/utils/utils-verify/action-key.d.ts +27 -0
- package/build/utils/utils-verify/action-key.js +292 -0
- package/build/utils/utils-verify/allow.d.ts +8 -1
- package/build/utils/utils-verify/allow.js +14 -1
- package/build/utils/utils-verify/call-sites.d.ts +76 -8
- package/build/utils/utils-verify/call-sites.js +256 -70
- package/build/utils/utils-verify/language-spec.d.ts +3 -2
- package/build/utils/utils-verify/parse.d.ts +22 -3
- package/build/utils/utils-verify/parse.js +123 -52
- package/build/utils/utils-verify/verify.d.ts +33 -3
- package/build/utils/utils-verify/verify.js +126 -12
- package/build/utils/workspaceAuth.d.ts +59 -19
- package/build/utils/workspaceAuth.js +228 -31
- package/package.json +1 -1
- package/plugin/prompts/generate-tests/execution-plan.md +3 -3
- package/plugin/prompts/generate-tests/generation.md +1 -0
- package/plugin/prompts/plan-tests.md +41 -15
- package/plugin/prompts/testbot-task1.md +3 -9
- package/build/prompts/testbot/planDeclarations.d.ts +0 -6
- package/build/prompts/testbot/planDeclarations.js +0 -9
- package/plugin/prompts/declaring-a-plan.md +0 -20
|
@@ -4,7 +4,7 @@ import * as fs from "fs";
|
|
|
4
4
|
import * as path from "path";
|
|
5
5
|
import yaml from "js-yaml";
|
|
6
6
|
import { logger } from "../../utils/logger.js";
|
|
7
|
-
import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWorkspaceConfigRaw } from "../../utils/workspaceAuth.js";
|
|
7
|
+
import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWorkspaceConfigRaw, resolveWorkspaceApiService, findWorkspaceAuthDisagreement } from "../../utils/workspaceAuth.js";
|
|
8
8
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
9
9
|
import { makeProgressReporter } from "../../utils/progress.js";
|
|
10
10
|
import { StateManager, registerSession, storeSessionData, rememberTestsRepoDir, getActiveRunStatePath, setActiveRunStatePath, } from "../../utils/AnalysisStateManager.js";
|
|
@@ -101,6 +101,20 @@ export const analyzeChangesInputSchema = {
|
|
|
101
101
|
.default(false)
|
|
102
102
|
.describe("When true, diffs the base ref against the working tree (captures uncommitted and unstaged changes). Use for local-dev workflows. Defaults to false (CI mode — committed changes only)."),
|
|
103
103
|
};
|
|
104
|
+
/** The shape an auth finding takes. Declared here beside the other output shapes;
|
|
105
|
+
* the assignment below fails to compile if it drifts from `AuthFinding`. */
|
|
106
|
+
const authFindingShape = z.object({
|
|
107
|
+
kind: z.literal("workspace-disagreement"),
|
|
108
|
+
message: z.string(),
|
|
109
|
+
serviceName: z.string(),
|
|
110
|
+
field: z.enum(["authType", "authHeader", "authScheme"]),
|
|
111
|
+
primary: z.string(),
|
|
112
|
+
analysed: z.string(),
|
|
113
|
+
primaryWorkspaceFile: z.string(),
|
|
114
|
+
analysedWorkspaceFile: z.string(),
|
|
115
|
+
});
|
|
116
|
+
const _authFindingShapeMatchesType = {};
|
|
117
|
+
void _authFindingShapeMatchesType;
|
|
104
118
|
/** Both halves of `uiContext` carry the same record, so the shape is declared once.
|
|
105
119
|
* It mirrors `RemovedUiElement`; the assignment below fails to compile if either drifts. */
|
|
106
120
|
const uiIdentifierShape = z.object({ attribute: z.string(), value: z.string(), file: z.string() });
|
|
@@ -135,6 +149,7 @@ export const analyzeChangesOutputSchema = {
|
|
|
135
149
|
authHeader: z.string().optional(),
|
|
136
150
|
openApiSpecPath: z.string().optional(),
|
|
137
151
|
openApiSpecLoaded: z.boolean(),
|
|
152
|
+
authFindings: z.array(authFindingShape).optional(),
|
|
138
153
|
})
|
|
139
154
|
.optional(),
|
|
140
155
|
uiContext: z
|
|
@@ -285,9 +300,12 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
285
300
|
let wsAuthMethod = "none";
|
|
286
301
|
let wsAuthType;
|
|
287
302
|
let wsAuthScheme;
|
|
303
|
+
// What the agent has to decide about this repository's auth. Empty on a
|
|
304
|
+
// workspace whose config and code agree, which is every clean run.
|
|
305
|
+
const authFindings = [];
|
|
288
306
|
const rawConfig = await readWorkspaceConfigRaw(params.repositoryPath);
|
|
289
307
|
if (rawConfig) {
|
|
290
|
-
const svc = rawConfig
|
|
308
|
+
const svc = resolveWorkspaceApiService(rawConfig);
|
|
291
309
|
if (svc?.api?.baseUrl)
|
|
292
310
|
wsBaseUrl = svc.api.baseUrl;
|
|
293
311
|
if (svc?.api?.authHeader !== undefined)
|
|
@@ -626,6 +644,40 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
626
644
|
if (params.repository)
|
|
627
645
|
setActiveRunStatePath(stateFile);
|
|
628
646
|
registerSession(sessionId, stateFile);
|
|
647
|
+
// A related repo reads its OWN .skyramp/workspace.yml, so it can disagree
|
|
648
|
+
// with the primary's unified entry for the same service and nothing would
|
|
649
|
+
// say so.
|
|
650
|
+
//
|
|
651
|
+
// Runs AFTER the state write, because both checkout paths come off the run's
|
|
652
|
+
// state file and this call's own path is not in it until then. Write order is
|
|
653
|
+
// not fixed: given a declared `primaryRepository`, a related repo may write
|
|
654
|
+
// first and the root is then created empty (AnalysisStateManager.writeRepoData).
|
|
655
|
+
// So whichever call arrives second holds both sides and is the one that
|
|
656
|
+
// reports — the primary against the related repos already recorded, a related
|
|
657
|
+
// repo against the primary.
|
|
658
|
+
if (rawConfig && params.repository) {
|
|
659
|
+
try {
|
|
660
|
+
const checkouts = await stateManager.listRepoCheckouts();
|
|
661
|
+
const analysingPrimary = checkouts.some((checkout) => checkout.primary && checkout.root === params.repositoryPath);
|
|
662
|
+
const others = checkouts.filter((checkout) => checkout.primary !== analysingPrimary &&
|
|
663
|
+
checkout.root !== params.repositoryPath);
|
|
664
|
+
for (const other of others) {
|
|
665
|
+
const otherConfig = await readWorkspaceConfigRaw(other.root);
|
|
666
|
+
const disagreement = analysingPrimary
|
|
667
|
+
? findWorkspaceAuthDisagreement(rawConfig, otherConfig, params.repositoryPath, other.root, other.repository)
|
|
668
|
+
: findWorkspaceAuthDisagreement(otherConfig, rawConfig, other.root, params.repositoryPath, params.repository);
|
|
669
|
+
if (disagreement) {
|
|
670
|
+
authFindings.push(disagreement);
|
|
671
|
+
break;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
catch (error) {
|
|
676
|
+
logger.debug("Could not compare this repository's workspace against the run's others", {
|
|
677
|
+
error: error instanceof Error ? error.message : String(error),
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
}
|
|
629
681
|
try {
|
|
630
682
|
await server.server.sendResourceListChanged();
|
|
631
683
|
}
|
|
@@ -672,6 +724,7 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
672
724
|
...(wsAuthHeader ? { authHeader: wsAuthHeader } : {}),
|
|
673
725
|
...(wsSchemaPath ? { openApiSpecPath: wsSchemaPath } : {}),
|
|
674
726
|
openApiSpecLoaded: specFetchSucceeded,
|
|
727
|
+
...(authFindings.length ? { authFindings } : {}),
|
|
675
728
|
},
|
|
676
729
|
// Removed half only. No prompt describes `retiredElements`, and beside the guard
|
|
677
730
|
// list it reads as more guards to record; the plan checks read the state file.
|
|
@@ -3,13 +3,14 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
3
3
|
import { StateManager, UnifiedAnalysisState } from "../../utils/AnalysisStateManager.js";
|
|
4
4
|
import { TestType } from "../../types/TestTypes.js";
|
|
5
5
|
import { PlanResult } from "../../recommendation/registerPlan.js";
|
|
6
|
-
import { ObjectionAnswer, PlanChange, PlanInput, VerifyContext } from "../../recommendation/types.js";
|
|
6
|
+
import { ObjectionAnswer, PlanChange, PlanDefect, PlanInput, PlanMaintenance, VerifyContext } from "../../recommendation/types.js";
|
|
7
7
|
/** The whole run's state file: the primary repository at the root, every related
|
|
8
8
|
* repository under `relatedRepos`. */
|
|
9
9
|
type RunState = Awaited<ReturnType<StateManager<UnifiedAnalysisState>["readFullState"]>>;
|
|
10
10
|
export declare const declarationFieldsSchema: z.ZodObject<{
|
|
11
11
|
changedFile: z.ZodOptional<z.ZodString>;
|
|
12
12
|
changes: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
13
|
+
defects: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
13
14
|
asserts: z.ZodString;
|
|
14
15
|
stepUnderTest: z.ZodOptional<z.ZodNumber>;
|
|
15
16
|
expected: z.ZodObject<{
|
|
@@ -121,11 +122,12 @@ export declare const declarationFieldsSchema: z.ZodObject<{
|
|
|
121
122
|
differsBy: string;
|
|
122
123
|
}[] | undefined;
|
|
123
124
|
changedFile?: string | undefined;
|
|
124
|
-
|
|
125
|
+
defects?: string[] | undefined;
|
|
125
126
|
routes?: {
|
|
126
127
|
file: string;
|
|
127
128
|
step: number;
|
|
128
129
|
}[] | undefined;
|
|
130
|
+
changes?: string[] | undefined;
|
|
129
131
|
elements?: {
|
|
130
132
|
items?: {
|
|
131
133
|
role: string;
|
|
@@ -158,11 +160,12 @@ export declare const declarationFieldsSchema: z.ZodObject<{
|
|
|
158
160
|
differsBy: string;
|
|
159
161
|
}[] | undefined;
|
|
160
162
|
changedFile?: string | undefined;
|
|
161
|
-
|
|
163
|
+
defects?: string[] | undefined;
|
|
162
164
|
routes?: {
|
|
163
165
|
file: string;
|
|
164
166
|
step: number;
|
|
165
167
|
}[] | undefined;
|
|
168
|
+
changes?: string[] | undefined;
|
|
166
169
|
elements?: {
|
|
167
170
|
items?: {
|
|
168
171
|
role: string;
|
|
@@ -250,6 +253,7 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
250
253
|
declarations: z.ZodOptional<z.ZodEffects<z.ZodObject<{
|
|
251
254
|
changedFile: z.ZodOptional<z.ZodString>;
|
|
252
255
|
changes: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
256
|
+
defects: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
253
257
|
asserts: z.ZodString;
|
|
254
258
|
stepUnderTest: z.ZodOptional<z.ZodNumber>;
|
|
255
259
|
expected: z.ZodObject<{
|
|
@@ -361,11 +365,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
361
365
|
differsBy: string;
|
|
362
366
|
}[] | undefined;
|
|
363
367
|
changedFile?: string | undefined;
|
|
364
|
-
|
|
368
|
+
defects?: string[] | undefined;
|
|
365
369
|
routes?: {
|
|
366
370
|
file: string;
|
|
367
371
|
step: number;
|
|
368
372
|
}[] | undefined;
|
|
373
|
+
changes?: string[] | undefined;
|
|
369
374
|
elements?: {
|
|
370
375
|
items?: {
|
|
371
376
|
role: string;
|
|
@@ -398,11 +403,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
398
403
|
differsBy: string;
|
|
399
404
|
}[] | undefined;
|
|
400
405
|
changedFile?: string | undefined;
|
|
401
|
-
|
|
406
|
+
defects?: string[] | undefined;
|
|
402
407
|
routes?: {
|
|
403
408
|
file: string;
|
|
404
409
|
step: number;
|
|
405
410
|
}[] | undefined;
|
|
411
|
+
changes?: string[] | undefined;
|
|
406
412
|
elements?: {
|
|
407
413
|
items?: {
|
|
408
414
|
role: string;
|
|
@@ -435,11 +441,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
435
441
|
differsBy: string;
|
|
436
442
|
}[] | undefined;
|
|
437
443
|
changedFile?: string | undefined;
|
|
438
|
-
|
|
444
|
+
defects?: string[] | undefined;
|
|
439
445
|
routes?: {
|
|
440
446
|
file: string;
|
|
441
447
|
step: number;
|
|
442
448
|
}[] | undefined;
|
|
449
|
+
changes?: string[] | undefined;
|
|
443
450
|
elements?: {
|
|
444
451
|
items?: {
|
|
445
452
|
role: string;
|
|
@@ -472,11 +479,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
472
479
|
differsBy: string;
|
|
473
480
|
}[] | undefined;
|
|
474
481
|
changedFile?: string | undefined;
|
|
475
|
-
|
|
482
|
+
defects?: string[] | undefined;
|
|
476
483
|
routes?: {
|
|
477
484
|
file: string;
|
|
478
485
|
step: number;
|
|
479
486
|
}[] | undefined;
|
|
487
|
+
changes?: string[] | undefined;
|
|
480
488
|
elements?: {
|
|
481
489
|
items?: {
|
|
482
490
|
role: string;
|
|
@@ -502,8 +510,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
502
510
|
}, "strict", z.ZodTypeAny, {
|
|
503
511
|
description: string;
|
|
504
512
|
testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
|
|
505
|
-
scenarioName: string;
|
|
506
513
|
category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
514
|
+
scenarioName: string;
|
|
507
515
|
steps: {
|
|
508
516
|
path: string;
|
|
509
517
|
method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
|
|
@@ -527,11 +535,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
527
535
|
differsBy: string;
|
|
528
536
|
}[] | undefined;
|
|
529
537
|
changedFile?: string | undefined;
|
|
530
|
-
|
|
538
|
+
defects?: string[] | undefined;
|
|
531
539
|
routes?: {
|
|
532
540
|
file: string;
|
|
533
541
|
step: number;
|
|
534
542
|
}[] | undefined;
|
|
543
|
+
changes?: string[] | undefined;
|
|
535
544
|
elements?: {
|
|
536
545
|
items?: {
|
|
537
546
|
role: string;
|
|
@@ -557,8 +566,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
557
566
|
}, {
|
|
558
567
|
description: string;
|
|
559
568
|
testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
|
|
560
|
-
scenarioName: string;
|
|
561
569
|
category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
570
|
+
scenarioName: string;
|
|
562
571
|
steps: {
|
|
563
572
|
path: string;
|
|
564
573
|
order: number;
|
|
@@ -582,11 +591,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
582
591
|
differsBy: string;
|
|
583
592
|
}[] | undefined;
|
|
584
593
|
changedFile?: string | undefined;
|
|
585
|
-
|
|
594
|
+
defects?: string[] | undefined;
|
|
586
595
|
routes?: {
|
|
587
596
|
file: string;
|
|
588
597
|
step: number;
|
|
589
598
|
}[] | undefined;
|
|
599
|
+
changes?: string[] | undefined;
|
|
590
600
|
elements?: {
|
|
591
601
|
items?: {
|
|
592
602
|
role: string;
|
|
@@ -612,8 +622,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
612
622
|
}>, {
|
|
613
623
|
description: string;
|
|
614
624
|
testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
|
|
615
|
-
scenarioName: string;
|
|
616
625
|
category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
626
|
+
scenarioName: string;
|
|
617
627
|
steps: {
|
|
618
628
|
path: string;
|
|
619
629
|
method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
|
|
@@ -637,11 +647,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
637
647
|
differsBy: string;
|
|
638
648
|
}[] | undefined;
|
|
639
649
|
changedFile?: string | undefined;
|
|
640
|
-
|
|
650
|
+
defects?: string[] | undefined;
|
|
641
651
|
routes?: {
|
|
642
652
|
file: string;
|
|
643
653
|
step: number;
|
|
644
654
|
}[] | undefined;
|
|
655
|
+
changes?: string[] | undefined;
|
|
645
656
|
elements?: {
|
|
646
657
|
items?: {
|
|
647
658
|
role: string;
|
|
@@ -667,8 +678,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
667
678
|
}, {
|
|
668
679
|
description: string;
|
|
669
680
|
testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
|
|
670
|
-
scenarioName: string;
|
|
671
681
|
category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
682
|
+
scenarioName: string;
|
|
672
683
|
steps: {
|
|
673
684
|
path: string;
|
|
674
685
|
order: number;
|
|
@@ -692,11 +703,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
692
703
|
differsBy: string;
|
|
693
704
|
}[] | undefined;
|
|
694
705
|
changedFile?: string | undefined;
|
|
695
|
-
|
|
706
|
+
defects?: string[] | undefined;
|
|
696
707
|
routes?: {
|
|
697
708
|
file: string;
|
|
698
709
|
step: number;
|
|
699
710
|
}[] | undefined;
|
|
711
|
+
changes?: string[] | undefined;
|
|
700
712
|
elements?: {
|
|
701
713
|
items?: {
|
|
702
714
|
role: string;
|
|
@@ -722,8 +734,8 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
722
734
|
}>, {
|
|
723
735
|
description: string;
|
|
724
736
|
testType: TestType.CONTRACT | TestType.INTEGRATION | TestType.E2E | TestType.UI;
|
|
725
|
-
scenarioName: string;
|
|
726
737
|
category: "new_endpoint" | "bug_caught" | "requirement_conflict" | "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
738
|
+
scenarioName: string;
|
|
727
739
|
steps: {
|
|
728
740
|
path: string;
|
|
729
741
|
method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS" | "type" | "TRACE" | "CONNECT" | "assert" | "click" | "drag" | "drag-hold" | "hover" | "inspect" | "navigate" | "press" | "release" | "tap" | "wait" | "OPERATION";
|
|
@@ -747,11 +759,12 @@ declare const registerPlannedTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
|
747
759
|
differsBy: string;
|
|
748
760
|
}[] | undefined;
|
|
749
761
|
changedFile?: string | undefined;
|
|
750
|
-
|
|
762
|
+
defects?: string[] | undefined;
|
|
751
763
|
routes?: {
|
|
752
764
|
file: string;
|
|
753
765
|
step: number;
|
|
754
766
|
}[] | undefined;
|
|
767
|
+
changes?: string[] | undefined;
|
|
755
768
|
elements?: {
|
|
756
769
|
items?: {
|
|
757
770
|
role: string;
|
|
@@ -779,7 +792,9 @@ type RegisterPlannedTestInput = z.infer<typeof registerPlannedTestSchema>;
|
|
|
779
792
|
export interface RegisterTestPlanParams {
|
|
780
793
|
stateFile?: string;
|
|
781
794
|
changes?: PlanChange[];
|
|
795
|
+
defects?: PlanDefect[];
|
|
782
796
|
plannedTests?: RegisterPlannedTestInput[];
|
|
797
|
+
maintains?: PlanMaintenance[];
|
|
783
798
|
answers?: ObjectionAnswer[];
|
|
784
799
|
}
|
|
785
800
|
/** Map the schema-validated input onto a `PlanInput`. `subjectEndpoints` is
|
|
@@ -3,6 +3,7 @@ import * as fs from "fs";
|
|
|
3
3
|
import * as path from "path";
|
|
4
4
|
import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
|
|
5
5
|
import { logger } from "../../utils/logger.js";
|
|
6
|
+
import { canonicalJson, isPlainObject } from "../../utils/canonicalJson.js";
|
|
6
7
|
import { StateManager, resolveRunStatePath } from "../../utils/AnalysisStateManager.js";
|
|
7
8
|
import { toolError } from "../../utils/utils.js";
|
|
8
9
|
import { TestType } from "../../types/TestTypes.js";
|
|
@@ -13,6 +14,7 @@ import { STEP_METHOD_DESCRIPTION, normalizeStepMethod, stepMethodSchema } from "
|
|
|
13
14
|
import { registerPlan } from "../../recommendation/registerPlan.js";
|
|
14
15
|
import { targetElementSchema } from "../submitReportTool.js";
|
|
15
16
|
import { capturedBlueprintDigests } from "../../playwright/blueprintDigestStore.js";
|
|
17
|
+
import { pullRequestText } from "../../recommendation/pullRequestText.js";
|
|
16
18
|
import { fileExistsWithinRoot } from "../../utils/containedPath.js";
|
|
17
19
|
import { createScreenRouteResolver } from "../../utils/screenRoutes.js";
|
|
18
20
|
import { categoryMenu, SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
|
|
@@ -62,7 +64,11 @@ export const declarationFieldsSchema = z.object({
|
|
|
62
64
|
changes: z
|
|
63
65
|
.array(z.string())
|
|
64
66
|
.optional()
|
|
65
|
-
.describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change."),
|
|
67
|
+
.describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change. CHECKED on a `requirement_conflict` test: at least one cited change must have been read from the pull request or a requirements file, never only from the diff."),
|
|
68
|
+
defects: z
|
|
69
|
+
.array(z.string())
|
|
70
|
+
.optional()
|
|
71
|
+
.describe("The ids of the defects that this test proves. Take the ids from your `defects` list. The server checks that each declared defect has a planned test that cites it and expects to fail, or a blocker naming what stopped the test. The server does not check that the test reaches the defect. CHECKED on a `requirement_conflict` test: it must cite at least one defect."),
|
|
66
72
|
asserts: z
|
|
67
73
|
.string()
|
|
68
74
|
.describe("What this test checks."),
|
|
@@ -75,7 +81,7 @@ export const declarationFieldsSchema = z.object({
|
|
|
75
81
|
.object({
|
|
76
82
|
outcome: z
|
|
77
83
|
.enum(["fail", "pass"])
|
|
78
|
-
.describe("Whether this test should fail or pass against the app as it stands."),
|
|
84
|
+
.describe("Whether this test should fail or pass against the app as it stands. Every case a `fail` test sends carries `expectedValue` or `derived`, never both."),
|
|
79
85
|
why: z
|
|
80
86
|
.string()
|
|
81
87
|
.describe("Why it has that outcome. A test that documents a defect asserts what the FIXED code returns — the rejection, the preserved value, the correct status — so it is red today and green after the fix, never today's defective response as correct."),
|
|
@@ -171,17 +177,6 @@ function namesAScreen(url) {
|
|
|
171
177
|
* missing `stepUnderTest`, so a stray one stayed at planned test level and zod
|
|
172
178
|
* dropped it. */
|
|
173
179
|
const DECLARATION_FIELDS = Object.keys(declarationFieldsSchema.shape);
|
|
174
|
-
function isPlainObject(value) {
|
|
175
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
176
|
-
}
|
|
177
|
-
/** One value as text with object keys sorted, so the same declaration written
|
|
178
|
-
* twice by hand is not a conflict merely because its keys are ordered
|
|
179
|
-
* differently. List order still counts: two orders are two different lists. */
|
|
180
|
-
function canonicalJson(value) {
|
|
181
|
-
return JSON.stringify(value, (_key, entry) => isPlainObject(entry)
|
|
182
|
-
? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]]))
|
|
183
|
-
: entry);
|
|
184
|
-
}
|
|
185
180
|
/** Move a declaration field written one level too high into `declarations`.
|
|
186
181
|
* MEASURED: about one planned test in five did this across 141 eval planned tests, and
|
|
187
182
|
* every stray planned test-level key was a declaration field. A field in BOTH places
|
|
@@ -294,9 +289,16 @@ const registerTestPlanSchema = {
|
|
|
294
289
|
.refine((value) => value.trim().length > 0, { message: "a change id must not be blank" })
|
|
295
290
|
.describe("Short slug for this change. A planned test cites it by this id."),
|
|
296
291
|
text: z.string().describe("The change in one sentence, as the pull request must make it."),
|
|
292
|
+
quote: z
|
|
293
|
+
.string()
|
|
294
|
+
.optional()
|
|
295
|
+
.describe("The requirement in the words the pull request uses, when `source` is `pr-title` or `pr-description`. CHECKED on a change a `requirement_conflict` test cites: the quote must appear in the title or description."),
|
|
297
296
|
source: z
|
|
298
297
|
.string()
|
|
299
|
-
.
|
|
298
|
+
.refine((value) => /^(pr-title|pr-description|diff|spec:\S.*)$/i.test(value.trim()), {
|
|
299
|
+
message: "source must be `pr-title`, `pr-description`, `diff`, or `spec:<path>`",
|
|
300
|
+
})
|
|
301
|
+
.describe("Where you read it: `pr-title`, `pr-description`, `spec:<path>` for a requirements file the description names, or `diff`."),
|
|
300
302
|
surfaces: z
|
|
301
303
|
.array(z.enum(["api", "page"]))
|
|
302
304
|
.min(1)
|
|
@@ -313,10 +315,31 @@ const registerTestPlanSchema = {
|
|
|
313
315
|
expect: z
|
|
314
316
|
.enum(["accept", "reject"])
|
|
315
317
|
.describe("Whether the route must accept this value or reject it."),
|
|
318
|
+
expectedValue: z
|
|
319
|
+
.union([z.string(), z.number(), z.boolean(), z.null()])
|
|
320
|
+
.optional()
|
|
321
|
+
.describe("What the response must carry when this case is sent — the value itself, not a description of it. Declare it here so the test asserts what the change is supposed to do; a value decided while the test is being written can only be the value the running app returned, which is the one value a defect cannot contradict. Leave it out when no source states the value, and the test will assert the shape instead. `expectedValue` and `derived` are alternatives: a case declares one of them, never both. A test that expects to fail declares one of them on every case it sends."),
|
|
322
|
+
derived: z
|
|
323
|
+
.string()
|
|
324
|
+
.optional()
|
|
325
|
+
.describe("The rule the value follows from, in the source's words, when the source states a rule and not the value: `10 percent of the subtotal`. A test computes the value from what it sent. Use this instead of `expectedValue` whenever the pull request states the rule but never writes the number."),
|
|
326
|
+
expectedFrom: z
|
|
327
|
+
.string()
|
|
328
|
+
.refine((value) => /^(pr-title|pr-description|code|spec:\S.*|convention:\S.*)$/i.test(value.trim()), {
|
|
329
|
+
message: "expectedFrom must be `pr-title`, `pr-description`, `code`, `spec:<path>`, or `convention:<file:line>`",
|
|
330
|
+
})
|
|
331
|
+
.optional()
|
|
332
|
+
.describe("Where you read `expectedValue` or `derived`: `pr-title`, `pr-description`, `spec:<path>` for a requirements file the description names, `convention:<file:line>` for a rule the application already follows in code this pull request did not change, or `code` for the code under test. `code` draws an objection — it is the one source that agrees with a defect — so name it only when nothing else states the value. A value read from the pull request has to be WRITTEN there — in the quoted sentence or elsewhere in the title or description. If the pull request states the rule and never writes the number, leave `expectedValue` out and put the rule in `derived`. A case that declares `expectedValue` or `derived` and names no source here draws an objection."),
|
|
316
333
|
})
|
|
317
334
|
.strict()
|
|
318
335
|
.refine((entry) => (entry.value !== undefined) !== (entry.absent === true), {
|
|
319
336
|
message: "a case states either `value` or `absent`, never both and never neither",
|
|
337
|
+
})
|
|
338
|
+
// The description called them alternatives and the schema took both,
|
|
339
|
+
// which left generation told to pin a number AND to compute it.
|
|
340
|
+
.refine((entry) => entry.expectedValue === undefined || String(entry.derived ?? "").trim().length === 0, {
|
|
341
|
+
message: "a case states either `expectedValue` or `derived`, never both: `derived` is the case saying no source writes the value",
|
|
342
|
+
path: ["derived"],
|
|
320
343
|
}))
|
|
321
344
|
.min(1, { message: "a `cases` list with no entry states no input — leave it out instead" })
|
|
322
345
|
.optional()
|
|
@@ -343,6 +366,43 @@ const registerTestPlanSchema = {
|
|
|
343
366
|
})
|
|
344
367
|
.default([])
|
|
345
368
|
.describe("The changes the pull request must make, read out of the pull request title and description, a requirements file they name, or the diff itself. Coverage per file is the basic minimum; coverage per change is the target, and anything more is a bonus."),
|
|
369
|
+
// `.default([])`, like `changes`: an empty list draws `defects:none`, which is
|
|
370
|
+
// exactly the objection a plan that reports no review should draw.
|
|
371
|
+
defects: z
|
|
372
|
+
.array(z
|
|
373
|
+
.object({
|
|
374
|
+
id: z
|
|
375
|
+
.string()
|
|
376
|
+
.refine((value) => value.trim().length > 0, { message: "a defect id must not be blank" })
|
|
377
|
+
.describe("Short id for this defect, such as `D1`. A planned test cites it by this id, and the report's `issuesFound` entry names it as `defectId`."),
|
|
378
|
+
file: z.string().describe("Repository-relative path of the file that holds the defect, spelled as the diff spells it."),
|
|
379
|
+
line: z.number().int().positive().optional().describe("1-based line of the wrong statement. Advisory; nothing checks it."),
|
|
380
|
+
description: z.string().describe("What is wrong, in one line."),
|
|
381
|
+
severity: z
|
|
382
|
+
.enum(["critical", "high", "medium", "low"])
|
|
383
|
+
.describe("critical = feature broken or data corrupted; high = wrong behaviour; medium = minor functional gap; low = cosmetic."),
|
|
384
|
+
})
|
|
385
|
+
.strict())
|
|
386
|
+
.superRefine((defects, ctx) => {
|
|
387
|
+
const first = new Map();
|
|
388
|
+
defects.forEach((defect, index) => {
|
|
389
|
+
const key = String(defect?.id ?? "").trim();
|
|
390
|
+
if (!key)
|
|
391
|
+
return;
|
|
392
|
+
const seen = first.get(key);
|
|
393
|
+
if (seen === undefined) {
|
|
394
|
+
first.set(key, index);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
ctx.addIssue({
|
|
398
|
+
code: z.ZodIssueCode.custom,
|
|
399
|
+
path: [index, "id"],
|
|
400
|
+
message: `defect id "${defect.id}" is already used by defect ${seen + 1}. Every defect needs its own id — planned tests cite it by that id.`,
|
|
401
|
+
});
|
|
402
|
+
});
|
|
403
|
+
})
|
|
404
|
+
.default([])
|
|
405
|
+
.describe("The defects your code review found in the code that serves each change, whether or not the diff contains the line. Every defect gets a planned test that cites it in `defects` and expects to fail, or a blocker naming what stopped the test. A pre-existing defect in a file this pull request touches belongs on this list. An empty list says the review found none; the check asks you to confirm that in one line. A requirement conflict is a defect: list it here, and the `requirement_conflict` test cites it."),
|
|
346
406
|
// REQUIRED, not `.default([])`. The MCP SDK wraps this shape in a plain
|
|
347
407
|
// `z.object`, which STRIPS an undeclared top-level key, so with a default
|
|
348
408
|
// `planned test:` for `planned tests:` stored an EMPTY plan and counted the
|
|
@@ -374,6 +434,20 @@ const registerTestPlanSchema = {
|
|
|
374
434
|
});
|
|
375
435
|
})
|
|
376
436
|
.describe("Your complete plan, unordered. Nothing ranks, dedups, reorders or trims it, and the server adds no planned test of its own."),
|
|
437
|
+
maintains: z
|
|
438
|
+
.array(z
|
|
439
|
+
.object({
|
|
440
|
+
file: z
|
|
441
|
+
.string()
|
|
442
|
+
.describe("Repository-relative path of the existing test this run edits. It has to resolve in the checkout: a change is credited as covered only when it does, and `existingCoverage` objects when it does not."),
|
|
443
|
+
differsBy: z.string().describe("What your edit changes in that test, in one sentence."),
|
|
444
|
+
changes: z
|
|
445
|
+
.array(z.string())
|
|
446
|
+
.describe("The ids of the declared changes this maintenance covers, spelled as your own `changes` list gives them."),
|
|
447
|
+
})
|
|
448
|
+
.strict())
|
|
449
|
+
.optional()
|
|
450
|
+
.describe("Existing tests this run edits instead of planning new ones. Coverage counts the changes these cite, so a change already covered by a spec you are updating needs no planned test — and a duplicate spec beside the one you edited is the mistake. Declare maintenance here even when the plan holds no planned tests at all: a maintenance-only plan is a complete plan."),
|
|
377
451
|
answers: z
|
|
378
452
|
.array(z.object({
|
|
379
453
|
objectionId: z.string().describe("The objection's id, exactly as this tool reported it. Answers are matched by id, never by position."),
|
|
@@ -381,7 +455,7 @@ const registerTestPlanSchema = {
|
|
|
381
455
|
blocker: z
|
|
382
456
|
.string()
|
|
383
457
|
.optional()
|
|
384
|
-
.describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change with no test closes only with this. A reason the change is not worth testing is not a blocker."),
|
|
458
|
+
.describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change or a defect with no test closes only with this. A reason the change is not worth testing is not a blocker."),
|
|
385
459
|
}).strict())
|
|
386
460
|
.default([])
|
|
387
461
|
.describe("One answer per objection from your previous registration that you do not intend to fix. There is no registration limit."),
|
|
@@ -394,7 +468,9 @@ const registerTestPlanSchema = {
|
|
|
394
468
|
export function buildRegistration(params, registrationNumber) {
|
|
395
469
|
return {
|
|
396
470
|
changes: params.changes ?? [],
|
|
471
|
+
defects: params.defects ?? [],
|
|
397
472
|
answers: params.answers ?? [],
|
|
473
|
+
...(params.maintains?.length ? { maintains: params.maintains } : {}),
|
|
398
474
|
// The tool counts the run's registrations; whatever the caller sent is ignored.
|
|
399
475
|
registrationNumber,
|
|
400
476
|
plannedTests: (params.plannedTests ?? []).map((input) => {
|
|
@@ -530,6 +606,16 @@ async function changedFilesForVerification(section) {
|
|
|
530
606
|
return undefined;
|
|
531
607
|
}
|
|
532
608
|
}
|
|
609
|
+
/** The pull request the prompt was rendered with. Blank on both fields only when
|
|
610
|
+
* no entry point recorded one, which is not a state the testbot produces: it
|
|
611
|
+
* fetches its prompt from this server. */
|
|
612
|
+
function recordedPullRequest() {
|
|
613
|
+
const recorded = pullRequestText();
|
|
614
|
+
if (recorded)
|
|
615
|
+
return recorded;
|
|
616
|
+
logger.warning("No pull request text was recorded before the plan was registered; the checks that read it will see an empty pull request.");
|
|
617
|
+
return { title: "", description: "" };
|
|
618
|
+
}
|
|
533
619
|
/** Every fact the verifiers check against, gathered once per registration. */
|
|
534
620
|
export async function buildVerifyContext(state) {
|
|
535
621
|
const sections = repoSections(state);
|
|
@@ -543,6 +629,9 @@ export async function buildVerifyContext(state) {
|
|
|
543
629
|
// From the process, not the state file: the browser tools and this one are
|
|
544
630
|
// registered on the same server, so captures and reader share a process.
|
|
545
631
|
uiCaptures: capturedBlueprintDigests(),
|
|
632
|
+
// From the process for the same reason: this server rendered the prompt, so it
|
|
633
|
+
// already holds the title and description the run was given.
|
|
634
|
+
pullRequest: recordedPullRequest(),
|
|
546
635
|
// From the analyze result on the state file, across every repository: a
|
|
547
636
|
// removal guard names an element that renders nowhere, so this list is the
|
|
548
637
|
// only thing that can tell an honest one from an invented element.
|
|
@@ -561,7 +650,7 @@ export async function buildVerifyContext(state) {
|
|
|
561
650
|
export function renderPlanResult(result) {
|
|
562
651
|
const lines = [];
|
|
563
652
|
lines.push(`Registration ${result.plan.registrationNumber} stored as this run's plan.`);
|
|
564
|
-
lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s).`);
|
|
653
|
+
lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s), ${(result.plan.defects ?? []).length} defect(s).`);
|
|
565
654
|
// A convenience: `differsFrom` also accepts a `scenarioName`.
|
|
566
655
|
for (const plannedTest of result.plan.plannedTests) {
|
|
567
656
|
lines.push(`- ${plannedTest.plannedTestId} — ${plannedTest.scenario?.scenarioName ?? "(unnamed)"}`);
|