@skyramp/mcp 0.3.8 → 0.4.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/commandLibrary.d.ts +1 -1
- package/build/commands/commandLibrary.js +3 -3
- package/build/commands/recommendTestsAndExecuteCommand.d.ts +1 -1
- package/build/commands/recommendTestsAndExecuteCommand.js +35 -20
- package/build/commands/testThisEndpointCommand.js +35 -19
- package/build/index.js +9 -3
- package/build/playwright/blueprintDigest.d.ts +15 -0
- package/build/playwright/blueprintDigest.js +152 -0
- package/build/playwright/blueprintDigestStore.d.ts +31 -0
- package/build/playwright/blueprintDigestStore.js +117 -0
- package/build/playwright/registerPlaywrightTools.js +60 -12
- package/build/playwright/traceRecordingPrompt.js +8 -7
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +9 -8
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -2
- package/build/prompts/promptAssets.d.ts +20 -0
- package/build/prompts/promptAssets.js +55 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +19 -5
- package/build/prompts/test-maintenance/actionsInstructions.d.ts +4 -0
- package/build/prompts/test-maintenance/actionsInstructions.js +14 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +0 -10
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +2 -11
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +8 -4
- package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +5 -22
- package/build/prompts/test-recommendation/diffExecutionPlan.js +37 -465
- package/build/prompts/test-recommendation/recommendationSections.d.ts +7 -17
- package/build/prompts/test-recommendation/recommendationSections.js +67 -309
- package/build/prompts/test-recommendation/recommendationShared.d.ts +19 -47
- package/build/prompts/test-recommendation/recommendationShared.js +49 -155
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +0 -5
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +10 -153
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -29
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +32 -457
- package/build/prompts/testbot/planDeclarations.d.ts +6 -0
- package/build/prompts/testbot/planDeclarations.js +9 -0
- package/build/prompts/testbot/testbot-prompts.d.ts +8 -0
- package/build/prompts/testbot/testbot-prompts.js +256 -381
- package/build/recommendation/answers.d.ts +35 -0
- package/build/recommendation/answers.js +96 -0
- package/build/recommendation/registerPlan.d.ts +49 -0
- package/build/recommendation/registerPlan.js +117 -0
- package/build/recommendation/runVerifiers.d.ts +10 -0
- package/build/recommendation/runVerifiers.js +49 -0
- package/build/recommendation/subjectStep.d.ts +42 -0
- package/build/recommendation/subjectStep.js +86 -0
- package/build/recommendation/types.d.ts +163 -0
- package/build/recommendation/types.js +20 -0
- package/build/recommendation/verifierContracts.d.ts +382 -0
- package/build/recommendation/verifierContracts.js +263 -0
- package/build/recommendation/verifiers/changedFile.d.ts +2 -0
- package/build/recommendation/verifiers/changedFile.js +82 -0
- package/build/recommendation/verifiers/citedPath.d.ts +12 -0
- package/build/recommendation/verifiers/citedPath.js +35 -0
- package/build/recommendation/verifiers/coverage.d.ts +7 -0
- package/build/recommendation/verifiers/coverage.js +617 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +11 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +33 -0
- package/build/recommendation/verifiers/endpointGrounded.d.ts +17 -0
- package/build/recommendation/verifiers/endpointGrounded.js +128 -0
- package/build/recommendation/verifiers/existingCoverage.d.ts +6 -0
- package/build/recommendation/verifiers/existingCoverage.js +51 -0
- package/build/recommendation/verifiers/expectedOutcome.d.ts +31 -0
- package/build/recommendation/verifiers/expectedOutcome.js +105 -0
- package/build/recommendation/verifiers/removedElementGuarded.d.ts +2 -0
- package/build/recommendation/verifiers/removedElementGuarded.js +57 -0
- package/build/recommendation/verifiers/reportedCategory.d.ts +26 -0
- package/build/recommendation/verifiers/reportedCategory.js +84 -0
- package/build/recommendation/verifiers/screenRoute.d.ts +10 -0
- package/build/recommendation/verifiers/screenRoute.js +118 -0
- package/build/recommendation/verifiers/statedDifference.d.ts +6 -0
- package/build/recommendation/verifiers/statedDifference.js +140 -0
- package/build/recommendation/verifiers/uiElementGrounded.d.ts +7 -0
- package/build/recommendation/verifiers/uiElementGrounded.js +318 -0
- package/build/resources/analysisResources.js +1 -114
- package/build/resources/testbotResource.js +23 -13
- package/build/services/ModularizationService.js +2 -1
- package/build/services/TestDiscoveryService.d.ts +3 -72
- package/build/services/TestDiscoveryService.js +10 -303
- package/build/services/containerEnv.d.ts +1 -1
- package/build/services/containerEnv.js +12 -0
- package/build/skills/fixTestImportErrorsSkill.d.ts +13 -0
- package/build/skills/fixTestImportErrorsSkill.js +20 -0
- package/build/toolNames.d.ts +1 -0
- package/build/toolNames.js +1 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +3 -3
- package/build/tools/code-refactor/modularizationTool.js +2 -1
- package/build/tools/executeSkyrampTestTool.d.ts +80 -0
- package/build/tools/executeSkyrampTestTool.js +246 -19
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +6 -0
- package/build/tools/generate-tests/generateContractRestTool.js +3 -3
- package/build/tools/generate-tests/planGuard.d.ts +2 -2
- package/build/tools/generate-tests/planGuard.js +78 -18
- package/build/tools/one-click/oneClickTool.d.ts +0 -1
- package/build/tools/one-click/oneClickTool.js +0 -5
- package/build/tools/submitReportTool.d.ts +48 -42
- package/build/tools/submitReportTool.js +576 -193
- package/build/tools/test-management/actionsTool.js +72 -4
- package/build/tools/test-management/analyzeChangesTool.d.ts +144 -48
- package/build/tools/test-management/analyzeChangesTool.js +212 -1219
- package/build/tools/test-management/analyzeTestHealthTool.js +13 -24
- package/build/tools/test-management/index.d.ts +1 -0
- package/build/tools/test-management/index.js +1 -0
- package/build/tools/test-management/registerTestPlanTool.d.ts +795 -172
- package/build/tools/test-management/registerTestPlanTool.js +609 -542
- package/build/tools/test-management/resolveScreenTool.d.ts +75 -0
- package/build/tools/test-management/resolveScreenTool.js +289 -0
- package/build/types/BlueprintDigest.d.ts +34 -0
- package/build/types/BlueprintDigest.js +1 -0
- package/build/types/RepositoryAnalysis.d.ts +20 -1559
- package/build/types/RepositoryAnalysis.js +2 -58
- package/build/types/StepMethod.d.ts +40 -0
- package/build/types/StepMethod.js +77 -0
- package/build/types/TestAnalysis.d.ts +12 -0
- package/build/types/TestExecution.d.ts +4 -0
- package/build/types/TestRecommendation.d.ts +24 -24
- package/build/types/TestRecommendation.js +91 -89
- package/build/types/TestbotPromptOptions.d.ts +0 -4
- package/build/types/TestbotReport.d.ts +64 -2
- package/build/utils/AnalysisStateManager.d.ts +79 -113
- package/build/utils/AnalysisStateManager.js +147 -57
- package/build/utils/assertion-verify/api-shared-lints.js +1 -1
- package/build/utils/assertion-verify/metrics.js +85 -36
- package/build/utils/assertion-verify/ui-lints.d.ts +0 -5
- package/build/utils/assertion-verify/ui-lints.js +32 -0
- package/build/utils/branchDiff.d.ts +63 -31
- package/build/utils/branchDiff.js +242 -94
- package/build/utils/containedPath.d.ts +18 -0
- package/build/utils/containedPath.js +73 -0
- package/build/utils/dartRouteExtractor.d.ts +18 -34
- package/build/utils/dartRouteExtractor.js +101 -173
- package/build/utils/featureFlags.d.ts +12 -0
- package/build/utils/featureFlags.js +14 -0
- package/build/utils/frontendSelectors.d.ts +48 -27
- package/build/utils/frontendSelectors.js +241 -80
- package/build/utils/pathMatching.d.ts +2 -4
- package/build/utils/pathMatching.js +2 -4
- package/build/utils/planMatchKeys.d.ts +38 -47
- package/build/utils/planMatchKeys.js +143 -81
- package/build/utils/rebaselineSnapshots.d.ts +24 -0
- package/build/utils/rebaselineSnapshots.js +65 -0
- package/build/utils/removedUiElements.d.ts +22 -0
- package/build/utils/removedUiElements.js +106 -0
- package/build/utils/reportVerification.d.ts +2 -6
- package/build/utils/reportVerification.js +61 -2
- package/build/utils/screenRoutes.d.ts +66 -0
- package/build/utils/screenRoutes.js +727 -0
- package/build/utils/sourceRouteExtractor.js +320 -112
- package/build/utils/testFileClassification.d.ts +11 -2
- package/build/utils/testFileClassification.js +44 -2
- package/build/utils/testFixtures.d.ts +5 -0
- package/build/utils/testFixtures.js +13 -0
- package/build/utils/utils.d.ts +0 -1
- package/build/utils/utils.js +0 -11
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/workspace/workspace.d.ts +12 -12
- package/node_modules/playwright/lib/mcp/skyramp/assertHiddenTool.js +56 -0
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +2 -1
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +4 -1
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +160 -1
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderSignalProcessor.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder.js +5 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.-Id052Lr.js → index.B7KbSQcC.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderSignalProcessor.ts +7 -0
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder.ts +6 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +4 -3
- package/plugin/.claude-plugin/plugin.json +8 -0
- package/plugin/plugin.json +6 -0
- package/plugin/prompts/declaring-a-plan.md +20 -0
- package/plugin/prompts/generate-tests/context-fetching.md +4 -0
- package/plugin/prompts/generate-tests/execution-plan.md +63 -0
- package/plugin/prompts/generate-tests/generation.md +108 -0
- package/plugin/prompts/generate-tests/path-parameters.md +1 -0
- package/plugin/prompts/generate-tests/reasoning-protocol.md +17 -0
- package/plugin/prompts/generate-tests/tool-workflow-variants.md +61 -0
- package/plugin/prompts/generate-tests/tool-workflows.md +65 -0
- package/plugin/prompts/plan-tests.md +42 -0
- package/plugin/prompts/testbot-task1.md +82 -0
- package/plugin/skills/fix-test-import-errors/SKILL.md +98 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.d.ts +0 -84
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +0 -369
- package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +0 -7
- package/build/prompts/test-recommendation/fullRepoCatalog.js +0 -283
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +0 -81
- package/build/prompts/test-recommendation/scopeAssessment.js +0 -359
- package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +0 -7
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +0 -105
- package/build/recommendation/budgeters/fixedNBudgeter.d.ts +0 -7
- package/build/recommendation/budgeters/fixedNBudgeter.js +0 -11
- package/build/recommendation/budgeters/shared.d.ts +0 -32
- package/build/recommendation/budgeters/shared.js +0 -246
- package/build/recommendation/discriminators.d.ts +0 -37
- package/build/recommendation/discriminators.js +0 -379
- package/build/recommendation/diversity.d.ts +0 -47
- package/build/recommendation/diversity.js +0 -101
- package/build/recommendation/planRanker.d.ts +0 -65
- package/build/recommendation/planRanker.js +0 -83
- package/build/recommendation/testFixtures.d.ts +0 -25
- package/build/recommendation/testFixtures.js +0 -45
- package/build/types/FrontendIntegration.d.ts +0 -28
- package/build/types/FrontendIntegration.js +0 -22
- package/build/types/Recommendation.d.ts +0 -146
- package/build/types/Recommendation.js +0 -74
- package/build/utils/changedRoutes.d.ts +0 -29
- package/build/utils/changedRoutes.js +0 -87
- package/build/utils/frontendIntegration.d.ts +0 -9
- package/build/utils/frontendIntegration.js +0 -243
- package/build/utils/importerHop.d.ts +0 -135
- package/build/utils/importerHop.js +0 -489
- package/build/utils/pathAffinityClassification.d.ts +0 -49
- package/build/utils/pathAffinityClassification.js +0 -180
- package/build/utils/pythonMountPrefixes.d.ts +0 -25
- package/build/utils/pythonMountPrefixes.js +0 -347
- package/build/utils/repoScanner.d.ts +0 -34
- package/build/utils/repoScanner.js +0 -300
- package/build/utils/routeParsers.d.ts +0 -95
- package/build/utils/routeParsers.js +0 -951
- package/build/utils/scenarioDrafting.d.ts +0 -92
- package/build/utils/scenarioDrafting.js +0 -951
- package/build/utils/subjectEndpoints.d.ts +0 -19
- package/build/utils/subjectEndpoints.js +0 -98
- package/build/utils/uiPageEnumerator.d.ts +0 -172
- package/build/utils/uiPageEnumerator.js +0 -474
|
@@ -1,608 +1,675 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import
|
|
2
|
+
import * as fs from "fs";
|
|
3
3
|
import * as path from "path";
|
|
4
|
-
import
|
|
4
|
+
import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
|
|
5
5
|
import { logger } from "../../utils/logger.js";
|
|
6
|
-
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
6
|
+
import { StateManager, resolveRunStatePath } from "../../utils/AnalysisStateManager.js";
|
|
7
7
|
import { toolError } from "../../utils/utils.js";
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
|
|
8
|
+
import { TestType } from "../../types/TestTypes.js";
|
|
9
|
+
import { changedFileNames, computeBranchDiff } from "../../utils/branchDiff.js";
|
|
10
|
+
import { LOCKFILE, NOT_SOURCE_DIR } from "../../recommendation/verifiers/endpointGrounded.js";
|
|
11
|
+
import { hasUITestTypeLabel, isHTTPStep } from "../../recommendation/subjectStep.js";
|
|
12
|
+
import { STEP_METHOD_DESCRIPTION, normalizeStepMethod, stepMethodSchema } from "../../types/StepMethod.js";
|
|
13
|
+
import { registerPlan } from "../../recommendation/registerPlan.js";
|
|
14
|
+
import { targetElementSchema } from "../submitReportTool.js";
|
|
15
|
+
import { capturedBlueprintDigests } from "../../playwright/blueprintDigestStore.js";
|
|
16
|
+
import { fileExistsWithinRoot } from "../../utils/containedPath.js";
|
|
17
|
+
import { createScreenRouteResolver } from "../../utils/screenRoutes.js";
|
|
18
|
+
import { categoryMenu, SCENARIO_CATEGORIES } from "../../types/TestRecommendation.js";
|
|
20
19
|
const TOOL_NAME = TOOL_REGISTER_TEST_PLAN;
|
|
21
20
|
// ── Zod schema ──────────────────────────────────────────────────────────────
|
|
22
|
-
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
const
|
|
26
|
-
TestType.CONTRACT,
|
|
27
|
-
TestType.INTEGRATION,
|
|
28
|
-
TestType.E2E,
|
|
29
|
-
TestType.UI,
|
|
30
|
-
]);
|
|
31
|
-
const chainingRefSchema = z.object({
|
|
32
|
-
sourceStep: z.number().int(),
|
|
33
|
-
sourceField: z.string(),
|
|
34
|
-
sourceLocation: z.enum(["body", "header", "cookie"]),
|
|
35
|
-
targetParam: z.string(),
|
|
36
|
-
targetLocation: z.enum(["path", "body", "query", "header", "cookie"]),
|
|
37
|
-
});
|
|
21
|
+
/** One sentence, shared by the step fields that carry a value and by the case
|
|
22
|
+
* that names one, so the two sides state the same convention. A plan that spells
|
|
23
|
+
* the same created value two different ways reads as a case no step sends. */
|
|
24
|
+
const CREATED_VALUE_PLACEHOLDER = "Write a value an earlier step creates as a placeholder in braces, and use the same name wherever else the test names it.";
|
|
38
25
|
const registerStepSchema = z.object({
|
|
39
|
-
order: z.number().int().
|
|
40
|
-
method:
|
|
41
|
-
path: z
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
.
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
.optional()
|
|
49
|
-
.describe("Request body fields with concrete values. Used to verify boundary_equality claims (two distinct fields with equal values)."),
|
|
26
|
+
order: z.number().int().positive().describe("This step's position in the scenario, from 1. `stepUnderTest` and `routes` name a step by this number."),
|
|
27
|
+
method: stepMethodSchema.describe(`${STEP_METHOD_DESCRIPTION} A \`ui\` or \`e2e\` step is a page interaction.`),
|
|
28
|
+
path: z
|
|
29
|
+
.string()
|
|
30
|
+
.refine((value) => value.trim().length > 0, { message: "path must not be blank" })
|
|
31
|
+
.describe("Request path as the running service serves it, mount prefix included — `/ghost/api/admin/site/stats`, not the `/site/stats` the router file declares. For a page interaction, its target instead (`checkout button`); for an `OPERATION` step, the operation's name."),
|
|
32
|
+
description: z.string().optional().describe("What this step does, in one line."),
|
|
33
|
+
interactionType: z.string().optional().describe("success, error, or another label for what this step exercises."),
|
|
34
|
+
requestBody: z.unknown().optional().describe(`Request payload for this step, when it sends one. ${CREATED_VALUE_PLACEHOLDER}`),
|
|
50
35
|
queryParams: z
|
|
51
|
-
.
|
|
52
|
-
.optional()
|
|
53
|
-
.describe("Query parameters with concrete values. Also scanned for boundary_equality and negative_match verification."),
|
|
54
|
-
responseBody: z
|
|
55
|
-
.union([z.record(z.any()), z.array(z.any())])
|
|
36
|
+
.unknown()
|
|
56
37
|
.optional()
|
|
57
|
-
.describe(
|
|
58
|
-
|
|
59
|
-
expectedResponseFields: z
|
|
38
|
+
.describe(`Query-string parameters for this step, when it sends any. ${CREATED_VALUE_PLACEHOLDER}`),
|
|
39
|
+
omits: z
|
|
60
40
|
.array(z.string())
|
|
41
|
+
.min(1)
|
|
42
|
+
.optional()
|
|
43
|
+
.describe("Parameters this step leaves out on purpose."),
|
|
44
|
+
expectedStatusCode: z.number().int().optional().describe("Status code this step expects."),
|
|
45
|
+
})
|
|
46
|
+
// A `navigate` step's path is the screen the runner opens, so it has to be a
|
|
47
|
+
// thing a browser can be pointed at. Runs have written "the login page" there,
|
|
48
|
+
// which no runner can open and no later check refused.
|
|
49
|
+
.refine((step) => {
|
|
50
|
+
if (normalizeStepMethod(step.method) !== "navigate")
|
|
51
|
+
return true;
|
|
52
|
+
const target = step.path.trim().toLowerCase();
|
|
53
|
+
return target.startsWith("/") || target.startsWith("http");
|
|
54
|
+
}, { message: "a navigate step's path is a URL path such as /login", path: ["path"] });
|
|
55
|
+
// The fields `declarations` accepts. Named on its own so one list of them can be
|
|
56
|
+
// derived from it — a hand-written second copy had already fallen behind.
|
|
57
|
+
export const declarationFieldsSchema = z.object({
|
|
58
|
+
changedFile: z
|
|
59
|
+
.string()
|
|
61
60
|
.optional()
|
|
62
|
-
.describe("
|
|
63
|
-
|
|
61
|
+
.describe("The changed file this test is about, repository-relative and spelled as the diff spells it — `src/routes/orders.ts`, not `./src/routes/orders.ts`. Checked: the name is one this pull request changed. Not checked: which line, so name the changed line in `description`."),
|
|
62
|
+
changes: z
|
|
64
63
|
.array(z.string())
|
|
65
64
|
.optional()
|
|
66
|
-
.describe("
|
|
67
|
-
|
|
68
|
-
});
|
|
69
|
-
const discriminatorClaimSchema = z.object({
|
|
70
|
-
kind: z.nativeEnum(DiscriminatorKind),
|
|
71
|
-
changedCodeAnchor: z
|
|
65
|
+
.describe("The ids of the changes that this test proves. Take the ids from your `changes` list. This test proves one behaviour: list the id of every change that the behaviour covers. The server checks that each declared change has a planned test that cites it, or an answer. The server does not check that the test exercises the change."),
|
|
66
|
+
asserts: z
|
|
72
67
|
.string()
|
|
73
|
-
.
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
68
|
+
.describe("What this test checks."),
|
|
69
|
+
stepUnderTest: z
|
|
70
|
+
.number()
|
|
71
|
+
.int()
|
|
72
|
+
.optional()
|
|
73
|
+
.describe("The `order` of the step this test is really about. Required whenever the planned test makes more than one call; one call is its own subject. In a create-then-verify pair the write is the subject, not the read-back. Two checks read it; nothing infers it."),
|
|
74
|
+
expected: z
|
|
75
|
+
.object({
|
|
76
|
+
outcome: z
|
|
77
|
+
.enum(["fail", "pass"])
|
|
78
|
+
.describe("Whether this test should fail or pass against the app as it stands."),
|
|
79
|
+
why: z
|
|
80
|
+
.string()
|
|
81
|
+
.describe("Why it has that outcome. A test that documents a defect asserts what the FIXED code returns — the rejection, the preserved value, the correct status — so it is red today and green after the fix, never today's defective response as correct."),
|
|
82
|
+
})
|
|
83
|
+
.strict()
|
|
84
|
+
.describe("What this test does against the app as it stands, and why."),
|
|
85
|
+
differsFrom: z
|
|
86
|
+
.array(z.object({
|
|
87
|
+
plannedTestId: z.string().describe("The other planned test this one is distinct from, named by its `scenarioName`."),
|
|
88
|
+
difference: z.string().describe("How the two differ."),
|
|
89
|
+
}).strict())
|
|
90
|
+
.optional()
|
|
91
|
+
.describe("For another planned test in THIS plan whose subject step calls the same method and path, what this one asserts that it does not."),
|
|
92
|
+
existingTests: z
|
|
93
|
+
.array(z.object({
|
|
94
|
+
file: z.string().describe("Repository-relative path of an existing test you read that exercises this change."),
|
|
95
|
+
differsBy: z.string().describe("What this planned test checks that the existing test does not."),
|
|
96
|
+
}).strict())
|
|
97
|
+
.optional()
|
|
98
|
+
.describe("The existing tests you read on this change. Naming none is an answer. Checked: each file named is in the checkout. Not checked: what the file contains."),
|
|
99
|
+
startState: z
|
|
100
|
+
.string()
|
|
101
|
+
.optional()
|
|
102
|
+
.describe("The state the record this test mutates is in when the mutation runs, in one sentence. A record with no rows yet is a state. The test puts the record in that state itself. Checked: present. Not checked: the sentence."),
|
|
103
|
+
routes: z
|
|
104
|
+
.array(z.object({
|
|
105
|
+
step: z.number().int().describe("The `order` of the step this file declares."),
|
|
106
|
+
file: z.string().describe("Repository-relative path of the file that declares what that step calls."),
|
|
107
|
+
}).strict())
|
|
86
108
|
.optional()
|
|
87
|
-
.describe("
|
|
109
|
+
.describe("One entry per step that calls the app, naming the repository file that declares what it calls. Checked: the file is in the checkout. Not checked: whether the route appears in it — frameworks build paths from scopes and prefixes, so it often does not."),
|
|
110
|
+
screenEvidence: z
|
|
111
|
+
.object({
|
|
112
|
+
file: z.string().describe("Repository-relative path of the changed file that renders this screen."),
|
|
113
|
+
})
|
|
114
|
+
.strict()
|
|
115
|
+
.optional()
|
|
116
|
+
.describe("UI planned tests only: the changed file that renders the screen `elements.pageUrl` opens. Checked: it is one of the files this pull request changed. Not checked: whether it really renders that screen."),
|
|
117
|
+
// The report's own schemas, reused so the two spellings cannot drift.
|
|
118
|
+
elements: z
|
|
119
|
+
.object({
|
|
120
|
+
pageUrl: z
|
|
121
|
+
.string()
|
|
122
|
+
.optional()
|
|
123
|
+
.describe("The page this test opens, spelled either as a route path starting with `/` or as an absolute `http` URL. Required on a `ui` or `e2e` planned test whose steps make no HTTP call."),
|
|
124
|
+
items: z
|
|
125
|
+
.array(targetElementSchema)
|
|
126
|
+
.nullish()
|
|
127
|
+
.describe("The elements this test targets, each one copied out of a `browser_blueprint` captured during this run. `null` when no capture holds them. Checked: each entry appears in a capture this run took of `pageUrl`. Not checked: whether the test needs that element."),
|
|
128
|
+
})
|
|
129
|
+
.strict()
|
|
130
|
+
.nullish()
|
|
131
|
+
.describe("UI planned tests only: the page this test opens and the elements it targets on it."),
|
|
132
|
+
});
|
|
133
|
+
// STRICT, here and on the planned test around it: zod strips an unrecognised key,
|
|
134
|
+
// so a misplaced declaration vanished and the check then objected to its absence.
|
|
135
|
+
const declarationsSchema = declarationFieldsSchema
|
|
136
|
+
.strict()
|
|
137
|
+
.superRefine((declared, ctx) => {
|
|
138
|
+
// Refuse a malformed shape; a judgment call gets an objection instead. An entry
|
|
139
|
+
// naming no element cannot be looked up, so there is nothing to report on.
|
|
140
|
+
const items = declared.elements?.items;
|
|
141
|
+
if (!Array.isArray(items))
|
|
142
|
+
return;
|
|
143
|
+
items.forEach((element, index) => {
|
|
144
|
+
const identifies = [element.accessibleName, element.testId, element.stableId].some((value) => typeof value === "string" && value.trim().length > 0);
|
|
145
|
+
if (identifies)
|
|
146
|
+
return;
|
|
147
|
+
ctx.addIssue({
|
|
148
|
+
code: z.ZodIssueCode.custom,
|
|
149
|
+
path: ["elements", "items", index],
|
|
150
|
+
message: "every `elements.items` entry needs at least one of `accessibleName`, `testId` or `stableId` with a value in it — an entry with none names no element that can be looked up in a capture.",
|
|
151
|
+
});
|
|
152
|
+
});
|
|
88
153
|
});
|
|
154
|
+
// A plan drafts only these four; `TestType`'s others are maintenance outcomes.
|
|
155
|
+
const PLAN_TEST_TYPES = [TestType.CONTRACT, TestType.INTEGRATION, TestType.E2E, TestType.UI];
|
|
156
|
+
/** Whether an `elements.pageUrl` names a screen `uiElementGrounded` can look up: a
|
|
157
|
+
* route path or an absolute http URL. Constrained here and never on the shared
|
|
158
|
+
* `pageContextSchema`, which the report fills from a capture carrying any url. */
|
|
159
|
+
function namesAScreen(url) {
|
|
160
|
+
if (url.startsWith("/"))
|
|
161
|
+
return true;
|
|
162
|
+
try {
|
|
163
|
+
return ["http:", "https:"].includes(new URL(url).protocol);
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/** The fields that belong under `declarations`. Read by the hoist below only.
|
|
170
|
+
* Derived from the schema, not written out again: the hand-written copy was
|
|
171
|
+
* missing `stepUnderTest`, so a stray one stayed at planned test level and zod
|
|
172
|
+
* dropped it. */
|
|
173
|
+
const DECLARATION_FIELDS = Object.keys(declarationFieldsSchema.shape);
|
|
174
|
+
function isPlainObject(value) {
|
|
175
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
176
|
+
}
|
|
177
|
+
/** One value as text with object keys sorted, so the same declaration written
|
|
178
|
+
* twice by hand is not a conflict merely because its keys are ordered
|
|
179
|
+
* differently. List order still counts: two orders are two different lists. */
|
|
180
|
+
function canonicalJson(value) {
|
|
181
|
+
return JSON.stringify(value, (_key, entry) => isPlainObject(entry)
|
|
182
|
+
? Object.fromEntries(Object.keys(entry).sort().map((key) => [key, entry[key]]))
|
|
183
|
+
: entry);
|
|
184
|
+
}
|
|
185
|
+
/** Move a declaration field written one level too high into `declarations`.
|
|
186
|
+
* MEASURED: about one planned test in five did this across 141 eval planned tests, and
|
|
187
|
+
* every stray planned test-level key was a declaration field. A field in BOTH places
|
|
188
|
+
* with different values stays put, so `.strict()` refuses it by name. */
|
|
189
|
+
export function hoistDeclarationFields(value) {
|
|
190
|
+
if (!isPlainObject(value))
|
|
191
|
+
return value;
|
|
192
|
+
// A non-object `declarations` is left as written: hoisting would discard it
|
|
193
|
+
// and the error would name a field the agent never wrote.
|
|
194
|
+
if (value.declarations !== undefined && !isPlainObject(value.declarations))
|
|
195
|
+
return value;
|
|
196
|
+
const plannedTest = { ...value };
|
|
197
|
+
const original = isPlainObject(plannedTest.declarations) ? plannedTest.declarations : undefined;
|
|
198
|
+
const declarations = { ...(original ?? {}) };
|
|
199
|
+
let hoisted = false;
|
|
200
|
+
for (const field of DECLARATION_FIELDS) {
|
|
201
|
+
if (!(field in plannedTest))
|
|
202
|
+
continue;
|
|
203
|
+
const atCandidateLevel = plannedTest[field];
|
|
204
|
+
if (declarations[field] === undefined) {
|
|
205
|
+
declarations[field] = atCandidateLevel;
|
|
206
|
+
delete plannedTest[field];
|
|
207
|
+
hoisted = true;
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
if (canonicalJson(declarations[field]) === canonicalJson(atCandidateLevel)) {
|
|
211
|
+
// The same value twice says the same thing; drop the copy.
|
|
212
|
+
delete plannedTest[field];
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (hoisted || original)
|
|
216
|
+
plannedTest.declarations = declarations;
|
|
217
|
+
return plannedTest;
|
|
218
|
+
}
|
|
219
|
+
const registerPlannedTestSchema = z.preprocess(hoistDeclarationFields, z.object({
|
|
220
|
+
scenarioName: z
|
|
221
|
+
.string()
|
|
222
|
+
.refine((value) => value.trim().length > 0, { message: "scenarioName must not be blank" })
|
|
223
|
+
.describe("Short name for this test, and its id: objections name it, `differsFrom` refers to it, and the report joins delivered tests on it. Keep it stable across registrations."),
|
|
224
|
+
repository: z
|
|
225
|
+
.string()
|
|
226
|
+
.optional()
|
|
227
|
+
.describe("On a multi-repo run, the `owner/repo` this test is for — the same value you passed to `skyramp_analyze_changes`."),
|
|
228
|
+
description: z.string().describe("What this test does, in one or two lines."),
|
|
229
|
+
category: z
|
|
230
|
+
.enum(SCENARIO_CATEGORIES)
|
|
231
|
+
.describe(`What kind of test this is. Write one of:\n${categoryMenu()}`),
|
|
232
|
+
testType: z
|
|
233
|
+
.enum(PLAN_TEST_TYPES)
|
|
234
|
+
.describe("contract, integration, e2e or ui."),
|
|
235
|
+
steps: z
|
|
236
|
+
.array(registerStepSchema)
|
|
237
|
+
.describe("The requests this test makes, in order. At least one, unless `testType` is `ui` or `e2e` and the test makes no HTTP call."),
|
|
238
|
+
declarations: declarationsSchema
|
|
239
|
+
.optional()
|
|
240
|
+
.describe("The changes that make this planned test checkable. Every change belongs under THIS key; one written beside `steps` is moved here for you and reported as `declarations.<field>`."),
|
|
241
|
+
})
|
|
242
|
+
.strict()
|
|
243
|
+
.superRefine((plannedTest, ctx) => {
|
|
244
|
+
// Three checks read the steps for the subject endpoint, so an empty list
|
|
245
|
+
// passed all three in silence.
|
|
246
|
+
const isUILabel = hasUITestTypeLabel(plannedTest.testType);
|
|
247
|
+
if (plannedTest.steps.length === 0 && !isUILabel) {
|
|
248
|
+
ctx.addIssue({
|
|
249
|
+
code: z.ZodIssueCode.custom,
|
|
250
|
+
path: ["steps"],
|
|
251
|
+
message: "steps must not be empty for a contract, integration or other non-UI planned test: the route, duplicate and existing-coverage checks all read the steps to find the endpoint this test is about. List the requests the test makes.",
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
// Ruling A: a ui or e2e planned test whose steps make no HTTP call opens a
|
|
255
|
+
// screen instead of a request, so it must name that screen. Empty steps
|
|
256
|
+
// stay legal for it — a planned test naming no request and no screen is not
|
|
257
|
+
// a plan the server can read either.
|
|
258
|
+
const hasHTTPStep = plannedTest.steps.some((step) => isHTTPStep(step));
|
|
259
|
+
const rawUrl = plannedTest.declarations?.elements?.pageUrl;
|
|
260
|
+
const url = typeof rawUrl === "string" ? rawUrl.trim() : "";
|
|
261
|
+
if (isUILabel && !hasHTTPStep && url.length === 0) {
|
|
262
|
+
ctx.addIssue({
|
|
263
|
+
code: z.ZodIssueCode.custom,
|
|
264
|
+
path: ["declarations", "elements", "pageUrl"],
|
|
265
|
+
message: "a ui or e2e planned test whose steps make no HTTP call must set `declarations.elements.pageUrl` to the screen it opens: a planned test naming no request and no screen is not a plan the server can read.",
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
// The other half of Ruling A. Naming a screen and naming SOMETHING are
|
|
269
|
+
// different obligations: a one-word url passed this rule, and the page
|
|
270
|
+
// match and the relatedness check both compare it as written, so it named
|
|
271
|
+
// no page either could look up and drew no objection from either.
|
|
272
|
+
if (isUILabel && url.length > 0 && !namesAScreen(url)) {
|
|
273
|
+
ctx.addIssue({
|
|
274
|
+
code: z.ZodIssueCode.custom,
|
|
275
|
+
path: ["declarations", "elements", "pageUrl"],
|
|
276
|
+
message: "`declarations.elements.pageUrl` is accepted in two spellings and no others: a route path starting with `/`, e.g. `/orders/1`, or an absolute `http` or `https` URL, e.g. `http://localhost:5173/orders/1`. Give the path when this run has no host to name.",
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}));
|
|
89
280
|
const registerTestPlanSchema = {
|
|
90
281
|
stateFile: z
|
|
91
282
|
.string()
|
|
92
283
|
.refine((p) => path.isAbsolute(p), { message: "stateFile must be an absolute path" })
|
|
93
|
-
.
|
|
94
|
-
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
//
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
284
|
+
.optional()
|
|
285
|
+
.describe("Optional. A Testbot run finds its own state file, so leave this out. Give an absolute path only to name a state file the run would not find by itself."),
|
|
286
|
+
// `.default([])`, where `plannedTests` is required: an empty list here cannot pass
|
|
287
|
+
// in silence the way an empty planned test list did. It raises `coverage:noChanges`,
|
|
288
|
+
// which is exactly the objection a plan that states no change should draw.
|
|
289
|
+
changes: z
|
|
290
|
+
.array(z
|
|
291
|
+
.object({
|
|
292
|
+
id: z
|
|
293
|
+
.string()
|
|
294
|
+
.refine((value) => value.trim().length > 0, { message: "a change id must not be blank" })
|
|
295
|
+
.describe("Short slug for this change. A planned test cites it by this id."),
|
|
296
|
+
text: z.string().describe("The change in one sentence, as the pull request must make it."),
|
|
297
|
+
source: z
|
|
298
|
+
.string()
|
|
299
|
+
.describe("Where you read it: `pr-description`, `spec:<path>` for a requirements file the description names, or `diff`."),
|
|
300
|
+
surfaces: z
|
|
301
|
+
.array(z.enum(["api", "page"]))
|
|
302
|
+
.min(1)
|
|
303
|
+
.describe("Where a user or caller meets this change: `api` for a request and response, `page` for something rendered, both when both."),
|
|
304
|
+
cases: z
|
|
305
|
+
.array(z
|
|
306
|
+
.object({
|
|
307
|
+
param: z.string().describe("The parameter this case sets."),
|
|
308
|
+
value: z
|
|
309
|
+
.union([z.string(), z.number(), z.boolean(), z.null()])
|
|
310
|
+
.optional()
|
|
311
|
+
.describe(`The value a test sends for that parameter. ${CREATED_VALUE_PLACEHOLDER}`),
|
|
312
|
+
absent: z.literal(true).optional().describe("The parameter is left out of the request."),
|
|
313
|
+
expect: z
|
|
314
|
+
.enum(["accept", "reject"])
|
|
315
|
+
.describe("Whether the route must accept this value or reject it."),
|
|
316
|
+
})
|
|
317
|
+
.strict()
|
|
318
|
+
.refine((entry) => (entry.value !== undefined) !== (entry.absent === true), {
|
|
319
|
+
message: "a case states either `value` or `absent`, never both and never neither",
|
|
320
|
+
}))
|
|
321
|
+
.min(1, { message: "a `cases` list with no entry states no input — leave it out instead" })
|
|
322
|
+
.optional()
|
|
323
|
+
.describe("The inputs that prove this change: for each parameter the change constrains, the values the route must accept and the values it must reject. A range gives the last accepted value and the first rejected one at each end; a set gives each allowed value and one outside it; a required field gives the field left out. A case names a query parameter, a body field, a path segment, or `method` for the HTTP verb; the steps of the test that cites this change say which route it is, and a rejection that depends on who calls belongs in the change text instead."),
|
|
324
|
+
})
|
|
325
|
+
.strict())
|
|
326
|
+
.superRefine((changes, ctx) => {
|
|
327
|
+
const first = new Map();
|
|
328
|
+
changes.forEach((change, index) => {
|
|
329
|
+
const key = String(change?.id ?? "").trim();
|
|
330
|
+
if (!key)
|
|
331
|
+
return;
|
|
332
|
+
const seen = first.get(key);
|
|
333
|
+
if (seen === undefined) {
|
|
334
|
+
first.set(key, index);
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
ctx.addIssue({
|
|
338
|
+
code: z.ZodIssueCode.custom,
|
|
339
|
+
path: [index, "id"],
|
|
340
|
+
message: `change id "${change.id}" is already used by change ${seen + 1}. Every change needs its own id — planned tests cite it by that id.`,
|
|
341
|
+
});
|
|
342
|
+
});
|
|
343
|
+
})
|
|
344
|
+
.default([])
|
|
345
|
+
.describe("The changes the pull request must make, read out of the pull request title and description, a requirements file they name, or the diff itself. Coverage per file is the basic minimum; coverage per change is the target, and anything more is a bonus."),
|
|
346
|
+
// REQUIRED, not `.default([])`. The MCP SDK wraps this shape in a plain
|
|
347
|
+
// `z.object`, which STRIPS an undeclared top-level key, so with a default
|
|
348
|
+
// `planned test:` for `planned tests:` stored an EMPTY plan and counted the
|
|
349
|
+
// registration. A run that warrants no test sends `[]` deliberately.
|
|
350
|
+
plannedTests: z
|
|
351
|
+
.array(registerPlannedTestSchema)
|
|
352
|
+
// A duplicated `scenarioName` is REFUSED here rather than objected to later,
|
|
353
|
+
// because the name is the planned test's id: two entries carrying one id share
|
|
354
|
+
// every objection about them and one answer closes both. Compared on the
|
|
355
|
+
// TRIMMED name and otherwise exactly — case is NOT folded, because the report
|
|
356
|
+
// joins on the exact id, and folding here would make "Login" and "LOGIN" one
|
|
357
|
+
// name to the plan and two ids to the report.
|
|
358
|
+
.superRefine((plannedTests, ctx) => {
|
|
359
|
+
const seen = new Map();
|
|
360
|
+
plannedTests.forEach((plannedTest, index) => {
|
|
361
|
+
const key = String(plannedTest?.scenarioName ?? "").trim();
|
|
362
|
+
if (!key)
|
|
363
|
+
return;
|
|
364
|
+
const first = seen.get(key);
|
|
365
|
+
if (first === undefined) {
|
|
366
|
+
seen.set(key, index);
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
ctx.addIssue({
|
|
370
|
+
code: z.ZodIssueCode.custom,
|
|
371
|
+
path: [index, "scenarioName"],
|
|
372
|
+
message: `scenarioName "${plannedTest.scenarioName}" is already used by planned test ${first + 1}. Every planned test needs its own name — the name is its id.`,
|
|
373
|
+
});
|
|
374
|
+
});
|
|
375
|
+
})
|
|
376
|
+
.describe("Your complete plan, unordered. Nothing ranks, dedups, reorders or trims it, and the server adds no planned test of its own."),
|
|
377
|
+
answers: z
|
|
378
|
+
.array(z.object({
|
|
379
|
+
objectionId: z.string().describe("The objection's id, exactly as this tool reported it. Answers are matched by id, never by position."),
|
|
380
|
+
answer: z.string().describe("Why the objection does not apply, or what you decided instead."),
|
|
381
|
+
blocker: z
|
|
382
|
+
.string()
|
|
383
|
+
.optional()
|
|
384
|
+
.describe("What stopped this run from writing the test: a service that is not running, a paired branch that no longer exists, the one credential the run holds. An objection about a change with no test closes only with this. A reason the change is not worth testing is not a blocker."),
|
|
385
|
+
}).strict())
|
|
102
386
|
.default([])
|
|
103
|
-
.describe("
|
|
387
|
+
.describe("One answer per objection from your previous registration that you do not intend to fix. There is no registration limit."),
|
|
104
388
|
};
|
|
105
|
-
// ──
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
function toScenarioStep(step) {
|
|
389
|
+
// ── Registration ──────────────────────────────────────────────────────────────
|
|
390
|
+
/** Map the schema-validated input onto a `PlanInput`. `subjectEndpoints` is
|
|
391
|
+
* deliberately not resolved: every verifier derives the subject from `steps`
|
|
392
|
+
* through one helper. The planned test id IS the scenario name, so an objection
|
|
393
|
+
* names the plan entry in the words the agent wrote for it. */
|
|
394
|
+
export function buildRegistration(params, registrationNumber) {
|
|
112
395
|
return {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
396
|
+
changes: params.changes ?? [],
|
|
397
|
+
answers: params.answers ?? [],
|
|
398
|
+
// The tool counts the run's registrations; whatever the caller sent is ignored.
|
|
399
|
+
registrationNumber,
|
|
400
|
+
plannedTests: (params.plannedTests ?? []).map((input) => {
|
|
401
|
+
const scenario = {
|
|
402
|
+
scenarioName: input.scenarioName,
|
|
403
|
+
description: input.description,
|
|
404
|
+
category: input.category,
|
|
405
|
+
steps: input.steps,
|
|
406
|
+
testType: input.testType,
|
|
407
|
+
};
|
|
408
|
+
const declared = input.declarations;
|
|
409
|
+
const { elements, ...rest } = declared ?? {};
|
|
410
|
+
return {
|
|
411
|
+
// TRIMMED: the report side joins on `plannedTestId.trim()`.
|
|
412
|
+
plannedTestId: input.scenarioName.trim(),
|
|
413
|
+
scenario,
|
|
414
|
+
...(input.repository ? { repository: input.repository } : {}),
|
|
415
|
+
// Passed through, never rebuilt field by field: a field the schema accepts
|
|
416
|
+
// and this function forgot to copy was dead once. Absent stays absent, so
|
|
417
|
+
// a field the agent did not write reads back as missing, not as a default
|
|
418
|
+
// the checks would then judge. `elements: null` is the model's "not
|
|
419
|
+
// applicable" habit and is dropped; `elements.items: null` MEANS capture
|
|
420
|
+
// failed and is kept.
|
|
421
|
+
declarations: { ...rest, asserts: declared?.asserts ?? "", ...(elements ? { elements } : {}) },
|
|
422
|
+
};
|
|
423
|
+
}),
|
|
125
424
|
};
|
|
126
425
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
// Resolved here, at the one boundary that has both the steps and the diff, so
|
|
140
|
-
// every later dedup stage reads one recorded value (SKYR-4214).
|
|
141
|
-
return { ...scenario, subjectEndpoints: resolveSubjectEndpoints(scenario, { changedRoutes }) };
|
|
426
|
+
/** The primary repository plus every related one, in that order. */
|
|
427
|
+
function repoSections(state) {
|
|
428
|
+
if (!state)
|
|
429
|
+
return [];
|
|
430
|
+
return [
|
|
431
|
+
{ repo: state.metadata?.repository ?? "the primary repository", repositoryPath: state.metadata?.repositoryPath, data: state },
|
|
432
|
+
...Object.entries(state.relatedRepos ?? {}).map(([repo, section]) => ({
|
|
433
|
+
repo,
|
|
434
|
+
repositoryPath: section?.repositoryPath,
|
|
435
|
+
data: section?.data,
|
|
436
|
+
})),
|
|
437
|
+
];
|
|
142
438
|
}
|
|
143
|
-
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
439
|
+
/**
|
|
440
|
+
* Whether a cited repository file EXISTS inside the checkout. Reads no content,
|
|
441
|
+
* and never throws: a bad citation is an objection, not a crashed tool.
|
|
442
|
+
*
|
|
443
|
+
* CONFINEMENT IS THIS FUNCTION'S JOB. The sibling verifier's string check misses
|
|
444
|
+
* `file://` and `http://` URLs, a drive-relative `C:routes.rb` and a
|
|
445
|
+
* percent-encoded `..`, so the containment in `containedPath` is the only thing
|
|
446
|
+
* between an agent-written path and a read outside the checkout.
|
|
447
|
+
*
|
|
448
|
+
* CASE IS THE FILESYSTEM'S RULE: a case-only misspelling passes on a
|
|
449
|
+
* case-insensitive checkout and is not found on a case-sensitive one. CI is the
|
|
450
|
+
* case-sensitive one, so that is the answer a real run gets.
|
|
451
|
+
*/
|
|
452
|
+
export function makeCitedFileCheck(state) {
|
|
453
|
+
const roots = [];
|
|
454
|
+
for (const section of repoSections(state)) {
|
|
455
|
+
// The literal "unknown" means unresolved; resolving against it would read
|
|
456
|
+
// the process working directory instead of a checkout.
|
|
457
|
+
if (!section.repositoryPath || section.repositoryPath === "unknown")
|
|
458
|
+
continue;
|
|
459
|
+
try {
|
|
460
|
+
const real = fs.realpathSync(section.repositoryPath);
|
|
461
|
+
if (!roots.includes(real))
|
|
462
|
+
roots.push(real);
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
// A repository path that no longer resolves holds no readable file.
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
return (relativePath) => {
|
|
469
|
+
if (typeof relativePath !== "string")
|
|
470
|
+
return false;
|
|
471
|
+
const cited = relativePath.trim();
|
|
472
|
+
if (!cited)
|
|
473
|
+
return false;
|
|
474
|
+
return roots.some((root) => fileExistsWithinRoot(root, cited));
|
|
475
|
+
};
|
|
161
476
|
}
|
|
162
477
|
/**
|
|
163
|
-
*
|
|
164
|
-
* the
|
|
478
|
+
* The route patterns that render a changed frontend file, across every repository
|
|
479
|
+
* of the run.
|
|
165
480
|
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
* steps, so a claim that only held for the agent's rewrite is correctly refused
|
|
171
|
-
* rather than carried across.
|
|
481
|
+
* Built like {@link makeCitedFileCheck}: one resolver per checkout, asked in turn,
|
|
482
|
+
* so a planned test does not have to say which repository its file lives in. The
|
|
483
|
+
* scan is lazy inside each resolver and reused across planned tests, so a plan with
|
|
484
|
+
* ten UI tests reads a repository once.
|
|
172
485
|
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
* earned it, on a submission that no longer makes that claim or whose new claim
|
|
178
|
-
* fails.
|
|
486
|
+
* A repository that CAN speak for the file wins: `undefined` from a resolver means
|
|
487
|
+
* "not my file, or not a framework I read", so it is skipped, and the first
|
|
488
|
+
* definite answer is returned. Every resolver saying `undefined` is the answer the
|
|
489
|
+
* check stays silent on.
|
|
179
490
|
*/
|
|
180
|
-
function
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
491
|
+
export function makeScreenRouteResolver(state) {
|
|
492
|
+
const resolvers = repoSections(state)
|
|
493
|
+
.filter((section) => section.repositoryPath && section.repositoryPath !== "unknown")
|
|
494
|
+
.map((section) => createScreenRouteResolver(section.repositoryPath));
|
|
495
|
+
return (relativePath) => {
|
|
496
|
+
let unmounted;
|
|
497
|
+
for (const resolve of resolvers) {
|
|
498
|
+
const routes = resolve(relativePath);
|
|
499
|
+
if (routes === undefined)
|
|
500
|
+
continue;
|
|
501
|
+
// A non-empty answer is a fact about a real mount; an empty one is only the
|
|
502
|
+
// answer when no repository found a mount at all.
|
|
503
|
+
if (routes.length > 0)
|
|
504
|
+
return routes;
|
|
505
|
+
unmounted = routes;
|
|
190
506
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
reason: result.reason ?? `${claim.kind} discriminator claim could not be verified.`,
|
|
194
|
-
});
|
|
195
|
-
return unmarked;
|
|
196
|
-
});
|
|
197
|
-
return { candidates: marked, demotions };
|
|
507
|
+
return unmounted;
|
|
508
|
+
};
|
|
198
509
|
}
|
|
199
|
-
/**
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
function
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
510
|
+
/** Which files each repository's branch diff changed. NAMES ONLY, never persisted.
|
|
511
|
+
* Lockfiles and non-source directories are dropped, so a planned test cannot cite
|
|
512
|
+
* `package-lock.json` and nothing demands a test for one. */
|
|
513
|
+
async function changedFilesForVerification(section) {
|
|
514
|
+
const repositoryPath = section.repositoryPath;
|
|
515
|
+
if (!repositoryPath || repositoryPath === "unknown")
|
|
516
|
+
return undefined;
|
|
517
|
+
const baseBranch = section.data?.repositoryAnalysis?.diff?.baseBranch;
|
|
518
|
+
try {
|
|
519
|
+
const files = changedFileNames(await computeBranchDiff(repositoryPath, baseBranch, false, true)).filter((file) => !(LOCKFILE.test(file.path) || NOT_SOURCE_DIR.test(file.path)));
|
|
520
|
+
return files.length > 0 ? files : undefined;
|
|
208
521
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
recovered.push({
|
|
218
|
-
scenario,
|
|
219
|
-
priority: derivePriorityTier(scenario),
|
|
220
|
-
novelty: Novelty.EXISTING,
|
|
221
|
-
source: CandidateSource.SERVER,
|
|
222
|
-
candidateId: item.candidateId,
|
|
223
|
-
...(item.verifiedDiscriminator ? { verifiedDiscriminator: item.verifiedDiscriminator } : {}),
|
|
522
|
+
catch (error) {
|
|
523
|
+
// A repository git cannot read yields no changed files, which
|
|
524
|
+
// `reposWithoutDiff` already reports to the agent — it must not fail the
|
|
525
|
+
// registration.
|
|
526
|
+
logger.warning("Verify-time changed-file listing failed; this repository has no diff for the checks", {
|
|
527
|
+
repositoryPath,
|
|
528
|
+
error: error instanceof Error ? error.message : String(error),
|
|
224
529
|
});
|
|
530
|
+
return undefined;
|
|
225
531
|
}
|
|
226
|
-
return recovered;
|
|
227
|
-
}
|
|
228
|
-
/** Whether a repo section's persisted diff classification contains any new,
|
|
229
|
-
* modified, or removed endpoint — the same signal `computeScoredCandidates`
|
|
230
|
-
* derives `hasApiChanges` from at analyze time. */
|
|
231
|
-
function sectionHasApiChanges(data) {
|
|
232
|
-
const diff = data?.repositoryAnalysis?.diff;
|
|
233
|
-
return ((diff?.newEndpoints?.length ?? 0) > 0 ||
|
|
234
|
-
(diff?.modifiedEndpoints?.length ?? 0) > 0 ||
|
|
235
|
-
(diff?.removedEndpoints?.length ?? 0) > 0);
|
|
236
532
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
externalCoverage: new Set(pbc?.externalCoverageKeys ?? []),
|
|
245
|
-
// `true` for the same reason `computeDiffChangesTestFiles` answers `true` with
|
|
246
|
-
// no diff: an unknown must shut the reserve, not open it. State written by an
|
|
247
|
-
// older MCP build carries no such field, and if that PR did add its own tests
|
|
248
|
-
// the reserve would recommend duplicates of them — the failure this flag
|
|
249
|
-
// exists to stop, reached by version skew alone. It is also the pre-reserve
|
|
250
|
-
// behaviour: covered candidates were dropped.
|
|
251
|
-
diffChangesTestFiles: pbc?.diffChangesTestFiles ?? true,
|
|
252
|
-
};
|
|
253
|
-
const relatedSections = Object.values(fullState?.relatedRepos ?? {}).map((section) => section.data);
|
|
254
|
-
if (relatedSections.length === 0)
|
|
255
|
-
return base;
|
|
256
|
-
// SKYR-4204: isUIOnlyPR/hasFrontendChanges are per-SECTION facts — each
|
|
257
|
-
// analyze call computes them from its own repo's diff. The approved plan is
|
|
258
|
-
// one run-wide decision, so a frontend-only PRIMARY diff must not put the
|
|
259
|
-
// budgeter on its all-UI branch (backendGenerateCount() = 0) while a related
|
|
260
|
-
// repo carries endpoint changes; that pinned GENERATE to UI placeholders and
|
|
261
|
-
// left every API candidate in ADDITIONAL regardless of maxGenerate. Derive
|
|
262
|
-
// the flags across ALL sections. A related repo checked out on its default
|
|
263
|
-
// branch classifies no endpoint changes, so it leaves a UI-only run UI-only.
|
|
264
|
-
// Budget numbers come from the root (primary) section: the testbot prompt
|
|
265
|
-
// passes topN/maxGenerate only to the primary analyze call, so related
|
|
266
|
-
// sections carry defaults.
|
|
267
|
-
const sections = [fullState ?? undefined, ...relatedSections];
|
|
268
|
-
const hasFrontendChanges = sections.some((s) => s?.planBudgetContext?.hasFrontendChanges ?? false);
|
|
269
|
-
const hasApiChanges = sections.some(sectionHasApiChanges);
|
|
270
|
-
const rootPbc = fullState?.planBudgetContext;
|
|
271
|
-
// SKYR-4211: external-test coverage is also a per-section fact — a backend
|
|
272
|
-
// candidate already covered by an external test recorded in a RELATED repo's
|
|
273
|
-
// section must dedup the same as one covered in the primary. Union the keys
|
|
274
|
-
// across all sections, like the flags above.
|
|
275
|
-
const externalCoverage = new Set(sections.flatMap((s) => s?.planBudgetContext?.externalCoverageKeys ?? []));
|
|
533
|
+
/** Every fact the verifiers check against, gathered once per registration. */
|
|
534
|
+
export async function buildVerifyContext(state) {
|
|
535
|
+
const sections = repoSections(state);
|
|
536
|
+
const listings = await Promise.all(sections.map((section) => changedFilesForVerification(section)));
|
|
537
|
+
// Per repository, so a planned test that names one is checked against it alone.
|
|
538
|
+
// A planned test that names none reads every entry, which is every planned test of a
|
|
539
|
+
// single-repository run.
|
|
276
540
|
return {
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
541
|
+
changedFiles: Object.fromEntries(sections.map((section, i) => [section.repo, listings[i] ?? []])),
|
|
542
|
+
reposWithoutDiff: sections.filter((_s, i) => !listings[i]?.length).map((s) => s.repo),
|
|
543
|
+
// From the process, not the state file: the browser tools and this one are
|
|
544
|
+
// registered on the same server, so captures and reader share a process.
|
|
545
|
+
uiCaptures: capturedBlueprintDigests(),
|
|
546
|
+
// From the analyze result on the state file, across every repository: a
|
|
547
|
+
// removal guard names an element that renders nowhere, so this list is the
|
|
548
|
+
// only thing that can tell an honest one from an invented element.
|
|
549
|
+
removedUiElements: sections.flatMap((section) => section.data?.uiContext?.removedElements ?? []),
|
|
550
|
+
citedFileExists: makeCitedFileCheck(state),
|
|
551
|
+
routesRenderingFile: makeScreenRouteResolver(state),
|
|
288
552
|
};
|
|
289
553
|
}
|
|
290
|
-
|
|
291
|
-
switch (item.testType) {
|
|
292
|
-
case "contract":
|
|
293
|
-
return item.scenarioName
|
|
294
|
-
? `call ${TOOL_CONTRACT_TEST_GENERATION} with scenarioName "${item.scenarioName}"`
|
|
295
|
-
: `call ${TOOL_CONTRACT_TEST_GENERATION}`;
|
|
296
|
-
case "integration":
|
|
297
|
-
return `call ${TOOL_BATCH_SCENARIO_TEST_GENERATION}, then ${TOOL_INTEGRATION_TEST_GENERATION} with the returned scenarioFile`;
|
|
298
|
-
case "e2e":
|
|
299
|
-
return `call ${TOOL_E2E_TEST_GENERATION}`;
|
|
300
|
-
case "ui":
|
|
301
|
-
return `call ${TOOL_UI_TEST_GENERATION}`;
|
|
302
|
-
default:
|
|
303
|
-
return "use the generation tool matching this testType";
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
function renderPlanItem(item, rank) {
|
|
307
|
-
const discriminatorNote = item.verifiedDiscriminator ? `, verified discriminator: ${item.verifiedDiscriminator}` : "";
|
|
308
|
-
return `${rank}. [${item.testType}] "${item.scenarioName}" (${item.category}${discriminatorNote}) — ${describeGenerationCall(item)}`;
|
|
309
|
-
}
|
|
554
|
+
// ── Output ──────────────────────────────────────────────────────────────────
|
|
310
555
|
/**
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
* conditions the server cannot know (app unreachable, unintegrated component);
|
|
315
|
-
* this directive owns WHETHER and HOW MANY.
|
|
556
|
+
* What the agent reads back. Every objection carries its id so it can be answered,
|
|
557
|
+
* and the text says the plan is stored either way — a objections list without that
|
|
558
|
+
* line reads as a rejection and invites a needless second registration.
|
|
316
559
|
*/
|
|
317
|
-
function
|
|
318
|
-
const uiCount = plan.generate.filter((item) => item.testType === TestType.UI || item.testType === TestType.E2E).length;
|
|
319
|
-
const nonUICount = plan.generate.length - uiCount;
|
|
320
|
-
const tests = (n) => `${n} test${n === 1 ? "" : "s"}`;
|
|
321
|
-
if (plan.generate.length === 0) {
|
|
322
|
-
return [
|
|
323
|
-
"### Generation directive: GENERATE NOTHING",
|
|
324
|
-
"Create zero new tests of any type this run and proceed to the report. An empty `newTestsCreated` is the CORRECT " +
|
|
325
|
-
"result for this PR — there is no new observable surface to cover. Do not record a browser trace and do not invent " +
|
|
326
|
-
"a spec to have something to report; report your maintenance work and the ADDITIONAL candidates as recommendations instead.",
|
|
327
|
-
];
|
|
328
|
-
}
|
|
329
|
-
if (uiCount === 0) {
|
|
330
|
-
return [
|
|
331
|
-
"### Generation directive: NO UI/E2E GENERATION",
|
|
332
|
-
`This plan allocates no UI/E2E generation. Generate the ${tests(nonUICount)} listed above and create zero UI tests ` +
|
|
333
|
-
"— do not record a browser trace and do not add a UI spec to fill `newTestsCreated`.",
|
|
334
|
-
];
|
|
335
|
-
}
|
|
336
|
-
return [
|
|
337
|
-
`### Generation directive: UI/E2E GENERATION REQUIRED (${uiCount})`,
|
|
338
|
-
`This plan allocates ${tests(uiCount)} of type UI/E2E. You MUST attempt to record and generate each one — do not ` +
|
|
339
|
-
"downgrade them to recommendations while the app is reachable." +
|
|
340
|
-
(nonUICount > 0 ? ` Generate the ${tests(nonUICount)} of other types as well.` : ""),
|
|
341
|
-
];
|
|
342
|
-
}
|
|
343
|
-
/** SKYR-4292: the server cannot count the diff's changed behaviors, so an
|
|
344
|
-
* under-budget plan is flagged (not rejected) with what a missing candidate looks
|
|
345
|
-
* like. Wipro PR #24 registered 3 candidates against a 20-slot UI-only budget and
|
|
346
|
-
* the tool answered "AUTHORITATIVE and FINAL" with no pushback. */
|
|
347
|
-
function renderBudgetCheck(plan, maxTotal) {
|
|
348
|
-
const filled = plan.generate.length + plan.additional.length;
|
|
349
|
-
if (filled >= maxTotal)
|
|
350
|
-
return [];
|
|
351
|
-
return [
|
|
352
|
-
"",
|
|
353
|
-
`### Budget check: ${filled} of ${maxTotal} slots filled`,
|
|
354
|
-
"Every observable behavior this diff changes — each option value added, removed, renamed or reordered; each label, " +
|
|
355
|
-
"element, route, state, validation rule, computed value or style value (spacing, size, color, radius, font weight, " +
|
|
356
|
-
"class name) that renders differently; each removed element, asserted absent — needs a candidate above, however " +
|
|
357
|
-
"small the change. A change is covered only when an existing test asserts the changed value, not when a test merely " +
|
|
358
|
-
"reaches the component. If any changed behavior has no candidate, call skyramp_register_test_plan again with your " +
|
|
359
|
-
"full updated list. If the diff truly changes fewer behaviors than the budget, this plan stands as-is.",
|
|
360
|
-
];
|
|
361
|
-
}
|
|
362
|
-
function renderPlanText(plan, dropped, maxTotal) {
|
|
560
|
+
export function renderPlanResult(result) {
|
|
363
561
|
const lines = [];
|
|
364
|
-
lines.push(
|
|
365
|
-
lines.push(
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
lines.push(`### GENERATE (${plan.generate.length}) — generate exactly these, in this order`);
|
|
370
|
-
if (plan.generate.length === 0) {
|
|
371
|
-
lines.push(" (none — nothing to generate this run)");
|
|
372
|
-
}
|
|
373
|
-
else {
|
|
374
|
-
plan.generate.forEach((item, i) => lines.push(renderPlanItem(item, i + 1)));
|
|
375
|
-
}
|
|
376
|
-
lines.push("");
|
|
377
|
-
lines.push(`### ADDITIONAL (${plan.additional.length}) — report as recommendations; only generate if backfilling a failed GENERATE item`);
|
|
378
|
-
if (plan.additional.length === 0) {
|
|
379
|
-
lines.push(" (none)");
|
|
380
|
-
}
|
|
381
|
-
else {
|
|
382
|
-
plan.additional.forEach((item, i) => lines.push(renderPlanItem(item, plan.generate.length + i + 1)));
|
|
562
|
+
lines.push(`Registration ${result.plan.registrationNumber} stored as this run's plan.`);
|
|
563
|
+
lines.push(`Plan: ${result.plan.plannedTests.length} planned test(s).`);
|
|
564
|
+
// A convenience: `differsFrom` also accepts a `scenarioName`.
|
|
565
|
+
for (const plannedTest of result.plan.plannedTests) {
|
|
566
|
+
lines.push(`- ${plannedTest.plannedTestId} — ${plannedTest.scenario?.scenarioName ?? "(unnamed)"}`);
|
|
383
567
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
if (registeredDemotions.length > 0) {
|
|
393
|
-
lines.push("");
|
|
394
|
-
lines.push(`### Demotions (${registeredDemotions.length}) — discriminator claims that did NOT verify (candidate still registered, claim dropped)`);
|
|
395
|
-
for (const demotion of registeredDemotions) {
|
|
396
|
-
lines.push(`- ${demotion.candidateId}: ${demotion.reason}`);
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
if (removedDemotions.length > 0) {
|
|
400
|
-
lines.push("");
|
|
401
|
-
lines.push(`### Demoted AND removed (${removedDemotions.length}) — the claim failed to verify AND the candidate left the plan. It is NOT registered.`);
|
|
402
|
-
for (const demotion of removedDemotions) {
|
|
403
|
-
lines.push(`- ${demotion.candidateId}: claim failed — ${demotion.reason}; removed — ${dropReasonByCandidateId.get(demotion.candidateId)}`);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
568
|
+
lines.push(result.summary);
|
|
569
|
+
// The agent sends the answers it wants read; these came off the stored plan, and
|
|
570
|
+
// the plan they answered has changed since.
|
|
571
|
+
if (result.carried) {
|
|
572
|
+
lines.push("");
|
|
573
|
+
lines.push(`Answers carried from registration ${result.carried.registrationNumber} — recheck each; revise it if the plan change made it wrong:`);
|
|
574
|
+
for (const { objectionId, answer } of result.carried.answers)
|
|
575
|
+
lines.push(`- ${objectionId} — ${answer}`);
|
|
406
576
|
}
|
|
407
|
-
|
|
408
|
-
// This section is independent of `demotions`: the two channels are unrelated,
|
|
409
|
-
// and the motivating case has drops and NO demotions. Nesting it inside the
|
|
410
|
-
// demotion block returned empty GENERATE/ADDITIONAL lists with no candidate id
|
|
411
|
-
// and no reason, which is the state item 5 exists to make visible. Measured on
|
|
412
|
-
// eval run 32586610615, fixture repo15-P14: 10 drops, 0 demotions.
|
|
413
|
-
const demotedIds = new Set(plan.demotions.map((d) => d.candidateId));
|
|
414
|
-
const removedWithoutDemotion = dropped.filter((d) => !demotedIds.has(d.candidateId));
|
|
415
|
-
if (removedWithoutDemotion.length > 0) {
|
|
577
|
+
if (result.objections.length > 0) {
|
|
416
578
|
lines.push("");
|
|
417
|
-
lines.push(
|
|
418
|
-
for (const
|
|
419
|
-
lines.push(`- ${
|
|
579
|
+
lines.push("Objections:");
|
|
580
|
+
for (const objection of result.objections) {
|
|
581
|
+
lines.push(`- ${objection.objectionId} — ${objection.message}`);
|
|
582
|
+
lines.push(` evidence: ${objection.evidence}`);
|
|
583
|
+
lines.push(` next: ${objection.suggestion}`);
|
|
420
584
|
}
|
|
585
|
+
lines.push("");
|
|
586
|
+
lines.push("The plan is stored either way. Revise it and call this tool again — there is no registration limit — " +
|
|
587
|
+
"or answer an objection through `answers` using the id above. Anything left open is rendered in the report.");
|
|
421
588
|
}
|
|
422
|
-
lines.push("");
|
|
423
|
-
lines.push(...renderGenerationDirective(plan));
|
|
424
|
-
lines.push(...renderBudgetCheck(plan, maxTotal));
|
|
425
589
|
return lines.join("\n");
|
|
426
590
|
}
|
|
427
591
|
// ── Tool registration ───────────────────────────────────────────────────────
|
|
592
|
+
/** One registration at a time per state file: the registration number is a
|
|
593
|
+
* read-modify-write with an await between the halves, so two overlapping calls
|
|
594
|
+
* returned one number for two plans. In-process serialisation is enough. */
|
|
595
|
+
const registrationsInFlight = new Map();
|
|
596
|
+
/** The one file two calls contend for, whatever they spelled. Keying on the raw
|
|
597
|
+
* string let `/a/b/state.json` and `/a/b//state.json` take different locks on one
|
|
598
|
+
* file. `realpathSync` throws before the file exists; the resolved path is the
|
|
599
|
+
* right key then. */
|
|
600
|
+
function lockKeyFor(stateFile) {
|
|
601
|
+
const resolved = path.resolve(stateFile);
|
|
602
|
+
try {
|
|
603
|
+
return fs.realpathSync(resolved);
|
|
604
|
+
}
|
|
605
|
+
catch {
|
|
606
|
+
return resolved;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
function oneAtATime(stateFile, work) {
|
|
610
|
+
const key = lockKeyFor(stateFile);
|
|
611
|
+
const queued = (registrationsInFlight.get(key) ?? Promise.resolve()).then(work, work);
|
|
612
|
+
// Never rejects, so one failed registration cannot break the chain.
|
|
613
|
+
const settled = queued.catch(() => undefined);
|
|
614
|
+
registrationsInFlight.set(key, settled);
|
|
615
|
+
// Dropped when nothing is queued, so the map does not grow without bound.
|
|
616
|
+
void settled.then(() => {
|
|
617
|
+
if (registrationsInFlight.get(key) === settled)
|
|
618
|
+
registrationsInFlight.delete(key);
|
|
619
|
+
});
|
|
620
|
+
return queued;
|
|
621
|
+
}
|
|
428
622
|
export function registerRegisterTestPlanTool(server) {
|
|
429
623
|
server.registerTool(TOOL_NAME, {
|
|
430
624
|
annotations: {
|
|
431
|
-
|
|
625
|
+
// It REPLACES the stored plan and the registration count.
|
|
626
|
+
destructiveHint: true,
|
|
432
627
|
readOnlyHint: false,
|
|
433
628
|
idempotentHint: false,
|
|
434
629
|
openWorldHint: false,
|
|
435
630
|
},
|
|
436
|
-
description: "
|
|
437
|
-
"
|
|
438
|
-
"
|
|
439
|
-
"
|
|
440
|
-
"
|
|
441
|
-
"
|
|
442
|
-
"
|
|
631
|
+
description: "Register your complete test plan for this run, with the changes that make each planned test checkable. " +
|
|
632
|
+
"The planning procedure in your prompt says how to decide and declare the plan. " +
|
|
633
|
+
"Every registration the schema accepts is stored, and no check ever refuses one. " +
|
|
634
|
+
"The schema refuses the whole call on ordinary field validation, and on three rules that read more than one field: a non-UI planned test with an empty `steps` list, a `ui` or `e2e` planned test whose steps make no HTTP call and names no `elements.pageUrl`, and an `elements.items` entry carrying no name and no identifier. " +
|
|
635
|
+
"An undeclared key is refused by name on a planned test and on `declarations`, so a declaration written one level too high is corrected rather than dropped in silence. " +
|
|
636
|
+
"Checks run against the stored plan and return objections. Fix one and register it again as often as you like, or answer it by id through `answers`. " +
|
|
637
|
+
"There is no fallback plan: if you never register, this run has no plan.",
|
|
443
638
|
inputSchema: registerTestPlanSchema,
|
|
444
639
|
}, async (params) => {
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
// section — there is no section selector (routing by one is what let a
|
|
453
|
-
// related-repo registration run with that section's context —
|
|
454
|
-
// SKYR-4204 reopen). Run-wide facts (diff union, budget flags,
|
|
455
|
-
// scenario lookup) come from fullState below.
|
|
456
|
-
const stateData = await stateManager.readData();
|
|
457
|
-
if (!stateData) {
|
|
458
|
-
errorResult = toolError(`State file is empty or invalid: ${params.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
|
|
459
|
-
return errorResult;
|
|
460
|
-
}
|
|
461
|
-
// SKYR-4203: one register call carries candidates for EVERY repo of the
|
|
462
|
-
// run, but `stateData` is a single repo's section — and the candidate
|
|
463
|
-
// schema has no per-candidate repository, so a related repo's anchors
|
|
464
|
-
// were unverifiable by construction. Verify anchors (and rank diff-hunk
|
|
465
|
-
// proximity) against the union of ALL sections' diffs: an anchor
|
|
466
|
-
// grounded in a related repo's change is as real as one in the
|
|
467
|
-
// primary's. Sections with no diff (a related repo checked out on its
|
|
468
|
-
// default branch) are collected so an unmatched anchor reads as
|
|
469
|
-
// uncheckable rather than ungrounded.
|
|
470
|
-
const fullState = await stateManager.readFullState();
|
|
471
|
-
const diffSections = [
|
|
472
|
-
{ repo: fullState?.metadata?.repository ?? "the primary repository", diff: fullState?.diffText },
|
|
473
|
-
...Object.entries(fullState?.relatedRepos ?? {}).map(([repo, section]) => ({
|
|
474
|
-
repo,
|
|
475
|
-
diff: section.data?.diffText,
|
|
476
|
-
})),
|
|
477
|
-
];
|
|
478
|
-
const diffText = diffSections.map((s) => s.diff).filter(Boolean).join("\n");
|
|
479
|
-
const reposWithoutDiff = diffSections.filter((s) => !s.diff).map((s) => s.repo);
|
|
480
|
-
// Scenario lookup for server-candidate recovery spans ALL sections —
|
|
481
|
-
// the primary's drafted scenarios must resolve even when `repository`
|
|
482
|
-
// targets a related repo (SKYR-4204 reopen: the same single-section
|
|
483
|
-
// shape as the diff union above).
|
|
484
|
-
const allScenarios = [
|
|
485
|
-
...(fullState?.repositoryAnalysis?.scenarios ?? []),
|
|
486
|
-
...Object.values(fullState?.relatedRepos ?? {}).flatMap((section) => section.data?.repositoryAnalysis?.scenarios ?? []),
|
|
487
|
-
];
|
|
488
|
-
const changedRoutes = collectChangedRouteLines(diffText);
|
|
489
|
-
const agentCandidates = buildAgentCandidates(params.candidates ?? [], changedRoutes);
|
|
490
|
-
// The approved plan is run-wide and persists at the ROOT (see the
|
|
491
|
-
// persist step below), so prior-plan recovery reads the root first;
|
|
492
|
-
// the section fallback covers state written by older builds.
|
|
493
|
-
const serverCandidates = recoverServerCandidates(fullState?.approvedPlan ?? stateData.approvedPlan, allScenarios);
|
|
494
|
-
// Merge by scenario-name identity, NOT the full content-hashed candidateId:
|
|
495
|
-
// the server (analyze_changes) and the agent frequently draft their own
|
|
496
|
-
// independent steps[] for "the same" scenario (identical scenarioName),
|
|
497
|
-
// which hash to different candidateIds — deduping on the full id let
|
|
498
|
-
// both survive as apparent duplicates (SKYR-4026). A pre-seeded server
|
|
499
|
-
// candidate wins over an agent submission for the same scenario name —
|
|
500
|
-
// source: "server" is preserved per the unified-plan design.
|
|
501
|
-
//
|
|
502
|
-
// Uses scenarioMergeKey (untruncated), not scenarioNameSlug — the
|
|
503
|
-
// latter's 48-char cap is fine for a short candidateId prefix combined
|
|
504
|
-
// with a content hash, but bare as a Map key it would silently collapse
|
|
505
|
-
// two distinct long scenario names sharing a common prefix. A missing
|
|
506
|
-
// scenarioName (schema requires one for agent candidates; only a
|
|
507
|
-
// malformed server-recovered scenario could lack one) falls back to
|
|
508
|
-
// the already-unique candidateId rather than a shared literal.
|
|
509
|
-
const mergeKey = (c) => scenarioMergeKey(c.scenario.scenarioName) || c.candidateId;
|
|
510
|
-
const merged = new Map();
|
|
511
|
-
for (const candidate of agentCandidates)
|
|
512
|
-
merged.set(mergeKey(candidate), candidate);
|
|
513
|
-
for (const candidate of serverCandidates)
|
|
514
|
-
merged.set(mergeKey(candidate), candidate);
|
|
515
|
-
// The agent declares a claim per scenario NAME, so key the claims the
|
|
516
|
-
// same way the merge does and verify against whichever copy survived.
|
|
517
|
-
const claimsByKey = new Map();
|
|
518
|
-
// Keyed through mergeKey, not scenarioMergeKey alone: a name that
|
|
519
|
-
// slugifies to nothing (e.g. an all-non-ASCII name) would be stored
|
|
520
|
-
// under "" while the lookup falls back to the candidateId, so the claim
|
|
521
|
-
// would be neither verified nor demoted.
|
|
522
|
-
(params.candidates ?? []).forEach((input, i) => {
|
|
523
|
-
if (input.discriminator)
|
|
524
|
-
claimsByKey.set(mergeKey(agentCandidates[i]), input.discriminator);
|
|
525
|
-
});
|
|
526
|
-
const { candidates: allCandidates, demotions } = applyDiscriminatorClaims([...merged.values()], claimsByKey, diffText, reposWithoutDiff);
|
|
527
|
-
// An empty union would persist an authoritative plan with an empty
|
|
528
|
-
// GENERATE list, which the generation gate then enforces — bricking
|
|
529
|
-
// the run until re-registration. Reject instead so the agent can
|
|
530
|
-
// self-correct in one step.
|
|
531
|
-
if (allCandidates.length === 0) {
|
|
532
|
-
errorResult = toolError("No candidates submitted and no server pre-ranked candidates exist for this run. " +
|
|
533
|
-
"Submit your complete candidate list (every test you would generate or recommend) in `candidates` — an empty plan would block all generation.");
|
|
534
|
-
return errorResult;
|
|
535
|
-
}
|
|
536
|
-
// Review (mcp#774): enforce ONE run-wide plan at the tool boundary. A
|
|
537
|
-
// re-registration whose pool shares no scenario with the active plan
|
|
538
|
-
// is the per-repo-partial signature (the demoshop rig's second,
|
|
539
|
-
// backend-only call) — accepting it would silently REPLACE the run's
|
|
540
|
-
// plan with a subset and drop every earlier agent candidate. Genuine
|
|
541
|
-
// full-list updates overlap the active plan and replace it as before.
|
|
542
|
-
// The overlap test spans GENERATE ∪ ADDITIONAL: with maxGenerate=0 the
|
|
543
|
-
// active plan's GENERATE is empty while ADDITIONAL carries every
|
|
544
|
-
// candidate, and a disjoint partial could otherwise still replace it
|
|
545
|
-
// silently (Copilot review on #774).
|
|
546
|
-
const priorItems = [
|
|
547
|
-
...(fullState?.approvedPlan?.generate ?? []),
|
|
548
|
-
...(fullState?.approvedPlan?.additional ?? []),
|
|
549
|
-
];
|
|
550
|
-
if (priorItems.length > 0) {
|
|
551
|
-
const poolKeys = new Set(allCandidates.map(mergeKey));
|
|
552
|
-
const overlaps = priorItems.some((item) => poolKeys.has(scenarioMergeKey(item.scenarioName) || item.candidateId));
|
|
553
|
-
if (!overlaps) {
|
|
554
|
-
errorResult = toolError(`A run-wide plan is already active (${fullState.approvedPlan.planId}: ` +
|
|
555
|
-
`${priorItems.map((i) => i.scenarioName).join(", ")}) and this submission shares none of its ` +
|
|
556
|
-
"scenarios. skyramp_register_test_plan REPLACES the whole run's plan — resubmit ONE pooled candidate " +
|
|
557
|
-
"list for the entire run (your previous candidates plus these changes), not a per-repo subset.");
|
|
558
|
-
return errorResult;
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
const budgetContext = resolveBudgetContext(stateData, fullState);
|
|
562
|
-
// SKYR-4204 reopen: a reserved UI slot is only fillable from the
|
|
563
|
-
// candidate pool. A pool with no UI candidate on a run with frontend
|
|
564
|
-
// changes would silently plan zero UI tests (runs 32419025266,
|
|
565
|
-
// 32431042133, 32431106864) — reject so the agent self-corrects in one
|
|
566
|
-
// step, exactly like the empty-candidates rejection above.
|
|
567
|
-
const uiFloor = generateFloors(budgetContext).ui;
|
|
568
|
-
if (uiFloor > 0 && !allCandidates.some((c) => inferScenarioType(c.scenario) === TestType.UI)) {
|
|
569
|
-
errorResult = toolError(`This run has frontend changes, so the plan MUST include UI tests — at least ${uiFloor} of the ` +
|
|
570
|
-
"GENERATE slots are held for them, and more are open to UI candidates on rank. No submitted or " +
|
|
571
|
-
"recovered candidate has testType 'ui'. Resubmit your full candidate list including UI candidates " +
|
|
572
|
-
"for the changed frontend (see uiContext.changedFrontendFiles).");
|
|
573
|
-
return errorResult;
|
|
574
|
-
}
|
|
575
|
-
const result = selectPlan(allCandidates, { ...budgetContext, demotions, diffText });
|
|
576
|
-
const approvedPlan = {
|
|
577
|
-
planId: crypto.randomUUID(),
|
|
578
|
-
createdAt: new Date().toISOString(),
|
|
579
|
-
generate: result.generate.map(buildApprovedPlanItem),
|
|
580
|
-
additional: result.additional.map(buildApprovedPlanItem),
|
|
581
|
-
demotions: result.demotions,
|
|
582
|
-
dropped: result.dropped,
|
|
583
|
-
};
|
|
584
|
-
// The approved plan is the ONE run-wide authority (the prompt mandates
|
|
585
|
-
// a single pooled registration on multi-repo runs), so it persists at
|
|
586
|
-
// the state-file ROOT regardless of `repository`. Per-section plans
|
|
587
|
-
// blinded every root-reading consumer — planGuard let generation run
|
|
588
|
-
// unguarded, submitReportTool's reconciliation was inert, and the eval
|
|
589
|
-
// extractor saw "no plan" (SKYR-4204 reopen). writeData preserves the
|
|
590
|
-
// relatedRepos sections and the root metadata.
|
|
640
|
+
// The run's own state file, unless the agent named one. Outside a run there
|
|
641
|
+
// is no path to fall back to, and no analysis to check a plan against.
|
|
642
|
+
const stateFile = resolveRunStatePath(params.stateFile);
|
|
643
|
+
if (!stateFile) {
|
|
644
|
+
return toolError("This run has no state file. Call skyramp_analyze_changes first — it writes the state file this plan is checked against.");
|
|
645
|
+
}
|
|
646
|
+
return oneAtATime(stateFile, async () => {
|
|
591
647
|
try {
|
|
592
|
-
const
|
|
593
|
-
|
|
648
|
+
const stateManager = StateManager.fromStatePath(stateFile);
|
|
649
|
+
// Run-wide: one registration covers every repository, so the facts come
|
|
650
|
+
// from the whole state file and the plan persists at its root.
|
|
651
|
+
const fullState = await stateManager.readFullState();
|
|
652
|
+
if (!fullState) {
|
|
653
|
+
return toolError(`State file is empty or invalid: ${stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
|
|
654
|
+
}
|
|
655
|
+
const registrationNumber = (fullState.registrationCount ?? 0) + 1;
|
|
656
|
+
const registration = buildRegistration(params, registrationNumber);
|
|
657
|
+
// The previous registration's answers are already on the stored plan, which
|
|
658
|
+
// the write below replaces.
|
|
659
|
+
const result = registerPlan(registration, await buildVerifyContext(fullState), fullState.plan);
|
|
660
|
+
try {
|
|
661
|
+
const { metadata: _metadata, relatedRepos: _sections, ...rootData } = fullState;
|
|
662
|
+
await stateManager.writeData({ ...rootData, plan: result.plan, registrationCount: registrationNumber }, { step: TOOL_NAME });
|
|
663
|
+
}
|
|
664
|
+
catch (error) {
|
|
665
|
+
return toolError(`Failed to persist the test plan to the state file: ${error.message}. Retry this call.`);
|
|
666
|
+
}
|
|
667
|
+
logger.info(`${TOOL_NAME}: registration ${registrationNumber}, ${result.plan.plannedTests.length} planned test(s), ${result.objections.length} objection(s)`);
|
|
668
|
+
return { content: [{ type: "text", text: renderPlanResult(result) }] };
|
|
594
669
|
}
|
|
595
670
|
catch (error) {
|
|
596
|
-
|
|
597
|
-
return errorResult;
|
|
671
|
+
return toolError(`${TOOL_NAME} failed: ${error?.message ?? String(error)}`);
|
|
598
672
|
}
|
|
599
|
-
|
|
600
|
-
content: [{ type: "text", text: renderPlanText(approvedPlan, result.dropped, budgetContext.maxTotal) }],
|
|
601
|
-
};
|
|
602
|
-
}
|
|
603
|
-
catch (error) {
|
|
604
|
-
errorResult = toolError(`skyramp_register_test_plan failed: ${error?.message ?? String(error)}`);
|
|
605
|
-
return errorResult;
|
|
606
|
-
}
|
|
673
|
+
});
|
|
607
674
|
});
|
|
608
675
|
}
|