@skyramp/mcp 0.3.8 → 0.4.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/commandLibrary.d.ts +1 -1
- package/build/commands/commandLibrary.js +3 -3
- package/build/commands/recommendTestsAndExecuteCommand.d.ts +1 -1
- package/build/commands/recommendTestsAndExecuteCommand.js +35 -20
- package/build/commands/testThisEndpointCommand.js +35 -19
- package/build/index.js +9 -3
- package/build/playwright/blueprintDigest.d.ts +15 -0
- package/build/playwright/blueprintDigest.js +152 -0
- package/build/playwright/blueprintDigestStore.d.ts +31 -0
- package/build/playwright/blueprintDigestStore.js +117 -0
- package/build/playwright/registerPlaywrightTools.js +60 -12
- package/build/playwright/traceRecordingPrompt.js +8 -7
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +9 -8
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -2
- package/build/prompts/promptAssets.d.ts +20 -0
- package/build/prompts/promptAssets.js +55 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +19 -5
- package/build/prompts/test-maintenance/actionsInstructions.d.ts +4 -0
- package/build/prompts/test-maintenance/actionsInstructions.js +14 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +0 -10
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +2 -11
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +8 -4
- package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +5 -22
- package/build/prompts/test-recommendation/diffExecutionPlan.js +37 -465
- package/build/prompts/test-recommendation/recommendationSections.d.ts +7 -17
- package/build/prompts/test-recommendation/recommendationSections.js +67 -309
- package/build/prompts/test-recommendation/recommendationShared.d.ts +19 -47
- package/build/prompts/test-recommendation/recommendationShared.js +49 -155
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.d.ts +0 -5
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +10 -153
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -29
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +32 -457
- package/build/prompts/testbot/planDeclarations.d.ts +6 -0
- package/build/prompts/testbot/planDeclarations.js +9 -0
- package/build/prompts/testbot/testbot-prompts.d.ts +8 -0
- package/build/prompts/testbot/testbot-prompts.js +256 -381
- package/build/recommendation/answers.d.ts +35 -0
- package/build/recommendation/answers.js +96 -0
- package/build/recommendation/registerPlan.d.ts +49 -0
- package/build/recommendation/registerPlan.js +117 -0
- package/build/recommendation/runVerifiers.d.ts +10 -0
- package/build/recommendation/runVerifiers.js +49 -0
- package/build/recommendation/subjectStep.d.ts +42 -0
- package/build/recommendation/subjectStep.js +86 -0
- package/build/recommendation/types.d.ts +163 -0
- package/build/recommendation/types.js +20 -0
- package/build/recommendation/verifierContracts.d.ts +382 -0
- package/build/recommendation/verifierContracts.js +263 -0
- package/build/recommendation/verifiers/changedFile.d.ts +2 -0
- package/build/recommendation/verifiers/changedFile.js +82 -0
- package/build/recommendation/verifiers/citedPath.d.ts +12 -0
- package/build/recommendation/verifiers/citedPath.js +35 -0
- package/build/recommendation/verifiers/coverage.d.ts +7 -0
- package/build/recommendation/verifiers/coverage.js +617 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.d.ts +11 -0
- package/build/recommendation/verifiers/deliveredMatchesPlan.js +33 -0
- package/build/recommendation/verifiers/endpointGrounded.d.ts +17 -0
- package/build/recommendation/verifiers/endpointGrounded.js +128 -0
- package/build/recommendation/verifiers/existingCoverage.d.ts +6 -0
- package/build/recommendation/verifiers/existingCoverage.js +51 -0
- package/build/recommendation/verifiers/expectedOutcome.d.ts +31 -0
- package/build/recommendation/verifiers/expectedOutcome.js +105 -0
- package/build/recommendation/verifiers/removedElementGuarded.d.ts +2 -0
- package/build/recommendation/verifiers/removedElementGuarded.js +57 -0
- package/build/recommendation/verifiers/reportedCategory.d.ts +26 -0
- package/build/recommendation/verifiers/reportedCategory.js +84 -0
- package/build/recommendation/verifiers/screenRoute.d.ts +10 -0
- package/build/recommendation/verifiers/screenRoute.js +118 -0
- package/build/recommendation/verifiers/statedDifference.d.ts +6 -0
- package/build/recommendation/verifiers/statedDifference.js +140 -0
- package/build/recommendation/verifiers/uiElementGrounded.d.ts +7 -0
- package/build/recommendation/verifiers/uiElementGrounded.js +318 -0
- package/build/resources/analysisResources.js +1 -114
- package/build/resources/testbotResource.js +23 -13
- package/build/services/ModularizationService.js +2 -1
- package/build/services/TestDiscoveryService.d.ts +3 -72
- package/build/services/TestDiscoveryService.js +10 -303
- package/build/services/containerEnv.d.ts +1 -1
- package/build/services/containerEnv.js +12 -0
- package/build/skills/fixTestImportErrorsSkill.d.ts +13 -0
- package/build/skills/fixTestImportErrorsSkill.js +20 -0
- package/build/toolNames.d.ts +1 -0
- package/build/toolNames.js +1 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +3 -3
- package/build/tools/code-refactor/modularizationTool.js +2 -1
- package/build/tools/executeSkyrampTestTool.d.ts +80 -0
- package/build/tools/executeSkyrampTestTool.js +246 -19
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +6 -0
- package/build/tools/generate-tests/generateContractRestTool.js +3 -3
- package/build/tools/generate-tests/planGuard.d.ts +2 -2
- package/build/tools/generate-tests/planGuard.js +78 -18
- package/build/tools/one-click/oneClickTool.d.ts +0 -1
- package/build/tools/one-click/oneClickTool.js +0 -5
- package/build/tools/submitReportTool.d.ts +48 -42
- package/build/tools/submitReportTool.js +576 -193
- package/build/tools/test-management/actionsTool.js +72 -4
- package/build/tools/test-management/analyzeChangesTool.d.ts +144 -48
- package/build/tools/test-management/analyzeChangesTool.js +212 -1219
- package/build/tools/test-management/analyzeTestHealthTool.js +13 -24
- package/build/tools/test-management/index.d.ts +1 -0
- package/build/tools/test-management/index.js +1 -0
- package/build/tools/test-management/registerTestPlanTool.d.ts +795 -172
- package/build/tools/test-management/registerTestPlanTool.js +609 -542
- package/build/tools/test-management/resolveScreenTool.d.ts +75 -0
- package/build/tools/test-management/resolveScreenTool.js +289 -0
- package/build/types/BlueprintDigest.d.ts +34 -0
- package/build/types/BlueprintDigest.js +1 -0
- package/build/types/RepositoryAnalysis.d.ts +20 -1559
- package/build/types/RepositoryAnalysis.js +2 -58
- package/build/types/StepMethod.d.ts +40 -0
- package/build/types/StepMethod.js +77 -0
- package/build/types/TestAnalysis.d.ts +12 -0
- package/build/types/TestExecution.d.ts +4 -0
- package/build/types/TestRecommendation.d.ts +24 -24
- package/build/types/TestRecommendation.js +91 -89
- package/build/types/TestbotPromptOptions.d.ts +0 -4
- package/build/types/TestbotReport.d.ts +64 -2
- package/build/utils/AnalysisStateManager.d.ts +79 -113
- package/build/utils/AnalysisStateManager.js +147 -57
- package/build/utils/assertion-verify/api-shared-lints.js +1 -1
- package/build/utils/assertion-verify/metrics.js +85 -36
- package/build/utils/assertion-verify/ui-lints.d.ts +0 -5
- package/build/utils/assertion-verify/ui-lints.js +32 -0
- package/build/utils/branchDiff.d.ts +63 -31
- package/build/utils/branchDiff.js +242 -94
- package/build/utils/containedPath.d.ts +18 -0
- package/build/utils/containedPath.js +73 -0
- package/build/utils/dartRouteExtractor.d.ts +18 -34
- package/build/utils/dartRouteExtractor.js +101 -173
- package/build/utils/featureFlags.d.ts +12 -0
- package/build/utils/featureFlags.js +14 -0
- package/build/utils/frontendSelectors.d.ts +48 -27
- package/build/utils/frontendSelectors.js +241 -80
- package/build/utils/pathMatching.d.ts +2 -4
- package/build/utils/pathMatching.js +2 -4
- package/build/utils/planMatchKeys.d.ts +38 -47
- package/build/utils/planMatchKeys.js +143 -81
- package/build/utils/rebaselineSnapshots.d.ts +24 -0
- package/build/utils/rebaselineSnapshots.js +65 -0
- package/build/utils/removedUiElements.d.ts +22 -0
- package/build/utils/removedUiElements.js +106 -0
- package/build/utils/reportVerification.d.ts +2 -6
- package/build/utils/reportVerification.js +61 -2
- package/build/utils/screenRoutes.d.ts +66 -0
- package/build/utils/screenRoutes.js +727 -0
- package/build/utils/sourceRouteExtractor.js +320 -112
- package/build/utils/testFileClassification.d.ts +11 -2
- package/build/utils/testFileClassification.js +44 -2
- package/build/utils/testFixtures.d.ts +5 -0
- package/build/utils/testFixtures.js +13 -0
- package/build/utils/utils.d.ts +0 -1
- package/build/utils/utils.js +0 -11
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/workspace/workspace.d.ts +12 -12
- package/node_modules/playwright/lib/mcp/skyramp/assertHiddenTool.js +56 -0
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +2 -1
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +4 -1
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +160 -1
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +1 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder/recorderSignalProcessor.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/recorder.js +5 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.-Id052Lr.js → index.B7KbSQcC.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder/recorderSignalProcessor.ts +7 -0
- package/node_modules/playwright/node_modules/playwright-core/src/server/recorder.ts +6 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +4 -3
- package/plugin/.claude-plugin/plugin.json +8 -0
- package/plugin/plugin.json +6 -0
- package/plugin/prompts/declaring-a-plan.md +20 -0
- package/plugin/prompts/generate-tests/context-fetching.md +4 -0
- package/plugin/prompts/generate-tests/execution-plan.md +63 -0
- package/plugin/prompts/generate-tests/generation.md +108 -0
- package/plugin/prompts/generate-tests/path-parameters.md +1 -0
- package/plugin/prompts/generate-tests/reasoning-protocol.md +17 -0
- package/plugin/prompts/generate-tests/tool-workflow-variants.md +61 -0
- package/plugin/prompts/generate-tests/tool-workflows.md +65 -0
- package/plugin/prompts/plan-tests.md +42 -0
- package/plugin/prompts/testbot-task1.md +82 -0
- package/plugin/skills/fix-test-import-errors/SKILL.md +98 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.d.ts +0 -84
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +0 -369
- package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +0 -7
- package/build/prompts/test-recommendation/fullRepoCatalog.js +0 -283
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +0 -81
- package/build/prompts/test-recommendation/scopeAssessment.js +0 -359
- package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +0 -7
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +0 -105
- package/build/recommendation/budgeters/fixedNBudgeter.d.ts +0 -7
- package/build/recommendation/budgeters/fixedNBudgeter.js +0 -11
- package/build/recommendation/budgeters/shared.d.ts +0 -32
- package/build/recommendation/budgeters/shared.js +0 -246
- package/build/recommendation/discriminators.d.ts +0 -37
- package/build/recommendation/discriminators.js +0 -379
- package/build/recommendation/diversity.d.ts +0 -47
- package/build/recommendation/diversity.js +0 -101
- package/build/recommendation/planRanker.d.ts +0 -65
- package/build/recommendation/planRanker.js +0 -83
- package/build/recommendation/testFixtures.d.ts +0 -25
- package/build/recommendation/testFixtures.js +0 -45
- package/build/types/FrontendIntegration.d.ts +0 -28
- package/build/types/FrontendIntegration.js +0 -22
- package/build/types/Recommendation.d.ts +0 -146
- package/build/types/Recommendation.js +0 -74
- package/build/utils/changedRoutes.d.ts +0 -29
- package/build/utils/changedRoutes.js +0 -87
- package/build/utils/frontendIntegration.d.ts +0 -9
- package/build/utils/frontendIntegration.js +0 -243
- package/build/utils/importerHop.d.ts +0 -135
- package/build/utils/importerHop.js +0 -489
- package/build/utils/pathAffinityClassification.d.ts +0 -49
- package/build/utils/pathAffinityClassification.js +0 -180
- package/build/utils/pythonMountPrefixes.d.ts +0 -25
- package/build/utils/pythonMountPrefixes.js +0 -347
- package/build/utils/repoScanner.d.ts +0 -34
- package/build/utils/repoScanner.js +0 -300
- package/build/utils/routeParsers.d.ts +0 -95
- package/build/utils/routeParsers.js +0 -951
- package/build/utils/scenarioDrafting.d.ts +0 -92
- package/build/utils/scenarioDrafting.js +0 -951
- package/build/utils/subjectEndpoints.d.ts +0 -19
- package/build/utils/subjectEndpoints.js +0 -98
- package/build/utils/uiPageEnumerator.d.ts +0 -172
- package/build/utils/uiPageEnumerator.js +0 -474
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ObjectionAnswer, Objection } from "./types.js";
|
|
2
|
+
/** The objection raised below carries this prefix and the id it objects to. */
|
|
3
|
+
export declare const UNKNOWN_ANSWER_PREFIX = "answers:unknown:";
|
|
4
|
+
/** The answer to one objection, matched by `objectionId` and never by position — the
|
|
5
|
+
* agent reorders its list between registrations. A whitespace-only answer closes
|
|
6
|
+
* nothing, and the FIRST NON-BLANK entry wins when one id is answered twice. */
|
|
7
|
+
export declare function answerFor(objection: Objection, acknowledged: ObjectionAnswer[]): string | undefined;
|
|
8
|
+
/** One objection per answer naming an objection this call did not raise — a stale
|
|
9
|
+
* id, a typo and a pre-emptive answer all used to pass in silence. An id is
|
|
10
|
+
* recognised BY ITS PREFIX and never draws an objection of its own. `scope` and
|
|
11
|
+
* `suggestion` are passed in: the two callers raise this about different lists. */
|
|
12
|
+
export declare function unknownAnswerObjections(objections: Objection[], acknowledged: ObjectionAnswer[], { scope, suggestion }: {
|
|
13
|
+
scope: string;
|
|
14
|
+
suggestion: string;
|
|
15
|
+
}): Objection[];
|
|
16
|
+
/** The one objection an answer does not close. Every other check in v2 can be
|
|
17
|
+
* answered, and this one cannot: the missed bugs it exists for were reached by
|
|
18
|
+
* prose twice and skipped twice, so an answer would close the objection and
|
|
19
|
+
* leave the bug. `verifierContracts.ts` carries the evidence. */
|
|
20
|
+
export declare const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
21
|
+
export declare function isNonAnswerable(objectionId: unknown): boolean;
|
|
22
|
+
/** The objection a sentence alone does not close either. A change with no test
|
|
23
|
+
* needs the test, unless this run could not write one — so the answer closes it
|
|
24
|
+
* only when it also names what stopped the run. Measured on run 34176038240:
|
|
25
|
+
* across eight fixtures 45 objections were raised and 45 were closed, and the one
|
|
26
|
+
* class that a sentence cannot close was never raised at all. Prose closed every
|
|
27
|
+
* uncovered change, including changes nothing prevented a test from reaching. */
|
|
28
|
+
export declare const BLOCKER_ONLY_PREFIX = "coverage:change:";
|
|
29
|
+
export declare function needsBlocker(objectionId: unknown): boolean;
|
|
30
|
+
/** The objection raised below carries this prefix and the id it objects to. */
|
|
31
|
+
export declare const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
32
|
+
/** One objection per answer to an objection only a planned test closes. The answered
|
|
33
|
+
* objection STAYS OPEN — this says so, rather than letting the answer pass in
|
|
34
|
+
* silence while the plan still owes the test. */
|
|
35
|
+
export declare function refusedAnswerObjections(objections: Objection[], acknowledged: ObjectionAnswer[]): Objection[];
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/** The objection raised below carries this prefix and the id it objects to. */
|
|
2
|
+
export const UNKNOWN_ANSWER_PREFIX = "answers:unknown:";
|
|
3
|
+
/** The answer to one objection, matched by `objectionId` and never by position — the
|
|
4
|
+
* agent reorders its list between registrations. A whitespace-only answer closes
|
|
5
|
+
* nothing, and the FIRST NON-BLANK entry wins when one id is answered twice. */
|
|
6
|
+
export function answerFor(objection, acknowledged) {
|
|
7
|
+
const wantsBlocker = needsBlocker(objection.objectionId);
|
|
8
|
+
for (const entry of acknowledged) {
|
|
9
|
+
if (entry?.objectionId !== objection.objectionId)
|
|
10
|
+
continue;
|
|
11
|
+
const rawAnswer = entry.answer;
|
|
12
|
+
const answer = typeof rawAnswer === "string" ? rawAnswer.trim() : "";
|
|
13
|
+
if (answer.length === 0)
|
|
14
|
+
continue;
|
|
15
|
+
if (wantsBlocker && String(entry?.blocker ?? "").trim().length === 0)
|
|
16
|
+
continue;
|
|
17
|
+
return answer;
|
|
18
|
+
}
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
/** One objection per answer naming an objection this call did not raise — a stale
|
|
22
|
+
* id, a typo and a pre-emptive answer all used to pass in silence. An id is
|
|
23
|
+
* recognised BY ITS PREFIX and never draws an objection of its own. `scope` and
|
|
24
|
+
* `suggestion` are passed in: the two callers raise this about different lists. */
|
|
25
|
+
export function unknownAnswerObjections(objections, acknowledged, { scope, suggestion }) {
|
|
26
|
+
const answered = acknowledged.map((entry) => entry?.objectionId).filter((id) => typeof id === "string");
|
|
27
|
+
const raised = new Set(objections.map((objection) => objection.objectionId));
|
|
28
|
+
const raisedHere = [];
|
|
29
|
+
const seen = new Set();
|
|
30
|
+
for (const id of answered) {
|
|
31
|
+
if (id.startsWith(UNKNOWN_ANSWER_PREFIX) || raised.has(id) || seen.has(id))
|
|
32
|
+
continue;
|
|
33
|
+
seen.add(id);
|
|
34
|
+
raisedHere.push({
|
|
35
|
+
objectionId: `${UNKNOWN_ANSWER_PREFIX}${id}`,
|
|
36
|
+
verifier: "answers",
|
|
37
|
+
message: "An answer names an objection this registration did not raise, so it closed nothing.",
|
|
38
|
+
evidence: `answered "${id}", which is not the id of any objection ${scope}`,
|
|
39
|
+
suggestion,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
return raisedHere;
|
|
43
|
+
}
|
|
44
|
+
/** The one objection an answer does not close. Every other check in v2 can be
|
|
45
|
+
* answered, and this one cannot: the missed bugs it exists for were reached by
|
|
46
|
+
* prose twice and skipped twice, so an answer would close the objection and
|
|
47
|
+
* leave the bug. `verifierContracts.ts` carries the evidence. */
|
|
48
|
+
export const NON_ANSWERABLE_PREFIX = "coverage:stateTest:";
|
|
49
|
+
export function isNonAnswerable(objectionId) {
|
|
50
|
+
return typeof objectionId === "string" && objectionId.startsWith(NON_ANSWERABLE_PREFIX);
|
|
51
|
+
}
|
|
52
|
+
/** The objection a sentence alone does not close either. A change with no test
|
|
53
|
+
* needs the test, unless this run could not write one — so the answer closes it
|
|
54
|
+
* only when it also names what stopped the run. Measured on run 34176038240:
|
|
55
|
+
* across eight fixtures 45 objections were raised and 45 were closed, and the one
|
|
56
|
+
* class that a sentence cannot close was never raised at all. Prose closed every
|
|
57
|
+
* uncovered change, including changes nothing prevented a test from reaching. */
|
|
58
|
+
export const BLOCKER_ONLY_PREFIX = "coverage:change:";
|
|
59
|
+
export function needsBlocker(objectionId) {
|
|
60
|
+
return typeof objectionId === "string" && objectionId.startsWith(BLOCKER_ONLY_PREFIX);
|
|
61
|
+
}
|
|
62
|
+
/** The objection raised below carries this prefix and the id it objects to. */
|
|
63
|
+
export const REFUSED_ANSWER_PREFIX = "answers:refused:";
|
|
64
|
+
/** One objection per answer to an objection only a planned test closes. The answered
|
|
65
|
+
* objection STAYS OPEN — this says so, rather than letting the answer pass in
|
|
66
|
+
* silence while the plan still owes the test. */
|
|
67
|
+
export function refusedAnswerObjections(objections, acknowledged) {
|
|
68
|
+
const raised = new Set(objections.map((objection) => objection.objectionId).filter((id) => isNonAnswerable(id) || needsBlocker(id)));
|
|
69
|
+
const refused = [];
|
|
70
|
+
const seen = new Set();
|
|
71
|
+
for (const entry of acknowledged) {
|
|
72
|
+
const id = entry?.objectionId;
|
|
73
|
+
if (typeof id !== "string" || !raised.has(id) || seen.has(id))
|
|
74
|
+
continue;
|
|
75
|
+
const answer = entry.answer;
|
|
76
|
+
if (typeof answer !== "string" || answer.trim().length === 0)
|
|
77
|
+
continue;
|
|
78
|
+
// An answer that names a blocker DOES close an untested change, so it is not
|
|
79
|
+
// refused. Only the sentence on its own is.
|
|
80
|
+
if (needsBlocker(id) && String(entry?.blocker ?? "").trim().length > 0)
|
|
81
|
+
continue;
|
|
82
|
+
seen.add(id);
|
|
83
|
+
refused.push({
|
|
84
|
+
objectionId: `${REFUSED_ANSWER_PREFIX}${id}`,
|
|
85
|
+
verifier: "answers",
|
|
86
|
+
message: needsBlocker(id)
|
|
87
|
+
? "The answer says why this change has no test, and it does not say what stopped this run from writing one, so the objection stays open."
|
|
88
|
+
: "This objection is closed only by a planned test, so the answer closed nothing and the objection stays open.",
|
|
89
|
+
evidence: `answered "${id}"${needsBlocker(id) ? " with no `blocker`" : ", which no answer closes"}`,
|
|
90
|
+
suggestion: needsBlocker(id)
|
|
91
|
+
? "Plan a test for this change and register the plan again. If this run cannot write one, send the same answer with `blocker` naming what stopped it — a service that is not running, a paired branch that no longer exists, the one credential the run holds. A reason the change is not worth testing is not a blocker."
|
|
92
|
+
: "Add a planned test that names the record's state as `startState` and runs this route's mutation on it, then register the plan again. Every other objection takes an answer; this one takes a test.",
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
return refused;
|
|
96
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Objection, ObjectionAnswer, PlanChange, PlanInput, PlannedTest, VerifyContext } from "./types.js";
|
|
2
|
+
/** The plan as stored for the run. Every registration produces one; the newest
|
|
3
|
+
* replaces the last. */
|
|
4
|
+
export interface Plan {
|
|
5
|
+
/** The changes the agent said the diff makes. Published in the report
|
|
6
|
+
* beside the tests delivered against each one. */
|
|
7
|
+
changes: PlanChange[];
|
|
8
|
+
plannedTests: PlannedTest[];
|
|
9
|
+
/** Objections the agent answered, with the answers. */
|
|
10
|
+
answeredObjections: Array<{
|
|
11
|
+
objection: Objection;
|
|
12
|
+
answer: string;
|
|
13
|
+
blocker?: string;
|
|
14
|
+
}>;
|
|
15
|
+
/** Closed by a blocker nothing checked. Its own bucket rather than a flag on
|
|
16
|
+
* `answeredObjections`, so a run's count of them is a number the report can
|
|
17
|
+
* carry: the server holds six facts about a run and can falsify almost no
|
|
18
|
+
* blocker, so the honest position is that these closes are UNCHECKED, not that
|
|
19
|
+
* they are wrong. A run whose objections all close this way looks, in the
|
|
20
|
+
* counts, exactly like a run that answered nothing. */
|
|
21
|
+
unverifiedCloses: Array<{
|
|
22
|
+
objection: Objection;
|
|
23
|
+
answer: string;
|
|
24
|
+
blocker: string;
|
|
25
|
+
}>;
|
|
26
|
+
/** Objections with no answer. Rendered in the report — this is the accountability. */
|
|
27
|
+
openObjections: Objection[];
|
|
28
|
+
/** Recorded, not owned. The caller counts the run's registrations and overwrites
|
|
29
|
+
* whatever the agent sent; this function is pure and holds no run state, so it
|
|
30
|
+
* stores the number it was handed. Nothing here enforces that. */
|
|
31
|
+
registrationNumber: number;
|
|
32
|
+
}
|
|
33
|
+
export interface PlanResult {
|
|
34
|
+
/** Always present. Once a registration reaches this function it is never refused;
|
|
35
|
+
* the schema, which runs before it, is the only thing that can reject a call. */
|
|
36
|
+
plan: Plan;
|
|
37
|
+
/** Every objection this registration raised, answered or not. */
|
|
38
|
+
objections: Objection[];
|
|
39
|
+
/** One line for the tool's output text. */
|
|
40
|
+
summary: string;
|
|
41
|
+
/** The answers taken from the previous registration, and the number it had.
|
|
42
|
+
* Absent when this registration carried none. Rendered so the agent rechecks
|
|
43
|
+
* each one against the plan it has just changed. */
|
|
44
|
+
carried?: {
|
|
45
|
+
registrationNumber: number;
|
|
46
|
+
answers: ObjectionAnswer[];
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
export declare function registerPlan(registration: PlanInput, ctx: VerifyContext, previous?: Plan): PlanResult;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { answerFor, isNonAnswerable, refusedAnswerObjections, unknownAnswerObjections } from "./answers.js";
|
|
2
|
+
import { runPlanTimeVerifiers } from "./runVerifiers.js";
|
|
3
|
+
import { logger } from "../utils/logger.js";
|
|
4
|
+
function storedRegistrationNumber(registration) {
|
|
5
|
+
const raw = registration.registrationNumber;
|
|
6
|
+
return typeof raw === "number" && Number.isFinite(raw) ? raw : 1;
|
|
7
|
+
}
|
|
8
|
+
/** Run the plan-time verifiers and store the registration as the run's plan. Every
|
|
9
|
+
* registration reaching here is stored and no objection blocks the agent; the schema
|
|
10
|
+
* runs first and CAN reject. Never throws, because a crash discards the plan —
|
|
11
|
+
* the strongest refusal there is. */
|
|
12
|
+
/** The answer the previous registration gave one objection, or nothing. Passed as
|
|
13
|
+
* a parameter rather than on `VerifyContext`: no verifier reads it, and the
|
|
14
|
+
* previous plan is state the caller holds, not a fact a check is run against. */
|
|
15
|
+
function carriedAnswerFor(objectionId, previous) {
|
|
16
|
+
// Both buckets: a blocker close is stored apart from an answer, and an objection
|
|
17
|
+
// that recurs still carries the one it was closed with.
|
|
18
|
+
const answered = [
|
|
19
|
+
...(Array.isArray(previous?.answeredObjections) ? previous.answeredObjections : []),
|
|
20
|
+
...(Array.isArray(previous?.unverifiedCloses) ? previous.unverifiedCloses : []),
|
|
21
|
+
];
|
|
22
|
+
if (answered.length === 0)
|
|
23
|
+
return undefined;
|
|
24
|
+
for (const entry of answered) {
|
|
25
|
+
if (entry?.objection?.objectionId !== objectionId)
|
|
26
|
+
continue;
|
|
27
|
+
const answer = typeof entry.answer === "string" ? entry.answer.trim() : "";
|
|
28
|
+
// The blocker travels with the answer it belongs to. Carrying the sentence
|
|
29
|
+
// alone would reopen every untested change the previous registration closed.
|
|
30
|
+
if (answer.length > 0)
|
|
31
|
+
return { objectionId, answer, ...(entry.blocker ? { blocker: entry.blocker } : {}) };
|
|
32
|
+
}
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
/** The previous registration's answers to the objections THIS one raised again. An
|
|
36
|
+
* answer sent in this call wins, a state-test objection is never carried, and an
|
|
37
|
+
* answer whose objection is gone is dropped in silence — the agent sends the new
|
|
38
|
+
* answers only, and the stored plan holds the rest. */
|
|
39
|
+
function carriedAnswers(objections, acknowledged, previous) {
|
|
40
|
+
const carried = [];
|
|
41
|
+
for (const objection of objections) {
|
|
42
|
+
if (isNonAnswerable(objection.objectionId))
|
|
43
|
+
continue;
|
|
44
|
+
if (answerFor(objection, acknowledged))
|
|
45
|
+
continue;
|
|
46
|
+
const carriedAnswer = carriedAnswerFor(objection.objectionId, previous);
|
|
47
|
+
if (carriedAnswer)
|
|
48
|
+
carried.push(carriedAnswer);
|
|
49
|
+
}
|
|
50
|
+
return carried;
|
|
51
|
+
}
|
|
52
|
+
export function registerPlan(registration, ctx, previous) {
|
|
53
|
+
const acknowledged = Array.isArray(registration.answers) ? registration.answers : [];
|
|
54
|
+
const plannedTests = Array.isArray(registration.plannedTests) ? registration.plannedTests : [];
|
|
55
|
+
const changes = Array.isArray(registration.changes) ? registration.changes : [];
|
|
56
|
+
const registrationNumber = storedRegistrationNumber(registration);
|
|
57
|
+
// The verifiers see the same normalised fields the STORED plan gets, not the raw
|
|
58
|
+
// registration: handing them the raw one let a malformed top-level field crash
|
|
59
|
+
// every check at once, giving eight `:crashed` objections and no real ones.
|
|
60
|
+
// Crash containment is for a malformed field INSIDE a planned test.
|
|
61
|
+
const verified = runPlanTimeVerifiers({ ...registration, plannedTests, changes, answers: acknowledged }, ctx);
|
|
62
|
+
const objections = [
|
|
63
|
+
...verified,
|
|
64
|
+
...unknownAnswerObjections(verified, acknowledged, {
|
|
65
|
+
scope: "on this plan",
|
|
66
|
+
suggestion: "Answer only the ids in the Objections list this registration returns. An objection you have since fixed is gone and needs no answer, and an objection raised after the tests run cannot be answered before them.",
|
|
67
|
+
}),
|
|
68
|
+
...refusedAnswerObjections(verified, acknowledged),
|
|
69
|
+
];
|
|
70
|
+
const carried = carriedAnswers(objections, acknowledged, previous);
|
|
71
|
+
// The carried answers go last: `answerFor` takes the first non-blank entry, so
|
|
72
|
+
// an answer this registration sends wins over the one it replaces.
|
|
73
|
+
const answers = carried.length === 0 ? acknowledged : [...acknowledged, ...carried];
|
|
74
|
+
const answeredObjections = [];
|
|
75
|
+
const unverifiedCloses = [];
|
|
76
|
+
const openObjections = [];
|
|
77
|
+
for (const objection of objections) {
|
|
78
|
+
// Two objections take more than a sentence, and nothing here refuses a plan.
|
|
79
|
+
// Only a planned test closes the state-test objection. An untested change takes
|
|
80
|
+
// a planned test too, unless the answer names what stopped this run from
|
|
81
|
+
// writing one. Either way an answer that does not close it leaves it open and
|
|
82
|
+
// draws the refusal objection raised above.
|
|
83
|
+
const answer = isNonAnswerable(objection.objectionId) ? undefined : answerFor(objection, answers);
|
|
84
|
+
if (answer) {
|
|
85
|
+
const blocker = answers.find((entry) => entry?.objectionId === objection.objectionId && String(entry?.blocker ?? "").trim().length > 0)?.blocker;
|
|
86
|
+
// A blocker closes the objection and nothing here can check it, so it is
|
|
87
|
+
// counted apart from an answer the plan itself makes checkable.
|
|
88
|
+
if (blocker)
|
|
89
|
+
unverifiedCloses.push({ objection, answer, blocker });
|
|
90
|
+
else
|
|
91
|
+
answeredObjections.push({ objection, answer });
|
|
92
|
+
}
|
|
93
|
+
else
|
|
94
|
+
openObjections.push(objection);
|
|
95
|
+
}
|
|
96
|
+
const summary = objections.length === 0
|
|
97
|
+
? "Plan stored. No objections."
|
|
98
|
+
: `Plan stored. ${openObjections.length} objection(s) open, ${answeredObjections.length} answered${unverifiedCloses.length > 0
|
|
99
|
+
? `, ${unverifiedCloses.length} closed by a blocker nothing checked`
|
|
100
|
+
: ""}. Revise and register it again, or answer them — open objections appear in the report.`;
|
|
101
|
+
logger.info(`Plan v2: registration ${registrationNumber} stored — ${summary}`);
|
|
102
|
+
return {
|
|
103
|
+
objections,
|
|
104
|
+
summary,
|
|
105
|
+
...(carried.length > 0 && previous
|
|
106
|
+
? { carried: { registrationNumber: previous.registrationNumber, answers: carried } }
|
|
107
|
+
: {}),
|
|
108
|
+
plan: {
|
|
109
|
+
changes,
|
|
110
|
+
plannedTests,
|
|
111
|
+
answeredObjections,
|
|
112
|
+
unverifiedCloses,
|
|
113
|
+
openObjections,
|
|
114
|
+
registrationNumber,
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Objection, PlanInput, Verifier, VerifyContext } from "./types.js";
|
|
2
|
+
/** Verifiers that need only the plan and the run's facts. The order is the order
|
|
3
|
+
* objections are reported in; it carries no priority. */
|
|
4
|
+
export declare const PLAN_TIME_VERIFIERS: Verifier[];
|
|
5
|
+
/** One verifier, crash contained: declared types hold only on the validated
|
|
6
|
+
* path, and a throw would discard a registration this design promises to store. */
|
|
7
|
+
export declare function runVerifier(verifier: Verifier, registration: PlanInput, ctx: VerifyContext): Objection[];
|
|
8
|
+
/** Every verifier runs on every registration: stopping at the first objection would
|
|
9
|
+
* hand the agent one objection per round trip. */
|
|
10
|
+
export declare function runPlanTimeVerifiers(registration: PlanInput, ctx: VerifyContext): Objection[];
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { changedFile } from "./verifiers/changedFile.js";
|
|
2
|
+
import { endpointGrounded } from "./verifiers/endpointGrounded.js";
|
|
3
|
+
import { statedDifference } from "./verifiers/statedDifference.js";
|
|
4
|
+
import { existingCoverage } from "./verifiers/existingCoverage.js";
|
|
5
|
+
import { expectedOutcomeAtPlanTime } from "./verifiers/expectedOutcome.js";
|
|
6
|
+
import { uiElementGrounded } from "./verifiers/uiElementGrounded.js";
|
|
7
|
+
import { screenRoute } from "./verifiers/screenRoute.js";
|
|
8
|
+
import { coverage } from "./verifiers/coverage.js";
|
|
9
|
+
import { removedElementGuarded } from "./verifiers/removedElementGuarded.js";
|
|
10
|
+
/** Verifiers that need only the plan and the run's facts. The order is the order
|
|
11
|
+
* objections are reported in; it carries no priority. */
|
|
12
|
+
export const PLAN_TIME_VERIFIERS = [
|
|
13
|
+
changedFile,
|
|
14
|
+
endpointGrounded,
|
|
15
|
+
statedDifference,
|
|
16
|
+
existingCoverage,
|
|
17
|
+
expectedOutcomeAtPlanTime,
|
|
18
|
+
uiElementGrounded,
|
|
19
|
+
screenRoute,
|
|
20
|
+
coverage,
|
|
21
|
+
removedElementGuarded,
|
|
22
|
+
];
|
|
23
|
+
/** How a crashed verifier reads back. The id is the verifier's name plus a fixed
|
|
24
|
+
* word, so it stays stable across registrations and can be answered. */
|
|
25
|
+
function crashedObjection(verifierName, error) {
|
|
26
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
27
|
+
return {
|
|
28
|
+
objectionId: `${verifierName}:crashed`,
|
|
29
|
+
verifier: verifierName,
|
|
30
|
+
message: `The ${verifierName} check could not run against this plan.`,
|
|
31
|
+
evidence: `${verifierName} threw: ${detail}`,
|
|
32
|
+
suggestion: "A field on one of the planned tests is probably not the type the schema describes — check `changedFile`, `expected`, `differsFrom` and `routes`, and that every entry of `plannedTests` is an object. The plan was stored and the other checks still ran.",
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/** One verifier, crash contained: declared types hold only on the validated
|
|
36
|
+
* path, and a throw would discard a registration this design promises to store. */
|
|
37
|
+
export function runVerifier(verifier, registration, ctx) {
|
|
38
|
+
try {
|
|
39
|
+
return verifier.run(registration, ctx);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
return [crashedObjection(verifier.name, error)];
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Every verifier runs on every registration: stopping at the first objection would
|
|
46
|
+
* hand the agent one objection per round trip. */
|
|
47
|
+
export function runPlanTimeVerifiers(registration, ctx) {
|
|
48
|
+
return PLAN_TIME_VERIFIERS.flatMap((verifier) => runVerifier(verifier, registration, ctx));
|
|
49
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { ScenarioStep } from "../types/RepositoryAnalysis.js";
|
|
2
|
+
import { PlannedTest } from "./types.js";
|
|
3
|
+
export declare const HTTP_METHODS: ReadonlySet<string>;
|
|
4
|
+
/** Whether the planned test carries a `ui` or `e2e` label. The label alone exempts
|
|
5
|
+
* nothing (see `isUICandidate`); the schema reads it alone only for a planned test
|
|
6
|
+
* with no steps to look at. */
|
|
7
|
+
export declare function hasUITestTypeLabel(rawTestType: unknown): boolean;
|
|
8
|
+
/** Whether a step is an HTTP call: a real request method AND a path, since a verb
|
|
9
|
+
* with no path names no route. `interactionType` is deliberately not consulted —
|
|
10
|
+
* it labels what a step exercises, not how it reaches the app. Applied PER STEP. */
|
|
11
|
+
export declare function isHTTPStep(step: ScenarioStep | undefined): boolean;
|
|
12
|
+
/** Whether a step calls the app rather than driving a screen: anything with a
|
|
13
|
+
* target whose method is not a page interaction, so an HTTP request and an
|
|
14
|
+
* `OPERATION` both count. The page interactions are the closed set. */
|
|
15
|
+
export declare function isCallStep(step: ScenarioStep | undefined): boolean;
|
|
16
|
+
/** Whether a planned test is exempt from the route checks. A UI test's endpoints are
|
|
17
|
+
* attributions the agent wrote, not what it asserts. THE LABEL DECIDES: inferring
|
|
18
|
+
* from method names read a fixed list of HTTP verbs, so a GraphQL or Thrift call
|
|
19
|
+
* looked like a browser click. Every verifier reads this one function. */
|
|
20
|
+
export declare function isUIPlannedTest(plannedTest: PlannedTest): boolean;
|
|
21
|
+
/** Every step that calls the app rather than driving a screen. */
|
|
22
|
+
export declare function callSteps(plannedTest: PlannedTest): ScenarioStep[];
|
|
23
|
+
/** The step a planned test's test is about: the one `stepUnderTest` names, or the only
|
|
24
|
+
* call the planned test makes. A planned test that makes several calls and names none
|
|
25
|
+
* has NO subject — the agent states it and the server asks when it is missing,
|
|
26
|
+
* rather than ranking write methods and path prefixes to guess. A `stepUnderTest`
|
|
27
|
+
* matching no call is undefined too: the declaration is wrong, not a hint. */
|
|
28
|
+
export declare function subjectStep(plannedTest: PlannedTest): ScenarioStep | undefined;
|
|
29
|
+
/** One step's route as a single key, `METHOD path`. The method is trimmed and
|
|
30
|
+
* upper-cased, because a method is case-insensitive by protocol; the PATH is
|
|
31
|
+
* compared AS WRITTEN, so two spellings of one route stay two routes. A step
|
|
32
|
+
* with no path names no route and gets no key. */
|
|
33
|
+
export declare function stepRouteKey(step: ScenarioStep | undefined): string | undefined;
|
|
34
|
+
/** One step's route as `METHOD path` with every record id folded to `{id}`.
|
|
35
|
+
*
|
|
36
|
+
* The state test asks whether the PLAN covers a route, and the id a step happened
|
|
37
|
+
* to write is not part of the route: `/orders/777` and `/orders/888` are one
|
|
38
|
+
* route tested twice, and reading them as two let a plan owe two state tests and
|
|
39
|
+
* close one by renumbering (run 33988304641, LB03). `stepRouteKey` keeps the path
|
|
40
|
+
* as written because `statedDifference` asks the opposite question — whether two
|
|
41
|
+
* planned tests test the same thing — and there two spellings must stay two. */
|
|
42
|
+
export declare function stateRouteKey(step: ScenarioStep | undefined): string | undefined;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { HTTP_STEP_METHODS, isUiStepVerb } from "../types/StepMethod.js";
|
|
2
|
+
export const HTTP_METHODS = new Set(HTTP_STEP_METHODS);
|
|
3
|
+
/** Whether the planned test carries a `ui` or `e2e` label. The label alone exempts
|
|
4
|
+
* nothing (see `isUICandidate`); the schema reads it alone only for a planned test
|
|
5
|
+
* with no steps to look at. */
|
|
6
|
+
export function hasUITestTypeLabel(rawTestType) {
|
|
7
|
+
// Trimmed as well as lower-cased: without the trim `"ui "` was checked and
|
|
8
|
+
// `"UI"` was not, so one trailing space decided which of three verifiers ran.
|
|
9
|
+
const testType = String(rawTestType ?? "").trim().toLowerCase();
|
|
10
|
+
return testType === "ui" || testType === "e2e";
|
|
11
|
+
}
|
|
12
|
+
/** Whether a step is an HTTP call: a real request method AND a path, since a verb
|
|
13
|
+
* with no path names no route. `interactionType` is deliberately not consulted —
|
|
14
|
+
* it labels what a step exercises, not how it reaches the app. Applied PER STEP. */
|
|
15
|
+
export function isHTTPStep(step) {
|
|
16
|
+
const method = String(step?.method ?? "").trim().toUpperCase();
|
|
17
|
+
return HTTP_METHODS.has(method) && String(step?.path ?? "").trim().length > 0;
|
|
18
|
+
}
|
|
19
|
+
/** Whether a step calls the app rather than driving a screen: anything with a
|
|
20
|
+
* target whose method is not a page interaction, so an HTTP request and an
|
|
21
|
+
* `OPERATION` both count. The page interactions are the closed set. */
|
|
22
|
+
export function isCallStep(step) {
|
|
23
|
+
return !isUiStepVerb(step?.method) && String(step?.path ?? "").trim().length > 0;
|
|
24
|
+
}
|
|
25
|
+
/** Whether a planned test is exempt from the route checks. A UI test's endpoints are
|
|
26
|
+
* attributions the agent wrote, not what it asserts. THE LABEL DECIDES: inferring
|
|
27
|
+
* from method names read a fixed list of HTTP verbs, so a GraphQL or Thrift call
|
|
28
|
+
* looked like a browser click. Every verifier reads this one function. */
|
|
29
|
+
export function isUIPlannedTest(plannedTest) {
|
|
30
|
+
// A null entry in `plannedTests` reaches here on the second, unvalidated
|
|
31
|
+
// caller the schema does not cover. Without this guard `planned test.scenario`
|
|
32
|
+
// throws, and `runVerifier` turns that throw into an acknowledgeable
|
|
33
|
+
// "crashed" objection — which hides the real objection instead of raising it.
|
|
34
|
+
if (!plannedTest)
|
|
35
|
+
return false;
|
|
36
|
+
return hasUITestTypeLabel(plannedTest.scenario?.testType);
|
|
37
|
+
}
|
|
38
|
+
/** Every step that calls the app rather than driving a screen. */
|
|
39
|
+
export function callSteps(plannedTest) {
|
|
40
|
+
return (plannedTest?.scenario?.steps ?? []).filter((step) => isCallStep(step));
|
|
41
|
+
}
|
|
42
|
+
/** The step a planned test's test is about: the one `stepUnderTest` names, or the only
|
|
43
|
+
* call the planned test makes. A planned test that makes several calls and names none
|
|
44
|
+
* has NO subject — the agent states it and the server asks when it is missing,
|
|
45
|
+
* rather than ranking write methods and path prefixes to guess. A `stepUnderTest`
|
|
46
|
+
* matching no call is undefined too: the declaration is wrong, not a hint. */
|
|
47
|
+
export function subjectStep(plannedTest) {
|
|
48
|
+
const calls = callSteps(plannedTest);
|
|
49
|
+
const declaredOrder = plannedTest?.declarations?.stepUnderTest;
|
|
50
|
+
if (typeof declaredOrder === "number")
|
|
51
|
+
return calls.find((step) => (step.order ?? 0) === declaredOrder);
|
|
52
|
+
return calls.length === 1 ? calls[0] : undefined;
|
|
53
|
+
}
|
|
54
|
+
/** One step's route as a single key, `METHOD path`. The method is trimmed and
|
|
55
|
+
* upper-cased, because a method is case-insensitive by protocol; the PATH is
|
|
56
|
+
* compared AS WRITTEN, so two spellings of one route stay two routes. A step
|
|
57
|
+
* with no path names no route and gets no key. */
|
|
58
|
+
export function stepRouteKey(step) {
|
|
59
|
+
const path = String(step?.path ?? "").trim();
|
|
60
|
+
if (!path)
|
|
61
|
+
return undefined;
|
|
62
|
+
const method = String(step?.method ?? "ANY").trim().toUpperCase() || "ANY";
|
|
63
|
+
return `${method} ${path}`;
|
|
64
|
+
}
|
|
65
|
+
/** A segment that identifies one record rather than naming part of the route: all
|
|
66
|
+
* digits, a UUID, or a `{param}` / `:param` placeholder. */
|
|
67
|
+
const RECORD_ID_SEGMENT = /^(?:\d+|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|\{[^}]*\}|:[^/]+)$/;
|
|
68
|
+
/** One step's route as `METHOD path` with every record id folded to `{id}`.
|
|
69
|
+
*
|
|
70
|
+
* The state test asks whether the PLAN covers a route, and the id a step happened
|
|
71
|
+
* to write is not part of the route: `/orders/777` and `/orders/888` are one
|
|
72
|
+
* route tested twice, and reading them as two let a plan owe two state tests and
|
|
73
|
+
* close one by renumbering (run 33988304641, LB03). `stepRouteKey` keeps the path
|
|
74
|
+
* as written because `statedDifference` asks the opposite question — whether two
|
|
75
|
+
* planned tests test the same thing — and there two spellings must stay two. */
|
|
76
|
+
export function stateRouteKey(step) {
|
|
77
|
+
const path = String(step?.path ?? "").trim();
|
|
78
|
+
if (!path)
|
|
79
|
+
return undefined;
|
|
80
|
+
const method = String(step?.method ?? "ANY").trim().toUpperCase() || "ANY";
|
|
81
|
+
const folded = path
|
|
82
|
+
.split("/")
|
|
83
|
+
.map((segment) => (RECORD_ID_SEGMENT.test(segment) ? "{id}" : segment))
|
|
84
|
+
.join("/");
|
|
85
|
+
return `${method} ${folded}`;
|
|
86
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { ChangedFileName } from "../utils/branchDiff.js";
|
|
2
|
+
import { PlanScenario } from "../types/RepositoryAnalysis.js";
|
|
3
|
+
import type { BlueprintCaptureDigest } from "../types/BlueprintDigest.js";
|
|
4
|
+
import type { RemovedUiElement } from "../utils/removedUiElements.js";
|
|
5
|
+
/** One element a UI planned test targets. EVERY field is optional and three are
|
|
6
|
+
* nullable: a declaration reaches a verifier unvalidated, so a missing or null
|
|
7
|
+
* field is an objection, not a crash. Not `Partial<BlueprintElementDigest>` —
|
|
8
|
+
* the capture's shape has no nulls and requires role and accessibleName. */
|
|
9
|
+
export interface DeclaredUiElement {
|
|
10
|
+
role?: string;
|
|
11
|
+
accessibleName?: string;
|
|
12
|
+
testId?: string | null;
|
|
13
|
+
stableId?: string | null;
|
|
14
|
+
contextText?: string[] | null;
|
|
15
|
+
/** Carried because the plan schema reuses the report's `targetElementSchema`.
|
|
16
|
+
* No plan check reads either one. */
|
|
17
|
+
mutability?: "mutable" | "immutable" | "unknown";
|
|
18
|
+
widgetType?: "native" | "custom" | "unknown";
|
|
19
|
+
}
|
|
20
|
+
/** One change the diff makes, as the agent read it. Nothing on the server parses
|
|
21
|
+
* a description. */
|
|
22
|
+
export interface PlanChange {
|
|
23
|
+
id: string;
|
|
24
|
+
text: string;
|
|
25
|
+
source: string;
|
|
26
|
+
/** Where a user or caller meets this change. Every change states it: an optional
|
|
27
|
+
* field the agent skips gives the check nothing to read. */
|
|
28
|
+
surfaces: Array<"api" | "page">;
|
|
29
|
+
/** The inputs that prove this change: what a route must accept and what it must
|
|
30
|
+
* reject. Exactly one of `value` and `absent` per entry. The coverage check reads
|
|
31
|
+
* it; nothing else on the server does. */
|
|
32
|
+
cases?: Array<{
|
|
33
|
+
param: string;
|
|
34
|
+
value?: string | number | boolean | null;
|
|
35
|
+
absent?: true;
|
|
36
|
+
expect: "accept" | "reject";
|
|
37
|
+
}>;
|
|
38
|
+
}
|
|
39
|
+
/** What a planned test states so a verifier has a checkable change. Kept in step with
|
|
40
|
+
* `declarationFieldsSchema`, which the plan tool asserts against this type. */
|
|
41
|
+
export interface PlannedTestDetails {
|
|
42
|
+
/** One of the files this run's diff changed. It is what makes a planned test count
|
|
43
|
+
* as covering that file. Absent on a full-repo run. */
|
|
44
|
+
changedFile?: string;
|
|
45
|
+
asserts: string;
|
|
46
|
+
/** The `order` of the step the test is about. Required whenever the planned test
|
|
47
|
+
* makes more than one call: nothing infers a subject. */
|
|
48
|
+
stepUnderTest?: number;
|
|
49
|
+
/** ABSENT means undeclared, which readers must tell apart from a declared red. */
|
|
50
|
+
expected?: {
|
|
51
|
+
outcome: "fail" | "pass";
|
|
52
|
+
why: string;
|
|
53
|
+
};
|
|
54
|
+
/** Required only when another planned test targets the same endpoint. */
|
|
55
|
+
differsFrom?: Array<{
|
|
56
|
+
plannedTestId: string;
|
|
57
|
+
difference: string;
|
|
58
|
+
}>;
|
|
59
|
+
/** Only each file's presence in the checkout is checked. */
|
|
60
|
+
existingTests?: Array<{
|
|
61
|
+
file: string;
|
|
62
|
+
differsBy: string;
|
|
63
|
+
}>;
|
|
64
|
+
/** The file that declares what a step calls, one entry per step, named by
|
|
65
|
+
* `order`. Nothing reads INSIDE the cited file: frameworks assemble paths from
|
|
66
|
+
* scopes and prefixes, so the path often appears nowhere. */
|
|
67
|
+
routes?: Array<{
|
|
68
|
+
step?: number;
|
|
69
|
+
file: string;
|
|
70
|
+
}>;
|
|
71
|
+
/** The changed file that renders the screen `elements.pageUrl` opens. */
|
|
72
|
+
screenEvidence?: {
|
|
73
|
+
file: string;
|
|
74
|
+
};
|
|
75
|
+
/** The ids of the declared changes this test proves. */
|
|
76
|
+
changes?: string[];
|
|
77
|
+
/** The state the mutated record is in before the mutation. Presence only. */
|
|
78
|
+
startState?: string;
|
|
79
|
+
/** UI planned tests only. `items` entries are COPIES of elements a capture of
|
|
80
|
+
* `pageUrl` holds. `items: null` MEANS capture failed and carries the
|
|
81
|
+
* `[no-blueprint-data]` obligation; `elements: null` is dropped at parse. */
|
|
82
|
+
elements?: {
|
|
83
|
+
pageUrl?: string;
|
|
84
|
+
items?: DeclaredUiElement[] | null;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
export interface PlannedTest {
|
|
88
|
+
plannedTestId: string;
|
|
89
|
+
scenario: PlanScenario;
|
|
90
|
+
declarations: PlannedTestDetails;
|
|
91
|
+
/** Deliberately NOT part of the planned test id: an id stays the same across
|
|
92
|
+
* registrations, and a single-repo plan names no repository. */
|
|
93
|
+
repository?: string;
|
|
94
|
+
}
|
|
95
|
+
export interface ObjectionAnswer {
|
|
96
|
+
objectionId: string;
|
|
97
|
+
answer: string;
|
|
98
|
+
/** What stopped this run from writing the test: a service that is not running,
|
|
99
|
+
* a paired branch that no longer exists, the one credential the run holds.
|
|
100
|
+
* An objection about an untested change closes only with one of these. */
|
|
101
|
+
blocker?: string;
|
|
102
|
+
}
|
|
103
|
+
export interface PlanInput {
|
|
104
|
+
/** The changes the diff makes. Each planned test cites the ones it tests. */
|
|
105
|
+
changes: PlanChange[];
|
|
106
|
+
plannedTests: PlannedTest[];
|
|
107
|
+
answers: ObjectionAnswer[];
|
|
108
|
+
/** The tool owns this and overwrites whatever the caller sends. */
|
|
109
|
+
registrationNumber: number;
|
|
110
|
+
}
|
|
111
|
+
/** A verifier's objection. Advisory: it never removes a planned test. */
|
|
112
|
+
export interface Objection {
|
|
113
|
+
/** UNIQUE within a run — answers match by id, so a shared id closes the wrong
|
|
114
|
+
* objection. `<verifier>:<plannedTestId>`, or `<verifier>:<objection>`. */
|
|
115
|
+
objectionId: string;
|
|
116
|
+
verifier: string;
|
|
117
|
+
/** Absent for a plan-level objection, and absent whenever the id is not one the
|
|
118
|
+
* plan holds. It is always in `evidence` regardless. */
|
|
119
|
+
plannedTestId?: string;
|
|
120
|
+
message: string;
|
|
121
|
+
evidence: string;
|
|
122
|
+
/** Required: an objection without a next move reads as a rejection. */
|
|
123
|
+
suggestion: string;
|
|
124
|
+
}
|
|
125
|
+
/** Facts the verifiers check against. Gathered once per run by the caller. */
|
|
126
|
+
export interface VerifyContext {
|
|
127
|
+
/** Every repository this run analyzed, and the files its diff changed there.
|
|
128
|
+
* NAMES ONLY, never contents. A single-repository run has one entry. */
|
|
129
|
+
changedFiles: Record<string, ChangedFileName[]>;
|
|
130
|
+
/** Repositories with no branch diff. Softens objections RUN-WIDE, because a
|
|
131
|
+
* planned test that names no repository is checked against every one of them. */
|
|
132
|
+
reposWithoutDiff: string[];
|
|
133
|
+
/** Every `browser_blueprint` capture this run took, in order. Evidence, not an
|
|
134
|
+
* allowlist: a capture shows the page as it stood when it was taken, so an
|
|
135
|
+
* element behind a modal the walk never opened is legitimately missing. */
|
|
136
|
+
uiCaptures: BlueprintCaptureDigest[];
|
|
137
|
+
/** The identifying `data-*` attributes the diff removed from a page that
|
|
138
|
+
* survives. A removal guard declares the element that is GONE, so no capture
|
|
139
|
+
* can hold it and this list is its only grounding. */
|
|
140
|
+
removedUiElements: RemovedUiElement[];
|
|
141
|
+
/** Whether a cited path names a file in any of the run's repositories. Injected
|
|
142
|
+
* to keep the verifiers filesystem-free. */
|
|
143
|
+
citedFileExists(relativePath: string): boolean;
|
|
144
|
+
/** The route patterns that mount a changed frontend file, as the framework
|
|
145
|
+
* spells them (`/secrets/$envSlug`, `/orders/:id`). Injected for the same
|
|
146
|
+
* reason as `citedFileExists`: the scan reads the repository, the verifier
|
|
147
|
+
* does not. UNDEFINED means the resolver cannot speak for this file — an
|
|
148
|
+
* unsupported framework, a route table it could not parse — and the check
|
|
149
|
+
* stays silent. An EMPTY array is a statement: nothing mounts it. */
|
|
150
|
+
routesRenderingFile?(relativePath: string): string[] | undefined;
|
|
151
|
+
}
|
|
152
|
+
/** Every changed file of the run, whatever repository it sits in. */
|
|
153
|
+
export declare function allChangedFiles(ctx: VerifyContext): ChangedFileName[];
|
|
154
|
+
export interface Verifier {
|
|
155
|
+
readonly name: string;
|
|
156
|
+
run(registration: PlanInput, ctx: VerifyContext): Objection[];
|
|
157
|
+
}
|
|
158
|
+
/** The changed files a planned test is checked against: its own repository's when it
|
|
159
|
+
* names one this run analyzed, every repository's when it names none. UNDEFINED
|
|
160
|
+
* when it names a repository this run does not know, which the caller objects to. */
|
|
161
|
+
export declare function changedFilesFor(ctx: VerifyContext, plannedTest: {
|
|
162
|
+
repository?: string;
|
|
163
|
+
}): ChangedFileName[] | undefined;
|