@skyramp/mcp 0.3.2-rc.pom-4 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/adapters/jestAdapter.d.ts +14 -0
- package/build/adapters/jestAdapter.js +113 -0
- package/build/adapters/mochaAdapter.d.ts +13 -0
- package/build/adapters/mochaAdapter.js +87 -0
- package/build/adapters/playwrightAdapter.d.ts +17 -0
- package/build/adapters/playwrightAdapter.js +182 -0
- package/build/adapters/pytestAdapter.d.ts +15 -0
- package/build/adapters/pytestAdapter.js +108 -0
- package/build/commands/commandLibrary.d.ts +1 -0
- package/build/commands/commandLibrary.js +19 -13
- package/build/commands/localDevTestChangesCommand.d.ts +15 -0
- package/build/commands/localDevTestChangesCommand.js +201 -0
- package/build/index.js +82 -6
- package/build/prompts/code-reuse.js +3 -0
- package/build/prompts/enhance-assertions/sharedAssertionRules.d.ts +1 -0
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +17 -0
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +11 -9
- package/build/prompts/local-dev/local-dev-plan.d.ts +35 -0
- package/build/prompts/local-dev/local-dev-plan.js +429 -0
- package/build/prompts/local-dev/local-dev-prompts.d.ts +4 -0
- package/build/prompts/local-dev/local-dev-prompts.js +190 -0
- package/build/prompts/prompt-utils.d.ts +8 -0
- package/build/prompts/prompt-utils.js +33 -0
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +70 -7
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +1 -1
- package/build/prompts/sut-setup/shared.js +19 -17
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +10 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +54 -1
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +21 -29
- package/build/prompts/test-recommendation/scopeAssessment.d.ts +5 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +78 -6
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +41 -4
- package/build/prompts/testbot/testbot-prompts.d.ts +0 -5
- package/build/prompts/testbot/testbot-prompts.js +39 -55
- package/build/recommendation/planRanker.d.ts +15 -2
- package/build/recommendation/planRanker.js +76 -5
- package/build/resources/testbotResource.js +2 -1
- package/build/services/AnalyticsService.d.ts +1 -1
- package/build/services/TestExecutionService.d.ts +2 -1
- package/build/services/TestExecutionService.js +8 -3
- package/build/services/TestGenerationService.d.ts +2 -2
- package/build/services/TestGenerationService.js +39 -21
- package/build/services/containerEnv.js +3 -1
- package/build/tool-phases.js +7 -0
- package/build/tools/code-refactor/codeReuseTool.js +43 -4
- package/build/tools/code-refactor/enhanceAssertionsTool.js +68 -18
- package/build/tools/code-refactor/reuse-outcome.d.ts +109 -0
- package/build/tools/code-refactor/reuse-outcome.js +158 -0
- package/build/tools/code-refactor/reuse-state.d.ts +45 -0
- package/build/tools/code-refactor/reuse-state.js +140 -0
- package/build/tools/enrichTestWithMocksTool.d.ts +28 -0
- package/build/tools/enrichTestWithMocksTool.js +726 -0
- package/build/tools/executeSkyrampTestTool.d.ts +11 -0
- package/build/tools/executeSkyrampTestTool.js +62 -21
- package/build/tools/generate-tests/batchMockGenerationTool.d.ts +106 -0
- package/build/tools/generate-tests/batchMockGenerationTool.js +545 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +25 -23
- package/build/tools/generate-tests/generateContractRestTool.js +2 -2
- package/build/tools/generate-tests/generateE2ERestTool.d.ts +1 -1
- package/build/tools/generate-tests/generateE2ERestTool.js +1 -1
- package/build/tools/generate-tests/generateLoadRestTool.d.ts +1 -1
- package/build/tools/generate-tests/generateLoadRestTool.js +1 -1
- package/build/tools/generate-tests/generateMockRestTool.d.ts +179 -6
- package/build/tools/generate-tests/generateMockRestTool.js +391 -22
- package/build/tools/generate-tests/loadTestSchema.d.ts +1 -1
- package/build/tools/generate-tests/planGuard.js +2 -22
- package/build/tools/generateEnrichedIntegrationTestTool.d.ts +25 -0
- package/build/tools/generateEnrichedIntegrationTestTool.js +235 -0
- package/build/tools/localDevWorkerComposeTool.d.ts +24 -0
- package/build/tools/localDevWorkerComposeTool.js +264 -0
- package/build/tools/one-click/oneClickTool.d.ts +13 -0
- package/build/tools/one-click/oneClickTool.js +195 -24
- package/build/tools/preflightMockCheckTool.d.ts +2 -0
- package/build/tools/preflightMockCheckTool.js +96 -0
- package/build/tools/queryProxyMocksTool.d.ts +70 -0
- package/build/tools/queryProxyMocksTool.js +522 -0
- package/build/tools/runExistingTestsTool.d.ts +138 -0
- package/build/tools/runExistingTestsTool.js +644 -0
- package/build/tools/submitReportTool.js +30 -1
- package/build/tools/test-management/analyzeChangesTool.d.ts +2 -1
- package/build/tools/test-management/analyzeChangesTool.js +63 -40
- package/build/tools/test-management/analyzeTestHealthTool.d.ts +11 -0
- package/build/tools/test-management/analyzeTestHealthTool.js +63 -1
- package/build/tools/test-management/registerTestPlanTool.js +55 -7
- package/build/tools/trace/startTraceCollectionTool.js +3 -3
- package/build/types/ExternalTestExecution.d.ts +67 -0
- package/build/types/ExternalTestExecution.js +8 -0
- package/build/types/OneClickCommands.d.ts +1 -1
- package/build/types/Recommendation.d.ts +20 -0
- package/build/types/Recommendation.js +32 -6
- package/build/types/RepositoryAnalysis.d.ts +131 -14
- package/build/types/RepositoryAnalysis.js +16 -2
- package/build/types/ReuseOutcome.d.ts +63 -0
- package/build/types/ReuseOutcome.js +33 -0
- package/build/types/TestExecution.d.ts +1 -0
- package/build/types/TestTypes.d.ts +25 -7
- package/build/types/TestTypes.js +25 -7
- package/build/types/TestbotReport.d.ts +7 -0
- package/build/types/index.d.ts +2 -0
- package/build/types/index.js +1 -0
- package/build/utils/AnalysisStateManager.d.ts +33 -0
- package/build/utils/AnalysisStateManager.js +36 -2
- package/build/utils/analyze-openapi.js +18 -1
- package/build/utils/branchDiff.d.ts +17 -1
- package/build/utils/branchDiff.js +99 -14
- package/build/utils/featureFlags.d.ts +31 -0
- package/build/utils/featureFlags.js +37 -0
- package/build/utils/grpcMockValidation.d.ts +1 -0
- package/build/utils/grpcMockValidation.js +49 -0
- package/build/utils/httpMethodValidation.d.ts +4 -0
- package/build/utils/httpMethodValidation.js +15 -0
- package/build/utils/logger.js +1 -1
- package/build/utils/mockCompatibility.d.ts +49 -0
- package/build/utils/mockCompatibility.js +82 -0
- package/build/utils/pom-verify/verify.d.ts +5 -0
- package/build/utils/pom-verify/verify.js +1 -0
- package/build/utils/progress.js +10 -5
- package/build/utils/proxy-terminal.js +3 -3
- package/build/utils/routeParsers.d.ts +3 -9
- package/build/utils/routeParsers.js +79 -4
- package/build/utils/utils.js +2 -2
- package/build/utils/versions.d.ts +4 -3
- package/build/utils/versions.js +3 -1
- package/build/utils/workspaceAuth.d.ts +46 -0
- package/build/utils/workspaceAuth.js +156 -1
- package/build/workspace/frameworks.d.ts +11 -0
- package/build/workspace/frameworks.js +22 -0
- package/build/workspace/testSuites.d.ts +20 -0
- package/build/workspace/testSuites.js +17 -0
- package/build/workspace/workspace.d.ts +206 -24
- package/build/workspace/workspace.js +52 -2
- package/package.json +5 -2
package/build/tool-phases.js
CHANGED
|
@@ -11,7 +11,11 @@ export const TOOL_PHASE_MAP = {
|
|
|
11
11
|
skyramp_ui_test_generation: "generating",
|
|
12
12
|
skyramp_batch_scenario_test_generation: "generating",
|
|
13
13
|
skyramp_mock_generation: "generating",
|
|
14
|
+
skyramp_batch_mock_generation: "generating",
|
|
15
|
+
skyramp_generate_enriched_integration_test: "generating",
|
|
16
|
+
skyramp_enrich_test_with_mocks: "generating",
|
|
14
17
|
skyramp_execute_test: { before: "maintaining", after: "executing" },
|
|
18
|
+
skyramp_run_existing_tests: { before: "maintaining", after: "executing" },
|
|
15
19
|
skyramp_analyze_test_health: "maintaining",
|
|
16
20
|
skyramp_submit_report: "reporting",
|
|
17
21
|
};
|
|
@@ -30,6 +34,7 @@ export const TOOLS_WITHOUT_PHASE = new Set([
|
|
|
30
34
|
"skyramp_init_scan",
|
|
31
35
|
"skyramp_init_workspace",
|
|
32
36
|
"skyramp_one_click_tool",
|
|
37
|
+
"skyramp_setup_local_dev_worker",
|
|
33
38
|
"skyramp_actions",
|
|
34
39
|
"skyramp_start_trace_collection",
|
|
35
40
|
"skyramp_stop_trace_collection",
|
|
@@ -37,4 +42,6 @@ export const TOOLS_WITHOUT_PHASE = new Set([
|
|
|
37
42
|
"skyramp_modularization",
|
|
38
43
|
"skyramp_reuse_code",
|
|
39
44
|
"skyramp_enhance_assertions",
|
|
45
|
+
"skyramp_preflight_mock_check",
|
|
46
|
+
"skyramp_query_proxy_mocks",
|
|
40
47
|
]);
|
|
@@ -4,9 +4,11 @@ import { getCodeReusePrompt, isPomAwareTarget } from "../../prompts/code-reuse.j
|
|
|
4
4
|
import { selectScopedPoms } from "../../utils/pom-scope/index.js";
|
|
5
5
|
import { verifyReuse } from "../../utils/pom-verify/index.js";
|
|
6
6
|
import { infraGateFailure, zeroReuseGateFailure, composeVerifyText } from "./verify-gates.js";
|
|
7
|
+
import { recordCandidates, recordNoPomLayer, recordVerifyOutcome } from "./reuse-state.js";
|
|
7
8
|
import { codeRefactoringSchema, languageSchema, } from "../../types/TestTypes.js";
|
|
8
9
|
import { SKYRAMP_UTILS_HEADER } from "../../utils/utils.js";
|
|
9
10
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
11
|
+
import { isPomReuseEnabled } from "../../utils/featureFlags.js";
|
|
10
12
|
import { stageGeneratedPaths } from "../../utils/gitStaging.js";
|
|
11
13
|
const codeReuseSchema = z.object({
|
|
12
14
|
testFile: z
|
|
@@ -25,6 +27,12 @@ const codeReuseSchema = z.object({
|
|
|
25
27
|
.describe("Verify a previously written refactored test's POM calls against source instead of returning the reuse prompt"),
|
|
26
28
|
});
|
|
27
29
|
const TOOL_NAME = "skyramp_reuse_code";
|
|
30
|
+
// Only advertised when SKYRAMP_FEATURE_POM_REUSE=1 — with the flag off this tool
|
|
31
|
+
// never takes the POM path, so describing it would promise behavior it won't do.
|
|
32
|
+
const POM_AWARE_MODE_DESCRIPTION = `
|
|
33
|
+
|
|
34
|
+
**POM-AWARE MODE (TypeScript/JavaScript + Playwright):**
|
|
35
|
+
For projects with \`language: "typescript" or "javascript"\` and \`framework: "playwright"\`, this tool first checks whether the project has an existing Page Object Model (POM) library (\`pageobjects/\` directories or \`*.page.ts\` / \`*.page.js\` files). If POMs are found, the test is refactored to use those POM classes and methods — replacing raw inline locators with the existing abstractions. SkyrampUtils consolidation is skipped on this path. If no POMs are found, the tool falls back to the standard SkyrampUtils path. New POM creation is out of scope — existing POMs are reused only. After POM-aware code reuse, do NOT call skyramp_modularization — on this path that supersedes WORKFLOW SUMMARY step 6.`;
|
|
28
36
|
export function registerCodeReuseTool(server) {
|
|
29
37
|
server.registerTool(TOOL_NAME, {
|
|
30
38
|
description: `Analyzes code for reuse opportunities and enforces code reuse principles.
|
|
@@ -57,10 +65,7 @@ export function registerCodeReuseTool(server) {
|
|
|
57
65
|
|
|
58
66
|
**MANDATORY**: ONLY ALLOW CODE REUSE IF THE IS TRACE BASED FLAG IS SET TO TRUE ELSE DO NOT ALLOW CODE REUSE AND LEAVE THE TEST FILE AS IS.
|
|
59
67
|
**CRITICAL**: NON TRACE BASED TESTS ARE ALREADY MODULARIZED AND DO NOT NEED CODE REUSE.
|
|
60
|
-
The tool will provide step-by-step instructions that MUST be followed exactly
|
|
61
|
-
|
|
62
|
-
**POM-AWARE MODE (TypeScript/JavaScript + Playwright):**
|
|
63
|
-
For projects with \`language: "typescript" or "javascript"\` and \`framework: "playwright"\`, this tool first checks whether the project has an existing Page Object Model (POM) library (\`pageobjects/\` directories or \`*.page.ts\` / \`*.page.js\` files). If POMs are found, the test is refactored to use those POM classes and methods — replacing raw inline locators with the existing abstractions. SkyrampUtils consolidation is skipped on this path. If no POMs are found, the tool falls back to the standard SkyrampUtils path. New POM creation is out of scope — existing POMs are reused only. After POM-aware code reuse, do NOT call skyramp_modularization.`,
|
|
68
|
+
The tool will provide step-by-step instructions that MUST be followed exactly.${isPomReuseEnabled() ? POM_AWARE_MODE_DESCRIPTION : ""}`,
|
|
64
69
|
inputSchema: codeReuseSchema.shape,
|
|
65
70
|
_meta: {
|
|
66
71
|
keywords: [
|
|
@@ -75,9 +80,26 @@ export function registerCodeReuseTool(server) {
|
|
|
75
80
|
let errorResult;
|
|
76
81
|
try {
|
|
77
82
|
if (params.verify) {
|
|
83
|
+
// The verify pass is POM-only, and it is the sole writer of the report's
|
|
84
|
+
// reuse summary — so with the flag off a stale instruction to verify must
|
|
85
|
+
// not run it, or a POM-less run would still emit POM reuse numbers.
|
|
86
|
+
if (!isPomReuseEnabled()) {
|
|
87
|
+
return {
|
|
88
|
+
content: [
|
|
89
|
+
{
|
|
90
|
+
type: "text",
|
|
91
|
+
text: "VERIFICATION SKIPPED — POM-aware code reuse is disabled. Nothing to verify; continue.",
|
|
92
|
+
},
|
|
93
|
+
],
|
|
94
|
+
};
|
|
95
|
+
}
|
|
78
96
|
try {
|
|
79
97
|
const r = await verifyReuse(params.testFile, params.language);
|
|
80
98
|
const gate = (await infraGateFailure(params, r)) ?? (await zeroReuseGateFailure(params, r));
|
|
99
|
+
// Record what this pass established before returning the text report:
|
|
100
|
+
// every number in it is already in hand here, so the agent is never
|
|
101
|
+
// asked to read one back out and retype it later.
|
|
102
|
+
await recordVerifyOutcome(params.testFile, r, gate !== undefined, params.language);
|
|
81
103
|
return { content: [{ type: "text", text: composeVerifyText(gate, r) }] };
|
|
82
104
|
}
|
|
83
105
|
catch (err) {
|
|
@@ -110,11 +132,28 @@ export function registerCodeReuseTool(server) {
|
|
|
110
132
|
});
|
|
111
133
|
if (tier1.length + tier2.length > 0) {
|
|
112
134
|
scopedPoms = { tier1, tier2 };
|
|
135
|
+
// Only report candidates found through the conventional POM globs.
|
|
136
|
+
//
|
|
137
|
+
// The selector-grep fallback also matches ordinary in-repo frontend
|
|
138
|
+
// source — app components legitimately contain the spec's test-ids by
|
|
139
|
+
// construction — so in a repo with no page-object layer at all it still
|
|
140
|
+
// yields "candidates". Reporting those would render, on a POM-less repo,
|
|
141
|
+
// "4 candidate POM files detected" next to zero reuse: each number true
|
|
142
|
+
// in isolation, but together they assert reusable page objects existed
|
|
143
|
+
// and were ignored. `zeroReuseGateFailure` already draws this exact line
|
|
144
|
+
// for the same reason; the report has to draw it too.
|
|
145
|
+
//
|
|
146
|
+
// The agent is still handed the grep-discovered files to consider —
|
|
147
|
+
// only the customer-facing count is withheld.
|
|
148
|
+
if (diagnostics.discovery === "globs") {
|
|
149
|
+
await recordCandidates(params.testFile, tier1.length);
|
|
150
|
+
}
|
|
113
151
|
}
|
|
114
152
|
else if (diagnostics.selectors > 0 && diagnostics.discovery !== "skipped-too-large") {
|
|
115
153
|
// A clean scan ran (glob or selector-grep) and found zero overlap — distinct from
|
|
116
154
|
// "scoping wasn't attempted" (no selectors) or "scan was skipped" (repo too large).
|
|
117
155
|
scopedPoms = { tier1: [], tier2: [], scannedNoOverlap: true };
|
|
156
|
+
await recordNoPomLayer(params.testFile);
|
|
118
157
|
}
|
|
119
158
|
logger.info("POM pre-scoping", {
|
|
120
159
|
...diagnostics,
|
|
@@ -5,6 +5,7 @@ import { getContractProviderAssertionsPrompt } from "../../prompts/enhance-asser
|
|
|
5
5
|
import { getIntegrationAssertionsPrompt } from "../../prompts/enhance-assertions/integrationAssertionsPrompt.js";
|
|
6
6
|
import { getUIAssertionsPrompt } from "../../prompts/enhance-assertions/uiAssertionsPrompt.js";
|
|
7
7
|
import { isTestbotEnabled } from "../../utils/featureFlags.js";
|
|
8
|
+
import { renderSharedAssertionRules } from "../../prompts/enhance-assertions/sharedAssertionRules.js";
|
|
8
9
|
import { stageGeneratedPaths } from "../../utils/gitStaging.js";
|
|
9
10
|
const TOOL_NAME = "skyramp_enhance_assertions";
|
|
10
11
|
const TESTBOT_UI_CHECKS = `
|
|
@@ -12,6 +13,37 @@ const TESTBOT_UI_CHECKS = `
|
|
|
12
13
|
- If no suitable selector exists in the generated file for an assertion you need to add, go back and call \`browser_assert\` on the live page to record it with a valid selector, then re-export and regenerate.
|
|
13
14
|
- **After executing a UI test that documents a bug from \`issuesFound\`**: if it passed when you expected it to fail, the assertions are too weak — add a stronger \`expect()\` that directly targets the buggy behavior. This counts as the single allowed retry under the 2-attempt cap — do NOT re-run more than once.
|
|
14
15
|
`;
|
|
16
|
+
function buildAssertionInstructions(testFile, testType, enhanceType) {
|
|
17
|
+
if (testType === TestType.UI) {
|
|
18
|
+
let instructions = getUIAssertionsPrompt(testFile, enhanceType);
|
|
19
|
+
if (isTestbotEnabled()) {
|
|
20
|
+
instructions += TESTBOT_UI_CHECKS;
|
|
21
|
+
}
|
|
22
|
+
return instructions;
|
|
23
|
+
}
|
|
24
|
+
if (testType === TestType.CONTRACT) {
|
|
25
|
+
return getContractProviderAssertionsPrompt(testFile, enhanceType);
|
|
26
|
+
}
|
|
27
|
+
if (testType === TestType.INTEGRATION) {
|
|
28
|
+
return getIntegrationAssertionsPrompt(testFile, enhanceType);
|
|
29
|
+
}
|
|
30
|
+
throw new Error(`Unsupported testType for ${TOOL_NAME}: ${testType}`);
|
|
31
|
+
}
|
|
32
|
+
function buildAutoApplyInstructions(testFile, testType, enhanceType) {
|
|
33
|
+
const typeSpecificInstructions = buildAssertionInstructions(testFile, testType, enhanceType);
|
|
34
|
+
return [
|
|
35
|
+
`Enhance response body assertions in: \`${testFile}\``,
|
|
36
|
+
`Test type: ${testType} | Context: ${enhanceType}`,
|
|
37
|
+
``,
|
|
38
|
+
`Read the file, apply the type-specific assertion guidance below, and write the file back directly.`,
|
|
39
|
+
`Add assertions after each send_request/sendRequest status-code assertion.`,
|
|
40
|
+
`Use SDK helper: Python \`skyramp.get_response_value(response, "json.path")\`, JS \`getValue(response, "json.path")\`.`,
|
|
41
|
+
`Do NOT restructure, reformat, add comments, or change imports. Only add assertion lines.`,
|
|
42
|
+
``,
|
|
43
|
+
`Type-specific assertion guidance:`,
|
|
44
|
+
typeSpecificInstructions,
|
|
45
|
+
].join("\n");
|
|
46
|
+
}
|
|
15
47
|
const enhanceAssertionsSchema = {
|
|
16
48
|
testFile: z
|
|
17
49
|
.string()
|
|
@@ -24,6 +56,12 @@ const enhanceAssertionsSchema = {
|
|
|
24
56
|
.describe("The context of the enhancement. " +
|
|
25
57
|
"Use 'generation' after generating a new test file (applies to every test function). " +
|
|
26
58
|
"Use 'maintenance' during drift UPDATE (applies only to new or diff-affected functions)."),
|
|
59
|
+
autoApply: z
|
|
60
|
+
.boolean()
|
|
61
|
+
.default(false)
|
|
62
|
+
.describe("When true, returns a compact instruction set (10 lines) that tells the agent to " +
|
|
63
|
+
"read the test file, enhance assertions per standard rules, and write it back directly. " +
|
|
64
|
+
"When false (default), returns the full verbose instruction set."),
|
|
27
65
|
};
|
|
28
66
|
export function registerEnhanceAssertionsTool(server) {
|
|
29
67
|
server.registerTool(TOOL_NAME, {
|
|
@@ -34,26 +72,33 @@ export function registerEnhanceAssertionsTool(server) {
|
|
|
34
72
|
- After updating an existing supported test file during maintenance`,
|
|
35
73
|
inputSchema: enhanceAssertionsSchema,
|
|
36
74
|
}, async (params) => {
|
|
37
|
-
const { testFile, testType, enhanceType } = params;
|
|
75
|
+
const { testFile, testType, enhanceType, autoApply } = params;
|
|
38
76
|
// Stage so testbot includes the generated files in its output commit.
|
|
39
77
|
await stageGeneratedPaths(testFile);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
78
|
+
if (autoApply) {
|
|
79
|
+
const compactInstructions = [
|
|
80
|
+
`Enhance response body assertions in: \`${testFile}\``,
|
|
81
|
+
`Test type: ${testType} | Context: ${enhanceType}`,
|
|
82
|
+
``,
|
|
83
|
+
`Read the file, add assertions after each send_request/sendRequest status-code assertion, write it back.`,
|
|
84
|
+
`Use SDK helper: Python \`skyramp.get_response_value(response, "json.path")\`, JS \`getValue(response, "json.path")\`.`,
|
|
85
|
+
``,
|
|
86
|
+
`Shared assertion rules (apply all that fit):`,
|
|
87
|
+
renderSharedAssertionRules(),
|
|
88
|
+
``,
|
|
89
|
+
`Do NOT restructure, reformat, add comments, or change imports. Only add assertion lines.`,
|
|
90
|
+
].join("\n");
|
|
91
|
+
const result = {
|
|
92
|
+
content: [{ type: "text", text: compactInstructions }],
|
|
93
|
+
isError: false,
|
|
94
|
+
};
|
|
95
|
+
AnalyticsService.pushMCPToolEvent(TOOL_NAME, undefined, { testFile: params.testFile, testType: params.testType, enhanceType: params.enhanceType, autoApply: Boolean(params.autoApply) }).catch(() => { });
|
|
96
|
+
return result;
|
|
56
97
|
}
|
|
98
|
+
const enhanceCtx = enhanceType;
|
|
99
|
+
const instructions = autoApply
|
|
100
|
+
? buildAutoApplyInstructions(testFile, testType, enhanceCtx)
|
|
101
|
+
: buildAssertionInstructions(testFile, testType, enhanceCtx);
|
|
57
102
|
const result = {
|
|
58
103
|
content: [
|
|
59
104
|
{
|
|
@@ -63,7 +108,12 @@ export function registerEnhanceAssertionsTool(server) {
|
|
|
63
108
|
],
|
|
64
109
|
isError: false,
|
|
65
110
|
};
|
|
66
|
-
AnalyticsService.pushMCPToolEvent(TOOL_NAME, undefined,
|
|
111
|
+
AnalyticsService.pushMCPToolEvent(TOOL_NAME, undefined, {
|
|
112
|
+
testFile: params.testFile,
|
|
113
|
+
testType: params.testType,
|
|
114
|
+
enhanceType: params.enhanceType,
|
|
115
|
+
autoApply: Boolean(params.autoApply),
|
|
116
|
+
}).catch(() => { });
|
|
67
117
|
return result;
|
|
68
118
|
});
|
|
69
119
|
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import type { VerifyResult } from "../../utils/pom-verify/index.js";
|
|
2
|
+
import { ReuseVerificationOutcome, type ReuseSkippedEntry } from "../../types/ReuseOutcome.js";
|
|
3
|
+
export { ReuseDeclinedBy, ReuseVerificationOutcome, type ReuseOutcome, type ReuseSkippedEntry, } from "../../types/ReuseOutcome.js";
|
|
4
|
+
/** A member the verifier flagged at some point in this run. NOT the same as "a
|
|
5
|
+
* member that was demoted": the workflow allows one remap attempt, so a flagged
|
|
6
|
+
* member may have been fixed rather than restored to raw inline. Only a
|
|
7
|
+
* `// kept inline:` marker proves a decline; this list can at most say which
|
|
8
|
+
* member a decline was about. */
|
|
9
|
+
export interface FlaggedMember {
|
|
10
|
+
pageObject: string;
|
|
11
|
+
member: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* What run state holds between the verify pass and the report step.
|
|
15
|
+
*
|
|
16
|
+
* Deliberately NOT an extension of {@link ReuseOutcome}: it is a disjoint type
|
|
17
|
+
* holding only what cannot be recomputed from the delivered spec. Two reasons.
|
|
18
|
+
*
|
|
19
|
+
* Anything recomputable must not be stored, because storing it makes it stale the
|
|
20
|
+
* moment the spec changes — and it does change: the execution fix-up may restore
|
|
21
|
+
* `<testFile>.raw.bak` over the spec by plain `cp`, invisible to this tool. The
|
|
22
|
+
* counts and the decline list are therefore derived at report time (see
|
|
23
|
+
* `rederiveReuseOutcome`), never read back from here.
|
|
24
|
+
*
|
|
25
|
+
* And keeping the two types disjoint makes the report boundary opt-IN. When the
|
|
26
|
+
* record extended the wire type, every field added here shipped to the
|
|
27
|
+
* customer-facing report unless someone remembered to widen a destructure — a
|
|
28
|
+
* default that quietly asserts something, which is the same shape as the
|
|
29
|
+
* file-level attribution bug this module already had once. Now adding a field is
|
|
30
|
+
* a type error at the construction site instead.
|
|
31
|
+
*/
|
|
32
|
+
export interface ReuseRecord {
|
|
33
|
+
/** Tier-1 POM candidate FILES detection found. Not recoverable from the spec:
|
|
34
|
+
* it describes what was available to reuse, not what was reused. */
|
|
35
|
+
candidatesDetected?: number;
|
|
36
|
+
/** Needs `gateFired`, which only the verify call knows. */
|
|
37
|
+
verification?: ReuseVerificationOutcome;
|
|
38
|
+
/** Members the verifier flagged, accumulated across this run's verify passes —
|
|
39
|
+
* by the time verification PASSES the violations are gone, so a member that was
|
|
40
|
+
* substituted and then restored survives only as a `// kept inline:` comment,
|
|
41
|
+
* indistinguishable from one the mapper never promoted.
|
|
42
|
+
*
|
|
43
|
+
* Named for what it holds, not for what it is used to infer: FLAGGED is weaker
|
|
44
|
+
* than "demoted" (see {@link FlaggedMember}). Calling it `demoted` is what
|
|
45
|
+
* licensed attributing a whole file's declines to one member — do not rename it
|
|
46
|
+
* back. */
|
|
47
|
+
flagged?: FlaggedMember[];
|
|
48
|
+
/** Identity of the spec to re-derive from, and the language to parse it as. */
|
|
49
|
+
testFilePath?: string;
|
|
50
|
+
language?: string;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Every well-formed `// kept inline: <basename> — <reason>` marker in a spec.
|
|
54
|
+
*
|
|
55
|
+
* Deliberately separate from `isKeptInlineDocumented` (verify-gates.ts) rather
|
|
56
|
+
* than replacing it: that one asks "is THIS known basename documented", so it can
|
|
57
|
+
* anchor on the basename and never has to guess where the name ends. Loosening the
|
|
58
|
+
* gate's predicate to share this regex would change when the zero-reuse escape
|
|
59
|
+
* hatch fires, which is not a side effect worth taking for deduplication — but see
|
|
60
|
+
* `findUnparsableKeptInline` for how the gap between them is kept visible.
|
|
61
|
+
*/
|
|
62
|
+
export declare function parseKeptInline(specContent: string): {
|
|
63
|
+
pageObject: string;
|
|
64
|
+
reason: string;
|
|
65
|
+
}[];
|
|
66
|
+
/**
|
|
67
|
+
* Lines that announce a kept-inline decline but that `parseKeptInline` cannot
|
|
68
|
+
* read — a missing reason clause, or a bare unspaced hyphen the zero-reuse gate's
|
|
69
|
+
* looser predicate would still accept.
|
|
70
|
+
*
|
|
71
|
+
* Such a marker satisfies the escape hatch (no gate failure, the agent proceeds)
|
|
72
|
+
* while the decline never reaches the report: silent signal loss, which is exactly
|
|
73
|
+
* the failure class this feature exists to close. Callers log these rather than
|
|
74
|
+
* dropping them quietly.
|
|
75
|
+
*/
|
|
76
|
+
export declare function findUnparsableKeptInline(specContent: string): string[];
|
|
77
|
+
/** Members the verifier flagged in one pass, keyed to the file a decline comment
|
|
78
|
+
* would name. */
|
|
79
|
+
export declare function flaggedFrom(r: VerifyResult): FlaggedMember[];
|
|
80
|
+
/** Union two flagged-member lists, deduplicated. */
|
|
81
|
+
export declare function mergeFlagged(prior: FlaggedMember[] | undefined, next: FlaggedMember[]): FlaggedMember[];
|
|
82
|
+
/**
|
|
83
|
+
* Build the `skipped` list for a spec: one entry per documented kept-inline
|
|
84
|
+
* marker — markers are the only ground truth for "this was actually left inline".
|
|
85
|
+
*
|
|
86
|
+
* Attribution is deliberately conservative, and only labels a decline when the
|
|
87
|
+
* correspondence is exact — one marker for that file and one flagged member for
|
|
88
|
+
* it:
|
|
89
|
+
*
|
|
90
|
+
* - No member of the file was ever flagged → the mapper never promoted it,
|
|
91
|
+
* `not-mapped`. Unambiguous: a flagged member implies it had been substituted.
|
|
92
|
+
* - Exactly one marker and one flagged member → that marker is that member,
|
|
93
|
+
* `demoted-by-verify`.
|
|
94
|
+
* - Anything else → the entry keeps its reason but carries no `member` and no
|
|
95
|
+
* `declinedBy`. Several markers on one file (S-01b's shape: a demote and an
|
|
96
|
+
* abstention on the same class) give no way to say which marker is which
|
|
97
|
+
* member, and a flagged member may have been remapped successfully rather than
|
|
98
|
+
* demoted, so a count mismatch is not even reliably a mixed-cause file.
|
|
99
|
+
*
|
|
100
|
+
* Guessing here is worse than omitting: a wrong member under a confident
|
|
101
|
+
* "demoted by verification" label sends a reviewer chasing a decline that never
|
|
102
|
+
* happened, which is the multi-hour log dig this field exists to end.
|
|
103
|
+
*/
|
|
104
|
+
export declare function buildSkipped(specContent: string, flagged: FlaggedMember[] | undefined): ReuseSkippedEntry[];
|
|
105
|
+
/** The verdict for a verify pass. A fired compliance gate is a failure regardless
|
|
106
|
+
* of the raw result; otherwise `r.ok` decides. `skipped-no-pom` is not reachable
|
|
107
|
+
* here — it is recorded on the prompt-generation call, where "no reusable POM
|
|
108
|
+
* layer detected" is determined. */
|
|
109
|
+
export declare function verificationFrom(gateFired: boolean, r: VerifyResult): ReuseVerificationOutcome;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import { stripStrings } from "../../utils/pom-scope/strip.js";
|
|
3
|
+
import { ReuseDeclinedBy, ReuseVerificationOutcome, } from "../../types/ReuseOutcome.js";
|
|
4
|
+
/**
|
|
5
|
+
* Server-side derivation of a UI test's POM code-reuse outcome.
|
|
6
|
+
*
|
|
7
|
+
* The counts and verdicts here are all computed in-process by `skyramp_reuse_code`
|
|
8
|
+
* already — tier-1 detection by `selectScopedPoms`, the call counts and violations
|
|
9
|
+
* by `verifyReuse`, the demote/abstain record by the `// kept inline:` comments in
|
|
10
|
+
* the spec the tool just read. Routing them through the agent (read four integers
|
|
11
|
+
* out of tool text, hold them across the rest of the run, retype them into a later
|
|
12
|
+
* call) is still prose-mediated and still droppable, which is the failure this
|
|
13
|
+
* whole feature exists to fix. So they are persisted to run state and merged into
|
|
14
|
+
* the report server-side, exactly as `beforeStatus`/`afterStatus` are.
|
|
15
|
+
*
|
|
16
|
+
* The agent supplies nothing here.
|
|
17
|
+
*/
|
|
18
|
+
/** Separator between composite-key parts. `\0` cannot occur in a file name or a
|
|
19
|
+
* reason clause, so it cannot collide — written as an escape, never as a literal
|
|
20
|
+
* control byte, or git classifies this source file as binary. */
|
|
21
|
+
const KEY_SEP = "\0";
|
|
22
|
+
// The wire shape lives in types/ (and is re-exported from types/index.ts for
|
|
23
|
+
// consumers). Re-exported here so this module stays the one place to look when
|
|
24
|
+
// working on the derivation.
|
|
25
|
+
export { ReuseDeclinedBy, ReuseVerificationOutcome, } from "../../types/ReuseOutcome.js";
|
|
26
|
+
/** Matches a well-formed `// kept inline: <basename> <sep> <reason>` marker.
|
|
27
|
+
*
|
|
28
|
+
* The separator must be an em/en dash or a *spaced* hyphen. Discovery has no
|
|
29
|
+
* anchor to tell where the file name ends, and a bare hyphen is ambiguous with
|
|
30
|
+
* hyphens inside file names (`alert-destinations.page.ts-reason`), so requiring
|
|
31
|
+
* whitespace is what makes the split decidable. `findUnparsableKeptInline` below
|
|
32
|
+
* reports anything that looks like a marker but does not match, so a malformed
|
|
33
|
+
* one is never silently lost. */
|
|
34
|
+
const KEPT_INLINE_RE = /\/\/\s*kept inline:\s*(\S.*?)\s*(?:[—–]+|\s+-+\s+)\s*(\S.*?)\s*$/;
|
|
35
|
+
/** Lines that look like a kept-inline marker. Deliberately looser than
|
|
36
|
+
* KEPT_INLINE_RE so the difference between the two is detectable. */
|
|
37
|
+
const KEPT_INLINE_LOOSE_RE = /\/\/\s*kept inline:/;
|
|
38
|
+
/** Comment lines only — a marker quoted inside a string literal is not a real
|
|
39
|
+
* marker, the same reasoning as the zero-reuse gate's predicate. */
|
|
40
|
+
function commentLines(specContent) {
|
|
41
|
+
return stripStrings(specContent).split("\n");
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Every well-formed `// kept inline: <basename> — <reason>` marker in a spec.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately separate from `isKeptInlineDocumented` (verify-gates.ts) rather
|
|
47
|
+
* than replacing it: that one asks "is THIS known basename documented", so it can
|
|
48
|
+
* anchor on the basename and never has to guess where the name ends. Loosening the
|
|
49
|
+
* gate's predicate to share this regex would change when the zero-reuse escape
|
|
50
|
+
* hatch fires, which is not a side effect worth taking for deduplication — but see
|
|
51
|
+
* `findUnparsableKeptInline` for how the gap between them is kept visible.
|
|
52
|
+
*/
|
|
53
|
+
export function parseKeptInline(specContent) {
|
|
54
|
+
const out = [];
|
|
55
|
+
const seen = new Set();
|
|
56
|
+
for (const line of commentLines(specContent)) {
|
|
57
|
+
const m = KEPT_INLINE_RE.exec(line);
|
|
58
|
+
if (!m)
|
|
59
|
+
continue;
|
|
60
|
+
const pageObject = path.basename(m[1].trim());
|
|
61
|
+
const reason = m[2].trim();
|
|
62
|
+
const key = `${pageObject}${KEY_SEP}${reason}`;
|
|
63
|
+
if (seen.has(key))
|
|
64
|
+
continue;
|
|
65
|
+
seen.add(key);
|
|
66
|
+
out.push({ pageObject, reason });
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Lines that announce a kept-inline decline but that `parseKeptInline` cannot
|
|
72
|
+
* read — a missing reason clause, or a bare unspaced hyphen the zero-reuse gate's
|
|
73
|
+
* looser predicate would still accept.
|
|
74
|
+
*
|
|
75
|
+
* Such a marker satisfies the escape hatch (no gate failure, the agent proceeds)
|
|
76
|
+
* while the decline never reaches the report: silent signal loss, which is exactly
|
|
77
|
+
* the failure class this feature exists to close. Callers log these rather than
|
|
78
|
+
* dropping them quietly.
|
|
79
|
+
*/
|
|
80
|
+
export function findUnparsableKeptInline(specContent) {
|
|
81
|
+
return commentLines(specContent)
|
|
82
|
+
.filter((line) => KEPT_INLINE_LOOSE_RE.test(line) && !KEPT_INLINE_RE.test(line))
|
|
83
|
+
.map((line) => line.trim());
|
|
84
|
+
}
|
|
85
|
+
/** Members the verifier flagged in one pass, keyed to the file a decline comment
|
|
86
|
+
* would name. */
|
|
87
|
+
export function flaggedFrom(r) {
|
|
88
|
+
const out = [];
|
|
89
|
+
for (const v of r.violations) {
|
|
90
|
+
const file = v.searched[0];
|
|
91
|
+
if (file)
|
|
92
|
+
out.push({ pageObject: path.basename(file), member: `${v.binding}.${v.member}` });
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Union two flagged-member lists, deduplicated. */
|
|
97
|
+
export function mergeFlagged(prior, next) {
|
|
98
|
+
const byKey = new Map();
|
|
99
|
+
for (const d of [...(prior ?? []), ...next]) {
|
|
100
|
+
byKey.set(`${d.pageObject}${KEY_SEP}${d.member}`, d);
|
|
101
|
+
}
|
|
102
|
+
return [...byKey.values()];
|
|
103
|
+
}
|
|
104
|
+
function countByFile(items) {
|
|
105
|
+
const counts = new Map();
|
|
106
|
+
for (const i of items)
|
|
107
|
+
counts.set(i.pageObject, (counts.get(i.pageObject) ?? 0) + 1);
|
|
108
|
+
return counts;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Build the `skipped` list for a spec: one entry per documented kept-inline
|
|
112
|
+
* marker — markers are the only ground truth for "this was actually left inline".
|
|
113
|
+
*
|
|
114
|
+
* Attribution is deliberately conservative, and only labels a decline when the
|
|
115
|
+
* correspondence is exact — one marker for that file and one flagged member for
|
|
116
|
+
* it:
|
|
117
|
+
*
|
|
118
|
+
* - No member of the file was ever flagged → the mapper never promoted it,
|
|
119
|
+
* `not-mapped`. Unambiguous: a flagged member implies it had been substituted.
|
|
120
|
+
* - Exactly one marker and one flagged member → that marker is that member,
|
|
121
|
+
* `demoted-by-verify`.
|
|
122
|
+
* - Anything else → the entry keeps its reason but carries no `member` and no
|
|
123
|
+
* `declinedBy`. Several markers on one file (S-01b's shape: a demote and an
|
|
124
|
+
* abstention on the same class) give no way to say which marker is which
|
|
125
|
+
* member, and a flagged member may have been remapped successfully rather than
|
|
126
|
+
* demoted, so a count mismatch is not even reliably a mixed-cause file.
|
|
127
|
+
*
|
|
128
|
+
* Guessing here is worse than omitting: a wrong member under a confident
|
|
129
|
+
* "demoted by verification" label sends a reviewer chasing a decline that never
|
|
130
|
+
* happened, which is the multi-hour log dig this field exists to end.
|
|
131
|
+
*/
|
|
132
|
+
export function buildSkipped(specContent, flagged) {
|
|
133
|
+
const markers = parseKeptInline(specContent);
|
|
134
|
+
const markerCounts = countByFile(markers);
|
|
135
|
+
const flaggedByFile = new Map();
|
|
136
|
+
for (const f of flagged ?? []) {
|
|
137
|
+
const members = flaggedByFile.get(f.pageObject) ?? [];
|
|
138
|
+
members.push(f.member);
|
|
139
|
+
flaggedByFile.set(f.pageObject, members);
|
|
140
|
+
}
|
|
141
|
+
return markers.map(({ pageObject, reason }) => {
|
|
142
|
+
const members = flaggedByFile.get(pageObject) ?? [];
|
|
143
|
+
if (members.length === 0) {
|
|
144
|
+
return { pageObject, declinedBy: ReuseDeclinedBy.NotMapped, reason };
|
|
145
|
+
}
|
|
146
|
+
if (members.length === 1 && markerCounts.get(pageObject) === 1) {
|
|
147
|
+
return { pageObject, member: members[0], declinedBy: ReuseDeclinedBy.DemotedByVerify, reason };
|
|
148
|
+
}
|
|
149
|
+
return { pageObject, reason };
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/** The verdict for a verify pass. A fired compliance gate is a failure regardless
|
|
153
|
+
* of the raw result; otherwise `r.ok` decides. `skipped-no-pom` is not reachable
|
|
154
|
+
* here — it is recorded on the prompt-generation call, where "no reusable POM
|
|
155
|
+
* layer detected" is determined. */
|
|
156
|
+
export function verificationFrom(gateFired, r) {
|
|
157
|
+
return !gateFired && r.ok ? ReuseVerificationOutcome.Passed : ReuseVerificationOutcome.Failed;
|
|
158
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type VerifyResult } from "../../utils/pom-verify/index.js";
|
|
2
|
+
import { type ReuseOutcome, type ReuseRecord } from "./reuse-outcome.js";
|
|
3
|
+
/** Record the tier-1 candidate count STEP 1 detected. */
|
|
4
|
+
export declare function recordCandidates(testFile: string, candidatesDetected: number, explicitStateFile?: string): Promise<void>;
|
|
5
|
+
/** The prompt-generation call found no reusable POM layer at all. No candidate
|
|
6
|
+
* count is recorded: "0 candidates" adds nothing to "no POM layer was found",
|
|
7
|
+
* and rendering both produces a redundant zero in the summary. */
|
|
8
|
+
export declare function recordNoPomLayer(testFile: string, explicitStateFile?: string): Promise<void>;
|
|
9
|
+
/**
|
|
10
|
+
* Record what one verify pass established: the call counts, the verdict, and the
|
|
11
|
+
* declines read out of the spec the verifier just parsed.
|
|
12
|
+
*
|
|
13
|
+
* `flagged` accumulates across passes because a PASSED verdict has no violations
|
|
14
|
+
* left to inspect — a member that was substituted and then restored survives only
|
|
15
|
+
* as a `// kept inline:` comment, indistinguishable from one the mapper never
|
|
16
|
+
* promoted. Accumulating is what keeps that distinction recoverable at all; it does
|
|
17
|
+
* not by itself establish it, since a flagged member may have been remapped rather
|
|
18
|
+
* than restored (see buildSkipped for how narrowly it is trusted).
|
|
19
|
+
*/
|
|
20
|
+
export declare function recordVerifyOutcome(testFile: string, r: VerifyResult, gateFired: boolean, language: string, explicitStateFile?: string): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Re-derive a recorded outcome from the spec as it stands NOW, so the report
|
|
23
|
+
* describes the delivered artifact rather than the state at verify time.
|
|
24
|
+
*
|
|
25
|
+
* The two can differ: the execution fix-up may restore `<testFile>.raw.bak` over
|
|
26
|
+
* the spec after verification, reverting every substitution with a plain `cp` that
|
|
27
|
+
* this tool never sees. Trusting the recorded counts there would publish
|
|
28
|
+
* `callsReused: 5, verification: "passed"` for a spec containing no POM calls —
|
|
29
|
+
* a confident false claim, strictly worse than the silence this feature replaced.
|
|
30
|
+
* Re-deriving here rather than requiring a final `verify: true` pass is deliberate:
|
|
31
|
+
* the report step always runs, so the agent cannot skip it.
|
|
32
|
+
*
|
|
33
|
+
* `candidatesDetected` is kept as recorded — it counts POM FILES that detection
|
|
34
|
+
* found, which a restore does not change. Everything else comes from the file.
|
|
35
|
+
*
|
|
36
|
+
* `verification` is dropped when the delivered spec reuses nothing: a PASSED
|
|
37
|
+
* verdict describes a spec that no longer exists, and pairing it with a zero count
|
|
38
|
+
* reads as though the empty result had been blessed.
|
|
39
|
+
*
|
|
40
|
+
* Returns the recorded outcome minus internals when re-derivation is impossible
|
|
41
|
+
* (no path recorded — e.g. only the candidate count was ever written), and
|
|
42
|
+
* `undefined` when it fails outright. Failing closed matters: falling back to the
|
|
43
|
+
* recorded counts is exactly the false claim this exists to prevent.
|
|
44
|
+
*/
|
|
45
|
+
export declare function rederiveReuseOutcome(record: ReuseRecord): Promise<ReuseOutcome | undefined>;
|