@skyramp/mcp 0.3.1 → 0.3.2-rc.pom-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +2 -1
- package/build/prompts/code-reuse.d.ts +7 -1
- package/build/prompts/code-reuse.js +8 -5
- package/build/prompts/code-reuse.test.d.ts +1 -0
- package/build/prompts/code-reuse.test.js +62 -0
- package/build/prompts/pom-aware-code-reuse.d.ts +6 -1
- package/build/prompts/pom-aware-code-reuse.js +98 -52
- package/build/prompts/pom-aware-code-reuse.test.d.ts +1 -0
- package/build/prompts/pom-aware-code-reuse.test.js +11 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +4 -2
- package/build/prompts/test-recommendation/diffExecutionPlan.js +11 -65
- package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +2 -2
- package/build/prompts/test-recommendation/recommendationSections.js +5 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +1 -1
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +26 -1
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +68 -56
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +48 -11
- package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
- package/build/prompts/testbot/testbot-prompts.js +67 -21
- package/build/prompts/testbot/testbot-prompts.test.js +44 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +7 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +71 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.test.d.ts +1 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.test.js +75 -0
- package/build/recommendation/budgeters/fixedNBudgeter.d.ts +7 -0
- package/build/recommendation/budgeters/fixedNBudgeter.js +11 -0
- package/build/recommendation/budgeters/fixedNBudgeter.test.d.ts +1 -0
- package/build/recommendation/budgeters/fixedNBudgeter.test.js +66 -0
- package/build/recommendation/budgeters/shared.d.ts +19 -0
- package/build/recommendation/budgeters/shared.js +66 -0
- package/build/recommendation/discriminators.d.ts +31 -0
- package/build/recommendation/discriminators.js +355 -0
- package/build/recommendation/discriminators.test.d.ts +1 -0
- package/build/recommendation/discriminators.test.js +324 -0
- package/build/recommendation/diversity.d.ts +47 -0
- package/build/recommendation/diversity.js +101 -0
- package/build/recommendation/diversity.test.d.ts +1 -0
- package/build/recommendation/diversity.test.js +77 -0
- package/build/recommendation/planRanker.d.ts +50 -0
- package/build/recommendation/planRanker.js +67 -0
- package/build/recommendation/planRanker.test.d.ts +1 -0
- package/build/recommendation/planRanker.test.js +110 -0
- package/build/recommendation/testFixtures.d.ts +25 -0
- package/build/recommendation/testFixtures.js +45 -0
- package/build/resources/testbotResource.js +4 -1
- package/build/services/ScenarioGenerationService.d.ts +5 -0
- package/build/services/ScenarioGenerationService.js +16 -1
- package/build/services/ScenarioGenerationService.test.js +44 -0
- package/build/services/TestExecutionService.d.ts +15 -1
- package/build/services/TestExecutionService.js +210 -55
- package/build/services/TestExecutionService.test.js +397 -0
- package/build/services/TestGenerationService.js +19 -1
- package/build/services/TestGenerationService.test.js +58 -0
- package/build/tool-phases.js +1 -0
- package/build/toolNames.d.ts +19 -0
- package/build/toolNames.js +19 -0
- package/build/tools/code-refactor/codeReuseTool.d.ts +3 -0
- package/build/tools/code-refactor/codeReuseTool.js +76 -4
- package/build/tools/code-refactor/codeReuseTool.test.d.ts +1 -0
- package/build/tools/code-refactor/codeReuseTool.test.js +51 -0
- package/build/tools/executeSkyrampTestTool.js +8 -2
- package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +6 -1
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +110 -17
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +147 -0
- package/build/tools/generate-tests/generateContractRestTool.js +11 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.js +24 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.test.d.ts +1 -0
- package/build/tools/generate-tests/generateIntegrationRestTool.test.js +159 -0
- package/build/tools/generate-tests/planGuard.d.ts +13 -0
- package/build/tools/generate-tests/planGuard.js +78 -0
- package/build/tools/generate-tests/planGuard.test.d.ts +1 -0
- package/build/tools/generate-tests/planGuard.test.js +185 -0
- package/build/tools/generate-tests/scenarioFileIdentity.d.ts +10 -0
- package/build/tools/generate-tests/scenarioFileIdentity.js +46 -0
- package/build/tools/generate-tests/scenarioLint.d.ts +30 -0
- package/build/tools/generate-tests/scenarioLint.js +150 -0
- package/build/tools/generate-tests/scenarioLint.test.d.ts +1 -0
- package/build/tools/generate-tests/scenarioLint.test.js +100 -0
- package/build/tools/submitReportTool.js +78 -0
- package/build/tools/submitReportTool.test.js +255 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/analyzeChangesTool.test.js +12 -0
- package/build/tools/test-management/index.d.ts +1 -0
- package/build/tools/test-management/index.js +1 -0
- package/build/tools/test-management/registerTestPlanTool.d.ts +2 -0
- package/build/tools/test-management/registerTestPlanTool.js +329 -0
- package/build/tools/test-management/registerTestPlanTool.test.d.ts +1 -0
- package/build/tools/test-management/registerTestPlanTool.test.js +296 -0
- package/build/types/Recommendation.d.ts +97 -0
- package/build/types/Recommendation.js +48 -0
- package/build/types/RepositoryAnalysis.d.ts +14 -14
- package/build/types/TestExecution.d.ts +2 -0
- package/build/types/TestRecommendation.d.ts +12 -1
- package/build/types/TestRecommendation.js +26 -11
- package/build/types/TestTypes.js +1 -1
- package/build/utils/AnalysisStateManager.d.ts +47 -0
- package/build/utils/docker.test.js +1 -1
- package/build/utils/planMatchKeys.d.ts +61 -0
- package/build/utils/planMatchKeys.js +125 -0
- package/build/utils/pom-scope/import-expansion.d.ts +5 -0
- package/build/utils/pom-scope/import-expansion.js +32 -0
- package/build/utils/pom-scope/index.d.ts +39 -0
- package/build/utils/pom-scope/index.js +120 -0
- package/build/utils/pom-scope/index.test.d.ts +1 -0
- package/build/utils/pom-scope/index.test.js +239 -0
- package/build/utils/pom-scope/pom-files.d.ts +3 -0
- package/build/utils/pom-scope/pom-files.js +48 -0
- package/build/utils/pom-scope/pom-files.test.d.ts +1 -0
- package/build/utils/pom-scope/pom-files.test.js +29 -0
- package/build/utils/pom-scope/scoring.d.ts +20 -0
- package/build/utils/pom-scope/scoring.js +45 -0
- package/build/utils/pom-scope/scoring.test.d.ts +1 -0
- package/build/utils/pom-scope/scoring.test.js +39 -0
- package/build/utils/pom-scope/selector-extractor.d.ts +7 -0
- package/build/utils/pom-scope/selector-extractor.js +57 -0
- package/build/utils/pom-scope/selector-extractor.test.d.ts +1 -0
- package/build/utils/pom-scope/selector-extractor.test.js +67 -0
- package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.d.ts +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.js +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.d.ts +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.js +9 -0
- package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.d.ts +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.js +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.d.ts +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.js +4 -0
- package/build/utils/pom-verify/bindings.d.ts +19 -0
- package/build/utils/pom-verify/bindings.js +161 -0
- package/build/utils/pom-verify/bindings.test.d.ts +1 -0
- package/build/utils/pom-verify/bindings.test.js +164 -0
- package/build/utils/pom-verify/calls.d.ts +16 -0
- package/build/utils/pom-verify/calls.js +42 -0
- package/build/utils/pom-verify/calls.test.d.ts +1 -0
- package/build/utils/pom-verify/calls.test.js +61 -0
- package/build/utils/pom-verify/index.d.ts +4 -0
- package/build/utils/pom-verify/index.js +4 -0
- package/build/utils/pom-verify/resolve.d.ts +7 -0
- package/build/utils/pom-verify/resolve.js +27 -0
- package/build/utils/pom-verify/resolve.test.d.ts +1 -0
- package/build/utils/pom-verify/resolve.test.js +68 -0
- package/build/utils/pom-verify/strip.d.ts +9 -0
- package/build/utils/pom-verify/strip.js +89 -0
- package/build/utils/pom-verify/verify.d.ts +14 -0
- package/build/utils/pom-verify/verify.js +158 -0
- package/build/utils/pom-verify/verify.test.d.ts +1 -0
- package/build/utils/pom-verify/verify.test.js +325 -0
- package/build/utils/reportVerification.d.ts +61 -0
- package/build/utils/reportVerification.js +104 -0
- package/build/utils/reportVerification.test.d.ts +1 -0
- package/build/utils/reportVerification.test.js +185 -0
- package/build/utils/scenarioDrafting.js +5 -5
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/utils/workspaceAuth.d.ts +9 -1
- package/build/utils/workspaceAuth.js +25 -5
- package/build/utils/workspaceAuth.test.js +48 -0
- package/build/workspace/workspace.d.ts +20 -0
- package/build/workspace/workspace.js +4 -0
- package/build/workspace/workspace.test.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +77 -8
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/pollingRecorderSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/utils/isomorphic/volatileDate.js +101 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/utils.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-aszq5EdG.js → codeMirrorModule-Bzd72-bG.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-BxS7Jm4s.js → defaultSettingsView-DzxTioTK.js} +101 -101
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.D4JTTy4R.js → index.BGc30U3S.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.DaRMQKOI.js → uiMode.IaDrb29A.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/pollingRecorderSource.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/utils/isomorphic/volatileDate.ts +131 -0
- package/node_modules/playwright/node_modules/playwright-core/src/utils.ts +1 -0
- package/node_modules/playwright/package.json +1 -1
- package/package.json +3 -3
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
|
-
import { getCodeReusePrompt } from "../../prompts/code-reuse.js";
|
|
3
|
+
import { getCodeReusePrompt, isPomAwareTarget } from "../../prompts/code-reuse.js";
|
|
4
|
+
import { selectScopedPoms } from "../../utils/pom-scope/index.js";
|
|
5
|
+
import { verifyReuse } from "../../utils/pom-verify/index.js";
|
|
4
6
|
import { codeRefactoringSchema, languageSchema, } from "../../types/TestTypes.js";
|
|
5
7
|
import { SKYRAMP_UTILS_HEADER } from "../../utils/utils.js";
|
|
6
8
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
@@ -14,8 +16,33 @@ const codeReuseSchema = z.object({
|
|
|
14
16
|
modularizeCode: codeRefactoringSchema.shape.modularizeCode,
|
|
15
17
|
prompt: z
|
|
16
18
|
.string()
|
|
19
|
+
.default("")
|
|
17
20
|
.describe("The code content to analyze and optimize for code reuse"),
|
|
21
|
+
verify: z
|
|
22
|
+
.boolean()
|
|
23
|
+
.default(false)
|
|
24
|
+
.describe("Verify a previously written refactored test's POM calls against source instead of returning the reuse prompt"),
|
|
18
25
|
});
|
|
26
|
+
/** Renders a `VerifyResult` (from `verifyReuse`) as the text report returned to the agent. */
|
|
27
|
+
export function formatVerifyReport(r) {
|
|
28
|
+
if (r.ok) {
|
|
29
|
+
const lines = [`VERIFICATION PASSED — ${r.checkedCalls} calls checked.`];
|
|
30
|
+
if (r.unverifiable.length > 0) {
|
|
31
|
+
lines.push(`Unverifiable (informational): ${r.unverifiable.join(", ")}`);
|
|
32
|
+
}
|
|
33
|
+
return lines.join("\n");
|
|
34
|
+
}
|
|
35
|
+
const lines = ["VERIFICATION FAILED"];
|
|
36
|
+
for (const v of r.violations) {
|
|
37
|
+
const location = v.searched.length > 1 ? `${v.searched[0]} (+1-hop)` : v.searched[0];
|
|
38
|
+
lines.push(`${v.binding}.${v.member} — not found in ${location}. Available members: ${v.availableMembers.join(", ")}`);
|
|
39
|
+
}
|
|
40
|
+
if (r.parseError) {
|
|
41
|
+
lines.push(`Parse error (fix the syntax and re-verify): ${r.parseError}`);
|
|
42
|
+
}
|
|
43
|
+
lines.push("Each violation may be remapped AT MOST ONCE, and only to a member listed above. If it is flagged again, or you are not confident, restore the raw inline sequence (from your mapping table or `<testFile>.raw.bak`) with a `// kept inline: no matching POM method` comment. Re-run this verification after every fix pass. Do not finish until it reports PASSED.");
|
|
44
|
+
return lines.join("\n\n");
|
|
45
|
+
}
|
|
19
46
|
const TOOL_NAME = "skyramp_reuse_code";
|
|
20
47
|
export function registerCodeReuseTool(server) {
|
|
21
48
|
server.registerTool(TOOL_NAME, {
|
|
@@ -51,8 +78,8 @@ export function registerCodeReuseTool(server) {
|
|
|
51
78
|
**CRITICAL**: NON TRACE BASED TESTS ARE ALREADY MODULARIZED AND DO NOT NEED CODE REUSE.
|
|
52
79
|
The tool will provide step-by-step instructions that MUST be followed exactly.
|
|
53
80
|
|
|
54
|
-
**POM-AWARE MODE (TypeScript/Playwright
|
|
55
|
-
For projects with \`language: "typescript"\` and \`framework: "playwright"\`, this tool first checks whether the project has an existing Page Object Model (POM) library (\`pageobjects/\` directories or \`*.page.ts\` files). If POMs are found, the test is refactored to use those POM classes and methods — replacing raw inline locators with the existing abstractions. SkyrampUtils consolidation is skipped on this path. If no POMs are found, the tool falls back to the standard SkyrampUtils path. New POM creation is out of scope — existing POMs are reused only. After POM-aware code reuse, do NOT call skyramp_modularization.`,
|
|
81
|
+
**POM-AWARE MODE (TypeScript/JavaScript + Playwright):**
|
|
82
|
+
For projects with \`language: "typescript" or "javascript"\` and \`framework: "playwright"\`, this tool first checks whether the project has an existing Page Object Model (POM) library (\`pageobjects/\` directories or \`*.page.ts\` / \`*.page.js\` files). If POMs are found, the test is refactored to use those POM classes and methods — replacing raw inline locators with the existing abstractions. SkyrampUtils consolidation is skipped on this path. If no POMs are found, the tool falls back to the standard SkyrampUtils path. New POM creation is out of scope — existing POMs are reused only. After POM-aware code reuse, do NOT call skyramp_modularization.`,
|
|
56
83
|
inputSchema: codeReuseSchema.shape,
|
|
57
84
|
_meta: {
|
|
58
85
|
keywords: [
|
|
@@ -66,6 +93,22 @@ export function registerCodeReuseTool(server) {
|
|
|
66
93
|
}, async (params) => {
|
|
67
94
|
let errorResult;
|
|
68
95
|
try {
|
|
96
|
+
if (params.verify) {
|
|
97
|
+
try {
|
|
98
|
+
const r = await verifyReuse(params.testFile, params.language);
|
|
99
|
+
return { content: [{ type: "text", text: formatVerifyReport(r) }] };
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
return {
|
|
103
|
+
content: [
|
|
104
|
+
{
|
|
105
|
+
type: "text",
|
|
106
|
+
text: `VERIFICATION ERROR (treat as unverified, do not retry more than once): ${String(err)}`,
|
|
107
|
+
},
|
|
108
|
+
],
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
69
112
|
logger.info("Analyzing code for reuse opportunities", {
|
|
70
113
|
testFile: params.testFile,
|
|
71
114
|
language: params.language,
|
|
@@ -73,7 +116,36 @@ export function registerCodeReuseTool(server) {
|
|
|
73
116
|
});
|
|
74
117
|
// Stage so testbot includes the generated files in its output commit.
|
|
75
118
|
await stageGeneratedPaths(params.testFile);
|
|
76
|
-
|
|
119
|
+
// Deterministic POM pre-scoping (TS/JS+Playwright only). Never blocks:
|
|
120
|
+
// any error or empty result falls back to the agent-driven STEP 1.
|
|
121
|
+
let scopedPoms;
|
|
122
|
+
if (isPomAwareTarget(params.language, params.framework)) {
|
|
123
|
+
try {
|
|
124
|
+
const t0 = Date.now();
|
|
125
|
+
const { tier1, tier2, diagnostics } = await selectScopedPoms({
|
|
126
|
+
testFile: params.testFile,
|
|
127
|
+
specContent: params.prompt,
|
|
128
|
+
});
|
|
129
|
+
if (tier1.length + tier2.length > 0) {
|
|
130
|
+
scopedPoms = { tier1, tier2 };
|
|
131
|
+
}
|
|
132
|
+
else if (diagnostics.selectors > 0 && diagnostics.discovery !== "skipped-too-large") {
|
|
133
|
+
// A clean scan ran (glob or selector-grep) and found zero overlap — distinct from
|
|
134
|
+
// "scoping wasn't attempted" (no selectors) or "scan was skipped" (repo too large).
|
|
135
|
+
scopedPoms = { tier1: [], tier2: [], scannedNoOverlap: true };
|
|
136
|
+
}
|
|
137
|
+
logger.info("POM pre-scoping", {
|
|
138
|
+
...diagnostics,
|
|
139
|
+
tier1: tier1.length,
|
|
140
|
+
tier2: tier2.length,
|
|
141
|
+
ms: Date.now() - t0,
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
logger.warning("POM pre-scoping failed — falling back to agent-driven detection", { error: String(err) });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const codeReusePrompt = getCodeReusePrompt(params.testFile, params.language, params.framework, scopedPoms);
|
|
77
149
|
return {
|
|
78
150
|
content: [
|
|
79
151
|
{
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// @ts-nocheck - Jest ESM type inference issues
|
|
2
|
+
import { jest } from "@jest/globals";
|
|
3
|
+
jest.unstable_mockModule("../../utils/logger.js", () => ({
|
|
4
|
+
logger: { info: jest.fn(), error: jest.fn(), warning: jest.fn(), debug: jest.fn() },
|
|
5
|
+
}));
|
|
6
|
+
jest.unstable_mockModule("../../services/AnalyticsService.js", () => ({
|
|
7
|
+
AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) },
|
|
8
|
+
}));
|
|
9
|
+
jest.unstable_mockModule("../../utils/gitStaging.js", () => ({
|
|
10
|
+
stageGeneratedPaths: jest.fn().mockResolvedValue(undefined),
|
|
11
|
+
}));
|
|
12
|
+
jest.unstable_mockModule("../../utils/pom-verify/index.js", () => ({
|
|
13
|
+
verifyReuse: jest.fn().mockResolvedValue({ ok: true, violations: [], unverifiable: [], checkedCalls: 3 }),
|
|
14
|
+
}));
|
|
15
|
+
const { registerCodeReuseTool, formatVerifyReport } = await import("./codeReuseTool.js");
|
|
16
|
+
const { stageGeneratedPaths } = await import("../../utils/gitStaging.js");
|
|
17
|
+
const { verifyReuse } = await import("../../utils/pom-verify/index.js");
|
|
18
|
+
function captureToolHandler() {
|
|
19
|
+
let handler;
|
|
20
|
+
registerCodeReuseTool({ registerTool: (_n, _o, fn) => { handler = fn; } });
|
|
21
|
+
return handler;
|
|
22
|
+
}
|
|
23
|
+
describe("skyramp_reuse_code — verify mode", () => {
|
|
24
|
+
it("returns the verification report and skips prompt generation/staging when verify:true", async () => {
|
|
25
|
+
const handler = captureToolHandler();
|
|
26
|
+
const result = await handler({
|
|
27
|
+
testFile: "t.spec.ts",
|
|
28
|
+
language: "typescript",
|
|
29
|
+
framework: "playwright",
|
|
30
|
+
modularizeCode: false,
|
|
31
|
+
prompt: "",
|
|
32
|
+
verify: true,
|
|
33
|
+
});
|
|
34
|
+
expect(result.content[0].text).toBe("VERIFICATION PASSED — 3 calls checked.");
|
|
35
|
+
expect(verifyReuse).toHaveBeenCalledWith("t.spec.ts", "typescript");
|
|
36
|
+
expect(stageGeneratedPaths).not.toHaveBeenCalled();
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
describe("formatVerifyReport", () => {
|
|
40
|
+
it("reports a violation block with the fix-rule paragraph, not the passed message", () => {
|
|
41
|
+
const text = formatVerifyReport({
|
|
42
|
+
ok: false,
|
|
43
|
+
violations: [{ binding: "pm", member: "fakeMethod", searched: ["a.ts", "b.ts"], availableMembers: ["realMethod"] }],
|
|
44
|
+
unverifiable: [],
|
|
45
|
+
checkedCalls: 1,
|
|
46
|
+
});
|
|
47
|
+
expect(text).toContain("VERIFICATION FAILED");
|
|
48
|
+
expect(text).toContain("pm.fakeMethod — not found in a.ts (+1-hop). Available members: realMethod");
|
|
49
|
+
expect(text).toContain("Do not finish until it reports PASSED.");
|
|
50
|
+
});
|
|
51
|
+
});
|
|
@@ -7,7 +7,7 @@ import { makeProgressReporter } from "../utils/progress.js";
|
|
|
7
7
|
import { TestExecutionStatus } from "../types/TestExecution.js";
|
|
8
8
|
import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
|
|
9
9
|
import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
|
|
10
|
-
import { StateManager } from "../utils/AnalysisStateManager.js";
|
|
10
|
+
import { StateManager, getTestsRepoDir } from "../utils/AnalysisStateManager.js";
|
|
11
11
|
import { TestSource } from "../types/TestAnalysis.js";
|
|
12
12
|
import { logger } from "../utils/logger.js";
|
|
13
13
|
import { toolError } from "../utils/utils.js";
|
|
@@ -108,10 +108,15 @@ export function registerExecuteSkyrampTestTool(server) {
|
|
|
108
108
|
}
|
|
109
109
|
// Send initial progress
|
|
110
110
|
await sendProgress(0, 100, "Starting test execution...");
|
|
111
|
+
// Cross-repo run (SKYR-3819): tests delivered under the run's testsRepoDir
|
|
112
|
+
// (set by skyramp_analyze_changes, run-scoped) live outside workspacePath.
|
|
113
|
+
// Thread it through so the executor mounts the test repo and service
|
|
114
|
+
// matching still resolves the SUT baseUrl/dockerNetwork.
|
|
115
|
+
const testRepoPath = getTestsRepoDir();
|
|
111
116
|
// Always resolve workspace config for dockerNetwork (host networking)
|
|
112
117
|
// and optionally inject SKYRAMP_TEST_BASE_URL if not already set.
|
|
113
118
|
if (params.workspacePath) {
|
|
114
|
-
const { baseUrl, dockerNetwork, candidates } = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language);
|
|
119
|
+
const { baseUrl, dockerNetwork, candidates } = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language, testRepoPath);
|
|
115
120
|
useHostNetwork = !!dockerNetwork;
|
|
116
121
|
if (!process.env.SKYRAMP_TEST_BASE_URL) {
|
|
117
122
|
if (baseUrl) {
|
|
@@ -142,6 +147,7 @@ export function registerExecuteSkyrampTestTool(server) {
|
|
|
142
147
|
const result = await executionService.executeTest({
|
|
143
148
|
testFile: params.testFile,
|
|
144
149
|
workspacePath: params.workspacePath,
|
|
150
|
+
testRepoPath,
|
|
145
151
|
language: params.language,
|
|
146
152
|
testType: params.testType,
|
|
147
153
|
token: effectiveToken,
|
|
@@ -6,9 +6,10 @@ export declare const stepSchema: z.ZodEffects<z.ZodObject<{
|
|
|
6
6
|
path: z.ZodString;
|
|
7
7
|
requestBody: z.ZodEffects<z.ZodOptional<z.ZodString>, string | undefined, string | undefined>;
|
|
8
8
|
queryParams: z.ZodEffects<z.ZodOptional<z.ZodString>, string | undefined, string | undefined>;
|
|
9
|
-
responseBody: z.ZodOptional<z.ZodString>;
|
|
9
|
+
responseBody: z.ZodEffects<z.ZodOptional<z.ZodString>, string | undefined, string | undefined>;
|
|
10
10
|
statusCode: z.ZodOptional<z.ZodNumber>;
|
|
11
11
|
responseHeaders: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString, "many">>>;
|
|
12
|
+
uniqueFields: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
12
13
|
}, "strip", z.ZodTypeAny, {
|
|
13
14
|
path: string;
|
|
14
15
|
method: HttpMethod;
|
|
@@ -17,6 +18,7 @@ export declare const stepSchema: z.ZodEffects<z.ZodObject<{
|
|
|
17
18
|
requestBody?: string | undefined;
|
|
18
19
|
responseBody?: string | undefined;
|
|
19
20
|
responseHeaders?: Record<string, string[]> | undefined;
|
|
21
|
+
uniqueFields?: string[] | undefined;
|
|
20
22
|
}, {
|
|
21
23
|
path: string;
|
|
22
24
|
method: HttpMethod;
|
|
@@ -25,6 +27,7 @@ export declare const stepSchema: z.ZodEffects<z.ZodObject<{
|
|
|
25
27
|
requestBody?: string | undefined;
|
|
26
28
|
responseBody?: string | undefined;
|
|
27
29
|
responseHeaders?: Record<string, string[]> | undefined;
|
|
30
|
+
uniqueFields?: string[] | undefined;
|
|
28
31
|
}>, {
|
|
29
32
|
path: string;
|
|
30
33
|
method: HttpMethod;
|
|
@@ -33,6 +36,7 @@ export declare const stepSchema: z.ZodEffects<z.ZodObject<{
|
|
|
33
36
|
requestBody?: string | undefined;
|
|
34
37
|
responseBody?: string | undefined;
|
|
35
38
|
responseHeaders?: Record<string, string[]> | undefined;
|
|
39
|
+
uniqueFields?: string[] | undefined;
|
|
36
40
|
}, {
|
|
37
41
|
path: string;
|
|
38
42
|
method: HttpMethod;
|
|
@@ -41,5 +45,6 @@ export declare const stepSchema: z.ZodEffects<z.ZodObject<{
|
|
|
41
45
|
requestBody?: string | undefined;
|
|
42
46
|
responseBody?: string | undefined;
|
|
43
47
|
responseHeaders?: Record<string, string[]> | undefined;
|
|
48
|
+
uniqueFields?: string[] | undefined;
|
|
44
49
|
}>;
|
|
45
50
|
export declare function registerBatchScenarioTestTool(server: McpServer): void;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { TOOL_BATCH_SCENARIO_TEST_GENERATION } from "../../toolNames.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import path from "path";
|
|
3
4
|
import { ScenarioGenerationService } from "../../services/ScenarioGenerationService.js";
|
|
@@ -7,6 +8,10 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
|
7
8
|
import { getWorkspaceAuthConfig, WorkspaceAuthType, getDefaultAuthHeader, isAuthorizationHeaderName, getAuthScheme } from "../../utils/workspaceAuth.js";
|
|
8
9
|
import yaml from "js-yaml";
|
|
9
10
|
import { logger } from "../../utils/logger.js";
|
|
11
|
+
import { guardScenarioApproved } from "./planGuard.js";
|
|
12
|
+
import { lintScenarioChaining } from "./scenarioLint.js";
|
|
13
|
+
import { scenarioFileNameFor } from "./scenarioFileIdentity.js";
|
|
14
|
+
import { toolError } from "../../utils/utils.js";
|
|
10
15
|
function isJsonValue(v) {
|
|
11
16
|
if (v === undefined || v === null)
|
|
12
17
|
return true;
|
|
@@ -18,6 +23,19 @@ function isJsonValue(v) {
|
|
|
18
23
|
return false;
|
|
19
24
|
}
|
|
20
25
|
}
|
|
26
|
+
/** Resolve a gjson-style dot path ("name", "nested.id", "items.0.sku") against parsed JSON.
|
|
27
|
+
* The Go generator silently skips uniqueFields paths that don't resolve, which would
|
|
28
|
+
* quietly disable the SKYR-3882 run-unique protection — so a typo must fail loudly here.
|
|
29
|
+
* `seg in cur` covers arrays too (in-bounds numeric indices are own properties). */
|
|
30
|
+
function resolvesInJson(obj, dotPath) {
|
|
31
|
+
let cur = obj;
|
|
32
|
+
for (const seg of dotPath.split(".")) {
|
|
33
|
+
if (cur === null || typeof cur !== "object" || !(seg in cur))
|
|
34
|
+
return false;
|
|
35
|
+
cur = cur[seg];
|
|
36
|
+
}
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
21
39
|
function isJsonObject(v) {
|
|
22
40
|
if (v === undefined || v === null)
|
|
23
41
|
return true;
|
|
@@ -67,6 +85,7 @@ CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&li
|
|
|
67
85
|
responseBody: z
|
|
68
86
|
.string()
|
|
69
87
|
.optional()
|
|
88
|
+
.refine(isJsonValue, { message: "responseBody must be valid JSON (e.g. '{\"id\": 1}'). The backend extracts chaining identifiers by parsing it — a malformed body silently disables chaining for every later step." })
|
|
70
89
|
.describe("JSON string of the expected response body. For steps that produce an id used by later steps, include that id field with a realistic UUID generated fresh for this scenario — the backend chains by value-matching any subsequent step path or requestBody that contains the same UUID."),
|
|
71
90
|
statusCode: z
|
|
72
91
|
.number()
|
|
@@ -76,26 +95,69 @@ CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&li
|
|
|
76
95
|
.record(z.array(z.string()))
|
|
77
96
|
.optional()
|
|
78
97
|
.describe("Response headers as a JSON object. Defaults to Content-Type: application/json."),
|
|
98
|
+
uniqueFields: z
|
|
99
|
+
.array(z.string().min(1, { message: "uniqueFields paths must be non-empty" }))
|
|
100
|
+
.min(1, { message: "uniqueFields, when provided, must list at least one field path" })
|
|
101
|
+
.optional()
|
|
102
|
+
.describe("For POST/PUT create steps: request-body field paths (gjson notation, e.g. \"name\", \"slug\", \"nested.id\") "
|
|
103
|
+
+ "that carry a UNIQUE constraint in the target's data model. The generated test injects a run-unique value "
|
|
104
|
+
+ "for each so re-runs don't fail with HTTP 409 'already exists'. Identify these from the source schema "
|
|
105
|
+
+ "(e.g. Django unique=True, SQL UNIQUE, Pydantic/DTO). Omit for GET/DELETE steps and non-unique fields."),
|
|
79
106
|
}).superRefine((step, ctx) => {
|
|
80
107
|
const isBodyMethod = ["POST", "PUT", "PATCH"].includes(step.method);
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
108
|
+
// Parse once for every body-shape check below; malformed JSON is already
|
|
109
|
+
// rejected by the isJsonValue refine, so an unparseable body just skips them.
|
|
110
|
+
let parsedBody;
|
|
111
|
+
try {
|
|
112
|
+
parsedBody = step.requestBody ? JSON.parse(step.requestBody) : undefined;
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
parsedBody = undefined;
|
|
116
|
+
}
|
|
117
|
+
// uniqueFields only applies to create-style requests that carry a body — the
|
|
118
|
+
// run-unique value is injected into a request-body field. Reject it on GET/DELETE
|
|
119
|
+
// (or a body method with no requestBody) so a misplaced hint fails loudly.
|
|
120
|
+
if (step.uniqueFields && step.uniqueFields.length > 0) {
|
|
121
|
+
if (!isBodyMethod) {
|
|
122
|
+
ctx.addIssue({
|
|
123
|
+
code: z.ZodIssueCode.custom,
|
|
124
|
+
path: ["uniqueFields"],
|
|
125
|
+
message: `uniqueFields is only valid on POST/PUT/PATCH steps (got ${step.method} ${step.path}). `
|
|
126
|
+
+ `Omit it for GET/DELETE steps.`,
|
|
127
|
+
});
|
|
89
128
|
}
|
|
90
|
-
if (
|
|
129
|
+
else if (!step.requestBody) {
|
|
91
130
|
ctx.addIssue({
|
|
92
131
|
code: z.ZodIssueCode.custom,
|
|
93
|
-
path: ["
|
|
94
|
-
message: `
|
|
95
|
-
+ `
|
|
96
|
-
+ `do not send an empty body for ${step.method} requests.`,
|
|
132
|
+
path: ["uniqueFields"],
|
|
133
|
+
message: `uniqueFields is set for ${step.method} ${step.path} but there is no requestBody `
|
|
134
|
+
+ `containing those fields.`,
|
|
97
135
|
});
|
|
98
136
|
}
|
|
137
|
+
else if (parsedBody !== undefined) {
|
|
138
|
+
for (const fieldPath of step.uniqueFields) {
|
|
139
|
+
if (!resolvesInJson(parsedBody, fieldPath)) {
|
|
140
|
+
ctx.addIssue({
|
|
141
|
+
code: z.ZodIssueCode.custom,
|
|
142
|
+
path: ["uniqueFields"],
|
|
143
|
+
message: `uniqueFields path "${fieldPath}" does not resolve in the requestBody of `
|
|
144
|
+
+ `${step.method} ${step.path}. The generator silently skips unresolvable paths, `
|
|
145
|
+
+ `disabling the run-unique protection — fix the path (gjson dot notation, e.g. `
|
|
146
|
+
+ `"name", "nested.id", "items.0.sku") to match an existing body field.`,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (isBodyMethod && parsedBody !== null && typeof parsedBody === "object"
|
|
153
|
+
&& !Array.isArray(parsedBody) && Object.keys(parsedBody).length === 0) {
|
|
154
|
+
ctx.addIssue({
|
|
155
|
+
code: z.ZodIssueCode.custom,
|
|
156
|
+
path: ["requestBody"],
|
|
157
|
+
message: `requestBody for ${step.method} ${step.path} is an empty object {}. `
|
|
158
|
+
+ `Provide actual field values from the source code schema (Pydantic model, Zod type, DTO) — `
|
|
159
|
+
+ `do not send an empty body for ${step.method} requests.`,
|
|
160
|
+
});
|
|
99
161
|
}
|
|
100
162
|
});
|
|
101
163
|
const batchScenarioSchema = {
|
|
@@ -141,7 +203,7 @@ const batchScenarioSchema = {
|
|
|
141
203
|
+ `${AUTH_PLACEHOLDER_TOKEN} for all auth headers (Authorization, X-API-Key, Cookie, etc.). `
|
|
142
204
|
+ `Only provide this when you need a custom value (e.g., 'session=${AUTH_PLACEHOLDER_TOKEN}' for Cookie).`),
|
|
143
205
|
};
|
|
144
|
-
const TOOL_NAME =
|
|
206
|
+
const TOOL_NAME = TOOL_BATCH_SCENARIO_TEST_GENERATION;
|
|
145
207
|
export function registerBatchScenarioTestTool(server) {
|
|
146
208
|
server.registerTool(TOOL_NAME, {
|
|
147
209
|
description: `Generate a synthetic scenario trace file for a multi-step API workflow scenario — use this before skyramp_integration_test_generation.
|
|
@@ -167,6 +229,13 @@ If a step fails, the error shows which step (N/total), method+path, and reason.
|
|
|
167
229
|
Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\` path. Do NOT also pass \`apiSchema\` or \`endpointURL\` — the scenario file already contains all endpoint information and passing both will cause test generation to fail.`,
|
|
168
230
|
inputSchema: batchScenarioSchema,
|
|
169
231
|
}, async (params) => {
|
|
232
|
+
const guardMessage = await guardScenarioApproved({
|
|
233
|
+
scenarioName: params.scenarioName,
|
|
234
|
+
testType: "integration",
|
|
235
|
+
});
|
|
236
|
+
if (guardMessage) {
|
|
237
|
+
return { content: [{ type: "text", text: guardMessage }], isError: true };
|
|
238
|
+
}
|
|
170
239
|
if (params.authHeader === undefined) {
|
|
171
240
|
try {
|
|
172
241
|
const repoPath = params.outputDir || process.cwd();
|
|
@@ -319,8 +388,7 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
319
388
|
}
|
|
320
389
|
const service = new ScenarioGenerationService();
|
|
321
390
|
const steps = params.steps;
|
|
322
|
-
const
|
|
323
|
-
const fileName = `scenario_${scenarioSlug}.json`;
|
|
391
|
+
const fileName = scenarioFileNameFor(params.scenarioName);
|
|
324
392
|
const filePath = path.join(params.outputDir, fileName);
|
|
325
393
|
const resolvedOut = path.resolve(params.outputDir);
|
|
326
394
|
if (!path.resolve(filePath).startsWith(resolvedOut + path.sep) && path.resolve(filePath) !== resolvedOut) {
|
|
@@ -350,6 +418,7 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
350
418
|
authHeader: params.authHeader ?? "",
|
|
351
419
|
authScheme: params.authScheme ?? "",
|
|
352
420
|
authToken: params.authToken,
|
|
421
|
+
uniqueFields: step.uniqueFields,
|
|
353
422
|
};
|
|
354
423
|
const traceRequest = service.generateTraceRequestFromInput(stepParams);
|
|
355
424
|
if (!traceRequest) {
|
|
@@ -375,6 +444,29 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
375
444
|
}
|
|
376
445
|
traceRequests.push(traceRequest);
|
|
377
446
|
}
|
|
447
|
+
// ── SKYR-3884: chaining lint ──
|
|
448
|
+
// The generated test chains FKs/path params by value-matching against
|
|
449
|
+
// PRIOR steps' responses; mis-ordered or missing "bait" values silently
|
|
450
|
+
// degrade to hardcoded literals. Lint the built TraceRequests — the
|
|
451
|
+
// artifact the Go reader actually consumes (post basePath/default
|
|
452
|
+
// transforms), not the raw tool input. Ordering mistakes are fixable →
|
|
453
|
+
// hard error before the file is written; unchainable/ambiguous
|
|
454
|
+
// references → warning on the success text.
|
|
455
|
+
const chainLint = lintScenarioChaining(traceRequests.map((t) => ({
|
|
456
|
+
method: t.Method,
|
|
457
|
+
path: t.Path,
|
|
458
|
+
requestBody: t.RequestBody,
|
|
459
|
+
responseBody: t.ResponseBody,
|
|
460
|
+
})));
|
|
461
|
+
if (chainLint.errors.length > 0) {
|
|
462
|
+
return toolError(`**Scenario step-ordering errors**\n\n`
|
|
463
|
+
+ chainLint.errors.map((e) => `- ${e}`).join("\n")
|
|
464
|
+
+ `\n\nReorder or fix the steps and resubmit the full batch.`);
|
|
465
|
+
}
|
|
466
|
+
const chainLintWarning = chainLint.warnings.length > 0
|
|
467
|
+
? `\n\n⚠️ **Chaining warnings** — the generated test may hardcode these instead of chaining:\n`
|
|
468
|
+
+ chainLint.warnings.map((w) => ` - ${w}`).join("\n")
|
|
469
|
+
: "";
|
|
378
470
|
try {
|
|
379
471
|
fs.writeFileSync(filePath, JSON.stringify(traceRequests, null, 2), "utf8");
|
|
380
472
|
}
|
|
@@ -395,7 +487,8 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
395
487
|
+ `**Steps:**\n${steps.map((s, i) => ` ${i + 1}. ${s.method} ${s.path} → ${s.statusCode ?? "default"}`).join("\n")}\n\n`
|
|
396
488
|
+ `**File:** ${filePath}\n\n`
|
|
397
489
|
+ `**Next:** Call \`skyramp_integration_test_generation\` with \`scenarioFile: "${filePath}"\``
|
|
398
|
-
+ specValidationWarning
|
|
490
|
+
+ specValidationWarning
|
|
491
|
+
+ chainLintWarning,
|
|
399
492
|
},
|
|
400
493
|
],
|
|
401
494
|
isError: false,
|
|
@@ -284,3 +284,150 @@ describe("generateBatchScenarioRestTool — spec URL fetch safety (comment 32030
|
|
|
284
284
|
delete global.fetch;
|
|
285
285
|
});
|
|
286
286
|
});
|
|
287
|
+
describe("stepSchema uniqueFields validation (SKYR-3882)", () => {
|
|
288
|
+
const base = { method: "POST", path: "/api/v1/providers/", requestBody: '{"name":"x"}' };
|
|
289
|
+
it("accepts uniqueFields on a POST with a body", () => {
|
|
290
|
+
expect(stepSchema.safeParse({ ...base, uniqueFields: ["name"] }).success).toBe(true);
|
|
291
|
+
});
|
|
292
|
+
it("rejects an empty uniqueFields array", () => {
|
|
293
|
+
expect(stepSchema.safeParse({ ...base, uniqueFields: [] }).success).toBe(false);
|
|
294
|
+
});
|
|
295
|
+
it("rejects empty-string paths in uniqueFields", () => {
|
|
296
|
+
expect(stepSchema.safeParse({ ...base, uniqueFields: [""] }).success).toBe(false);
|
|
297
|
+
});
|
|
298
|
+
it("rejects uniqueFields on a non-body method (GET)", () => {
|
|
299
|
+
expect(stepSchema.safeParse({ method: "GET", path: "/x", uniqueFields: ["name"] }).success).toBe(false);
|
|
300
|
+
});
|
|
301
|
+
it("rejects uniqueFields on a POST with no requestBody", () => {
|
|
302
|
+
expect(stepSchema.safeParse({ method: "POST", path: "/api/v1/providers/", uniqueFields: ["name"] }).success).toBe(false);
|
|
303
|
+
});
|
|
304
|
+
});
|
|
305
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
306
|
+
// SKYR-3884 write-side validations
|
|
307
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
308
|
+
describe("stepSchema — responseBody JSON validation (SKYR-3884)", () => {
|
|
309
|
+
it("rejects a responseBody that is not valid JSON", () => {
|
|
310
|
+
const result = stepSchema.safeParse({
|
|
311
|
+
method: "POST",
|
|
312
|
+
path: "/api/v1/products",
|
|
313
|
+
requestBody: JSON.stringify({ name: "W" }),
|
|
314
|
+
responseBody: "{id: 1, oops",
|
|
315
|
+
});
|
|
316
|
+
expect(result.success).toBe(false);
|
|
317
|
+
const issue = result.error?.issues.find((i) => i.path.includes("responseBody"));
|
|
318
|
+
expect(issue?.message).toContain("valid JSON");
|
|
319
|
+
});
|
|
320
|
+
it("accepts a valid JSON responseBody and an omitted one", () => {
|
|
321
|
+
expect(stepSchema.safeParse({
|
|
322
|
+
method: "POST",
|
|
323
|
+
path: "/api/v1/products",
|
|
324
|
+
requestBody: JSON.stringify({ name: "W" }),
|
|
325
|
+
responseBody: JSON.stringify({ id: 1 }),
|
|
326
|
+
}).success).toBe(true);
|
|
327
|
+
expect(stepSchema.safeParse({ method: "GET", path: "/api/v1/products" }).success).toBe(true);
|
|
328
|
+
});
|
|
329
|
+
});
|
|
330
|
+
describe("stepSchema — uniqueFields paths must resolve in requestBody (SKYR-3884)", () => {
|
|
331
|
+
it("rejects a uniqueFields path that does not resolve in the requestBody", () => {
|
|
332
|
+
const result = stepSchema.safeParse({
|
|
333
|
+
method: "POST",
|
|
334
|
+
path: "/api/v1/products",
|
|
335
|
+
requestBody: JSON.stringify({ name: "W", slug: "w" }),
|
|
336
|
+
uniqueFields: ["nmae"],
|
|
337
|
+
});
|
|
338
|
+
expect(result.success).toBe(false);
|
|
339
|
+
const issue = result.error?.issues.find((i) => i.path.includes("uniqueFields"));
|
|
340
|
+
expect(issue?.message).toContain("nmae");
|
|
341
|
+
});
|
|
342
|
+
it("accepts top-level, nested, and array-indexed uniqueFields paths that resolve", () => {
|
|
343
|
+
const result = stepSchema.safeParse({
|
|
344
|
+
method: "POST",
|
|
345
|
+
path: "/api/v1/products",
|
|
346
|
+
requestBody: JSON.stringify({ name: "W", nested: { slug: "w" }, items: [{ sku: "S1" }] }),
|
|
347
|
+
uniqueFields: ["name", "nested.slug", "items.0.sku"],
|
|
348
|
+
});
|
|
349
|
+
expect(result.success).toBe(true);
|
|
350
|
+
});
|
|
351
|
+
});
|
|
352
|
+
describe("generateBatchScenarioRestTool — chaining lint (SKYR-3884)", () => {
|
|
353
|
+
let handler;
|
|
354
|
+
let writeFileSyncSpy;
|
|
355
|
+
const baseParams = {
|
|
356
|
+
scenarioName: "test_scenario",
|
|
357
|
+
destination: "localhost",
|
|
358
|
+
outputDir: "/tmp/skyramp-test",
|
|
359
|
+
};
|
|
360
|
+
beforeEach(() => {
|
|
361
|
+
jest.clearAllMocks();
|
|
362
|
+
handler = captureHandler();
|
|
363
|
+
writeFileSyncSpy = jest.spyOn(fs, "writeFileSync").mockImplementation(() => { });
|
|
364
|
+
// The lint inspects the BUILT TraceRequests (the artifact the Go reader
|
|
365
|
+
// consumes), so the mock must pass the step through in TraceRequest shape.
|
|
366
|
+
ScenarioGenerationService.mockImplementation(() => ({
|
|
367
|
+
generateTraceRequestFromInput: jest.fn((p) => ({
|
|
368
|
+
Method: p.method,
|
|
369
|
+
Path: p.path,
|
|
370
|
+
RequestBody: p.requestBody ?? "",
|
|
371
|
+
ResponseBody: p.responseBody ?? "",
|
|
372
|
+
})),
|
|
373
|
+
}));
|
|
374
|
+
});
|
|
375
|
+
afterEach(() => {
|
|
376
|
+
writeFileSyncSpy.mockRestore();
|
|
377
|
+
});
|
|
378
|
+
it("hard-rejects a scenario whose FK reference is produced only by a later step", async () => {
|
|
379
|
+
const result = await handler({
|
|
380
|
+
...baseParams,
|
|
381
|
+
steps: [
|
|
382
|
+
{
|
|
383
|
+
method: "POST",
|
|
384
|
+
path: "/api/circuits/provider-accounts/",
|
|
385
|
+
requestBody: JSON.stringify({ provider: 1, name: "A" }),
|
|
386
|
+
responseBody: JSON.stringify({ id: 7 }),
|
|
387
|
+
},
|
|
388
|
+
{
|
|
389
|
+
method: "POST",
|
|
390
|
+
path: "/api/circuits/providers/",
|
|
391
|
+
requestBody: JSON.stringify({ name: "P" }),
|
|
392
|
+
responseBody: JSON.stringify({ id: 1 }),
|
|
393
|
+
},
|
|
394
|
+
],
|
|
395
|
+
});
|
|
396
|
+
expect(result.isError).toBe(true);
|
|
397
|
+
expect(result.content[0].text).toContain("later step");
|
|
398
|
+
expect(writeFileSyncSpy).not.toHaveBeenCalled();
|
|
399
|
+
});
|
|
400
|
+
it("appends a chaining warning (does not reject) for unchainable path values", async () => {
|
|
401
|
+
const result = await handler({
|
|
402
|
+
...baseParams,
|
|
403
|
+
steps: [
|
|
404
|
+
{
|
|
405
|
+
method: "POST",
|
|
406
|
+
path: "/api/circuits/providers/",
|
|
407
|
+
requestBody: JSON.stringify({ name: "P" }),
|
|
408
|
+
responseBody: JSON.stringify({ id: 1 }),
|
|
409
|
+
},
|
|
410
|
+
{ method: "GET", path: "/api/circuits/providers/999/summary/" },
|
|
411
|
+
],
|
|
412
|
+
});
|
|
413
|
+
expect(result.isError).toBeFalsy();
|
|
414
|
+
expect(result.content[0].text).toContain("Chaining warning");
|
|
415
|
+
expect(result.content[0].text).toContain("999");
|
|
416
|
+
});
|
|
417
|
+
it("emits no chaining output for a well-chained scenario", async () => {
|
|
418
|
+
const result = await handler({
|
|
419
|
+
...baseParams,
|
|
420
|
+
steps: [
|
|
421
|
+
{
|
|
422
|
+
method: "POST",
|
|
423
|
+
path: "/api/circuits/providers/",
|
|
424
|
+
requestBody: JSON.stringify({ name: "P" }),
|
|
425
|
+
responseBody: JSON.stringify({ id: 1 }),
|
|
426
|
+
},
|
|
427
|
+
{ method: "GET", path: "/api/circuits/providers/1/summary/" },
|
|
428
|
+
],
|
|
429
|
+
});
|
|
430
|
+
expect(result.isError).toBeFalsy();
|
|
431
|
+
expect(result.content[0].text).not.toContain("Chaining warning");
|
|
432
|
+
});
|
|
433
|
+
});
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import path from "path";
|
|
2
|
+
import { TOOL_CONTRACT_TEST_GENERATION } from "../../toolNames.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { baseTestSchema, TestType } from "../../types/TestTypes.js";
|
|
4
5
|
import { TestGenerationService, } from "../../services/TestGenerationService.js";
|
|
5
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
6
7
|
import { getPersonaPrefix } from "../../prompts/personas.js";
|
|
7
8
|
import { isContractConsumerModeEnabled, isTestbotEnabled } from "../../utils/featureFlags.js";
|
|
9
|
+
import { guardScenarioApproved, pathFromEndpointURL } from "./planGuard.js";
|
|
10
|
+
import { toolError } from "../../utils/utils.js";
|
|
8
11
|
// SKYRAMP_FEATURE_CONTRACT_CONSUMER_MODE gates BOTH:
|
|
9
12
|
// 1. Consumer-side contract test generation (`consumerMode` /
|
|
10
13
|
// `consumerOutput` schema fields, validation, post-gen instructions,
|
|
@@ -353,7 +356,7 @@ The generated consumer contract test contains a stub test function that uses Sky
|
|
|
353
356
|
return baseOptions;
|
|
354
357
|
}
|
|
355
358
|
}
|
|
356
|
-
const TOOL_NAME =
|
|
359
|
+
const TOOL_NAME = TOOL_CONTRACT_TEST_GENERATION;
|
|
357
360
|
const MODES_SECTION = CONSUMER_MODE_ENABLED
|
|
358
361
|
? `**Modes:**
|
|
359
362
|
- Default (no mode set): both \`providerMode\` and \`consumerMode\` default to false. This generates both provider and consumer contract tests — equivalent to setting both modes to true.
|
|
@@ -413,6 +416,13 @@ ${CHAINING_SECTION}
|
|
|
413
416
|
`,
|
|
414
417
|
inputSchema: contractTestSchema,
|
|
415
418
|
}, async (params) => {
|
|
419
|
+
const guardMessage = await guardScenarioApproved({
|
|
420
|
+
testType: "contract",
|
|
421
|
+
method: params.method || undefined,
|
|
422
|
+
path: pathFromEndpointURL(params.endpointURL),
|
|
423
|
+
});
|
|
424
|
+
if (guardMessage)
|
|
425
|
+
return toolError(guardMessage);
|
|
416
426
|
const service = new ContractTestService();
|
|
417
427
|
const result = await service.generateTest(params);
|
|
418
428
|
AnalyticsService.pushTestGenerationToolEvent(TOOL_NAME, result, params).catch(() => {
|