@skyramp/mcp 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +56 -9
- package/build/prompts/startTraceCollectionPrompts.js +20 -2
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +42 -9
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +91 -85
- package/build/prompts/testbot/testbot-prompts.js +85 -18
- package/build/services/AnalyticsService.js +31 -12
- package/build/services/DriftAnalysisService.js +139 -13
- package/build/services/DriftAnalysisService.test.js +168 -0
- package/build/services/TestExecutionService.js +1 -1
- package/build/services/TestGenerationService.js +7 -2
- package/build/services/TestHealthService.js +38 -3
- package/build/services/TestHealthService.test.js +211 -0
- package/build/tools/submitReportTool.js +12 -3
- package/build/tools/submitReportTool.test.js +59 -1
- package/build/tools/test-maintenance/actionsTool.js +115 -9
- package/build/tools/test-maintenance/actionsTool.test.js +93 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +221 -14
- package/build/tools/test-recommendation/mapTestsTool.js +44 -22
- package/build/tools/test-recommendation/recommendTestsTool.js +87 -33
- package/build/tools/trace/startTraceCollectionTool.js +5 -2
- package/build/tools/trace/stopTraceCollectionTool.js +2 -2
- package/build/tools/workspace/initializeWorkspaceTool.js +222 -0
- package/build/types/RepositoryAnalysis.js +20 -0
- package/build/types/TestTypes.js +6 -2
- package/build/utils/initAgent.js +2 -1
- package/build/utils/scoring-engine.js +136 -18
- package/build/utils/telemetry.js +23 -0
- package/build/utils/utils.js +0 -1
- package/package.json +2 -2
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { TestHealthService } from "./TestHealthService.js";
|
|
2
|
+
describe("TestHealthService", () => {
|
|
3
|
+
let service;
|
|
4
|
+
beforeEach(() => {
|
|
5
|
+
service = new TestHealthService();
|
|
6
|
+
});
|
|
7
|
+
describe("identifyIssues - endpoint rename detection", () => {
|
|
8
|
+
function identifyIssues(execution, drift) {
|
|
9
|
+
return service["identifyIssues"](execution, drift);
|
|
10
|
+
}
|
|
11
|
+
it("should create an endpoints_renamed issue when drift has endpoint_renamed changes", () => {
|
|
12
|
+
const drift = {
|
|
13
|
+
testFile: "products_smoke_test.py",
|
|
14
|
+
lastCommit: "abc123",
|
|
15
|
+
currentCommit: "def456",
|
|
16
|
+
driftScore: 30,
|
|
17
|
+
changes: [
|
|
18
|
+
{
|
|
19
|
+
type: "endpoint_renamed",
|
|
20
|
+
file: "API Schema",
|
|
21
|
+
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
22
|
+
severity: "high",
|
|
23
|
+
details: "Path changed from /api/v1/products to /api/v1/items",
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
affectedFiles: { files: ["src/routers/product.py"] },
|
|
27
|
+
analysisTimestamp: new Date().toISOString(),
|
|
28
|
+
};
|
|
29
|
+
const issues = identifyIssues(undefined, drift);
|
|
30
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
31
|
+
expect(renameIssue).toBeDefined();
|
|
32
|
+
expect(renameIssue?.severity).toBe("high");
|
|
33
|
+
expect(renameIssue?.description).toContain("1 API endpoint(s) renamed");
|
|
34
|
+
});
|
|
35
|
+
it("should not create endpoints_renamed issue when no renames in drift", () => {
|
|
36
|
+
const drift = {
|
|
37
|
+
testFile: "products_smoke_test.py",
|
|
38
|
+
lastCommit: "abc123",
|
|
39
|
+
currentCommit: "def456",
|
|
40
|
+
driftScore: 15,
|
|
41
|
+
changes: [
|
|
42
|
+
{
|
|
43
|
+
type: "endpoint_removed",
|
|
44
|
+
file: "API Schema",
|
|
45
|
+
description: "1 endpoint(s) removed",
|
|
46
|
+
severity: "high",
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
affectedFiles: { files: [] },
|
|
50
|
+
analysisTimestamp: new Date().toISOString(),
|
|
51
|
+
};
|
|
52
|
+
const issues = identifyIssues(undefined, drift);
|
|
53
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
54
|
+
expect(renameIssue).toBeUndefined();
|
|
55
|
+
const removeIssue = issues.find((i) => i.type === "endpoints_removed");
|
|
56
|
+
expect(removeIssue).toBeDefined();
|
|
57
|
+
});
|
|
58
|
+
it("should handle multiple rename changes", () => {
|
|
59
|
+
const drift = {
|
|
60
|
+
testFile: "products_smoke_test.py",
|
|
61
|
+
lastCommit: "abc123",
|
|
62
|
+
currentCommit: "def456",
|
|
63
|
+
driftScore: 40,
|
|
64
|
+
changes: [
|
|
65
|
+
{
|
|
66
|
+
type: "endpoint_renamed",
|
|
67
|
+
file: "API Schema",
|
|
68
|
+
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
69
|
+
severity: "high",
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
type: "endpoint_renamed",
|
|
73
|
+
file: "API Schema",
|
|
74
|
+
description: "Endpoint renamed: post /api/v1/products -> /api/v1/items",
|
|
75
|
+
severity: "high",
|
|
76
|
+
},
|
|
77
|
+
],
|
|
78
|
+
affectedFiles: { files: [] },
|
|
79
|
+
analysisTimestamp: new Date().toISOString(),
|
|
80
|
+
};
|
|
81
|
+
const issues = identifyIssues(undefined, drift);
|
|
82
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
83
|
+
expect(renameIssue).toBeDefined();
|
|
84
|
+
expect(renameIssue?.description).toContain("2 API endpoint(s) renamed");
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
describe("generateRecommendation - endpoint rename handling", () => {
|
|
88
|
+
function generateRecommendation(testFile, driftScore, execution, issues, apiEndpoint, apiSchemaChanges) {
|
|
89
|
+
const healthScore = service["calculateHealthScore"](execution
|
|
90
|
+
? service["calculateExecutionScore"](execution).score
|
|
91
|
+
: undefined, driftScore);
|
|
92
|
+
return service["generateRecommendation"](testFile, healthScore, driftScore, execution, issues, apiEndpoint, apiSchemaChanges);
|
|
93
|
+
}
|
|
94
|
+
it("should return UPDATE action for endpoint renames regardless of drift score", () => {
|
|
95
|
+
const issues = [
|
|
96
|
+
{
|
|
97
|
+
type: "endpoints_renamed",
|
|
98
|
+
severity: "high",
|
|
99
|
+
description: "1 API endpoint(s) renamed",
|
|
100
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
const apiSchemaChanges = {
|
|
104
|
+
endpointsRemoved: [],
|
|
105
|
+
endpointsRenamed: [
|
|
106
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
107
|
+
],
|
|
108
|
+
endpointsModified: [],
|
|
109
|
+
authenticationChanged: false,
|
|
110
|
+
};
|
|
111
|
+
// Even with low drift score, renames should trigger UPDATE
|
|
112
|
+
const rec = generateRecommendation("products_smoke_test.py", 12, // low drift
|
|
113
|
+
undefined, issues, undefined, apiSchemaChanges);
|
|
114
|
+
expect(rec.action).toBe("UPDATE");
|
|
115
|
+
expect(rec.priority).toBe("HIGH");
|
|
116
|
+
expect(rec.rationale).toContain("renamed");
|
|
117
|
+
expect(rec.estimatedWork).toBe("SMALL");
|
|
118
|
+
});
|
|
119
|
+
it("should include renamedEndpoints in recommendation details", () => {
|
|
120
|
+
const issues = [
|
|
121
|
+
{
|
|
122
|
+
type: "endpoints_renamed",
|
|
123
|
+
severity: "high",
|
|
124
|
+
description: "1 API endpoint(s) renamed",
|
|
125
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
126
|
+
},
|
|
127
|
+
];
|
|
128
|
+
const apiSchemaChanges = {
|
|
129
|
+
endpointsRemoved: [],
|
|
130
|
+
endpointsRenamed: [
|
|
131
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
132
|
+
],
|
|
133
|
+
endpointsModified: [],
|
|
134
|
+
authenticationChanged: false,
|
|
135
|
+
};
|
|
136
|
+
const rec = generateRecommendation("products_smoke_test.py", 30, undefined, issues, undefined, apiSchemaChanges);
|
|
137
|
+
expect(rec.details?.endpointStatus).toBe("renamed");
|
|
138
|
+
expect(rec.details?.renamedEndpoints).toBeDefined();
|
|
139
|
+
expect(rec.details?.renamedEndpoints).toHaveLength(1);
|
|
140
|
+
expect(rec.details?.renamedEndpoints?.[0]).toEqual({
|
|
141
|
+
oldPath: "/api/v1/products",
|
|
142
|
+
newPath: "/api/v1/items",
|
|
143
|
+
method: "get",
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
it("should mention test failure in rationale when test is failing due to rename", () => {
|
|
147
|
+
const issues = [
|
|
148
|
+
{
|
|
149
|
+
type: "endpoints_renamed",
|
|
150
|
+
severity: "high",
|
|
151
|
+
description: "1 API endpoint(s) renamed",
|
|
152
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
153
|
+
},
|
|
154
|
+
];
|
|
155
|
+
const execution = {
|
|
156
|
+
testFile: "products_smoke_test.py",
|
|
157
|
+
executedAt: new Date().toISOString(),
|
|
158
|
+
passed: false,
|
|
159
|
+
duration: 10000,
|
|
160
|
+
errors: ["404 Not Found"],
|
|
161
|
+
warnings: [],
|
|
162
|
+
crashed: false,
|
|
163
|
+
};
|
|
164
|
+
const apiSchemaChanges = {
|
|
165
|
+
endpointsRemoved: [],
|
|
166
|
+
endpointsRenamed: [
|
|
167
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
168
|
+
],
|
|
169
|
+
endpointsModified: [],
|
|
170
|
+
authenticationChanged: false,
|
|
171
|
+
};
|
|
172
|
+
const rec = generateRecommendation("products_smoke_test.py", 30, execution, issues, undefined, apiSchemaChanges);
|
|
173
|
+
expect(rec.action).toBe("UPDATE");
|
|
174
|
+
expect(rec.rationale).toContain("failing");
|
|
175
|
+
});
|
|
176
|
+
it("should not set renamedEndpoints when there are no renames", () => {
|
|
177
|
+
const rec = generateRecommendation("orders_smoke_test.py", 5, undefined, [], { exists: true }, undefined);
|
|
178
|
+
expect(rec.action).toBe("VERIFY");
|
|
179
|
+
expect(rec.details?.renamedEndpoints).toBeUndefined();
|
|
180
|
+
expect(rec.details?.endpointStatus).toBe("exists");
|
|
181
|
+
});
|
|
182
|
+
it("should prefer rename handling over high-drift REGENERATE", () => {
|
|
183
|
+
// If drift is > 70 but it's caused by a rename, we should UPDATE not REGENERATE
|
|
184
|
+
const issues = [
|
|
185
|
+
{
|
|
186
|
+
type: "endpoints_renamed",
|
|
187
|
+
severity: "high",
|
|
188
|
+
description: "5 API endpoint(s) renamed",
|
|
189
|
+
details: "Multiple renames",
|
|
190
|
+
},
|
|
191
|
+
];
|
|
192
|
+
const apiSchemaChanges = {
|
|
193
|
+
endpointsRemoved: [],
|
|
194
|
+
endpointsRenamed: [
|
|
195
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
196
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "post" },
|
|
197
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "get" },
|
|
198
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "put" },
|
|
199
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "delete" },
|
|
200
|
+
],
|
|
201
|
+
endpointsModified: [],
|
|
202
|
+
authenticationChanged: false,
|
|
203
|
+
};
|
|
204
|
+
const rec = generateRecommendation("products_smoke_test.py", 75, // would normally trigger REGENERATE
|
|
205
|
+
undefined, issues, undefined, apiSchemaChanges);
|
|
206
|
+
// Rename detection should take priority over drift threshold
|
|
207
|
+
expect(rec.action).toBe("UPDATE");
|
|
208
|
+
expect(rec.estimatedWork).toBe("SMALL"); // renames are simple substitutions
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
});
|
|
@@ -4,6 +4,7 @@ import * as fs from "fs/promises";
|
|
|
4
4
|
import * as path from "path";
|
|
5
5
|
import { AnalyticsService } from "../services/AnalyticsService.js";
|
|
6
6
|
const TOOL_NAME = "skyramp_submit_report";
|
|
7
|
+
const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
|
|
7
8
|
const testResultSchema = z.object({
|
|
8
9
|
testType: z.string().describe("Type of test: Smoke, Contract, Integration, Fuzz, E2E, Load, etc."),
|
|
9
10
|
endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
|
|
@@ -41,6 +42,12 @@ export function registerSubmitReportTool(server) {
|
|
|
41
42
|
issuesFound: z
|
|
42
43
|
.array(descriptionSchema)
|
|
43
44
|
.describe("List of issues, failures, or bugs found. Use empty array [] if none."),
|
|
45
|
+
commitMessage: z
|
|
46
|
+
.string()
|
|
47
|
+
.optional()
|
|
48
|
+
.default(DEFAULT_COMMIT_MESSAGE)
|
|
49
|
+
.describe("Succinct commit message (under 72 chars) summarizing what the testbot did, " +
|
|
50
|
+
"e.g. 'add contract tests for /products endpoint' or 'update smoke tests for order API changes'"),
|
|
44
51
|
},
|
|
45
52
|
_meta: {
|
|
46
53
|
keywords: ["report", "summary", "testbot", "submit"],
|
|
@@ -54,6 +61,7 @@ export function registerSubmitReportTool(server) {
|
|
|
54
61
|
testMaintenance: params.testMaintenance,
|
|
55
62
|
testResults: params.testResults,
|
|
56
63
|
issuesFound: params.issuesFound,
|
|
64
|
+
commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim().slice(0, 72) || DEFAULT_COMMIT_MESSAGE,
|
|
57
65
|
}, null, 2);
|
|
58
66
|
logger.info("Submitting testbot report", {
|
|
59
67
|
outputFile: params.summaryOutputFile,
|
|
@@ -88,10 +96,11 @@ export function registerSubmitReportTool(server) {
|
|
|
88
96
|
return errorResult;
|
|
89
97
|
}
|
|
90
98
|
finally {
|
|
91
|
-
|
|
99
|
+
AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
|
|
92
100
|
summary_output_file: params.summaryOutputFile,
|
|
93
|
-
|
|
94
|
-
|
|
101
|
+
testResultCount: String(params.testResults.length),
|
|
102
|
+
payloadBytes: String(reportJson.length),
|
|
103
|
+
}).catch(() => { });
|
|
95
104
|
}
|
|
96
105
|
});
|
|
97
106
|
}
|
|
@@ -7,7 +7,7 @@ jest.mock("../utils/logger.js", () => ({
|
|
|
7
7
|
logger: { info: jest.fn(), error: jest.fn() },
|
|
8
8
|
}));
|
|
9
9
|
jest.mock("../services/AnalyticsService.js", () => ({
|
|
10
|
-
AnalyticsService: { pushMCPToolEvent: jest.fn() },
|
|
10
|
+
AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) },
|
|
11
11
|
}));
|
|
12
12
|
function captureToolHandler() {
|
|
13
13
|
let handler;
|
|
@@ -104,6 +104,64 @@ describe("registerSubmitReportTool", () => {
|
|
|
104
104
|
expect(result.isError).toBe(true);
|
|
105
105
|
expect(result.content[0].text).toContain("Failed to write report");
|
|
106
106
|
});
|
|
107
|
+
it("should write commitMessage when provided", async () => {
|
|
108
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
109
|
+
tmpDirs.push(tmpDir);
|
|
110
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
111
|
+
const result = await handler({
|
|
112
|
+
...sampleReportParams(outputFile),
|
|
113
|
+
commitMessage: "add contract tests for /products endpoint",
|
|
114
|
+
});
|
|
115
|
+
expect(result.isError).toBeUndefined();
|
|
116
|
+
const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
|
|
117
|
+
expect(written.commitMessage).toBe("add contract tests for /products endpoint");
|
|
118
|
+
});
|
|
119
|
+
it("should use default commitMessage when omitted", async () => {
|
|
120
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
121
|
+
tmpDirs.push(tmpDir);
|
|
122
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
123
|
+
const result = await handler(sampleReportParams(outputFile));
|
|
124
|
+
expect(result.isError).toBeUndefined();
|
|
125
|
+
const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
|
|
126
|
+
expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
|
|
127
|
+
});
|
|
128
|
+
it("should sanitize commitMessage (newlines, length)", async () => {
|
|
129
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
130
|
+
tmpDirs.push(tmpDir);
|
|
131
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
132
|
+
const result = await handler({
|
|
133
|
+
...sampleReportParams(outputFile),
|
|
134
|
+
commitMessage: " line one\nline two\r\nline three ",
|
|
135
|
+
});
|
|
136
|
+
expect(result.isError).toBeUndefined();
|
|
137
|
+
const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
|
|
138
|
+
expect(written.commitMessage).toBe("line one line two line three");
|
|
139
|
+
expect(written.commitMessage.length).toBeLessThanOrEqual(72);
|
|
140
|
+
});
|
|
141
|
+
it("should use default commitMessage when provided as empty string", async () => {
|
|
142
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
143
|
+
tmpDirs.push(tmpDir);
|
|
144
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
145
|
+
const result = await handler({
|
|
146
|
+
...sampleReportParams(outputFile),
|
|
147
|
+
commitMessage: "",
|
|
148
|
+
});
|
|
149
|
+
expect(result.isError).toBeUndefined();
|
|
150
|
+
const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
|
|
151
|
+
expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
|
|
152
|
+
});
|
|
153
|
+
it("should use default commitMessage when whitespace-only", async () => {
|
|
154
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
155
|
+
tmpDirs.push(tmpDir);
|
|
156
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
157
|
+
const result = await handler({
|
|
158
|
+
...sampleReportParams(outputFile),
|
|
159
|
+
commitMessage: " \n\r\n ",
|
|
160
|
+
});
|
|
161
|
+
expect(result.isError).toBeUndefined();
|
|
162
|
+
const written = JSON.parse(await fs.readFile(outputFile, "utf-8"));
|
|
163
|
+
expect(written.commitMessage).toBe("Added recommendations by Skyramp Testbot.");
|
|
164
|
+
});
|
|
107
165
|
it("should produce valid JSON with pretty formatting", async () => {
|
|
108
166
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
109
167
|
tmpDirs.push(tmpDir);
|
|
@@ -2,7 +2,60 @@ import { z } from "zod";
|
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
4
4
|
import * as fs from "fs";
|
|
5
|
+
import * as path from "path";
|
|
5
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
|
+
/**
|
|
8
|
+
* Compute a suggested new filename when an endpoint is renamed.
|
|
9
|
+
*
|
|
10
|
+
* Extracts the differing static segments between oldPath and newPath,
|
|
11
|
+
* then replaces occurrences in the filename.
|
|
12
|
+
*
|
|
13
|
+
* Example:
|
|
14
|
+
* testFile: "/repo/tests/python/products_smoke_test.py"
|
|
15
|
+
* oldPath: "/api/v1/products"
|
|
16
|
+
* newPath: "/api/v1/items"
|
|
17
|
+
* result: "/repo/tests/python/items_smoke_test.py"
|
|
18
|
+
*/
|
|
19
|
+
export function computeRenamedTestFile(testFile, renames) {
|
|
20
|
+
const basename = path.basename(testFile);
|
|
21
|
+
let newBasename = basename;
|
|
22
|
+
for (const rename of renames) {
|
|
23
|
+
const oldSegments = rename.oldPath.split("/").filter((s) => s.length > 0);
|
|
24
|
+
const newSegments = rename.newPath.split("/").filter((s) => s.length > 0);
|
|
25
|
+
if (oldSegments.length !== newSegments.length)
|
|
26
|
+
continue;
|
|
27
|
+
const paramPattern = /^\{[^}]+\}$/;
|
|
28
|
+
for (let i = 0; i < oldSegments.length; i++) {
|
|
29
|
+
if (paramPattern.test(oldSegments[i]))
|
|
30
|
+
continue;
|
|
31
|
+
if (oldSegments[i] !== newSegments[i]) {
|
|
32
|
+
// Replace the old segment name in the filename with the new one
|
|
33
|
+
// Handle both exact matches and common variations:
|
|
34
|
+
// "products" in "products_smoke_test.py"
|
|
35
|
+
// "product" in "product_smoke_test.py" (singular)
|
|
36
|
+
const oldName = oldSegments[i].toLowerCase();
|
|
37
|
+
const newName = newSegments[i].toLowerCase();
|
|
38
|
+
if (newBasename.toLowerCase().includes(oldName)) {
|
|
39
|
+
// Case-preserving replace
|
|
40
|
+
const idx = newBasename.toLowerCase().indexOf(oldName);
|
|
41
|
+
newBasename =
|
|
42
|
+
newBasename.substring(0, idx) +
|
|
43
|
+
newName +
|
|
44
|
+
newBasename.substring(idx + oldName.length);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (newBasename === basename)
|
|
50
|
+
return null; // No change needed
|
|
51
|
+
const newFilePath = path.join(path.dirname(testFile), newBasename);
|
|
52
|
+
// Don't suggest a rename if the target file already exists
|
|
53
|
+
if (fs.existsSync(newFilePath)) {
|
|
54
|
+
logger.info(`Skipping file rename suggestion: ${newFilePath} already exists`);
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
return newFilePath;
|
|
58
|
+
}
|
|
6
59
|
const actionsSchema = {
|
|
7
60
|
stateFile: z
|
|
8
61
|
.string()
|
|
@@ -70,6 +123,7 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
70
123
|
rationale: test.recommendation.rationale,
|
|
71
124
|
estimatedWork: test.recommendation.estimatedWork,
|
|
72
125
|
issues: test.issues || [],
|
|
126
|
+
renamedEndpoints: test.recommendation.details?.renamedEndpoints || [],
|
|
73
127
|
});
|
|
74
128
|
}
|
|
75
129
|
});
|
|
@@ -130,11 +184,33 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
130
184
|
logger.error(`Failed to read test file ${rec.testFile}: ${error.message}`);
|
|
131
185
|
continue;
|
|
132
186
|
}
|
|
187
|
+
// Check if this is a rename-driven update
|
|
188
|
+
const renames = rec.renamedEndpoints || [];
|
|
189
|
+
const isRenameUpdate = renames.length > 0;
|
|
133
190
|
// Build update instructions
|
|
134
191
|
let instruction = `\n### ${rec.testFile}\n\n`;
|
|
135
192
|
instruction += `**Priority:** ${rec.priority} | `;
|
|
136
193
|
instruction += `**Estimated Effort:** ${rec.estimatedWork || "Small"}\n\n`;
|
|
137
194
|
instruction += `**Why Update Needed:** ${rec.rationale}\n\n`;
|
|
195
|
+
if (isRenameUpdate) {
|
|
196
|
+
instruction += `**🔄 Endpoint Rename Detected — Path Substitution Required:**\n\n`;
|
|
197
|
+
instruction += `| Old Path | New Path | Method |\n`;
|
|
198
|
+
instruction += `|----------|----------|--------|\n`;
|
|
199
|
+
for (const rename of renames) {
|
|
200
|
+
instruction += `| \`${rename.oldPath}\` | \`${rename.newPath}\` | ${rename.method} |\n`;
|
|
201
|
+
}
|
|
202
|
+
instruction += `\n`;
|
|
203
|
+
instruction += `**Action:** Find-and-replace all occurrences of the old path with the new path in this test file. `;
|
|
204
|
+
instruction += `Do NOT change any test logic, assertions, or structure — only update the URL paths.\n\n`;
|
|
205
|
+
// Compute suggested file rename
|
|
206
|
+
const suggestedNewFile = computeRenamedTestFile(rec.testFile, renames);
|
|
207
|
+
if (suggestedNewFile) {
|
|
208
|
+
instruction += `**📁 File Rename:** After updating the paths, rename this file:\n`;
|
|
209
|
+
instruction += `- From: \`${path.basename(rec.testFile)}\`\n`;
|
|
210
|
+
instruction += `- To: \`${path.basename(suggestedNewFile)}\`\n\n`;
|
|
211
|
+
rec._suggestedNewFile = suggestedNewFile;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
138
214
|
if (driftData) {
|
|
139
215
|
instruction += `**Analysis:**\n`;
|
|
140
216
|
instruction += `- Drift Score: ${driftData.driftScore ?? "N/A"}\n`;
|
|
@@ -184,16 +260,46 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
184
260
|
responseText += `4. Show you the changes made\n\n`;
|
|
185
261
|
responseText += `5. At the end of the tool execution, MUST display the below message\n`;
|
|
186
262
|
responseText += `**This tool is currently in Early Preview stage. Please verify the results.**\n\n`;
|
|
263
|
+
// Collect all rename mappings across recommendations
|
|
264
|
+
const allRenames = [];
|
|
265
|
+
for (const rec of updateRecommendations) {
|
|
266
|
+
if (rec.renamedEndpoints && rec.renamedEndpoints.length > 0) {
|
|
267
|
+
allRenames.push(...rec.renamedEndpoints);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// Deduplicate renames
|
|
271
|
+
const uniqueRenames = allRenames.filter((r, i, arr) => arr.findIndex((x) => x.oldPath === r.oldPath &&
|
|
272
|
+
x.newPath === r.newPath &&
|
|
273
|
+
x.method === r.method) === i);
|
|
187
274
|
// Build LLM-only instructions (hidden from users)
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
275
|
+
const llmInstructionsObj = {
|
|
276
|
+
workflow: "test_maintenance",
|
|
277
|
+
action: "execute_updates",
|
|
278
|
+
auto_proceed: true,
|
|
279
|
+
files_to_update: testFilesToUpdate,
|
|
280
|
+
update_count: updateRecommendations.length,
|
|
281
|
+
};
|
|
282
|
+
if (uniqueRenames.length > 0) {
|
|
283
|
+
llmInstructionsObj.endpoint_renames = uniqueRenames;
|
|
284
|
+
llmInstructionsObj.rename_strategy =
|
|
285
|
+
"For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.";
|
|
286
|
+
// Collect file rename suggestions
|
|
287
|
+
const fileRenames = [];
|
|
288
|
+
for (const rec of updateRecommendations) {
|
|
289
|
+
if (rec._suggestedNewFile) {
|
|
290
|
+
fileRenames.push({
|
|
291
|
+
from: rec.testFile,
|
|
292
|
+
to: rec._suggestedNewFile,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (fileRenames.length > 0) {
|
|
297
|
+
llmInstructionsObj.file_renames = fileRenames;
|
|
298
|
+
llmInstructionsObj.file_rename_strategy =
|
|
299
|
+
"After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.";
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
const llmInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(llmInstructionsObj, null, 2)}\n-->\n`;
|
|
197
303
|
return {
|
|
198
304
|
content: [
|
|
199
305
|
{
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Mock modules that use ESM-only features (import.meta) before importing actionsTool
|
|
2
|
+
jest.mock("../../services/AnalyticsService.js", () => ({
|
|
3
|
+
AnalyticsService: { pushMCPToolEvent: jest.fn() },
|
|
4
|
+
}));
|
|
5
|
+
jest.mock("../../utils/logger.js", () => ({
|
|
6
|
+
logger: { info: jest.fn(), warning: jest.fn(), error: jest.fn(), debug: jest.fn() },
|
|
7
|
+
}));
|
|
8
|
+
jest.mock("../../utils/AnalysisStateManager.js", () => ({
|
|
9
|
+
StateManager: { fromStatePath: jest.fn() },
|
|
10
|
+
}));
|
|
11
|
+
jest.mock("fs");
|
|
12
|
+
// @ts-ignore
|
|
13
|
+
import { computeRenamedTestFile } from "./actionsTool.js";
|
|
14
|
+
import * as fs from "fs";
|
|
15
|
+
const mockExistsSync = fs.existsSync;
|
|
16
|
+
describe("computeRenamedTestFile", () => {
|
|
17
|
+
beforeEach(() => {
|
|
18
|
+
// Default: target file does not exist
|
|
19
|
+
mockExistsSync.mockReturnValue(false);
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
jest.restoreAllMocks();
|
|
23
|
+
});
|
|
24
|
+
// --- Basic renames ---
|
|
25
|
+
it("should rename products_smoke_test.py to items_smoke_test.py", () => {
|
|
26
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
27
|
+
expect(result).toBe("/repo/tests/python/items_smoke_test.py");
|
|
28
|
+
});
|
|
29
|
+
it("should rename products_contract_test.py to items_contract_test.py", () => {
|
|
30
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_contract_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
31
|
+
expect(result).toBe("/repo/tests/python/items_contract_test.py");
|
|
32
|
+
});
|
|
33
|
+
it("should rename products_integration_test.py to items_integration_test.py", () => {
|
|
34
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_integration_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "post" }]);
|
|
35
|
+
expect(result).toBe("/repo/tests/python/items_integration_test.py");
|
|
36
|
+
});
|
|
37
|
+
it("should rename products_fuzz_test.py to items_fuzz_test.py", () => {
|
|
38
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_fuzz_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
39
|
+
expect(result).toBe("/repo/tests/python/items_fuzz_test.py");
|
|
40
|
+
});
|
|
41
|
+
it("should rename products_load_test.py to items_load_test.py", () => {
|
|
42
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_load_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
43
|
+
expect(result).toBe("/repo/tests/python/items_load_test.py");
|
|
44
|
+
});
|
|
45
|
+
// --- Different file extensions ---
|
|
46
|
+
it("should work with .ts test files", () => {
|
|
47
|
+
const result = computeRenamedTestFile("/repo/tests/products.test.ts", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
48
|
+
expect(result).toBe("/repo/tests/items.test.ts");
|
|
49
|
+
});
|
|
50
|
+
it("should work with .js test files", () => {
|
|
51
|
+
const result = computeRenamedTestFile("/repo/tests/products_smoke.test.js", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
52
|
+
expect(result).toBe("/repo/tests/items_smoke.test.js");
|
|
53
|
+
});
|
|
54
|
+
// --- Returns null when no rename needed ---
|
|
55
|
+
it("should return null when filename does not contain old segment", () => {
|
|
56
|
+
const result = computeRenamedTestFile("/repo/tests/python/orders_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
57
|
+
expect(result).toBeNull();
|
|
58
|
+
});
|
|
59
|
+
it("should return null when target file already exists", () => {
|
|
60
|
+
mockExistsSync.mockReturnValue(true);
|
|
61
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
62
|
+
expect(result).toBeNull();
|
|
63
|
+
});
|
|
64
|
+
it("should return null when segments have different lengths", () => {
|
|
65
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v2/catalog/items", method: "get" }]);
|
|
66
|
+
// Different segment counts — no substitution attempted
|
|
67
|
+
expect(result).toBeNull();
|
|
68
|
+
});
|
|
69
|
+
it("should return null with empty renames array", () => {
|
|
70
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", []);
|
|
71
|
+
expect(result).toBeNull();
|
|
72
|
+
});
|
|
73
|
+
// --- Multiple renames ---
|
|
74
|
+
it("should apply multiple rename mappings", () => {
|
|
75
|
+
// Unlikely but possible: two segments change
|
|
76
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [
|
|
77
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
78
|
+
{ oldPath: "/api/v1/products/{product_id}", newPath: "/api/v1/items/{item_id}", method: "get" },
|
|
79
|
+
]);
|
|
80
|
+
expect(result).toBe("/repo/tests/python/items_smoke_test.py");
|
|
81
|
+
});
|
|
82
|
+
// --- Preserves directory structure ---
|
|
83
|
+
it("should preserve the directory path", () => {
|
|
84
|
+
const result = computeRenamedTestFile("/home/runner/work/api-insight/api-insight/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
85
|
+
expect(result).toBe("/home/runner/work/api-insight/api-insight/tests/python/items_smoke_test.py");
|
|
86
|
+
});
|
|
87
|
+
// --- Version bump rename ---
|
|
88
|
+
it("should handle version segment rename in filename if present", () => {
|
|
89
|
+
const result = computeRenamedTestFile("/repo/tests/v1_products_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v2/products", method: "get" }]);
|
|
90
|
+
// "v1" in filename gets replaced with "v2"
|
|
91
|
+
expect(result).toBe("/repo/tests/v2_products_test.py");
|
|
92
|
+
});
|
|
93
|
+
});
|