@skyramp/mcp 0.0.57 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/testbot/testbot-prompts.js +16 -8
- package/build/services/DriftAnalysisService.js +139 -13
- package/build/services/DriftAnalysisService.test.js +168 -0
- package/build/services/TestExecutionService.js +1 -1
- package/build/services/TestHealthService.js +38 -3
- package/build/services/TestHealthService.test.js +211 -0
- package/build/tools/test-maintenance/actionsTool.js +115 -9
- package/build/tools/test-maintenance/actionsTool.test.js +93 -0
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +24 -15
- package/package.json +2 -2
|
@@ -2,7 +2,7 @@ import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
4
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
5
|
-
function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath) {
|
|
5
|
+
function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath, baseBranch) {
|
|
6
6
|
return `<TITLE>${prTitle}</TITLE>
|
|
7
7
|
<DESCRIPTION>${prDescription}</DESCRIPTION>
|
|
8
8
|
<CODE CHANGES>${diffFile}</CODE CHANGES>
|
|
@@ -24,12 +24,14 @@ Read the diff at \`${diffFile}\`. Classify each changed file. A file is applicat
|
|
|
24
24
|
|
|
25
25
|
1. Call \`skyramp_analyze_repository\` with:
|
|
26
26
|
- \`repositoryPath\`: "${repositoryPath}"
|
|
27
|
-
- \`analysisScope\`: "current_branch_diff"
|
|
27
|
+
- \`analysisScope\`: "current_branch_diff"${baseBranch ? `\n - \`baseBranch\`: "${baseBranch}"` : ''}
|
|
28
28
|
2. MANDATORY: Call \`skyramp_map_tests\` with \`stateFile\` (the state file path returned above) and \`analysisScope: "current_branch_diff"\`.
|
|
29
29
|
3. MANDATORY: Call \`skyramp_recommend_tests\` with the \`stateFile\` returned by \`skyramp_map_tests\`. Use the priority summary and the specific endpoints/files that changed to determine exactly what to test.
|
|
30
30
|
4. Generate tests using the Skyramp MCP generate tools, in priority order (minimum 3 test types).
|
|
31
31
|
5. Use Skyramp MCP to execute the generated tests and validate the results.
|
|
32
32
|
|
|
33
|
+
**IMPORTANT — Endpoint Renames:** If the diff shows an endpoint path was renamed (e.g. \`/products\` changed to \`/items\`) and existing tests already cover that endpoint under the old name, do NOT generate new tests for the renamed endpoint. The existing tests will be updated with the new path in Task 2 (Test Maintenance). Only generate new tests for genuinely new endpoints that have no existing test coverage under any name.
|
|
34
|
+
|
|
33
35
|
## Task 2: Existing Test Maintenance (MANDATORY)
|
|
34
36
|
|
|
35
37
|
You MUST always run steps 1–4 below. Do NOT skip this task based on your own assessment of whether tests exist or are relevant — use the tools to determine that.
|
|
@@ -38,6 +40,8 @@ You MUST always run steps 1–4 below. Do NOT skip this task based on your own a
|
|
|
38
40
|
2. Call \`skyramp_analyze_test_drift\` with the \`stateFile\` returned by \`skyramp_discover_tests\`.
|
|
39
41
|
3. Call \`skyramp_calculate_health_scores\` with the \`stateFile\` from the previous step.
|
|
40
42
|
4. Call \`skyramp_actions\` with the updated \`stateFile\` to apply recommended updates.
|
|
43
|
+
- If \`skyramp_actions\` returns endpoint rename mappings (old path → new path), apply them as simple find-and-replace on the test file URLs. Do NOT regenerate or restructure the test — only update the paths.
|
|
44
|
+
- If \`skyramp_actions\` suggests file renames (e.g. \`products_smoke_test.py\` → \`items_smoke_test.py\`), rename the files using \`git mv\` after updating their content.
|
|
41
45
|
5. Execute any updated or affected tests using Skyramp MCP and validate the results.
|
|
42
46
|
6. You may skip this task ONLY if \`skyramp_discover_tests\` explicitly returns zero Skyramp-generated tests.
|
|
43
47
|
|
|
@@ -70,9 +74,7 @@ export function registerTestbotPrompt(server) {
|
|
|
70
74
|
description: "Run Skyramp TestBot to generate test recommendations and perform test maintenance for a pull request.",
|
|
71
75
|
argsSchema: {
|
|
72
76
|
prTitle: z.string().describe("Pull request title"),
|
|
73
|
-
prDescription: z
|
|
74
|
-
.string()
|
|
75
|
-
.describe("Pull request description/body"),
|
|
77
|
+
prDescription: z.string().describe("Pull request description/body"),
|
|
76
78
|
diffFile: z.string().describe("Path to the git diff file"),
|
|
77
79
|
testDirectory: z
|
|
78
80
|
.string()
|
|
@@ -85,9 +87,13 @@ export function registerTestbotPrompt(server) {
|
|
|
85
87
|
.string()
|
|
86
88
|
.default(".")
|
|
87
89
|
.describe("Absolute path to the repository being analyzed"),
|
|
90
|
+
baseBranch: z
|
|
91
|
+
.string()
|
|
92
|
+
.optional()
|
|
93
|
+
.describe("PR base branch name (e.g. 'main' or 'develop'). When provided, analyzeRepository diffs against this branch instead of auto-detecting."),
|
|
88
94
|
},
|
|
89
95
|
}, (args) => {
|
|
90
|
-
const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath);
|
|
96
|
+
const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath, args.baseBranch);
|
|
91
97
|
AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
|
|
92
98
|
return {
|
|
93
99
|
messages: [
|
|
@@ -109,14 +115,16 @@ export function registerTestbotResource(server) {
|
|
|
109
115
|
// fails on empty query-param values (e.g. prDescription=).
|
|
110
116
|
// We then parse query params from the URL object which handles URL-decoding
|
|
111
117
|
// and empty values correctly.
|
|
112
|
-
const template = new ResourceTemplate("skyramp://prompts/testbot{+rest}", {
|
|
118
|
+
const template = new ResourceTemplate("skyramp://prompts/testbot{+rest}", {
|
|
119
|
+
list: undefined,
|
|
120
|
+
});
|
|
113
121
|
server.registerResource("skyramp_testbot", template, {
|
|
114
122
|
title: "Skyramp TestBot Prompt",
|
|
115
123
|
description: "Returns task instructions for PR test analysis, generation, and maintenance.",
|
|
116
124
|
mimeType: "text/plain",
|
|
117
125
|
}, (uri) => {
|
|
118
126
|
const param = (name, fallback) => uri.searchParams.get(name) ?? fallback;
|
|
119
|
-
const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."));
|
|
127
|
+
const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."), uri.searchParams.get("baseBranch") || undefined);
|
|
120
128
|
AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
|
|
121
129
|
return {
|
|
122
130
|
contents: [
|
|
@@ -394,29 +394,71 @@ export class EnhancedDriftAnalysisService {
|
|
|
394
394
|
const newParsed = JSON.parse(newSchema);
|
|
395
395
|
const changes = {
|
|
396
396
|
endpointsRemoved: [],
|
|
397
|
+
endpointsRenamed: [],
|
|
397
398
|
endpointsModified: [],
|
|
398
399
|
authenticationChanged: false,
|
|
399
400
|
};
|
|
400
401
|
const oldPaths = oldParsed.paths || {};
|
|
401
402
|
const newPaths = newParsed.paths || {};
|
|
402
|
-
//
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
403
|
+
// Collect removed endpoints (old path not in new schema)
|
|
404
|
+
const removedEndpoints = [];
|
|
405
|
+
for (const pathStr in oldPaths) {
|
|
406
|
+
if (!newPaths[pathStr]) {
|
|
407
|
+
for (const method in oldPaths[pathStr]) {
|
|
408
|
+
removedEndpoints.push({ path: pathStr, method });
|
|
407
409
|
}
|
|
408
410
|
}
|
|
409
411
|
}
|
|
412
|
+
// Collect added endpoints (new path not in old schema)
|
|
413
|
+
const addedEndpoints = [];
|
|
414
|
+
for (const pathStr in newPaths) {
|
|
415
|
+
if (!oldPaths[pathStr]) {
|
|
416
|
+
for (const method in newPaths[pathStr]) {
|
|
417
|
+
addedEndpoints.push({ path: pathStr, method });
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
// Detect renames: match removed endpoints to added endpoints
|
|
422
|
+
const matchedRemoved = new Set();
|
|
423
|
+
const matchedAdded = new Set();
|
|
424
|
+
for (const removed of removedEndpoints) {
|
|
425
|
+
const removedKey = `${removed.path}::${removed.method}`;
|
|
426
|
+
if (matchedRemoved.has(removedKey))
|
|
427
|
+
continue;
|
|
428
|
+
for (const added of addedEndpoints) {
|
|
429
|
+
const addedKey = `${added.path}::${added.method}`;
|
|
430
|
+
if (matchedAdded.has(addedKey))
|
|
431
|
+
continue;
|
|
432
|
+
if (this.isEndpointRename(removed.path, added.path, removed.method, added.method, oldPaths, newPaths)) {
|
|
433
|
+
changes.endpointsRenamed.push({
|
|
434
|
+
oldPath: removed.path,
|
|
435
|
+
newPath: added.path,
|
|
436
|
+
method: removed.method,
|
|
437
|
+
});
|
|
438
|
+
matchedRemoved.add(removedKey);
|
|
439
|
+
matchedAdded.add(addedKey);
|
|
440
|
+
logger.info(`Detected endpoint rename: ${removed.method} ${removed.path} -> ${added.path}`);
|
|
441
|
+
break;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
// Remaining unmatched removals are true removals
|
|
446
|
+
for (const removed of removedEndpoints) {
|
|
447
|
+
const removedKey = `${removed.path}::${removed.method}`;
|
|
448
|
+
if (!matchedRemoved.has(removedKey)) {
|
|
449
|
+
changes.endpointsRemoved.push(removed);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
410
452
|
// Find modified endpoints and removed methods from existing paths
|
|
411
|
-
for (const
|
|
412
|
-
if (newPaths[
|
|
413
|
-
for (const method in oldPaths[
|
|
414
|
-
if (newPaths[
|
|
415
|
-
const oldEndpoint = JSON.stringify(oldPaths[
|
|
416
|
-
const newEndpoint = JSON.stringify(newPaths[
|
|
453
|
+
for (const pathStr in oldPaths) {
|
|
454
|
+
if (newPaths[pathStr]) {
|
|
455
|
+
for (const method in oldPaths[pathStr]) {
|
|
456
|
+
if (newPaths[pathStr][method]) {
|
|
457
|
+
const oldEndpoint = JSON.stringify(oldPaths[pathStr][method]);
|
|
458
|
+
const newEndpoint = JSON.stringify(newPaths[pathStr][method]);
|
|
417
459
|
if (oldEndpoint !== newEndpoint) {
|
|
418
460
|
changes.endpointsModified.push({
|
|
419
|
-
path,
|
|
461
|
+
path: pathStr,
|
|
420
462
|
method,
|
|
421
463
|
changes: ["Parameters or response modified"],
|
|
422
464
|
});
|
|
@@ -424,7 +466,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
424
466
|
}
|
|
425
467
|
else {
|
|
426
468
|
// Method exists in old schema but not in new schema
|
|
427
|
-
changes.endpointsRemoved.push({ path, method });
|
|
469
|
+
changes.endpointsRemoved.push({ path: pathStr, method });
|
|
428
470
|
}
|
|
429
471
|
}
|
|
430
472
|
}
|
|
@@ -448,6 +490,73 @@ export class EnhancedDriftAnalysisService {
|
|
|
448
490
|
return undefined;
|
|
449
491
|
}
|
|
450
492
|
}
|
|
493
|
+
/**
|
|
494
|
+
* Determine if a removed endpoint and an added endpoint represent a rename.
|
|
495
|
+
*
|
|
496
|
+
* Heuristics:
|
|
497
|
+
* 1. Must have the same HTTP method
|
|
498
|
+
* 2. Must have the same path structure (same number of segments, same param names)
|
|
499
|
+
* 3. The operations must be structurally similar (same response codes, similar params)
|
|
500
|
+
*/
|
|
501
|
+
isEndpointRename(oldPath, newPath, oldMethod, newMethod, oldPaths, newPaths) {
|
|
502
|
+
// Must be the same HTTP method
|
|
503
|
+
if (oldMethod !== newMethod)
|
|
504
|
+
return false;
|
|
505
|
+
const oldSegments = oldPath.split("/").filter((s) => s.length > 0);
|
|
506
|
+
const newSegments = newPath.split("/").filter((s) => s.length > 0);
|
|
507
|
+
// Must have same number of path segments
|
|
508
|
+
if (oldSegments.length !== newSegments.length)
|
|
509
|
+
return false;
|
|
510
|
+
// Path parameters (e.g., {product_id}) must be in the same positions
|
|
511
|
+
const paramPattern = /^\{[^}]+\}$/;
|
|
512
|
+
let staticDiffs = 0;
|
|
513
|
+
for (let i = 0; i < oldSegments.length; i++) {
|
|
514
|
+
const oldIsParam = paramPattern.test(oldSegments[i]);
|
|
515
|
+
const newIsParam = paramPattern.test(newSegments[i]);
|
|
516
|
+
if (oldIsParam !== newIsParam)
|
|
517
|
+
return false; // Structural mismatch
|
|
518
|
+
if (oldIsParam && newIsParam) {
|
|
519
|
+
// Both are params — param names may differ but structure matches
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
522
|
+
if (oldSegments[i] !== newSegments[i]) {
|
|
523
|
+
staticDiffs++;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
// At least one static segment must differ (otherwise paths are identical)
|
|
527
|
+
// But not too many — more than half differing suggests unrelated endpoints
|
|
528
|
+
if (staticDiffs === 0)
|
|
529
|
+
return false;
|
|
530
|
+
const staticSegments = oldSegments.filter((s) => !paramPattern.test(s));
|
|
531
|
+
if (staticDiffs > Math.max(1, Math.ceil(staticSegments.length / 2))) {
|
|
532
|
+
return false;
|
|
533
|
+
}
|
|
534
|
+
// Compare operation structure: same response codes is a strong signal
|
|
535
|
+
const oldOp = oldPaths[oldPath]?.[oldMethod];
|
|
536
|
+
const newOp = newPaths[newPath]?.[newMethod];
|
|
537
|
+
if (oldOp && newOp) {
|
|
538
|
+
const oldResponses = Object.keys(oldOp.responses || {}).sort();
|
|
539
|
+
const newResponses = Object.keys(newOp.responses || {}).sort();
|
|
540
|
+
if (oldResponses.length > 0 &&
|
|
541
|
+
newResponses.length > 0 &&
|
|
542
|
+
JSON.stringify(oldResponses) === JSON.stringify(newResponses)) {
|
|
543
|
+
return true; // Same method, similar structure, same response codes
|
|
544
|
+
}
|
|
545
|
+
// Fallback: if response codes differ, check parameter count similarity
|
|
546
|
+
const oldParamCount = (oldOp.parameters || []).length;
|
|
547
|
+
const newParamCount = (newOp.parameters || []).length;
|
|
548
|
+
const hasOldBody = !!oldOp.requestBody;
|
|
549
|
+
const hasNewBody = !!newOp.requestBody;
|
|
550
|
+
if (oldParamCount === newParamCount && hasOldBody === hasNewBody) {
|
|
551
|
+
return true;
|
|
552
|
+
}
|
|
553
|
+
// Operation data exists but doesn't match — not a rename
|
|
554
|
+
return false;
|
|
555
|
+
}
|
|
556
|
+
// If we can't access operations, rely on structural match alone
|
|
557
|
+
// (same segments, same params, only 1 static segment differs)
|
|
558
|
+
return staticDiffs === 1;
|
|
559
|
+
}
|
|
451
560
|
/**
|
|
452
561
|
* Extract API schema path from test file comments/metadata
|
|
453
562
|
*/
|
|
@@ -635,6 +744,17 @@ export class EnhancedDriftAnalysisService {
|
|
|
635
744
|
severity: "high",
|
|
636
745
|
});
|
|
637
746
|
}
|
|
747
|
+
if (apiSchemaChanges.endpointsRenamed.length > 0) {
|
|
748
|
+
for (const renamed of apiSchemaChanges.endpointsRenamed) {
|
|
749
|
+
changes.push({
|
|
750
|
+
type: "endpoint_renamed",
|
|
751
|
+
file: "API Schema",
|
|
752
|
+
description: `Endpoint renamed: ${renamed.method} ${renamed.oldPath} -> ${renamed.newPath}`,
|
|
753
|
+
severity: "high",
|
|
754
|
+
details: `Path changed from ${renamed.oldPath} to ${renamed.newPath}. Test endpoint URLs must be updated.`,
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
}
|
|
638
758
|
if (apiSchemaChanges.endpointsModified.length > 0) {
|
|
639
759
|
changes.push({
|
|
640
760
|
type: "endpoint_modified",
|
|
@@ -828,6 +948,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
828
948
|
// API schema changes
|
|
829
949
|
if (apiSchemaChanges) {
|
|
830
950
|
score += apiSchemaChanges.endpointsRemoved.length * 15;
|
|
951
|
+
score += apiSchemaChanges.endpointsRenamed.length * 12;
|
|
831
952
|
score += apiSchemaChanges.endpointsModified.length * 10;
|
|
832
953
|
if (apiSchemaChanges.authenticationChanged)
|
|
833
954
|
score += 25;
|
|
@@ -870,6 +991,11 @@ export class EnhancedDriftAnalysisService {
|
|
|
870
991
|
}
|
|
871
992
|
// Specific recommendations
|
|
872
993
|
if (apiSchemaChanges) {
|
|
994
|
+
if (apiSchemaChanges.endpointsRenamed.length > 0) {
|
|
995
|
+
for (const renamed of apiSchemaChanges.endpointsRenamed) {
|
|
996
|
+
recommendations.push(`🔄 Endpoint renamed: ${renamed.method} ${renamed.oldPath} -> ${renamed.newPath} — update test URL paths`);
|
|
997
|
+
}
|
|
998
|
+
}
|
|
873
999
|
if (apiSchemaChanges.endpointsRemoved.length > 0) {
|
|
874
1000
|
recommendations.push(`⚠️ ${apiSchemaChanges.endpointsRemoved.length} API endpoint(s) removed - update test`);
|
|
875
1001
|
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { EnhancedDriftAnalysisService } from "./DriftAnalysisService.js";
|
|
2
|
+
describe("DriftAnalysisService", () => {
|
|
3
|
+
let service;
|
|
4
|
+
beforeEach(() => {
|
|
5
|
+
service = new EnhancedDriftAnalysisService();
|
|
6
|
+
});
|
|
7
|
+
describe("isEndpointRename", () => {
|
|
8
|
+
// Helper to call the private method
|
|
9
|
+
function isRename(oldPath, newPath, oldMethod, newMethod, oldPaths = {}, newPaths = {}) {
|
|
10
|
+
return service["isEndpointRename"](oldPath, newPath, oldMethod, newMethod, oldPaths, newPaths);
|
|
11
|
+
}
|
|
12
|
+
// --- Basic rename detection ---
|
|
13
|
+
it("should detect a simple prefix rename", () => {
|
|
14
|
+
const oldPaths = {
|
|
15
|
+
"/api/v1/products": {
|
|
16
|
+
get: { responses: { "200": {}, "404": {} } },
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
const newPaths = {
|
|
20
|
+
"/api/v1/items": {
|
|
21
|
+
get: { responses: { "200": {}, "404": {} } },
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
it("should detect rename with path parameters", () => {
|
|
27
|
+
const oldPaths = {
|
|
28
|
+
"/api/v1/products/{product_id}": {
|
|
29
|
+
get: { responses: { "200": {}, "404": {} } },
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
const newPaths = {
|
|
33
|
+
"/api/v1/items/{product_id}": {
|
|
34
|
+
get: { responses: { "200": {}, "404": {} } },
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
expect(isRename("/api/v1/products/{product_id}", "/api/v1/items/{product_id}", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
38
|
+
});
|
|
39
|
+
it("should detect version bump as rename", () => {
|
|
40
|
+
const oldPaths = {
|
|
41
|
+
"/api/v1/products": {
|
|
42
|
+
get: { responses: { "200": {} } },
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
const newPaths = {
|
|
46
|
+
"/api/v2/products": {
|
|
47
|
+
get: { responses: { "200": {} } },
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
expect(isRename("/api/v1/products", "/api/v2/products", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
it("should detect rename across multiple HTTP methods independently", () => {
|
|
53
|
+
const oldPaths = {
|
|
54
|
+
"/api/v1/products": {
|
|
55
|
+
post: { responses: { "201": {} }, requestBody: {} },
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
const newPaths = {
|
|
59
|
+
"/api/v1/items": {
|
|
60
|
+
post: { responses: { "201": {} }, requestBody: {} },
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "post", "post", oldPaths, newPaths)).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
// --- Should NOT match ---
|
|
66
|
+
it("should not match different HTTP methods", () => {
|
|
67
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "get", "post")).toBe(false);
|
|
68
|
+
});
|
|
69
|
+
it("should not match paths with different segment counts", () => {
|
|
70
|
+
expect(isRename("/api/v1/products", "/api/v1/items/catalog", "get", "get")).toBe(false);
|
|
71
|
+
});
|
|
72
|
+
it("should not match identical paths", () => {
|
|
73
|
+
expect(isRename("/api/v1/products", "/api/v1/products", "get", "get")).toBe(false);
|
|
74
|
+
});
|
|
75
|
+
it("should not match when a static segment becomes a parameter", () => {
|
|
76
|
+
expect(isRename("/api/v1/products", "/api/v1/{resource}", "get", "get")).toBe(false);
|
|
77
|
+
});
|
|
78
|
+
it("should not match paths where too many segments differ", () => {
|
|
79
|
+
// 3 out of 3 static segments differ — clearly unrelated endpoints
|
|
80
|
+
const oldPaths = {
|
|
81
|
+
"/api/v1/products": {
|
|
82
|
+
get: { responses: { "200": {} } },
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
const newPaths = {
|
|
86
|
+
"/rest/v2/orders": {
|
|
87
|
+
get: { responses: { "200": {} } },
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
expect(isRename("/api/v1/products", "/rest/v2/orders", "get", "get", oldPaths, newPaths)).toBe(false);
|
|
91
|
+
});
|
|
92
|
+
it("should not match when response codes differ and params differ", () => {
|
|
93
|
+
const oldPaths = {
|
|
94
|
+
"/api/v1/products": {
|
|
95
|
+
get: { responses: { "200": {} }, parameters: [{ name: "limit" }] },
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
const newPaths = {
|
|
99
|
+
"/api/v1/items": {
|
|
100
|
+
get: { responses: { "201": {} } },
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "get", "get", oldPaths, newPaths)).toBe(false);
|
|
104
|
+
});
|
|
105
|
+
// --- Structural fallback (no operation data) ---
|
|
106
|
+
it("should match with single static segment diff when no operation data", () => {
|
|
107
|
+
// Only 1 segment differs, no operation data — should match on structure alone
|
|
108
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "get", "get", {}, {})).toBe(true);
|
|
109
|
+
});
|
|
110
|
+
it("should not match with 2+ static segment diffs when no operation data", () => {
|
|
111
|
+
expect(isRename("/api/v1/products", "/api/v2/items", "get", "get", {}, {})).toBe(false);
|
|
112
|
+
});
|
|
113
|
+
// --- Edge cases ---
|
|
114
|
+
it("should handle root-level path rename", () => {
|
|
115
|
+
const oldPaths = { "/products": { get: { responses: { "200": {} } } } };
|
|
116
|
+
const newPaths = { "/items": { get: { responses: { "200": {} } } } };
|
|
117
|
+
expect(isRename("/products", "/items", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
118
|
+
});
|
|
119
|
+
it("should handle deeply nested paths", () => {
|
|
120
|
+
const oldPaths = {
|
|
121
|
+
"/api/v1/store/products/{id}/reviews": {
|
|
122
|
+
get: { responses: { "200": {} } },
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
const newPaths = {
|
|
126
|
+
"/api/v1/store/items/{id}/reviews": {
|
|
127
|
+
get: { responses: { "200": {} } },
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
expect(isRename("/api/v1/store/products/{id}/reviews", "/api/v1/store/items/{id}/reviews", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
131
|
+
});
|
|
132
|
+
it("should match when param names differ but positions match", () => {
|
|
133
|
+
const oldPaths = {
|
|
134
|
+
"/api/v1/products/{product_id}": {
|
|
135
|
+
get: { responses: { "200": {} } },
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
const newPaths = {
|
|
139
|
+
"/api/v1/items/{item_id}": {
|
|
140
|
+
get: { responses: { "200": {} } },
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
expect(isRename("/api/v1/products/{product_id}", "/api/v1/items/{item_id}", "get", "get", oldPaths, newPaths)).toBe(true);
|
|
144
|
+
});
|
|
145
|
+
it("should match based on parameter count and request body similarity", () => {
|
|
146
|
+
const oldPaths = {
|
|
147
|
+
"/api/v1/products": {
|
|
148
|
+
post: {
|
|
149
|
+
responses: { "201": {} },
|
|
150
|
+
parameters: [{ name: "x" }],
|
|
151
|
+
requestBody: { content: {} },
|
|
152
|
+
},
|
|
153
|
+
},
|
|
154
|
+
};
|
|
155
|
+
const newPaths = {
|
|
156
|
+
"/api/v1/items": {
|
|
157
|
+
post: {
|
|
158
|
+
responses: { "200": {} }, // Different response code
|
|
159
|
+
parameters: [{ name: "y" }],
|
|
160
|
+
requestBody: { content: {} },
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
// Response codes differ, but param count and body presence match
|
|
165
|
+
expect(isRename("/api/v1/products", "/api/v1/items", "post", "post", oldPaths, newPaths)).toBe(true);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
});
|
|
@@ -6,7 +6,7 @@ import { stripVTControlCharacters } from "util";
|
|
|
6
6
|
import { logger } from "../utils/logger.js";
|
|
7
7
|
const DEFAULT_TIMEOUT = 300000; // 5 minutes
|
|
8
8
|
const MAX_CONCURRENT_EXECUTIONS = 5;
|
|
9
|
-
export const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.
|
|
9
|
+
export const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.11";
|
|
10
10
|
const DOCKER_PLATFORM = "linux/amd64";
|
|
11
11
|
const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
|
|
12
12
|
// Files and directories to exclude when mounting workspace to Docker container
|
|
@@ -68,7 +68,7 @@ export class TestHealthService {
|
|
|
68
68
|
? await this.extractEndpointFromTest(testFile, apiSchema)
|
|
69
69
|
: undefined;
|
|
70
70
|
// Generate recommendation
|
|
71
|
-
const recommendation = this.generateRecommendation(testFile, healthScore, drift?.driftScore, execution, issues, apiEndpoint);
|
|
71
|
+
const recommendation = this.generateRecommendation(testFile, healthScore, drift?.driftScore, execution, issues, apiEndpoint, drift?.apiSchemaChanges);
|
|
72
72
|
return {
|
|
73
73
|
testFile,
|
|
74
74
|
healthScore,
|
|
@@ -265,6 +265,15 @@ export class TestHealthService {
|
|
|
265
265
|
details: `${drift.affectedFiles?.files.length || 0} file(s) changed`,
|
|
266
266
|
});
|
|
267
267
|
}
|
|
268
|
+
const endpointsRenamed = drift.changes.filter((c) => c.type === "endpoint_renamed");
|
|
269
|
+
if (endpointsRenamed.length > 0) {
|
|
270
|
+
issues.push({
|
|
271
|
+
type: "endpoints_renamed",
|
|
272
|
+
severity: "high",
|
|
273
|
+
description: `${endpointsRenamed.length} API endpoint(s) renamed`,
|
|
274
|
+
details: endpointsRenamed.map((c) => c.description).join("; "),
|
|
275
|
+
});
|
|
276
|
+
}
|
|
268
277
|
const endpointsRemoved = drift.changes.filter((c) => c.type === "endpoint_removed");
|
|
269
278
|
if (endpointsRemoved.length > 0) {
|
|
270
279
|
issues.push({
|
|
@@ -306,7 +315,7 @@ export class TestHealthService {
|
|
|
306
315
|
*
|
|
307
316
|
* Execution failures enhance rationale but don't change primary action
|
|
308
317
|
*/
|
|
309
|
-
generateRecommendation(testFile, healthScore, driftScore, execution, issues, apiEndpoint) {
|
|
318
|
+
generateRecommendation(testFile, healthScore, driftScore, execution, issues, apiEndpoint, apiSchemaChanges) {
|
|
310
319
|
const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
|
|
311
320
|
let action;
|
|
312
321
|
let priority;
|
|
@@ -352,6 +361,21 @@ export class TestHealthService {
|
|
|
352
361
|
estimatedWork = "SMALL";
|
|
353
362
|
}
|
|
354
363
|
}
|
|
364
|
+
else if (issues && issues.some((i) => i.type === "endpoints_renamed")) {
|
|
365
|
+
// Endpoint renamed -> UPDATE with path substitution (regardless of drift score)
|
|
366
|
+
action = "UPDATE";
|
|
367
|
+
priority = "HIGH";
|
|
368
|
+
rationale =
|
|
369
|
+
"Endpoint path renamed - test URLs must be updated to match new path";
|
|
370
|
+
estimatedWork = "SMALL";
|
|
371
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
372
|
+
if (renameIssue?.details) {
|
|
373
|
+
rationale += `. ${renameIssue.details}`;
|
|
374
|
+
}
|
|
375
|
+
if (execution && !execution.passed) {
|
|
376
|
+
rationale += ". Test is currently failing due to the path change";
|
|
377
|
+
}
|
|
378
|
+
}
|
|
355
379
|
else if (drift > 70) {
|
|
356
380
|
// High drift -> REGENERATE
|
|
357
381
|
action = "REGENERATE";
|
|
@@ -391,6 +415,7 @@ export class TestHealthService {
|
|
|
391
415
|
const schemaChanges = issues?.filter((i) => [
|
|
392
416
|
"schema_changes",
|
|
393
417
|
"endpoints_removed",
|
|
418
|
+
"endpoints_renamed",
|
|
394
419
|
"authentication_changed",
|
|
395
420
|
].includes(i.type));
|
|
396
421
|
if (schemaChanges && schemaChanges.length > 0) {
|
|
@@ -439,7 +464,11 @@ export class TestHealthService {
|
|
|
439
464
|
}
|
|
440
465
|
// Determine endpoint status
|
|
441
466
|
let endpointStatus;
|
|
442
|
-
|
|
467
|
+
const renameIssues = issues?.filter((i) => i.type === "endpoints_renamed");
|
|
468
|
+
if (renameIssues && renameIssues.length > 0) {
|
|
469
|
+
endpointStatus = "renamed";
|
|
470
|
+
}
|
|
471
|
+
else if (apiEndpoint === undefined) {
|
|
443
472
|
endpointStatus = undefined;
|
|
444
473
|
}
|
|
445
474
|
else if (apiEndpoint.exists) {
|
|
@@ -448,6 +477,11 @@ export class TestHealthService {
|
|
|
448
477
|
else {
|
|
449
478
|
endpointStatus = "missing";
|
|
450
479
|
}
|
|
480
|
+
// Extract rename mappings from apiSchemaChanges for downstream tools
|
|
481
|
+
const renamedEndpoints = apiSchemaChanges?.endpointsRenamed &&
|
|
482
|
+
apiSchemaChanges.endpointsRenamed.length > 0
|
|
483
|
+
? apiSchemaChanges.endpointsRenamed
|
|
484
|
+
: undefined;
|
|
451
485
|
return {
|
|
452
486
|
testFile,
|
|
453
487
|
action,
|
|
@@ -459,6 +493,7 @@ export class TestHealthService {
|
|
|
459
493
|
driftScore: drift,
|
|
460
494
|
executionPassed: execution?.passed,
|
|
461
495
|
endpointStatus,
|
|
496
|
+
renamedEndpoints,
|
|
462
497
|
},
|
|
463
498
|
};
|
|
464
499
|
}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { TestHealthService } from "./TestHealthService.js";
|
|
2
|
+
describe("TestHealthService", () => {
|
|
3
|
+
let service;
|
|
4
|
+
beforeEach(() => {
|
|
5
|
+
service = new TestHealthService();
|
|
6
|
+
});
|
|
7
|
+
describe("identifyIssues - endpoint rename detection", () => {
|
|
8
|
+
function identifyIssues(execution, drift) {
|
|
9
|
+
return service["identifyIssues"](execution, drift);
|
|
10
|
+
}
|
|
11
|
+
it("should create an endpoints_renamed issue when drift has endpoint_renamed changes", () => {
|
|
12
|
+
const drift = {
|
|
13
|
+
testFile: "products_smoke_test.py",
|
|
14
|
+
lastCommit: "abc123",
|
|
15
|
+
currentCommit: "def456",
|
|
16
|
+
driftScore: 30,
|
|
17
|
+
changes: [
|
|
18
|
+
{
|
|
19
|
+
type: "endpoint_renamed",
|
|
20
|
+
file: "API Schema",
|
|
21
|
+
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
22
|
+
severity: "high",
|
|
23
|
+
details: "Path changed from /api/v1/products to /api/v1/items",
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
affectedFiles: { files: ["src/routers/product.py"] },
|
|
27
|
+
analysisTimestamp: new Date().toISOString(),
|
|
28
|
+
};
|
|
29
|
+
const issues = identifyIssues(undefined, drift);
|
|
30
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
31
|
+
expect(renameIssue).toBeDefined();
|
|
32
|
+
expect(renameIssue?.severity).toBe("high");
|
|
33
|
+
expect(renameIssue?.description).toContain("1 API endpoint(s) renamed");
|
|
34
|
+
});
|
|
35
|
+
it("should not create endpoints_renamed issue when no renames in drift", () => {
|
|
36
|
+
const drift = {
|
|
37
|
+
testFile: "products_smoke_test.py",
|
|
38
|
+
lastCommit: "abc123",
|
|
39
|
+
currentCommit: "def456",
|
|
40
|
+
driftScore: 15,
|
|
41
|
+
changes: [
|
|
42
|
+
{
|
|
43
|
+
type: "endpoint_removed",
|
|
44
|
+
file: "API Schema",
|
|
45
|
+
description: "1 endpoint(s) removed",
|
|
46
|
+
severity: "high",
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
affectedFiles: { files: [] },
|
|
50
|
+
analysisTimestamp: new Date().toISOString(),
|
|
51
|
+
};
|
|
52
|
+
const issues = identifyIssues(undefined, drift);
|
|
53
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
54
|
+
expect(renameIssue).toBeUndefined();
|
|
55
|
+
const removeIssue = issues.find((i) => i.type === "endpoints_removed");
|
|
56
|
+
expect(removeIssue).toBeDefined();
|
|
57
|
+
});
|
|
58
|
+
it("should handle multiple rename changes", () => {
|
|
59
|
+
const drift = {
|
|
60
|
+
testFile: "products_smoke_test.py",
|
|
61
|
+
lastCommit: "abc123",
|
|
62
|
+
currentCommit: "def456",
|
|
63
|
+
driftScore: 40,
|
|
64
|
+
changes: [
|
|
65
|
+
{
|
|
66
|
+
type: "endpoint_renamed",
|
|
67
|
+
file: "API Schema",
|
|
68
|
+
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
69
|
+
severity: "high",
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
type: "endpoint_renamed",
|
|
73
|
+
file: "API Schema",
|
|
74
|
+
description: "Endpoint renamed: post /api/v1/products -> /api/v1/items",
|
|
75
|
+
severity: "high",
|
|
76
|
+
},
|
|
77
|
+
],
|
|
78
|
+
affectedFiles: { files: [] },
|
|
79
|
+
analysisTimestamp: new Date().toISOString(),
|
|
80
|
+
};
|
|
81
|
+
const issues = identifyIssues(undefined, drift);
|
|
82
|
+
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
83
|
+
expect(renameIssue).toBeDefined();
|
|
84
|
+
expect(renameIssue?.description).toContain("2 API endpoint(s) renamed");
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
describe("generateRecommendation - endpoint rename handling", () => {
|
|
88
|
+
function generateRecommendation(testFile, driftScore, execution, issues, apiEndpoint, apiSchemaChanges) {
|
|
89
|
+
const healthScore = service["calculateHealthScore"](execution
|
|
90
|
+
? service["calculateExecutionScore"](execution).score
|
|
91
|
+
: undefined, driftScore);
|
|
92
|
+
return service["generateRecommendation"](testFile, healthScore, driftScore, execution, issues, apiEndpoint, apiSchemaChanges);
|
|
93
|
+
}
|
|
94
|
+
it("should return UPDATE action for endpoint renames regardless of drift score", () => {
|
|
95
|
+
const issues = [
|
|
96
|
+
{
|
|
97
|
+
type: "endpoints_renamed",
|
|
98
|
+
severity: "high",
|
|
99
|
+
description: "1 API endpoint(s) renamed",
|
|
100
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
const apiSchemaChanges = {
|
|
104
|
+
endpointsRemoved: [],
|
|
105
|
+
endpointsRenamed: [
|
|
106
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
107
|
+
],
|
|
108
|
+
endpointsModified: [],
|
|
109
|
+
authenticationChanged: false,
|
|
110
|
+
};
|
|
111
|
+
// Even with low drift score, renames should trigger UPDATE
|
|
112
|
+
const rec = generateRecommendation("products_smoke_test.py", 12, // low drift
|
|
113
|
+
undefined, issues, undefined, apiSchemaChanges);
|
|
114
|
+
expect(rec.action).toBe("UPDATE");
|
|
115
|
+
expect(rec.priority).toBe("HIGH");
|
|
116
|
+
expect(rec.rationale).toContain("renamed");
|
|
117
|
+
expect(rec.estimatedWork).toBe("SMALL");
|
|
118
|
+
});
|
|
119
|
+
it("should include renamedEndpoints in recommendation details", () => {
|
|
120
|
+
const issues = [
|
|
121
|
+
{
|
|
122
|
+
type: "endpoints_renamed",
|
|
123
|
+
severity: "high",
|
|
124
|
+
description: "1 API endpoint(s) renamed",
|
|
125
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
126
|
+
},
|
|
127
|
+
];
|
|
128
|
+
const apiSchemaChanges = {
|
|
129
|
+
endpointsRemoved: [],
|
|
130
|
+
endpointsRenamed: [
|
|
131
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
132
|
+
],
|
|
133
|
+
endpointsModified: [],
|
|
134
|
+
authenticationChanged: false,
|
|
135
|
+
};
|
|
136
|
+
const rec = generateRecommendation("products_smoke_test.py", 30, undefined, issues, undefined, apiSchemaChanges);
|
|
137
|
+
expect(rec.details?.endpointStatus).toBe("renamed");
|
|
138
|
+
expect(rec.details?.renamedEndpoints).toBeDefined();
|
|
139
|
+
expect(rec.details?.renamedEndpoints).toHaveLength(1);
|
|
140
|
+
expect(rec.details?.renamedEndpoints?.[0]).toEqual({
|
|
141
|
+
oldPath: "/api/v1/products",
|
|
142
|
+
newPath: "/api/v1/items",
|
|
143
|
+
method: "get",
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
it("should mention test failure in rationale when test is failing due to rename", () => {
|
|
147
|
+
const issues = [
|
|
148
|
+
{
|
|
149
|
+
type: "endpoints_renamed",
|
|
150
|
+
severity: "high",
|
|
151
|
+
description: "1 API endpoint(s) renamed",
|
|
152
|
+
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
153
|
+
},
|
|
154
|
+
];
|
|
155
|
+
const execution = {
|
|
156
|
+
testFile: "products_smoke_test.py",
|
|
157
|
+
executedAt: new Date().toISOString(),
|
|
158
|
+
passed: false,
|
|
159
|
+
duration: 10000,
|
|
160
|
+
errors: ["404 Not Found"],
|
|
161
|
+
warnings: [],
|
|
162
|
+
crashed: false,
|
|
163
|
+
};
|
|
164
|
+
const apiSchemaChanges = {
|
|
165
|
+
endpointsRemoved: [],
|
|
166
|
+
endpointsRenamed: [
|
|
167
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
168
|
+
],
|
|
169
|
+
endpointsModified: [],
|
|
170
|
+
authenticationChanged: false,
|
|
171
|
+
};
|
|
172
|
+
const rec = generateRecommendation("products_smoke_test.py", 30, execution, issues, undefined, apiSchemaChanges);
|
|
173
|
+
expect(rec.action).toBe("UPDATE");
|
|
174
|
+
expect(rec.rationale).toContain("failing");
|
|
175
|
+
});
|
|
176
|
+
it("should not set renamedEndpoints when there are no renames", () => {
|
|
177
|
+
const rec = generateRecommendation("orders_smoke_test.py", 5, undefined, [], { exists: true }, undefined);
|
|
178
|
+
expect(rec.action).toBe("VERIFY");
|
|
179
|
+
expect(rec.details?.renamedEndpoints).toBeUndefined();
|
|
180
|
+
expect(rec.details?.endpointStatus).toBe("exists");
|
|
181
|
+
});
|
|
182
|
+
it("should prefer rename handling over high-drift REGENERATE", () => {
|
|
183
|
+
// If drift is > 70 but it's caused by a rename, we should UPDATE not REGENERATE
|
|
184
|
+
const issues = [
|
|
185
|
+
{
|
|
186
|
+
type: "endpoints_renamed",
|
|
187
|
+
severity: "high",
|
|
188
|
+
description: "5 API endpoint(s) renamed",
|
|
189
|
+
details: "Multiple renames",
|
|
190
|
+
},
|
|
191
|
+
];
|
|
192
|
+
const apiSchemaChanges = {
|
|
193
|
+
endpointsRemoved: [],
|
|
194
|
+
endpointsRenamed: [
|
|
195
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
196
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "post" },
|
|
197
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "get" },
|
|
198
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "put" },
|
|
199
|
+
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "delete" },
|
|
200
|
+
],
|
|
201
|
+
endpointsModified: [],
|
|
202
|
+
authenticationChanged: false,
|
|
203
|
+
};
|
|
204
|
+
const rec = generateRecommendation("products_smoke_test.py", 75, // would normally trigger REGENERATE
|
|
205
|
+
undefined, issues, undefined, apiSchemaChanges);
|
|
206
|
+
// Rename detection should take priority over drift threshold
|
|
207
|
+
expect(rec.action).toBe("UPDATE");
|
|
208
|
+
expect(rec.estimatedWork).toBe("SMALL"); // renames are simple substitutions
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
});
|
|
@@ -2,7 +2,60 @@ import { z } from "zod";
|
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
4
4
|
import * as fs from "fs";
|
|
5
|
+
import * as path from "path";
|
|
5
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
|
+
/**
|
|
8
|
+
* Compute a suggested new filename when an endpoint is renamed.
|
|
9
|
+
*
|
|
10
|
+
* Extracts the differing static segments between oldPath and newPath,
|
|
11
|
+
* then replaces occurrences in the filename.
|
|
12
|
+
*
|
|
13
|
+
* Example:
|
|
14
|
+
* testFile: "/repo/tests/python/products_smoke_test.py"
|
|
15
|
+
* oldPath: "/api/v1/products"
|
|
16
|
+
* newPath: "/api/v1/items"
|
|
17
|
+
* result: "/repo/tests/python/items_smoke_test.py"
|
|
18
|
+
*/
|
|
19
|
+
export function computeRenamedTestFile(testFile, renames) {
|
|
20
|
+
const basename = path.basename(testFile);
|
|
21
|
+
let newBasename = basename;
|
|
22
|
+
for (const rename of renames) {
|
|
23
|
+
const oldSegments = rename.oldPath.split("/").filter((s) => s.length > 0);
|
|
24
|
+
const newSegments = rename.newPath.split("/").filter((s) => s.length > 0);
|
|
25
|
+
if (oldSegments.length !== newSegments.length)
|
|
26
|
+
continue;
|
|
27
|
+
const paramPattern = /^\{[^}]+\}$/;
|
|
28
|
+
for (let i = 0; i < oldSegments.length; i++) {
|
|
29
|
+
if (paramPattern.test(oldSegments[i]))
|
|
30
|
+
continue;
|
|
31
|
+
if (oldSegments[i] !== newSegments[i]) {
|
|
32
|
+
// Replace the old segment name in the filename with the new one
|
|
33
|
+
// Handle both exact matches and common variations:
|
|
34
|
+
// "products" in "products_smoke_test.py"
|
|
35
|
+
// "product" in "product_smoke_test.py" (singular)
|
|
36
|
+
const oldName = oldSegments[i].toLowerCase();
|
|
37
|
+
const newName = newSegments[i].toLowerCase();
|
|
38
|
+
if (newBasename.toLowerCase().includes(oldName)) {
|
|
39
|
+
// Case-preserving replace
|
|
40
|
+
const idx = newBasename.toLowerCase().indexOf(oldName);
|
|
41
|
+
newBasename =
|
|
42
|
+
newBasename.substring(0, idx) +
|
|
43
|
+
newName +
|
|
44
|
+
newBasename.substring(idx + oldName.length);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (newBasename === basename)
|
|
50
|
+
return null; // No change needed
|
|
51
|
+
const newFilePath = path.join(path.dirname(testFile), newBasename);
|
|
52
|
+
// Don't suggest a rename if the target file already exists
|
|
53
|
+
if (fs.existsSync(newFilePath)) {
|
|
54
|
+
logger.info(`Skipping file rename suggestion: ${newFilePath} already exists`);
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
return newFilePath;
|
|
58
|
+
}
|
|
6
59
|
const actionsSchema = {
|
|
7
60
|
stateFile: z
|
|
8
61
|
.string()
|
|
@@ -70,6 +123,7 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
70
123
|
rationale: test.recommendation.rationale,
|
|
71
124
|
estimatedWork: test.recommendation.estimatedWork,
|
|
72
125
|
issues: test.issues || [],
|
|
126
|
+
renamedEndpoints: test.recommendation.details?.renamedEndpoints || [],
|
|
73
127
|
});
|
|
74
128
|
}
|
|
75
129
|
});
|
|
@@ -130,11 +184,33 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
130
184
|
logger.error(`Failed to read test file ${rec.testFile}: ${error.message}`);
|
|
131
185
|
continue;
|
|
132
186
|
}
|
|
187
|
+
// Check if this is a rename-driven update
|
|
188
|
+
const renames = rec.renamedEndpoints || [];
|
|
189
|
+
const isRenameUpdate = renames.length > 0;
|
|
133
190
|
// Build update instructions
|
|
134
191
|
let instruction = `\n### ${rec.testFile}\n\n`;
|
|
135
192
|
instruction += `**Priority:** ${rec.priority} | `;
|
|
136
193
|
instruction += `**Estimated Effort:** ${rec.estimatedWork || "Small"}\n\n`;
|
|
137
194
|
instruction += `**Why Update Needed:** ${rec.rationale}\n\n`;
|
|
195
|
+
if (isRenameUpdate) {
|
|
196
|
+
instruction += `**🔄 Endpoint Rename Detected — Path Substitution Required:**\n\n`;
|
|
197
|
+
instruction += `| Old Path | New Path | Method |\n`;
|
|
198
|
+
instruction += `|----------|----------|--------|\n`;
|
|
199
|
+
for (const rename of renames) {
|
|
200
|
+
instruction += `| \`${rename.oldPath}\` | \`${rename.newPath}\` | ${rename.method} |\n`;
|
|
201
|
+
}
|
|
202
|
+
instruction += `\n`;
|
|
203
|
+
instruction += `**Action:** Find-and-replace all occurrences of the old path with the new path in this test file. `;
|
|
204
|
+
instruction += `Do NOT change any test logic, assertions, or structure — only update the URL paths.\n\n`;
|
|
205
|
+
// Compute suggested file rename
|
|
206
|
+
const suggestedNewFile = computeRenamedTestFile(rec.testFile, renames);
|
|
207
|
+
if (suggestedNewFile) {
|
|
208
|
+
instruction += `**📁 File Rename:** After updating the paths, rename this file:\n`;
|
|
209
|
+
instruction += `- From: \`${path.basename(rec.testFile)}\`\n`;
|
|
210
|
+
instruction += `- To: \`${path.basename(suggestedNewFile)}\`\n\n`;
|
|
211
|
+
rec._suggestedNewFile = suggestedNewFile;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
138
214
|
if (driftData) {
|
|
139
215
|
instruction += `**Analysis:**\n`;
|
|
140
216
|
instruction += `- Drift Score: ${driftData.driftScore ?? "N/A"}\n`;
|
|
@@ -184,16 +260,46 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
184
260
|
responseText += `4. Show you the changes made\n\n`;
|
|
185
261
|
responseText += `5. At the end of the tool execution, MUST display the below message\n`;
|
|
186
262
|
responseText += `**This tool is currently in Early Preview stage. Please verify the results.**\n\n`;
|
|
263
|
+
// Collect all rename mappings across recommendations
|
|
264
|
+
const allRenames = [];
|
|
265
|
+
for (const rec of updateRecommendations) {
|
|
266
|
+
if (rec.renamedEndpoints && rec.renamedEndpoints.length > 0) {
|
|
267
|
+
allRenames.push(...rec.renamedEndpoints);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// Deduplicate renames
|
|
271
|
+
const uniqueRenames = allRenames.filter((r, i, arr) => arr.findIndex((x) => x.oldPath === r.oldPath &&
|
|
272
|
+
x.newPath === r.newPath &&
|
|
273
|
+
x.method === r.method) === i);
|
|
187
274
|
// Build LLM-only instructions (hidden from users)
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
275
|
+
const llmInstructionsObj = {
|
|
276
|
+
workflow: "test_maintenance",
|
|
277
|
+
action: "execute_updates",
|
|
278
|
+
auto_proceed: true,
|
|
279
|
+
files_to_update: testFilesToUpdate,
|
|
280
|
+
update_count: updateRecommendations.length,
|
|
281
|
+
};
|
|
282
|
+
if (uniqueRenames.length > 0) {
|
|
283
|
+
llmInstructionsObj.endpoint_renames = uniqueRenames;
|
|
284
|
+
llmInstructionsObj.rename_strategy =
|
|
285
|
+
"For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.";
|
|
286
|
+
// Collect file rename suggestions
|
|
287
|
+
const fileRenames = [];
|
|
288
|
+
for (const rec of updateRecommendations) {
|
|
289
|
+
if (rec._suggestedNewFile) {
|
|
290
|
+
fileRenames.push({
|
|
291
|
+
from: rec.testFile,
|
|
292
|
+
to: rec._suggestedNewFile,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (fileRenames.length > 0) {
|
|
297
|
+
llmInstructionsObj.file_renames = fileRenames;
|
|
298
|
+
llmInstructionsObj.file_rename_strategy =
|
|
299
|
+
"After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.";
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
const llmInstructions = `<!-- LLM_INSTRUCTIONS:\n${JSON.stringify(llmInstructionsObj, null, 2)}\n-->\n`;
|
|
197
303
|
return {
|
|
198
304
|
content: [
|
|
199
305
|
{
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Mock modules that use ESM-only features (import.meta) before importing actionsTool
|
|
2
|
+
jest.mock("../../services/AnalyticsService.js", () => ({
|
|
3
|
+
AnalyticsService: { pushMCPToolEvent: jest.fn() },
|
|
4
|
+
}));
|
|
5
|
+
jest.mock("../../utils/logger.js", () => ({
|
|
6
|
+
logger: { info: jest.fn(), warning: jest.fn(), error: jest.fn(), debug: jest.fn() },
|
|
7
|
+
}));
|
|
8
|
+
jest.mock("../../utils/AnalysisStateManager.js", () => ({
|
|
9
|
+
StateManager: { fromStatePath: jest.fn() },
|
|
10
|
+
}));
|
|
11
|
+
jest.mock("fs");
|
|
12
|
+
// @ts-ignore
|
|
13
|
+
import { computeRenamedTestFile } from "./actionsTool.js";
|
|
14
|
+
import * as fs from "fs";
|
|
15
|
+
const mockExistsSync = fs.existsSync;
|
|
16
|
+
describe("computeRenamedTestFile", () => {
|
|
17
|
+
beforeEach(() => {
|
|
18
|
+
// Default: target file does not exist
|
|
19
|
+
mockExistsSync.mockReturnValue(false);
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
jest.restoreAllMocks();
|
|
23
|
+
});
|
|
24
|
+
// --- Basic renames ---
|
|
25
|
+
it("should rename products_smoke_test.py to items_smoke_test.py", () => {
|
|
26
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
27
|
+
expect(result).toBe("/repo/tests/python/items_smoke_test.py");
|
|
28
|
+
});
|
|
29
|
+
it("should rename products_contract_test.py to items_contract_test.py", () => {
|
|
30
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_contract_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
31
|
+
expect(result).toBe("/repo/tests/python/items_contract_test.py");
|
|
32
|
+
});
|
|
33
|
+
it("should rename products_integration_test.py to items_integration_test.py", () => {
|
|
34
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_integration_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "post" }]);
|
|
35
|
+
expect(result).toBe("/repo/tests/python/items_integration_test.py");
|
|
36
|
+
});
|
|
37
|
+
it("should rename products_fuzz_test.py to items_fuzz_test.py", () => {
|
|
38
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_fuzz_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
39
|
+
expect(result).toBe("/repo/tests/python/items_fuzz_test.py");
|
|
40
|
+
});
|
|
41
|
+
it("should rename products_load_test.py to items_load_test.py", () => {
|
|
42
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_load_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
43
|
+
expect(result).toBe("/repo/tests/python/items_load_test.py");
|
|
44
|
+
});
|
|
45
|
+
// --- Different file extensions ---
|
|
46
|
+
it("should work with .ts test files", () => {
|
|
47
|
+
const result = computeRenamedTestFile("/repo/tests/products.test.ts", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
48
|
+
expect(result).toBe("/repo/tests/items.test.ts");
|
|
49
|
+
});
|
|
50
|
+
it("should work with .js test files", () => {
|
|
51
|
+
const result = computeRenamedTestFile("/repo/tests/products_smoke.test.js", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
52
|
+
expect(result).toBe("/repo/tests/items_smoke.test.js");
|
|
53
|
+
});
|
|
54
|
+
// --- Returns null when no rename needed ---
|
|
55
|
+
it("should return null when filename does not contain old segment", () => {
|
|
56
|
+
const result = computeRenamedTestFile("/repo/tests/python/orders_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
57
|
+
expect(result).toBeNull();
|
|
58
|
+
});
|
|
59
|
+
it("should return null when target file already exists", () => {
|
|
60
|
+
mockExistsSync.mockReturnValue(true);
|
|
61
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
62
|
+
expect(result).toBeNull();
|
|
63
|
+
});
|
|
64
|
+
it("should return null when segments have different lengths", () => {
|
|
65
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v2/catalog/items", method: "get" }]);
|
|
66
|
+
// Different segment counts — no substitution attempted
|
|
67
|
+
expect(result).toBeNull();
|
|
68
|
+
});
|
|
69
|
+
it("should return null with empty renames array", () => {
|
|
70
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", []);
|
|
71
|
+
expect(result).toBeNull();
|
|
72
|
+
});
|
|
73
|
+
// --- Multiple renames ---
|
|
74
|
+
it("should apply multiple rename mappings", () => {
|
|
75
|
+
// Unlikely but possible: two segments change
|
|
76
|
+
const result = computeRenamedTestFile("/repo/tests/python/products_smoke_test.py", [
|
|
77
|
+
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
78
|
+
{ oldPath: "/api/v1/products/{product_id}", newPath: "/api/v1/items/{item_id}", method: "get" },
|
|
79
|
+
]);
|
|
80
|
+
expect(result).toBe("/repo/tests/python/items_smoke_test.py");
|
|
81
|
+
});
|
|
82
|
+
// --- Preserves directory structure ---
|
|
83
|
+
it("should preserve the directory path", () => {
|
|
84
|
+
const result = computeRenamedTestFile("/home/runner/work/api-insight/api-insight/tests/python/products_smoke_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" }]);
|
|
85
|
+
expect(result).toBe("/home/runner/work/api-insight/api-insight/tests/python/items_smoke_test.py");
|
|
86
|
+
});
|
|
87
|
+
// --- Version bump rename ---
|
|
88
|
+
it("should handle version segment rename in filename if present", () => {
|
|
89
|
+
const result = computeRenamedTestFile("/repo/tests/v1_products_test.py", [{ oldPath: "/api/v1/products", newPath: "/api/v2/products", method: "get" }]);
|
|
90
|
+
// "v1" in filename gets replaced with "v2"
|
|
91
|
+
expect(result).toBe("/repo/tests/v2_products_test.py");
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -119,7 +119,7 @@ function parseEndpointsFromDiff(diffData) {
|
|
|
119
119
|
affectedServices,
|
|
120
120
|
};
|
|
121
121
|
}
|
|
122
|
-
async function computeBranchDiff(repositoryPath) {
|
|
122
|
+
async function computeBranchDiff(repositoryPath, providedBaseBranch) {
|
|
123
123
|
const git = simpleGit(repositoryPath);
|
|
124
124
|
const isRepo = await git.checkIsRepo();
|
|
125
125
|
if (!isRepo) {
|
|
@@ -127,22 +127,27 @@ async function computeBranchDiff(repositoryPath) {
|
|
|
127
127
|
}
|
|
128
128
|
const branchInfo = await git.branch();
|
|
129
129
|
const currentBranch = branchInfo.current || "HEAD";
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
130
|
+
let baseBranch;
|
|
131
|
+
if (providedBaseBranch) {
|
|
132
|
+
// Use the PR's base branch when explicitly provided (e.g. from testbot)
|
|
133
|
+
baseBranch = `origin/${providedBaseBranch}`;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
// Fall back to auto-detecting origin/main or origin/master
|
|
137
|
+
baseBranch = "origin/main";
|
|
138
|
+
try {
|
|
139
|
+
const remoteBranches = await git.branch(["-r"]);
|
|
140
|
+
if (remoteBranches.all.some((b) => b.endsWith("/main"))) {
|
|
141
|
+
baseBranch = "origin/main";
|
|
142
|
+
}
|
|
143
|
+
else if (remoteBranches.all.some((b) => b.endsWith("/master"))) {
|
|
144
|
+
baseBranch = "origin/master";
|
|
145
|
+
}
|
|
138
146
|
}
|
|
139
|
-
|
|
140
|
-
|
|
147
|
+
catch {
|
|
148
|
+
logger.debug("Could not determine remote default branch, falling back to origin/main");
|
|
141
149
|
}
|
|
142
150
|
}
|
|
143
|
-
catch {
|
|
144
|
-
logger.debug("Could not determine remote default branch, falling back to origin/main");
|
|
145
|
-
}
|
|
146
151
|
const changedFilesRaw = await git.diff([
|
|
147
152
|
`${baseBranch}...HEAD`,
|
|
148
153
|
"--name-only",
|
|
@@ -180,6 +185,10 @@ const analyzeRepositorySchema = z.object({
|
|
|
180
185
|
.array(z.string())
|
|
181
186
|
.optional()
|
|
182
187
|
.describe("Optional: Specific areas to focus on (e.g., ['api', 'frontend', 'infrastructure'])"),
|
|
188
|
+
baseBranch: z
|
|
189
|
+
.string()
|
|
190
|
+
.optional()
|
|
191
|
+
.describe("Optional: PR base branch name (e.g. 'main', 'develop'). When provided, the diff is computed against origin/<baseBranch> instead of auto-detecting the default branch. Useful when the PR targets a non-default branch."),
|
|
183
192
|
});
|
|
184
193
|
const TOOL_NAME = "skyramp_analyze_repository";
|
|
185
194
|
export function registerAnalyzeRepositoryTool(server) {
|
|
@@ -240,7 +249,7 @@ Output: Detailed RepositoryAnalysis JSON object with all repository characterist
|
|
|
240
249
|
let diffData;
|
|
241
250
|
if (analysisScope === "current_branch_diff") {
|
|
242
251
|
try {
|
|
243
|
-
diffData = await computeBranchDiff(params.repositoryPath);
|
|
252
|
+
diffData = await computeBranchDiff(params.repositoryPath, params.baseBranch);
|
|
244
253
|
logger.info("Branch diff computed via git", {
|
|
245
254
|
currentBranch: diffData.currentBranch,
|
|
246
255
|
baseBranch: diffData.baseBranch,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skyramp/mcp",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.58",
|
|
4
4
|
"main": "build/index.js",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"@modelcontextprotocol/sdk": "^1.24.3",
|
|
48
48
|
"@playwright/test": "^1.55.0",
|
|
49
|
-
"@skyramp/skyramp": "1.3.
|
|
49
|
+
"@skyramp/skyramp": "1.3.11",
|
|
50
50
|
"dockerode": "^4.0.6",
|
|
51
51
|
"fast-glob": "^3.3.3",
|
|
52
52
|
"simple-git": "^3.30.0",
|