@skyramp/mcp 0.2.7 → 0.2.150-rc.mntnc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +5 -2
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +24 -5
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.d.ts +2 -0
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +114 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.d.ts +2 -0
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +292 -0
- package/build/prompts/sut-setup/shared.d.ts +39 -0
- package/build/prompts/sut-setup/shared.js +132 -0
- package/build/prompts/test-maintenance/driftAnalysisSections.js +9 -3
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
- package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
- package/build/prompts/testbot/testbot-prompts.js +23 -16
- package/build/prompts/testbot/testbot-prompts.test.js +0 -44
- package/build/resources/sutSetupResource.d.ts +2 -0
- package/build/resources/sutSetupResource.js +45 -0
- package/build/resources/testbotResource.js +1 -1
- package/build/services/TestDiscoveryService.d.ts +2 -2
- package/build/services/TestDiscoveryService.js +22 -12
- package/build/services/TestExecutionService.d.ts +9 -0
- package/build/services/TestExecutionService.js +158 -26
- package/build/services/TestExecutionService.test.js +306 -0
- package/build/tools/executeSkyrampTestTool.js +17 -4
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +1 -1
- package/build/tools/submitReportTool.js +81 -35
- package/build/tools/submitReportTool.test.js +264 -50
- package/build/tools/test-management/actionsTool.js +10 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +0 -1
- package/build/tools/test-management/analyzeChangesTool.js +84 -41
- package/build/tools/test-management/analyzeChangesTool.test.js +4 -34
- package/build/tools/test-management/analyzeTestHealthTool.js +6 -3
- package/build/types/TestAnalysis.d.ts +2 -1
- package/build/types/TestExecution.d.ts +12 -2
- package/build/types/TestExecution.js +11 -1
- package/build/types/TestbotReport.d.ts +68 -0
- package/build/types/TestbotReport.js +1 -0
- package/build/types/index.d.ts +4 -0
- package/build/types/index.js +3 -0
- package/build/utils/AnalysisStateManager.js +10 -1
- package/build/utils/routeParsers.d.ts +0 -10
- package/build/utils/routeParsers.js +31 -38
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +23 -1
- package/build/utils/utils.d.ts +7 -1
- package/build/utils/utils.js +10 -0
- package/node_modules/playwright/LICENSE +202 -0
- package/node_modules/playwright/NOTICE +5 -0
- package/node_modules/playwright/README.md +168 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +13 -9
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +4 -3
- package/node_modules/playwright/node_modules/playwright-core/LICENSE +202 -0
- package/node_modules/playwright/node_modules/playwright-core/NOTICE +5 -0
- package/node_modules/playwright/node_modules/playwright-core/ThirdPartyNotices.txt +23 -126
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-B0JOjboO.js → codeMirrorModule-B5kqh2EV.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-1anWeyDf.js → defaultSettingsView-CZ9npQ3N.js} +79 -79
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Cc5029a3.js → index.BCnxj-_b.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.Wo5yvvVh.js → uiMode.1Ym0Ivn8.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +9 -2
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
- package/node_modules/playwright/node_modules/playwright-core/bundles/mcp/node_modules/.bin/node-which +0 -52
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/is-docker +0 -5
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/mime +0 -46
- package/node_modules/playwright/node_modules/playwright-core/bundles/utils/node_modules/.bin/yaml +0 -11
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.CP3Y8Zsb.js +0 -2
|
@@ -24,6 +24,32 @@ import { buildAnalysisOutputText } from "../../prompts/test-recommendation/analy
|
|
|
24
24
|
import { parseTraceFile, discoverTraceFiles, discoverPlaywrightZips, } from "../../utils/trace-parser.js";
|
|
25
25
|
import { TestSource } from "../../types/TestAnalysis.js";
|
|
26
26
|
import { parsePRComments } from "../../utils/pr-comment-parser.js";
|
|
27
|
+
// Generic module names that appear as imports in virtually every router file.
|
|
28
|
+
// Matching against these would extract ALL prefix= tokens from mount files,
|
|
29
|
+
// flooding changedResources with unrelated resources.
|
|
30
|
+
const GENERIC_MODULE_NAMES = new Set([
|
|
31
|
+
"models", "utils", "helpers", "schemas", "types", "constants",
|
|
32
|
+
"config", "settings", "exceptions", "errors", "base", "common",
|
|
33
|
+
"app", "main", "index", "server", "init", "deps", "dependencies",
|
|
34
|
+
]);
|
|
35
|
+
function resolveMountPrefixTokens(mountFileContents, changedFiles) {
|
|
36
|
+
const tokens = changedFiles.flatMap((file) => {
|
|
37
|
+
const mod = path.basename(file).replace(/\.[^.]+$/, "").toLowerCase();
|
|
38
|
+
if (GENERIC_MODULE_NAMES.has(mod))
|
|
39
|
+
return [];
|
|
40
|
+
const modRe = new RegExp(`\\b${mod.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`);
|
|
41
|
+
return mountFileContents.flatMap((content) => {
|
|
42
|
+
if (!modRe.test(content.toLowerCase()))
|
|
43
|
+
return [];
|
|
44
|
+
const matches = [...content.matchAll(/prefix\s*=\s*["'](\/?[^"']+)/g)];
|
|
45
|
+
return matches.flatMap((m) => {
|
|
46
|
+
const token = extractResourceFromPath(m[1]);
|
|
47
|
+
return token !== "unknown" ? [token] : [];
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
return [...new Set(tokens)];
|
|
52
|
+
}
|
|
27
53
|
/** Exported for testing: maps a parsed trace result to a TraceFile. */
|
|
28
54
|
export function buildTraceFileEntry(tracePath, result) {
|
|
29
55
|
return {
|
|
@@ -339,11 +365,6 @@ export const analyzeChangesInputSchema = {
|
|
|
339
365
|
.number()
|
|
340
366
|
.optional()
|
|
341
367
|
.describe("GitHub PR number. When provided, fetches previous TestBot comments on this PR and skips re-recommending tests already suggested in earlier commits — reduces duplicate recommendations across multiple pushes to the same PR."),
|
|
342
|
-
stateOutputFile: z
|
|
343
|
-
.string()
|
|
344
|
-
.refine((v) => path.isAbsolute(v), { message: "stateOutputFile must be an absolute path" })
|
|
345
|
-
.optional()
|
|
346
|
-
.describe("Absolute path where the state file should be written. When provided, overrides the default auto-generated temp path so the caller can locate it without log parsing."),
|
|
347
368
|
testsRepoDir: z
|
|
348
369
|
.string()
|
|
349
370
|
.refine((v) => path.isAbsolute(v), { message: "testsRepoDir must be an absolute path" })
|
|
@@ -568,6 +589,30 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
568
589
|
// testsRepoDir is a cross-repo path override — honour it when set.
|
|
569
590
|
// Otherwise always scan the full repo root.
|
|
570
591
|
const testDir = params.testsRepoDir ?? undefined;
|
|
592
|
+
const routerMountContext = grepRouterMountingContext(params.repositoryPath);
|
|
593
|
+
// Read contents once — used for token extraction and LLM prompt injection.
|
|
594
|
+
const ROUTER_INLINE_LIMIT = 4096;
|
|
595
|
+
const routerMountContextContents = routerMountContext.map((f) => {
|
|
596
|
+
try {
|
|
597
|
+
const abs = path.isAbsolute(f) ? f : path.join(params.repositoryPath, f);
|
|
598
|
+
if (fs.statSync(abs).size > ROUTER_INLINE_LIMIT * 10)
|
|
599
|
+
return ""; // skip very large files
|
|
600
|
+
return fs.readFileSync(abs, "utf-8");
|
|
601
|
+
}
|
|
602
|
+
catch {
|
|
603
|
+
return "";
|
|
604
|
+
}
|
|
605
|
+
});
|
|
606
|
+
// Size-capped subset for LLM prompt injection.
|
|
607
|
+
const ROUTER_INLINE_MAX_FILES = 3;
|
|
608
|
+
const routerFileContents = routerMountContext
|
|
609
|
+
.slice(0, ROUTER_INLINE_MAX_FILES)
|
|
610
|
+
.flatMap((f, i) => {
|
|
611
|
+
const content = routerMountContextContents[i] ?? "";
|
|
612
|
+
if (!content || content.length > ROUTER_INLINE_LIMIT)
|
|
613
|
+
return [];
|
|
614
|
+
return [{ file: f, content: content.trimEnd() }];
|
|
615
|
+
});
|
|
571
616
|
// Compute changedResources from classified endpoints for test discovery filtering.
|
|
572
617
|
// undefined → full-repo mode (no diff context)
|
|
573
618
|
// [] → PR mode, no endpoints found → skip external tests
|
|
@@ -596,14 +641,22 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
596
641
|
}
|
|
597
642
|
else {
|
|
598
643
|
// All endpoints resolved to "unknown" (e.g. NestJS relative paths) —
|
|
599
|
-
// extract resource names from
|
|
600
|
-
const
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
644
|
+
// first try to extract resource names from router mount prefixes.
|
|
645
|
+
const changedSourceFiles = classifiedEndpoints.changedFiles ?? [];
|
|
646
|
+
const mountedResources = resolveMountPrefixTokens(routerMountContextContents, changedSourceFiles);
|
|
647
|
+
if (mountedResources.length > 0) {
|
|
648
|
+
changedResources = mountedResources;
|
|
649
|
+
}
|
|
650
|
+
else {
|
|
651
|
+
// Fall back to extracting resource names from changed file paths.
|
|
652
|
+
const candidateFiles = classifiedEndpoints.unmatchedFiles.length > 0
|
|
653
|
+
? classifiedEndpoints.unmatchedFiles
|
|
654
|
+
: classifiedEndpoints.changedFiles ?? [];
|
|
655
|
+
const fromFiles = candidateFiles
|
|
656
|
+
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
657
|
+
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
658
|
+
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
659
|
+
}
|
|
607
660
|
}
|
|
608
661
|
}
|
|
609
662
|
else if (classifiedEndpoints.unmatchedFiles.length > 0) {
|
|
@@ -622,7 +675,15 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
622
675
|
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
623
676
|
}
|
|
624
677
|
else {
|
|
625
|
-
|
|
678
|
+
// No endpoints and no unmatched files — the diff touches only non-route
|
|
679
|
+
// files (e.g. schema, model, DTO). Extract resource tokens from the
|
|
680
|
+
// changed file paths so tests that import the changed symbol are still
|
|
681
|
+
// surfaced for drift assessment. Fall back to [] only if extraction
|
|
682
|
+
// yields nothing (e.g. a file with a non-resource name like utils.py).
|
|
683
|
+
const fromChangedFiles = (classifiedEndpoints.changedFiles ?? [])
|
|
684
|
+
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
685
|
+
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
686
|
+
changedResources = fromChangedFiles.length > 0 ? fromChangedFiles : [];
|
|
626
687
|
}
|
|
627
688
|
}
|
|
628
689
|
let existingTests = [];
|
|
@@ -903,10 +964,15 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
903
964
|
sourceFile: m.sourceFile ?? "",
|
|
904
965
|
})))
|
|
905
966
|
: [];
|
|
967
|
+
const removedEndpointsForDrafting = classifiedEndpoints?.removedEndpoints.flatMap((ep) => ep.methods.map((m) => ({
|
|
968
|
+
method: m,
|
|
969
|
+
path: ep.path,
|
|
970
|
+
sourceFile: ep.sourceFile,
|
|
971
|
+
}))) ?? [];
|
|
906
972
|
const codeInferredScenarios = draftScenariosFromEndpoints(skeletonEndpoints, scenarioDraftSeed, wsAuthType, {
|
|
907
973
|
changedEndpoints: changedEndpointsForSecurityExpansion,
|
|
908
974
|
securityRelevantDiff,
|
|
909
|
-
});
|
|
975
|
+
}, removedEndpointsForDrafting);
|
|
910
976
|
let allDraftedScenarios = codeInferredScenarios;
|
|
911
977
|
if (traceResult && traceResult.userFlows.length > 0) {
|
|
912
978
|
const traceScenarios = traceResult.userFlows
|
|
@@ -1109,7 +1175,6 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1109
1175
|
// can be persisted to the state file for downstream tools (health, drift).
|
|
1110
1176
|
// Without them, analyzeTestHealth would work only off the static catalog
|
|
1111
1177
|
// which has wrong paths for nested resources and unsupported frameworks.
|
|
1112
|
-
const routerMountContext = grepRouterMountingContext(params.repositoryPath);
|
|
1113
1178
|
const routeLikeUnmatchedFiles = [];
|
|
1114
1179
|
for (const file of classifiedEndpoints?.unmatchedFiles ?? []) {
|
|
1115
1180
|
const routeLike = SOURCE_EXTS.test(file) &&
|
|
@@ -1138,25 +1203,6 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1138
1203
|
diffFilePath = path.join(os.tmpdir(), `skyramp-diff-${sessionId}.diff`);
|
|
1139
1204
|
await fs.promises.writeFile(diffFilePath, diffData.diffContent, { encoding: "utf-8", mode: 0o600 });
|
|
1140
1205
|
}
|
|
1141
|
-
// Read router mount files server-side (size-capped) so the LLM has them
|
|
1142
|
-
// inline and doesn't need an extra read step when no spec is available.
|
|
1143
|
-
const ROUTER_INLINE_LIMIT = 4096; // bytes — skip files larger than ~4 KB
|
|
1144
|
-
const ROUTER_INLINE_MAX_FILES = 3;
|
|
1145
|
-
const routerFileContents = routerMountContext
|
|
1146
|
-
.slice(0, ROUTER_INLINE_MAX_FILES)
|
|
1147
|
-
.flatMap((f) => {
|
|
1148
|
-
try {
|
|
1149
|
-
const absPath = path.isAbsolute(f) ? f : path.join(params.repositoryPath, f);
|
|
1150
|
-
const stat = fs.statSync(absPath);
|
|
1151
|
-
if (stat.size > ROUTER_INLINE_LIMIT)
|
|
1152
|
-
return [];
|
|
1153
|
-
const content = fs.readFileSync(absPath, "utf-8").trimEnd();
|
|
1154
|
-
return [{ file: f, content }];
|
|
1155
|
-
}
|
|
1156
|
-
catch {
|
|
1157
|
-
return [];
|
|
1158
|
-
}
|
|
1159
|
-
});
|
|
1160
1206
|
// Compute UI context from the diff's changed files using the shared
|
|
1161
1207
|
// `isFrontendFile` classifier. Persisting this in the stateFile lets
|
|
1162
1208
|
// skyramp_analyze_test_health and the recommendation prompt consume the
|
|
@@ -1252,17 +1298,14 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1252
1298
|
relevantExternalTestPaths,
|
|
1253
1299
|
},
|
|
1254
1300
|
};
|
|
1255
|
-
// Clean up old state files (>24 hours) before creating new one
|
|
1256
|
-
const stateDir = params.stateOutputFile
|
|
1257
|
-
? path.dirname(path.resolve(params.stateOutputFile))
|
|
1258
|
-
: undefined;
|
|
1301
|
+
// Clean up old state files (>24 hours) before creating new one.
|
|
1259
1302
|
try {
|
|
1260
|
-
await StateManager.cleanupOldFiles(24
|
|
1303
|
+
await StateManager.cleanupOldFiles(24);
|
|
1261
1304
|
}
|
|
1262
1305
|
catch (error) {
|
|
1263
1306
|
logger.warning(`Failed to cleanup old state files: ${error.message}`);
|
|
1264
1307
|
}
|
|
1265
|
-
const stateManager = new StateManager("analysis", sessionId
|
|
1308
|
+
const stateManager = new StateManager("analysis", sessionId);
|
|
1266
1309
|
await stateManager.writeData(unifiedState, {
|
|
1267
1310
|
repositoryPath: params.repositoryPath,
|
|
1268
1311
|
step: "analyze_changes",
|
|
@@ -99,28 +99,6 @@ describe("isSecurityRelevantDiff", () => {
|
|
|
99
99
|
expect(isSecurityRelevantDiff("move compression middleware to server setup")).toBe(false);
|
|
100
100
|
});
|
|
101
101
|
});
|
|
102
|
-
describe("analyzeChangesInputSchema — stateOutputFile validation", () => {
|
|
103
|
-
it("accepts a valid absolute path", () => {
|
|
104
|
-
const result = schema.safeParse({
|
|
105
|
-
repositoryPath: "/repo",
|
|
106
|
-
stateOutputFile: "/tmp/analyze-changes-state.json",
|
|
107
|
-
});
|
|
108
|
-
expect(result.success).toBe(true);
|
|
109
|
-
});
|
|
110
|
-
it("rejects a relative path for stateOutputFile", () => {
|
|
111
|
-
// stateOutputFile must be absolute so the caller can guarantee the file location.
|
|
112
|
-
// Relative paths are silently ambiguous and should be rejected.
|
|
113
|
-
const result = schema.safeParse({
|
|
114
|
-
repositoryPath: "/repo",
|
|
115
|
-
stateOutputFile: "relative/path/state.json",
|
|
116
|
-
});
|
|
117
|
-
expect(result.success).toBe(false);
|
|
118
|
-
});
|
|
119
|
-
it("accepts absence of stateOutputFile (optional field)", () => {
|
|
120
|
-
const result = schema.safeParse({ repositoryPath: "/repo" });
|
|
121
|
-
expect(result.success).toBe(true);
|
|
122
|
-
});
|
|
123
|
-
});
|
|
124
102
|
describe("automatic old files cleanup", () => {
|
|
125
103
|
let cleanupOldFilesSpy;
|
|
126
104
|
beforeEach(() => {
|
|
@@ -128,31 +106,23 @@ describe("automatic old files cleanup", () => {
|
|
|
128
106
|
cleanupOldFilesSpy = jest.fn().mockResolvedValue(0);
|
|
129
107
|
StateManager.cleanupOldFiles = cleanupOldFilesSpy;
|
|
130
108
|
});
|
|
131
|
-
it("calls cleanupOldFiles with default temp dir
|
|
132
|
-
// This test verifies the cleanup call is made without stateDir when using default location
|
|
109
|
+
it("calls cleanupOldFiles with default temp dir", async () => {
|
|
133
110
|
expect(StateManager.cleanupOldFiles).toBeDefined();
|
|
134
111
|
await StateManager.cleanupOldFiles(24, undefined);
|
|
135
112
|
expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, undefined);
|
|
136
113
|
});
|
|
137
|
-
it("calls cleanupOldFiles with custom dir
|
|
138
|
-
// This test verifies the cleanup call is made with the directory of stateOutputFile
|
|
114
|
+
it("calls cleanupOldFiles with custom dir", async () => {
|
|
139
115
|
const customPath = "/custom/dir";
|
|
140
116
|
await StateManager.cleanupOldFiles(24, customPath);
|
|
141
117
|
expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, customPath);
|
|
142
118
|
});
|
|
143
|
-
it("calls cleanupOldFiles with empty stateTypes
|
|
144
|
-
// analyzeTestHealthTool passes [] so state files (skyramp-analysis-*, skyramp-recommendation-*)
|
|
145
|
-
// are never deleted — the caller still needs args.stateFile for skyramp_actions.
|
|
119
|
+
it("calls cleanupOldFiles with empty stateTypes", async () => {
|
|
146
120
|
await StateManager.cleanupOldFiles(24, undefined, []);
|
|
147
121
|
expect(cleanupOldFilesSpy).toHaveBeenCalledWith(24, undefined, []);
|
|
148
122
|
});
|
|
149
123
|
it("continues execution if cleanup fails", async () => {
|
|
150
|
-
// Cleanup failures should not crash the analyze flow
|
|
151
124
|
cleanupOldFilesSpy.mockRejectedValue(new Error("permission denied"));
|
|
152
|
-
await expect(StateManager.cleanupOldFiles(24).catch(() =>
|
|
153
|
-
// In the real code, this is caught and logged as a warning
|
|
154
|
-
return Promise.resolve();
|
|
155
|
-
})).resolves.toBeUndefined();
|
|
125
|
+
await expect(StateManager.cleanupOldFiles(24).catch(() => Promise.resolve())).resolves.toBeUndefined();
|
|
156
126
|
});
|
|
157
127
|
});
|
|
158
128
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -15,11 +15,11 @@ export function registerAnalyzeTestHealthTool(server) {
|
|
|
15
15
|
idempotentHint: true,
|
|
16
16
|
openWorldHint: false,
|
|
17
17
|
},
|
|
18
|
-
description: `Generate drift assessment instructions for existing tests — second step of the unified Test Health Analysis Flow
|
|
18
|
+
description: `Generate drift assessment instructions for existing tests — second step of the unified Test Health Analysis Flow.`,
|
|
19
19
|
inputSchema: {
|
|
20
20
|
stateFile: z
|
|
21
21
|
.string()
|
|
22
|
-
.describe("Path to state file
|
|
22
|
+
.describe("Path to the analyze-changes-state.json file returned by skyramp_analyze_changes (the stateFile field in its output). Do NOT pass summaryOutputFile or testbot-result.txt here."),
|
|
23
23
|
},
|
|
24
24
|
outputSchema: {
|
|
25
25
|
prompt: z.string().describe("LLM drift analysis prompt and assessment instructions."),
|
|
@@ -37,7 +37,10 @@ export function registerAnalyzeTestHealthTool(server) {
|
|
|
37
37
|
return toolError(`State file is empty or invalid: ${args.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
|
|
38
38
|
}
|
|
39
39
|
if (!repositoryPath || typeof repositoryPath !== "string") {
|
|
40
|
-
return toolError(`repositoryPath not found in state file metadata. The
|
|
40
|
+
return toolError(`repositoryPath not found in state file metadata. The stateFile parameter must be the analyze-changes-state.json path returned by skyramp_analyze_changes — not summaryOutputFile or testbot-result.txt. Re-run skyramp_analyze_changes to regenerate it.`);
|
|
41
|
+
}
|
|
42
|
+
if (!Array.isArray(stateData.existingTests)) {
|
|
43
|
+
return toolError(`stateFile does not contain test discovery data (existingTests missing). The stateFile parameter must be the analyze-changes-state.json path returned by skyramp_analyze_changes — not summaryOutputFile or testbot-result.txt.`);
|
|
41
44
|
}
|
|
42
45
|
// Skyramp tests: full drift analysis + all actions permitted.
|
|
43
46
|
// Relevant external tests (user-written, relevant to this PR's endpoints): drift analysis
|
|
@@ -3,18 +3,28 @@
|
|
|
3
3
|
*
|
|
4
4
|
* These types define the structure for test execution results
|
|
5
5
|
*/
|
|
6
|
+
/** Execution outcome for a test run recorded in the maintenance report. */
|
|
7
|
+
export declare enum TestExecutionStatus {
|
|
8
|
+
Pass = "Pass",
|
|
9
|
+
Fail = "Fail",
|
|
10
|
+
Error = "Error",
|
|
11
|
+
/** Deliberately not executed — VERIFY/IGNORE actions or DELETE (file removed). */
|
|
12
|
+
Skipped = "Skipped",
|
|
13
|
+
/** Execution was expected (UPDATE/REGENERATE) but no result was recorded. */
|
|
14
|
+
Unknown = "Unknown"
|
|
15
|
+
}
|
|
6
16
|
export interface TestExecutionData {
|
|
7
|
-
|
|
17
|
+
status: TestExecutionStatus;
|
|
8
18
|
executedAt: string;
|
|
9
19
|
duration: number;
|
|
10
20
|
errors: string[];
|
|
11
21
|
warnings: string[];
|
|
12
|
-
crashed: boolean;
|
|
13
22
|
output?: string;
|
|
14
23
|
exitCode?: number;
|
|
15
24
|
}
|
|
16
25
|
export interface TestExecutionResult extends TestExecutionData {
|
|
17
26
|
testFile: string;
|
|
27
|
+
videoPath?: string;
|
|
18
28
|
}
|
|
19
29
|
export interface BatchExecutionResult {
|
|
20
30
|
totalTests: number;
|
|
@@ -3,4 +3,14 @@
|
|
|
3
3
|
*
|
|
4
4
|
* These types define the structure for test execution results
|
|
5
5
|
*/
|
|
6
|
-
|
|
6
|
+
/** Execution outcome for a test run recorded in the maintenance report. */
|
|
7
|
+
export var TestExecutionStatus;
|
|
8
|
+
(function (TestExecutionStatus) {
|
|
9
|
+
TestExecutionStatus["Pass"] = "Pass";
|
|
10
|
+
TestExecutionStatus["Fail"] = "Fail";
|
|
11
|
+
TestExecutionStatus["Error"] = "Error";
|
|
12
|
+
/** Deliberately not executed — VERIFY/IGNORE actions or DELETE (file removed). */
|
|
13
|
+
TestExecutionStatus["Skipped"] = "Skipped";
|
|
14
|
+
/** Execution was expected (UPDATE/REGENERATE) but no result was recorded. */
|
|
15
|
+
TestExecutionStatus["Unknown"] = "Unknown";
|
|
16
|
+
})(TestExecutionStatus || (TestExecutionStatus = {}));
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { DriftAction } from "./TestAnalysis.js";
|
|
2
|
+
import type { TestExecutionStatus } from "./TestExecution.js";
|
|
3
|
+
import type { HttpMethod, TestType } from "./TestTypes.js";
|
|
4
|
+
/**
|
|
5
|
+
* Shape of the JSON report written by skyramp_submit_report and read by testbot
|
|
6
|
+
* for rendering as Markdown. All fields mirror the corresponding Zod schemas in
|
|
7
|
+
* submitReportTool.ts — keep the two in sync.
|
|
8
|
+
*/
|
|
9
|
+
export interface TestbotReport {
|
|
10
|
+
businessCaseAnalysis: string;
|
|
11
|
+
newTestsCreated: {
|
|
12
|
+
testId: string;
|
|
13
|
+
testType: TestType;
|
|
14
|
+
endpoint: string;
|
|
15
|
+
fileName: string;
|
|
16
|
+
reasoning: string;
|
|
17
|
+
description?: string;
|
|
18
|
+
scenarioFile?: string;
|
|
19
|
+
traceFile?: string;
|
|
20
|
+
frontendTrace?: string;
|
|
21
|
+
}[];
|
|
22
|
+
/** Omitted when no existing tests were found (agent omits the field). */
|
|
23
|
+
testMaintenance?: {
|
|
24
|
+
testType: TestType;
|
|
25
|
+
endpoint: string;
|
|
26
|
+
fileName: string;
|
|
27
|
+
description: string;
|
|
28
|
+
action: DriftAction;
|
|
29
|
+
beforeStatus: TestExecutionStatus;
|
|
30
|
+
beforeDetails: string;
|
|
31
|
+
afterStatus: TestExecutionStatus;
|
|
32
|
+
afterDetails: string;
|
|
33
|
+
}[];
|
|
34
|
+
testResults: {
|
|
35
|
+
testType: TestType;
|
|
36
|
+
endpoint: string;
|
|
37
|
+
status: "Pass" | "Fail" | "Skipped";
|
|
38
|
+
details: string;
|
|
39
|
+
videoPath?: string;
|
|
40
|
+
}[];
|
|
41
|
+
additionalRecommendations?: {
|
|
42
|
+
testId: string;
|
|
43
|
+
testType: TestType;
|
|
44
|
+
category?: string;
|
|
45
|
+
primaryEndpoint?: string;
|
|
46
|
+
scenarioName?: string;
|
|
47
|
+
steps: {
|
|
48
|
+
method?: HttpMethod;
|
|
49
|
+
path?: string;
|
|
50
|
+
description: string;
|
|
51
|
+
expectedStatusCode?: number;
|
|
52
|
+
requestBody?: Record<string, unknown>;
|
|
53
|
+
responseBody?: Record<string, unknown>;
|
|
54
|
+
}[];
|
|
55
|
+
description: string;
|
|
56
|
+
priority: "high" | "medium" | "low";
|
|
57
|
+
reasoning: string;
|
|
58
|
+
openApiSpec?: string;
|
|
59
|
+
backendTrace?: string;
|
|
60
|
+
frontendTrace?: string;
|
|
61
|
+
}[];
|
|
62
|
+
issuesFound: {
|
|
63
|
+
description: string;
|
|
64
|
+
severity?: "critical" | "high" | "medium" | "low";
|
|
65
|
+
}[];
|
|
66
|
+
nextSteps: string[];
|
|
67
|
+
commitMessage: string;
|
|
68
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -119,6 +119,15 @@ export class StateManager {
|
|
|
119
119
|
if (stateFilePath) {
|
|
120
120
|
this.stateFile = stateFilePath;
|
|
121
121
|
}
|
|
122
|
+
else if (stateType === "analysis" && process.env.RUNNER_TEMP) {
|
|
123
|
+
// In CI (testbot action), RUNNER_TEMP is a job-isolated dir that both the
|
|
124
|
+
// action and this tool can independently derive — no LLM input needed.
|
|
125
|
+
// TODO: interim step — remove this branch when stateFile and summaryOutputFile are merged
|
|
126
|
+
// (see getTestbotPrompt TODO). RUNNER_TEMP is GitHub Actions-specific; other CI systems
|
|
127
|
+
// (Jenkins, GitLab, Buildkite) must set it explicitly. Once merged, path flows through
|
|
128
|
+
// summaryOutputFile which is already testbot-controlled and CI-agnostic.
|
|
129
|
+
this.stateFile = path.join(process.env.RUNNER_TEMP, "skyramp", "analyze-changes-state.json");
|
|
130
|
+
}
|
|
122
131
|
else {
|
|
123
132
|
const baseDir = stateDir || os.tmpdir();
|
|
124
133
|
const prefix = STATE_FILE_PREFIXES[stateType];
|
|
@@ -260,7 +269,7 @@ export class StateManager {
|
|
|
260
269
|
* @returns Number of files deleted
|
|
261
270
|
*/
|
|
262
271
|
static async cleanupOldFiles(maxAgeHours = 24, stateDir, stateTypes) {
|
|
263
|
-
const baseDir = stateDir || os.tmpdir();
|
|
272
|
+
const baseDir = stateDir || (process.env.RUNNER_TEMP ? path.join(process.env.RUNNER_TEMP, "skyramp") : os.tmpdir());
|
|
264
273
|
const files = await fs.promises.readdir(baseDir).catch(() => []);
|
|
265
274
|
const statePrefixes = stateTypes
|
|
266
275
|
? stateTypes.map((t) => STATE_FILE_PREFIXES[t])
|
|
@@ -50,14 +50,4 @@ export interface ClassifiedEndpoints {
|
|
|
50
50
|
baseBranch: string;
|
|
51
51
|
affectedServices: string[];
|
|
52
52
|
}
|
|
53
|
-
/**
|
|
54
|
-
* Classify endpoints by cross-referencing `changedFiles` against `scannedEndpoints[].sourceFile`.
|
|
55
|
-
*
|
|
56
|
-
* This replaces `parseEndpointsFromDiff` — instead of regex-parsing diff hunks for route
|
|
57
|
-
* annotations (fragile for 15+ frameworks), it uses the already-scanned endpoint catalog
|
|
58
|
-
* which has full, resolved paths and concrete HTTP methods.
|
|
59
|
-
*
|
|
60
|
-
* For deleted files, the caller must supply endpoints recovered from the base branch
|
|
61
|
-
* (via `parseFileEndpoints` on `git show base:<file>`) in `deletedFileEndpoints`.
|
|
62
|
-
*/
|
|
63
53
|
export declare function classifyEndpointsByChangedFiles(diffData: BranchDiffData, scannedEndpoints: ScannedEndpoint[], deletedFileEndpoints?: ScannedEndpoint[]): ClassifiedEndpoints;
|
|
@@ -406,21 +406,36 @@ const GENERIC_ROUTE_BASENAMES = new Set(["api", "app", "handler", "handlers", "i
|
|
|
406
406
|
function normalizeSourcePath(filePath) {
|
|
407
407
|
return filePath.replace(/\\/g, "/");
|
|
408
408
|
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
409
|
+
function staticPathSegments(urlPath) {
|
|
410
|
+
return urlPath.split("/").filter(s => s && !/^[:{[]/.test(s)).map(s => s.toLowerCase());
|
|
411
|
+
}
|
|
412
|
+
function endpointSourceResource(ep) {
|
|
413
|
+
const sourceSegments = ep.sourceFile.split(/[\\/]/).filter(Boolean);
|
|
414
|
+
const filename = sourceSegments.at(-1)?.replace(/\.[^.]+$/, "") ?? "";
|
|
415
|
+
const normalizedFilename = filename.replace(/^(?:old|new)[-_]/i, "").toLowerCase();
|
|
416
|
+
const isDynamic = /^\[.*\]$/.test(normalizedFilename) || /^\{.*\}$/.test(normalizedFilename);
|
|
417
|
+
if (!isDynamic && !GENERIC_ROUTE_BASENAMES.has(normalizedFilename) && normalizedFilename) {
|
|
418
|
+
return normalizedFilename;
|
|
419
|
+
}
|
|
420
|
+
else if (GENERIC_ROUTE_BASENAMES.has(normalizedFilename)) {
|
|
421
|
+
// Generic filename: walk up the directory tree for a more specific ancestor name.
|
|
422
|
+
const parentSegments = sourceSegments.slice(0, -1).reverse();
|
|
423
|
+
const contextSegment = parentSegments
|
|
424
|
+
.map(s => s.replace(/\.[^.]+$/, "").replace(/^(?:old|new)[-_]/i, "").toLowerCase())
|
|
425
|
+
.find(s => s && !GENERIC_ROUTE_BASENAMES.has(s) && !/^\[.*\]$/.test(s) && !/^\{.*\}$/.test(s));
|
|
426
|
+
if (contextSegment)
|
|
427
|
+
return `${contextSegment}/${normalizedFilename}`;
|
|
428
|
+
}
|
|
429
|
+
// Fall back to the last static segment of the endpoint's URL path.
|
|
430
|
+
const segments = staticPathSegments(ep.path);
|
|
431
|
+
return segments.at(-1) ?? normalizedFilename;
|
|
432
|
+
}
|
|
433
|
+
function endpointMovedKey(path, resource, method) {
|
|
434
|
+
return method ? `${path}::${resource}::${method}` : `${path}::${resource}`;
|
|
435
|
+
}
|
|
419
436
|
export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, deletedFileEndpoints) {
|
|
420
437
|
const newFileSet = new Set(diffData.newFiles.map(normalizeSourcePath));
|
|
421
438
|
const deletedFileSet = new Set(diffData.deletedFiles.map(normalizeSourcePath));
|
|
422
|
-
// Build sourceFile -> ScannedEndpoint[] map from the post-change catalog.
|
|
423
|
-
// Git diffs use POSIX separators; scanner output can reflect OS-native paths.
|
|
424
439
|
const bySourceFile = new Map();
|
|
425
440
|
for (const ep of scannedEndpoints) {
|
|
426
441
|
const sourceKey = normalizeSourcePath(ep.sourceFile);
|
|
@@ -435,7 +450,6 @@ export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, dele
|
|
|
435
450
|
const unmatchedFiles = [];
|
|
436
451
|
for (const file of diffData.changedFiles) {
|
|
437
452
|
const fileKey = normalizeSourcePath(file);
|
|
438
|
-
// Deleted files won't appear in scanned catalog — handled separately below
|
|
439
453
|
if (deletedFileSet.has(fileKey))
|
|
440
454
|
continue;
|
|
441
455
|
const eps = bySourceFile.get(fileKey);
|
|
@@ -454,47 +468,26 @@ export function classifyEndpointsByChangedFiles(diffData, scannedEndpoints, dele
|
|
|
454
468
|
// Filter out endpoints that still exist in the current catalog as moved/refactored.
|
|
455
469
|
// Same router-relative paths can exist in sibling files, so only treat them as moved
|
|
456
470
|
// when the source resource also matches.
|
|
457
|
-
function endpointSourceResource(ep) {
|
|
458
|
-
const sourceSegments = ep.sourceFile.split(/[\\/]/).filter(Boolean);
|
|
459
|
-
const base = sourceSegments.at(-1)?.replace(/\.[^.]+$/, "") ?? "";
|
|
460
|
-
const normalizedBase = base.replace(/^(?:old|new)[-_]/i, "").toLowerCase();
|
|
461
|
-
const isDynamicBase = /^\[.*\]$/.test(normalizedBase) || /^\{.*\}$/.test(normalizedBase);
|
|
462
|
-
if (normalizedBase && !isDynamicBase && !GENERIC_ROUTE_BASENAMES.has(normalizedBase)) {
|
|
463
|
-
return normalizedBase;
|
|
464
|
-
}
|
|
465
|
-
if (normalizedBase && GENERIC_ROUTE_BASENAMES.has(normalizedBase)) {
|
|
466
|
-
const contextSegment = [...sourceSegments]
|
|
467
|
-
.slice(0, -1)
|
|
468
|
-
.reverse()
|
|
469
|
-
.map((segment) => segment.replace(/\.[^.]+$/, "").replace(/^(?:old|new)[-_]/i, "").toLowerCase())
|
|
470
|
-
.find((segment) => segment && !GENERIC_ROUTE_BASENAMES.has(segment) && !/^\[.*\]$/.test(segment) && !/^\{.*\}$/.test(segment));
|
|
471
|
-
if (contextSegment)
|
|
472
|
-
return `${contextSegment}/${normalizedBase}`;
|
|
473
|
-
}
|
|
474
|
-
const staticSegments = ep.path.split("/").filter(Boolean).filter((segment) => !/^[:{[]/.test(segment));
|
|
475
|
-
return staticSegments[staticSegments.length - 1]?.toLowerCase() ?? normalizedBase;
|
|
476
|
-
}
|
|
477
471
|
const currentMethodKeys = new Set();
|
|
478
472
|
const currentPathResourceKeys = new Set();
|
|
479
473
|
const currentMultiKeys = new Set();
|
|
480
|
-
const movedKey = (path, resource, method) => method ? `${path}::${resource}::${method}` : `${path}::${resource}`;
|
|
481
474
|
for (const current of scannedEndpoints) {
|
|
482
475
|
const currentResource = endpointSourceResource(current);
|
|
483
|
-
const pathResourceKey =
|
|
476
|
+
const pathResourceKey = endpointMovedKey(current.path, currentResource);
|
|
484
477
|
currentPathResourceKeys.add(pathResourceKey);
|
|
485
478
|
for (const method of current.methods) {
|
|
486
479
|
if (method === "MULTI")
|
|
487
480
|
currentMultiKeys.add(pathResourceKey);
|
|
488
|
-
currentMethodKeys.add(
|
|
481
|
+
currentMethodKeys.add(endpointMovedKey(current.path, currentResource, method));
|
|
489
482
|
}
|
|
490
483
|
}
|
|
491
484
|
function isMovedEndpoint(deleted, method) {
|
|
492
485
|
const deletedResource = endpointSourceResource(deleted);
|
|
493
|
-
const pathResourceKey =
|
|
486
|
+
const pathResourceKey = endpointMovedKey(deleted.path, deletedResource);
|
|
494
487
|
if (method === "MULTI")
|
|
495
488
|
return currentPathResourceKeys.has(pathResourceKey);
|
|
496
489
|
return currentMultiKeys.has(pathResourceKey) ||
|
|
497
|
-
currentMethodKeys.has(
|
|
490
|
+
currentMethodKeys.has(endpointMovedKey(deleted.path, deletedResource, method));
|
|
498
491
|
}
|
|
499
492
|
const removedEndpoints = (deletedFileEndpoints ?? [])
|
|
500
493
|
.map((ep) => ({
|
|
@@ -41,7 +41,7 @@ export type ResourceRelationships = Map<string, Set<string>>;
|
|
|
41
41
|
* the caller to fall back to heuristic pairing.
|
|
42
42
|
*/
|
|
43
43
|
export declare function inferResourceRelationships(endpoints: Array<EndpointInput>): ResourceRelationships;
|
|
44
|
-
export declare function draftScenariosFromEndpoints(endpoints: Array<EndpointInput>, newEndpoints?: Array<EndpointRef>, wsAuthType?: WorkspaceAuthType, options?: ScenarioDraftOptions): DraftedScenario[];
|
|
44
|
+
export declare function draftScenariosFromEndpoints(endpoints: Array<EndpointInput>, newEndpoints?: Array<EndpointRef>, wsAuthType?: WorkspaceAuthType, options?: ScenarioDraftOptions, removedEndpoints?: Array<EndpointRef>): DraftedScenario[];
|
|
45
45
|
/**
|
|
46
46
|
* Draft scenarios that directly test each new endpoint in the branch diff.
|
|
47
47
|
*
|
|
@@ -142,7 +142,7 @@ export function inferResourceRelationships(endpoints) {
|
|
|
142
142
|
}
|
|
143
143
|
return relationships;
|
|
144
144
|
}
|
|
145
|
-
export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuthType, options = {}) {
|
|
145
|
+
export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuthType, options = {}, removedEndpoints = []) {
|
|
146
146
|
const scenarios = [];
|
|
147
147
|
const resourceGroups = new Map();
|
|
148
148
|
for (const ep of endpoints) {
|
|
@@ -191,6 +191,28 @@ export function draftScenariosFromEndpoints(endpoints, newEndpoints = [], wsAuth
|
|
|
191
191
|
}
|
|
192
192
|
scenarios.push(...draftDiffDirectScenarios(newEndpoints, resourceGroups, wsAuthType));
|
|
193
193
|
scenarios.push(...draftAttackSurfaceExpansionScenarios(endpoints, options.changedEndpoints ?? [], wsAuthType, options.securityRelevantDiff ?? false));
|
|
194
|
+
// 404-guard scenarios for removed endpoints — absence assertions confirming
|
|
195
|
+
// the endpoint is gone. Single-step contract tests, no chaining needed.
|
|
196
|
+
for (const ep of removedEndpoints) {
|
|
197
|
+
scenarios.push({
|
|
198
|
+
scenarioName: `verify-removed-${ep.method.toLowerCase()}-${ep.path.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "")}`,
|
|
199
|
+
description: `Verify ${ep.method} ${ep.path} returns 404 — endpoint was removed in this PR`,
|
|
200
|
+
category: "breaking_change",
|
|
201
|
+
priority: "high",
|
|
202
|
+
steps: [{
|
|
203
|
+
order: 1,
|
|
204
|
+
method: ep.method,
|
|
205
|
+
path: ep.path,
|
|
206
|
+
description: `${ep.method} ${ep.path} → 404 (endpoint removed)`,
|
|
207
|
+
interactionType: "error",
|
|
208
|
+
expectedStatusCode: 404,
|
|
209
|
+
}],
|
|
210
|
+
chainingKeys: [],
|
|
211
|
+
requiresAuth: false,
|
|
212
|
+
estimatedComplexity: "simple",
|
|
213
|
+
source: ScenarioSource.CodeInferred,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
194
216
|
return capScenarios(scenarios);
|
|
195
217
|
}
|
|
196
218
|
const MAX_TOTAL_SCENARIOS = 30;
|