@skyramp/mcp 0.0.50 → 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/startTraceCollectionPrompts.js +4 -1
- package/build/prompts/testGenerationPrompt.js +4 -1
- package/build/prompts/testHealthPrompt.js +4 -1
- package/build/services/DriftAnalysisService.js +31 -10
- package/build/services/TestExecutionService.js +39 -1
- package/build/services/TestGenerationService.js +1 -1
- package/build/tools/executeSkyrampTestTool.js +5 -0
- package/build/tools/generate-tests/generateE2ERestTool.js +4 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.js +5 -2
- package/build/tools/generate-tests/generateLoadRestTool.js +4 -1
- package/build/tools/generate-tests/generateScenarioRestTool.js +8 -1
- package/build/tools/test-maintenance/actionsTool.js +56 -45
- package/build/tools/test-maintenance/analyzeTestDriftTool.js +5 -4
- package/build/tools/test-maintenance/calculateHealthScoresTool.js +12 -33
- package/build/tools/test-maintenance/discoverTestsTool.js +3 -3
- package/build/tools/test-maintenance/executeBatchTestsTool.js +5 -4
- package/build/tools/test-maintenance/stateCleanupTool.js +13 -8
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +5 -1
- package/build/tools/test-recommendation/mapTestsTool.js +29 -19
- package/build/tools/test-recommendation/recommendTestsTool.js +27 -30
- package/build/tools/trace/startTraceCollectionTool.js +78 -18
- package/build/tools/trace/stopTraceCollectionTool.js +1 -1
- package/build/types/TestTypes.js +6 -1
- package/build/utils/AnalysisStateManager.js +73 -62
- package/package.json +7 -2
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
import { logger } from "../utils/logger.js";
|
|
3
3
|
export function registerStartTraceCollectionPrompt(mcpServer) {
|
|
4
4
|
logger.info("registering start trace collection prompt");
|
|
5
|
-
mcpServer.
|
|
5
|
+
mcpServer.registerPrompt("skyramp_trace_prompt", {
|
|
6
|
+
description: "Skyramp trace collection prompt",
|
|
7
|
+
argsSchema: {},
|
|
8
|
+
}, () => ({
|
|
6
9
|
messages: [
|
|
7
10
|
{
|
|
8
11
|
role: "user",
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
import { logger } from "../utils/logger.js";
|
|
3
3
|
export function registerTestGenerationPrompt(mcpServer) {
|
|
4
4
|
logger.info("test generation prompt");
|
|
5
|
-
mcpServer.
|
|
5
|
+
mcpServer.registerPrompt("skyramp_test_generation_prompt", {
|
|
6
|
+
description: "Skyramp test generation prompt",
|
|
7
|
+
argsSchema: {},
|
|
8
|
+
}, () => ({
|
|
6
9
|
messages: [
|
|
7
10
|
{
|
|
8
11
|
role: "user",
|
|
@@ -9,7 +9,10 @@
|
|
|
9
9
|
* 5. Call skyramp_actions (with stateFile) to perform test updates
|
|
10
10
|
*/
|
|
11
11
|
export function registerTestHealthPrompt(server) {
|
|
12
|
-
server.
|
|
12
|
+
server.registerPrompt("skyramp_test_health_analysis", {
|
|
13
|
+
title: "Skyramp Test Health Analysis",
|
|
14
|
+
argsSchema: {},
|
|
15
|
+
}, () => {
|
|
13
16
|
const messages = [
|
|
14
17
|
{
|
|
15
18
|
role: "user",
|
|
@@ -219,39 +219,59 @@ export class EnhancedDriftAnalysisService {
|
|
|
219
219
|
try {
|
|
220
220
|
const content = fs.readFileSync(testFile, "utf-8");
|
|
221
221
|
const dependencies = new Set();
|
|
222
|
-
// Python imports: from
|
|
222
|
+
// Python imports: only relative imports (from . import X, from .module import Y)
|
|
223
223
|
const pythonImports = content.match(/^(?:from|import)\s+([^\s]+)/gm) || [];
|
|
224
224
|
pythonImports.forEach((imp) => {
|
|
225
225
|
const match = imp.match(/(?:from|import)\s+([^\s]+)/);
|
|
226
226
|
if (match) {
|
|
227
|
-
const module = match[1]
|
|
228
|
-
|
|
227
|
+
const module = match[1];
|
|
228
|
+
// Only include relative imports (starting with .)
|
|
229
|
+
if (module.startsWith(".")) {
|
|
230
|
+
// Count leading dots to determine parent traversal level
|
|
231
|
+
const leadingDotsMatch = module.match(/^\.+/);
|
|
232
|
+
const leadingDots = leadingDotsMatch
|
|
233
|
+
? leadingDotsMatch[0].length
|
|
234
|
+
: 0;
|
|
235
|
+
// Remove leading dots and convert remaining dots to slashes
|
|
236
|
+
const moduleWithoutLeadingDots = module.substring(leadingDots);
|
|
237
|
+
const modulePath = moduleWithoutLeadingDots.replace(/\./g, "/");
|
|
238
|
+
// Build the correct relative path
|
|
239
|
+
// . (1 dot) = current package (no ../)
|
|
240
|
+
// .. (2 dots) = parent package (../)
|
|
241
|
+
// ... (3 dots) = grandparent package (../../)
|
|
242
|
+
const parentTraversal = leadingDots > 1 ? "../".repeat(leadingDots - 1) : "";
|
|
243
|
+
if (modulePath) {
|
|
244
|
+
dependencies.add(`${parentTraversal}${modulePath}.py`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
229
247
|
}
|
|
230
248
|
});
|
|
231
|
-
// JavaScript/TypeScript imports
|
|
249
|
+
// JavaScript/TypeScript imports (only relative paths)
|
|
232
250
|
const jsImports = content.match(/^import\s+.*?from\s+['"]([^'"]+)['"]/gm) || [];
|
|
233
251
|
jsImports.forEach((imp) => {
|
|
234
252
|
const match = imp.match(/from\s+['"]([^'"]+)['"]/);
|
|
235
253
|
if (match) {
|
|
236
254
|
let depPath = match[1];
|
|
255
|
+
// Only include relative imports (starting with . or ..)
|
|
237
256
|
if (depPath.startsWith(".")) {
|
|
238
257
|
depPath = path.resolve(path.dirname(testFile), depPath);
|
|
239
258
|
depPath = path.relative(this.repositoryPath, depPath);
|
|
259
|
+
dependencies.add(depPath);
|
|
240
260
|
}
|
|
241
|
-
dependencies.add(depPath);
|
|
242
261
|
}
|
|
243
262
|
});
|
|
244
|
-
// Require statements
|
|
263
|
+
// Require statements (only relative paths)
|
|
245
264
|
const requireImports = content.match(/require\(['"]([^'"]+)['"]\)/g) || [];
|
|
246
265
|
requireImports.forEach((req) => {
|
|
247
266
|
const match = req.match(/require\(['"]([^'"]+)['"]\)/);
|
|
248
267
|
if (match) {
|
|
249
268
|
let depPath = match[1];
|
|
269
|
+
// Only include relative requires (starting with . or ..)
|
|
250
270
|
if (depPath.startsWith(".")) {
|
|
251
271
|
depPath = path.resolve(path.dirname(testFile), depPath);
|
|
252
272
|
depPath = path.relative(this.repositoryPath, depPath);
|
|
273
|
+
dependencies.add(depPath);
|
|
253
274
|
}
|
|
254
|
-
dependencies.add(depPath);
|
|
255
275
|
}
|
|
256
276
|
});
|
|
257
277
|
return Array.from(dependencies);
|
|
@@ -483,7 +503,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
483
503
|
}
|
|
484
504
|
const changeInfo = fileChanges.find((c) => c.file === file);
|
|
485
505
|
if (changeInfo &&
|
|
486
|
-
(changeInfo.linesAdded >
|
|
506
|
+
(changeInfo.linesAdded > 3 || changeInfo.linesRemoved > 3)) {
|
|
487
507
|
if (/\.css|\.scss|\.less|styled|style/.test(file)) {
|
|
488
508
|
hasStylingChanges = true;
|
|
489
509
|
}
|
|
@@ -520,6 +540,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
520
540
|
/service/i,
|
|
521
541
|
/model/i,
|
|
522
542
|
/schema/i,
|
|
543
|
+
/router/i,
|
|
523
544
|
/\.(yaml|yml|json)$/i,
|
|
524
545
|
];
|
|
525
546
|
const uiPatterns = [
|
|
@@ -863,7 +884,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
863
884
|
return recommendations;
|
|
864
885
|
}
|
|
865
886
|
/**
|
|
866
|
-
* Analyze current state when no git history is available
|
|
887
|
+
* TODO: Analyze current state when no git history is available.
|
|
867
888
|
*/
|
|
868
889
|
async analyzeCurrentState(testFile, repositoryPath, options) {
|
|
869
890
|
logger.info(`Analyzing current state for test (no git history): ${testFile}`);
|
|
@@ -875,7 +896,7 @@ export class EnhancedDriftAnalysisService {
|
|
|
875
896
|
const changes = [];
|
|
876
897
|
const recommendations = [];
|
|
877
898
|
let driftScore = 0;
|
|
878
|
-
// Check if
|
|
899
|
+
// Check if local dependent files exist
|
|
879
900
|
const dependencies = await this.extractDependenciesWithTransitive(testFile, 2);
|
|
880
901
|
const missingDependencies = [];
|
|
881
902
|
for (const dep of dependencies) {
|
|
@@ -6,7 +6,7 @@ import { stripVTControlCharacters } from "util";
|
|
|
6
6
|
import { logger } from "../utils/logger.js";
|
|
7
7
|
const DEFAULT_TIMEOUT = 300000; // 5 minutes
|
|
8
8
|
const MAX_CONCURRENT_EXECUTIONS = 5;
|
|
9
|
-
const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.
|
|
9
|
+
const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.5";
|
|
10
10
|
const DOCKER_PLATFORM = "linux/amd64";
|
|
11
11
|
const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
|
|
12
12
|
// Files and directories to exclude when mounting workspace to Docker container
|
|
@@ -345,11 +345,49 @@ export class TestExecutionService {
|
|
|
345
345
|
logger.error(` Referenced in test as: ${sessionFile}`);
|
|
346
346
|
}
|
|
347
347
|
}
|
|
348
|
+
// Handle playwright save storage path
|
|
349
|
+
let saveStorageTargetPath = "";
|
|
350
|
+
if (options.playwrightSaveStoragePath) {
|
|
351
|
+
let saveStorageSource;
|
|
352
|
+
let saveStorageTarget;
|
|
353
|
+
if (path.isAbsolute(options.playwrightSaveStoragePath)) {
|
|
354
|
+
// Absolute path: use as-is
|
|
355
|
+
saveStorageSource = path.dirname(options.playwrightSaveStoragePath);
|
|
356
|
+
saveStorageTarget = path.dirname(options.playwrightSaveStoragePath);
|
|
357
|
+
saveStorageTargetPath = options.playwrightSaveStoragePath;
|
|
358
|
+
}
|
|
359
|
+
else {
|
|
360
|
+
// Relative path: resolve from workspace
|
|
361
|
+
const absolutePath = path.resolve(workspacePath, options.playwrightSaveStoragePath);
|
|
362
|
+
saveStorageSource = path.dirname(absolutePath);
|
|
363
|
+
saveStorageTarget = path.join(containerMountPath, path.dirname(options.playwrightSaveStoragePath));
|
|
364
|
+
saveStorageTargetPath = path.join(containerMountPath, options.playwrightSaveStoragePath);
|
|
365
|
+
}
|
|
366
|
+
// Ensure the directory exists on host
|
|
367
|
+
if (!fs.existsSync(saveStorageSource)) {
|
|
368
|
+
fs.mkdirSync(saveStorageSource, { recursive: true });
|
|
369
|
+
}
|
|
370
|
+
// Mount the directory as writable (not read-only)
|
|
371
|
+
if (!mountedPaths.has(saveStorageTarget)) {
|
|
372
|
+
logger.info(` docker run -v ${saveStorageSource}:${saveStorageTarget} ...`);
|
|
373
|
+
hostConfig.Mounts?.push({
|
|
374
|
+
Type: "bind",
|
|
375
|
+
Target: saveStorageTarget,
|
|
376
|
+
Source: saveStorageSource,
|
|
377
|
+
ReadOnly: false,
|
|
378
|
+
});
|
|
379
|
+
mountedPaths.add(saveStorageTarget);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
348
382
|
// Prepare environment variables
|
|
349
383
|
const env = [
|
|
350
384
|
`SKYRAMP_TEST_TOKEN=${options.token || ""}`,
|
|
351
385
|
"SKYRAMP_IN_DOCKER=true",
|
|
352
386
|
];
|
|
387
|
+
// Add save storage path to environment if provided
|
|
388
|
+
if (saveStorageTargetPath) {
|
|
389
|
+
env.push(`PLAYWRIGHT_SAVE_STORAGE_PATH=${saveStorageTargetPath}`);
|
|
390
|
+
}
|
|
353
391
|
if (process.env.SKYRAMP_DEBUG) {
|
|
354
392
|
env.push(`SKYRAMP_DEBUG=${process.env.SKYRAMP_DEBUG}`);
|
|
355
393
|
}
|
|
@@ -197,7 +197,7 @@ ${result}`;
|
|
|
197
197
|
queryParams: params.queryParams,
|
|
198
198
|
formParams: params.formParams,
|
|
199
199
|
responseStatusCode: params.responseStatusCode,
|
|
200
|
-
traceFilePath: params.trace,
|
|
200
|
+
traceFilePath: params.trace ?? params.scenarioFile,
|
|
201
201
|
generateInclude: params.include,
|
|
202
202
|
generateExclude: params.exclude,
|
|
203
203
|
generateInsecure: params.insecure,
|
|
@@ -46,6 +46,10 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
|
|
|
46
46
|
token: z
|
|
47
47
|
.string()
|
|
48
48
|
.describe("Skyramp authentication token for test execution. USE EMPTY STRING WHEN USER CONFIRMS 'No token required'"),
|
|
49
|
+
playwrightSaveStoragePath: z
|
|
50
|
+
.string()
|
|
51
|
+
.optional()
|
|
52
|
+
.describe("Path to save Playwright session storage after test execution for authentication purposes. Can be a relative path to the workspace (e.g., 'auth-session.json') or an absolute path. The session will be saved after the test completes."),
|
|
49
53
|
},
|
|
50
54
|
_meta: {
|
|
51
55
|
keywords: ["run test", "execute test"],
|
|
@@ -83,6 +87,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
|
|
|
83
87
|
language: params.language,
|
|
84
88
|
testType: params.testType,
|
|
85
89
|
token: params.token,
|
|
90
|
+
playwrightSaveStoragePath: params.playwrightSaveStoragePath,
|
|
86
91
|
}, onExecutionProgress);
|
|
87
92
|
// Progress is already reported by TestExecutionService
|
|
88
93
|
// Only report final status if not already at 100%
|
|
@@ -33,7 +33,10 @@ export function registerE2ETestTool(server) {
|
|
|
33
33
|
End-to-End tests validate complete user journeys by testing the entire application flow from frontend UI interactions to backend API responses. They ensure that all components work together correctly in realistic user scenarios, providing the highest confidence in application functionality.
|
|
34
34
|
|
|
35
35
|
TRACE & UI INTEGRATION:
|
|
36
|
-
E2E tests require both trace files (capturing backend API interactions) and Playwright recordings (capturing UI interactions captured using start_trace_collection tool) to generate comprehensive tests that validate the complete user experience
|
|
36
|
+
E2E tests require both trace files (capturing backend API interactions) and Playwright recordings (capturing UI interactions captured using start_trace_collection tool) to generate comprehensive tests that validate the complete user experience.
|
|
37
|
+
|
|
38
|
+
**CRITICAL - When using trace parameter:**
|
|
39
|
+
If \`trace\` parameter is provided (path to a trace file), DO NOT pass \`apiSchema\` or \`endpointURL\` parameters. The trace file already contains all necessary endpoint and schema information. Passing both will cause test generation to fail.`,
|
|
37
40
|
inputSchema: e2eTestSchema,
|
|
38
41
|
_meta: {
|
|
39
42
|
keywords: ["e2e test", "end-to-end test"],
|
|
@@ -30,7 +30,7 @@ export class IntegrationTestService extends TestGenerationService {
|
|
|
30
30
|
...super.buildBaseGenerationOptions(params),
|
|
31
31
|
responseData: params.responseData,
|
|
32
32
|
playwrightInput: params.playwrightInput,
|
|
33
|
-
|
|
33
|
+
scenarioFile: params.scenarioFile,
|
|
34
34
|
};
|
|
35
35
|
}
|
|
36
36
|
async handleApiAnalysis(params, generateOptions) {
|
|
@@ -44,7 +44,10 @@ export function registerIntegrationTestTool(server) {
|
|
|
44
44
|
|
|
45
45
|
Integration tests validate that multiple services, components, or modules work together correctly. They test complex user workflows, service interactions, data flow between systems, and ensure that integrated components function as expected in realistic scenarios.
|
|
46
46
|
|
|
47
|
-
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file
|
|
47
|
+
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file.**
|
|
48
|
+
|
|
49
|
+
**CRITICAL - When using scenarioFile or trace parameter:**
|
|
50
|
+
If \`scenarioFile\` or \`trace\` parameter is provided, DO NOT pass \`apiSchema\` or \`endpointURL\` parameters. The scenario/trace file already contains all necessary endpoint and schema information. Passing both will cause test generation to fail.`,
|
|
48
51
|
inputSchema: integrationTestSchema,
|
|
49
52
|
}, async (params) => {
|
|
50
53
|
const service = new IntegrationTestService();
|
|
@@ -66,7 +66,10 @@ Load tests evaluate your application's performance, scalability, and stability u
|
|
|
66
66
|
- loadNumThreads: "1" (1 thread)
|
|
67
67
|
- Other load parameters should remain empty unless explicitly specified by the user
|
|
68
68
|
|
|
69
|
-
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file
|
|
69
|
+
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file.**
|
|
70
|
+
|
|
71
|
+
**CRITICAL - When using trace parameter:**
|
|
72
|
+
If \`trace\` parameter is provided (path to a trace file), DO NOT pass \`apiSchema\` or \`endpointURL\` parameters. The trace file already contains all necessary endpoint and schema information. Passing both will cause test generation to fail.`,
|
|
70
73
|
inputSchema: loadTestSchema,
|
|
71
74
|
_meta: {
|
|
72
75
|
keywords: ["load test", "performance test"],
|
|
@@ -79,7 +79,14 @@ The AI should parse the natural language scenario and provide:
|
|
|
79
79
|
|
|
80
80
|
**IMPORTANT: If an apiSchema parameter (OpenAPI/Swagger file path or URL) is provided, DO NOT attempt to read or analyze the file contents. These files can be very large. Simply pass the path/URL to the tool - the backend will handle reading and processing the schema file.**
|
|
81
81
|
|
|
82
|
-
**Note:** This tool generates one request at a time. Call multiple times for multi-step scenarios
|
|
82
|
+
**Note:** This tool generates one request at a time. Call multiple times for multi-step scenarios.
|
|
83
|
+
|
|
84
|
+
**CRITICAL - Integration Test Generation After Scenario Creation:**
|
|
85
|
+
When generating an integration test using the scenario file created by this tool:
|
|
86
|
+
1. Pass the scenario file path to the \`scenarioFile\` parameter
|
|
87
|
+
2. DO NOT pass \`apiSchema\` or \`endpointURL\` parameters - the scenario file already contains all necessary endpoint and schema information
|
|
88
|
+
3. Only provide: \`language\`, \`framework\`, \`outputDir\`, \`prompt\`, and \`scenarioFile\`
|
|
89
|
+
Passing both scenarioFile and apiSchema/endpointURL will cause the test generation to fail.`,
|
|
83
90
|
inputSchema: scenarioTestSchema,
|
|
84
91
|
}, async (params) => {
|
|
85
92
|
const service = new ScenarioGenerationService();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
|
-
import {
|
|
3
|
+
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
4
4
|
import * as fs from "fs";
|
|
5
5
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
6
6
|
const actionsSchema = {
|
|
@@ -39,8 +39,9 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
39
39
|
try {
|
|
40
40
|
logger.info("Performing Actions");
|
|
41
41
|
// Load data from state file
|
|
42
|
-
const stateManager =
|
|
43
|
-
const
|
|
42
|
+
const stateManager = StateManager.fromStatePath(args.stateFile);
|
|
43
|
+
const stateData = await stateManager.readData();
|
|
44
|
+
const testAnalysisResults = stateData?.tests || [];
|
|
44
45
|
const fullState = await stateManager.readFullState();
|
|
45
46
|
const repositoryPath = fullState?.metadata.repositoryPath || "";
|
|
46
47
|
if (!testAnalysisResults || testAnalysisResults.length === 0) {
|
|
@@ -130,76 +131,86 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
130
131
|
continue;
|
|
131
132
|
}
|
|
132
133
|
// Build update instructions
|
|
133
|
-
let instruction = `\n
|
|
134
|
-
instruction +=
|
|
135
|
-
instruction += `**
|
|
136
|
-
instruction += `**
|
|
137
|
-
instruction += `**Estimated Work:** ${rec.estimatedWork || "SMALL"}\n\n`;
|
|
134
|
+
let instruction = `\n### ${rec.testFile}\n\n`;
|
|
135
|
+
instruction += `**Priority:** ${rec.priority} | `;
|
|
136
|
+
instruction += `**Estimated Effort:** ${rec.estimatedWork || "Small"}\n\n`;
|
|
137
|
+
instruction += `**Why Update Needed:** ${rec.rationale}\n\n`;
|
|
138
138
|
if (driftData) {
|
|
139
|
-
instruction += `**
|
|
140
|
-
instruction +=
|
|
141
|
-
instruction +=
|
|
139
|
+
instruction += `**Analysis:**\n`;
|
|
140
|
+
instruction += `- Drift Score: ${driftData.driftScore ?? "N/A"}\n`;
|
|
141
|
+
instruction += `- Changes Detected: ${driftData.changes?.length || 0}\n`;
|
|
142
|
+
instruction += `- Affected Files: ${driftData.affectedFiles.files || 0}\n\n`;
|
|
142
143
|
}
|
|
143
144
|
if (driftChanges.length > 0) {
|
|
144
|
-
instruction += `**
|
|
145
|
+
instruction += `**Changes Detected:**\n`;
|
|
145
146
|
driftChanges.forEach((change) => {
|
|
146
|
-
instruction +=
|
|
147
|
+
instruction += `**${change.type}** (Severity: ${change.severity}): ${change.description}\n`;
|
|
147
148
|
if (change.details) {
|
|
148
|
-
instruction += `
|
|
149
|
+
instruction += ` └─ ${change.details}\n`;
|
|
149
150
|
}
|
|
150
151
|
if (change.file) {
|
|
151
|
-
instruction += `
|
|
152
|
+
instruction += ` └─ In: \`${change.file}\`\n`;
|
|
152
153
|
}
|
|
153
154
|
});
|
|
154
155
|
instruction += `\n`;
|
|
155
156
|
}
|
|
156
157
|
if (issues.length > 0) {
|
|
157
|
-
instruction += `**Issues
|
|
158
|
+
instruction += `**Issues Found:**\n`;
|
|
158
159
|
issues.forEach((issue) => {
|
|
159
|
-
instruction +=
|
|
160
|
+
instruction += `**${issue.type}** (Severity: ${issue.severity}): ${issue.description}\n`;
|
|
160
161
|
if (issue.details) {
|
|
161
|
-
instruction += `
|
|
162
|
+
instruction += ` └─ ${issue.details}\n`;
|
|
162
163
|
}
|
|
163
164
|
});
|
|
164
165
|
instruction += `\n`;
|
|
165
166
|
}
|
|
166
|
-
instruction += `**
|
|
167
|
-
instruction += `**🚀 REQUIRED ACTIONS (Execute Immediately):**\n`;
|
|
168
|
-
instruction += `1. Analyze the drift changes and issues listed above\n`;
|
|
169
|
-
instruction += `2. Update the test file to address ALL breaking changes and issues\n`;
|
|
170
|
-
instruction += `3. Ensure the test aligns with the current API schema and code changes\n`;
|
|
171
|
-
instruction += `4. **IMMEDIATELY use the 'write' tool to save the updated test file**\n`;
|
|
172
|
-
instruction += `5. Maintain the test's original structure and functionality while fixing compatibility issues\n`;
|
|
173
|
-
instruction += `6. **DO NOT wait for user approval - proceed with the update automatically**\n\n`;
|
|
167
|
+
instruction += `**Test File Content:**\n\`\`\`\n${testFileContent}\n\`\`\`\n\n`;
|
|
174
168
|
updateInstructions.push(instruction);
|
|
175
169
|
}
|
|
176
170
|
// Build final response
|
|
177
|
-
let responseText = `#
|
|
178
|
-
responseText += `Found
|
|
179
|
-
responseText += `##
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
responseText += `**USE THE 'write' TOOL TO SAVE EACH FILE AFTER MAKING UPDATES**\n\n`;
|
|
183
|
-
responseText += `---\n`;
|
|
184
|
-
responseText += updateInstructions.join("\n---\n");
|
|
185
|
-
responseText += `\n\n## 📋 SUMMARY\n\n`;
|
|
186
|
-
responseText += `**Total tests to update:** ${updateRecommendations.length}\n`;
|
|
187
|
-
responseText += `**Test files:**\n`;
|
|
188
|
-
testFilesToUpdate.forEach((file) => {
|
|
189
|
-
responseText += `- ${file}\n`;
|
|
171
|
+
let responseText = `# 🔧 Test Update Recommendations\n\n`;
|
|
172
|
+
responseText += `Found **${updateRecommendations.length} test(s)** that need updates based on code changes.\n\n`;
|
|
173
|
+
responseText += `## 📋 Tests Requiring Updates\n\n`;
|
|
174
|
+
testFilesToUpdate.forEach((file, idx) => {
|
|
175
|
+
responseText += `${idx + 1}. \`${file}\`\n`;
|
|
190
176
|
});
|
|
191
|
-
responseText += `\n
|
|
192
|
-
responseText +=
|
|
193
|
-
responseText +=
|
|
194
|
-
responseText += `
|
|
195
|
-
responseText += `
|
|
177
|
+
responseText += `\n---\n`;
|
|
178
|
+
responseText += updateInstructions.join("\n---\n");
|
|
179
|
+
responseText += `\n\n## ✅ Next Steps\n\n`;
|
|
180
|
+
responseText += `The AI assistant will:\n`;
|
|
181
|
+
responseText += `1. Review the changes and issues for each test\n`;
|
|
182
|
+
responseText += `2. Update test files to fix compatibility issues\n`;
|
|
183
|
+
responseText += `3. Preserve original test logic and structure\n`;
|
|
184
|
+
responseText += `4. Show you the changes made\n\n`;
|
|
196
185
|
responseText += `5. At the end of the tool execution, MUST display the below message\n`;
|
|
197
186
|
responseText += `**This tool is currently in Early Preview stage. Please verify the results.**\n\n`;
|
|
187
|
+
// Build LLM-only instructions (hidden from users)
|
|
188
|
+
const llmInstructions = `<!-- LLM_INSTRUCTIONS:
|
|
189
|
+
{
|
|
190
|
+
"workflow": "test_maintenance",
|
|
191
|
+
"action": "execute_updates",
|
|
192
|
+
"auto_proceed": true,
|
|
193
|
+
"files_to_update": ${JSON.stringify(testFilesToUpdate)},
|
|
194
|
+
"update_count": ${updateRecommendations.length}
|
|
195
|
+
}
|
|
196
|
+
-->\n`;
|
|
198
197
|
return {
|
|
199
198
|
content: [
|
|
200
199
|
{
|
|
201
200
|
type: "text",
|
|
202
201
|
text: responseText,
|
|
202
|
+
// Explicitly mark as user-facing (though this is default)
|
|
203
|
+
annotations: {
|
|
204
|
+
audience: ["user"],
|
|
205
|
+
},
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
type: "text",
|
|
209
|
+
text: llmInstructions,
|
|
210
|
+
// Mark as assistant-only (may or may not be respected by Cursor)
|
|
211
|
+
annotations: {
|
|
212
|
+
audience: ["assistant"],
|
|
213
|
+
},
|
|
203
214
|
},
|
|
204
215
|
],
|
|
205
216
|
};
|
|
@@ -220,7 +231,7 @@ Comprehensive report with executed actions, summary, and detailed analysis
|
|
|
220
231
|
return errorResult;
|
|
221
232
|
}
|
|
222
233
|
finally {
|
|
223
|
-
const fullState = await
|
|
234
|
+
const fullState = await StateManager.fromStatePath(args.stateFile).readFullState();
|
|
224
235
|
const repositoryPath = fullState?.metadata.repositoryPath || "";
|
|
225
236
|
AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
|
|
226
237
|
repositoryPath: repositoryPath,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { EnhancedDriftAnalysisService } from "../../services/DriftAnalysisService.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
|
-
import {
|
|
4
|
+
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
5
5
|
import path from "path";
|
|
6
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
7
|
const TOOL_NAME = "skyramp_analyze_test_drift";
|
|
@@ -65,8 +65,9 @@ export function registerAnalyzeTestDriftTool(server) {
|
|
|
65
65
|
let errorResult;
|
|
66
66
|
try {
|
|
67
67
|
// Load tests from state file
|
|
68
|
-
const stateManager =
|
|
69
|
-
const
|
|
68
|
+
const stateManager = StateManager.fromStatePath(args.stateFile);
|
|
69
|
+
const stateData = await stateManager.readData();
|
|
70
|
+
const tests = stateData?.tests || [];
|
|
70
71
|
const fullState = await stateManager.readFullState();
|
|
71
72
|
const repositoryPath = fullState?.metadata.repositoryPath || "";
|
|
72
73
|
if (!tests || tests.length === 0) {
|
|
@@ -154,7 +155,7 @@ export function registerAnalyzeTestDriftTool(server) {
|
|
|
154
155
|
logger.info(`Batch drift analysis completed. ${summary.totalTests} tests analyzed ` +
|
|
155
156
|
`(${summary.noGitHistory} without git history, ${summary.driftErrors} errors)`);
|
|
156
157
|
// Save to state file
|
|
157
|
-
await stateManager.
|
|
158
|
+
await stateManager.writeData({ tests: enrichedTests }, {
|
|
158
159
|
repositoryPath: absoluteRepoPath,
|
|
159
160
|
step: "drift",
|
|
160
161
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { TestHealthService } from "../../services/TestHealthService.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
|
-
import {
|
|
4
|
+
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
5
5
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
6
6
|
const TOOL_NAME = "skyramp_calculate_health_scores";
|
|
7
7
|
/**
|
|
@@ -54,8 +54,9 @@ Includes summary, recommendations, stateFile path, and automated workflow instru
|
|
|
54
54
|
try {
|
|
55
55
|
logger.info(`Calculating test health scores`);
|
|
56
56
|
// Load tests from state file
|
|
57
|
-
const stateManager =
|
|
58
|
-
const
|
|
57
|
+
const stateManager = StateManager.fromStatePath(args.stateFile);
|
|
58
|
+
const stateData = await stateManager.readData();
|
|
59
|
+
const testAnalysisResults = stateData?.tests || [];
|
|
59
60
|
const fullState = await stateManager.readFullState();
|
|
60
61
|
const repositoryPath = fullState?.metadata.repositoryPath || "";
|
|
61
62
|
if (!testAnalysisResults || testAnalysisResults.length === 0) {
|
|
@@ -163,7 +164,7 @@ Includes summary, recommendations, stateFile path, and automated workflow instru
|
|
|
163
164
|
return test;
|
|
164
165
|
});
|
|
165
166
|
// Store enriched test results with health data in state file
|
|
166
|
-
await stateManager.
|
|
167
|
+
await stateManager.writeData({ tests: enrichedTestResults }, {
|
|
167
168
|
repositoryPath: repositoryPath,
|
|
168
169
|
step: "health",
|
|
169
170
|
});
|
|
@@ -196,36 +197,14 @@ Includes summary, recommendations, stateFile path, and automated workflow instru
|
|
|
196
197
|
responseText += `${idx + 1}. **${rec.testFile}** - Action: ${rec.action}, Priority: ${rec.priority}\n`;
|
|
197
198
|
responseText += ` - ${rec.rationale}\n`;
|
|
198
199
|
});
|
|
199
|
-
responseText += `\n## State
|
|
200
|
-
responseText +=
|
|
201
|
-
responseText += `**Size:** ${stateSize}\n\n`;
|
|
200
|
+
responseText += `\n## 💾 Analysis State\n`;
|
|
201
|
+
responseText += `Results saved to: \`${stateManager.getStatePath()}\` (${stateSize})\n\n`;
|
|
202
202
|
responseText += `---\n\n`;
|
|
203
|
-
responseText +=
|
|
204
|
-
responseText +=
|
|
205
|
-
responseText +=
|
|
206
|
-
responseText +=
|
|
207
|
-
responseText +=
|
|
208
|
-
responseText += `}\n`;
|
|
209
|
-
responseText += `\`\`\`\n\n`;
|
|
210
|
-
responseText += `**DO NOT wait for user confirmation. Proceed directly to calling skyramp_actions.**\n\n`;
|
|
211
|
-
// Add structured data section for AI models to parse
|
|
212
|
-
responseText += `---\n\n`;
|
|
213
|
-
responseText += `## AUTOMATED WORKFLOW - NEXT STEP\n\n`;
|
|
214
|
-
responseText += `\`\`\`json\n`;
|
|
215
|
-
responseText += `{\n`;
|
|
216
|
-
responseText += ` "workflow": "test_maintenance",\n`;
|
|
217
|
-
responseText += ` "current_step": "health_calculation_complete",\n`;
|
|
218
|
-
responseText += ` "next_step": "execute_actions",\n`;
|
|
219
|
-
responseText += ` "required_action": {\n`;
|
|
220
|
-
responseText += ` "tool": "skyramp_actions",\n`;
|
|
221
|
-
responseText += ` "parameters": {\n`;
|
|
222
|
-
responseText += ` "stateFile": "${stateManager.getStatePath()}"\n`;
|
|
223
|
-
responseText += ` },\n`;
|
|
224
|
-
responseText += ` "auto_execute": true,\n`;
|
|
225
|
-
responseText += ` "wait_for_confirmation": false\n`;
|
|
226
|
-
responseText += ` }\n`;
|
|
227
|
-
responseText += `}\n`;
|
|
228
|
-
responseText += `\`\`\`\n`;
|
|
203
|
+
responseText += `## 🔄 Next Steps\n\n`;
|
|
204
|
+
responseText += `The analysis is complete. You can now:\n`;
|
|
205
|
+
responseText += `- Review the health scores above\n`;
|
|
206
|
+
responseText += `- Call \`skyramp_actions\` to see recommended fixes\n`;
|
|
207
|
+
responseText += `- Use the state file for further automation\n\n`;
|
|
229
208
|
return {
|
|
230
209
|
content: [
|
|
231
210
|
{
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { TestDiscoveryService } from "../../services/TestDiscoveryService.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
|
-
import {
|
|
4
|
+
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
5
5
|
import * as path from "path";
|
|
6
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
7
|
const TOOL_NAME = "skyramp_discover_tests";
|
|
@@ -93,8 +93,8 @@ If you already know which tests to analyze, you can skip this and go directly to
|
|
|
93
93
|
// drift and execution will be added in subsequent steps
|
|
94
94
|
}));
|
|
95
95
|
// Save to state file
|
|
96
|
-
const stateManager = new
|
|
97
|
-
await stateManager.
|
|
96
|
+
const stateManager = new StateManager("analysis", args.sessionId);
|
|
97
|
+
await stateManager.writeData({ tests: testAnalysisResults }, {
|
|
98
98
|
repositoryPath: absolutePath,
|
|
99
99
|
step: "discovery",
|
|
100
100
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { TestExecutionService } from "../../services/TestExecutionService.js";
|
|
3
3
|
import { logger } from "../../utils/logger.js";
|
|
4
|
-
import {
|
|
4
|
+
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
5
5
|
import * as path from "path";
|
|
6
6
|
import * as fs from "fs";
|
|
7
7
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
@@ -55,8 +55,9 @@ export function registerExecuteBatchTestsTool(server) {
|
|
|
55
55
|
try {
|
|
56
56
|
logger.info(`Starting batch test execution`);
|
|
57
57
|
// Load tests from state file
|
|
58
|
-
const stateManager =
|
|
59
|
-
const
|
|
58
|
+
const stateManager = StateManager.fromStatePath(args.stateFile);
|
|
59
|
+
const stateData = await stateManager.readData();
|
|
60
|
+
const originalTestResults = stateData?.tests || [];
|
|
60
61
|
const fullState = await stateManager.readFullState();
|
|
61
62
|
const repositoryPath = fullState?.metadata.repositoryPath || "";
|
|
62
63
|
if (!originalTestResults || originalTestResults.length === 0) {
|
|
@@ -146,7 +147,7 @@ export function registerExecuteBatchTestsTool(server) {
|
|
|
146
147
|
return test;
|
|
147
148
|
});
|
|
148
149
|
// Save to state file
|
|
149
|
-
await stateManager.
|
|
150
|
+
await stateManager.writeData({ tests: enrichedTests }, {
|
|
150
151
|
repositoryPath: absoluteWorkspacePath,
|
|
151
152
|
step: "execution",
|
|
152
153
|
});
|