@skyramp/mcp 0.0.37 → 0.0.39

package/build/prompts/testGenerationPrompt.js

@@ -18,6 +18,13 @@ export function registerTestGenerationPrompt(mcpServer) {
 - CRITICAL: UI, INTEGRATION, E2E TESTS MUST BE MODULARIZED USING skyramp_modularization TOOL. ALWAYS ADD A TASK TO MODULARIZE THE TEST USING skyramp_modularization TOOL AFTER GENERATING THESE(UI, INTEGRATION, E2E) TESTS.
 - **CRITICAL: skyramp_reuse_code TOOL MUST BE CALLED IF DURING THE TEST GENERATION THE CODE REUSE FLAG IS SET TO TRUE EXPLICITLY BY THE USER.**
 
+**MANDATORY RULES**:
+1. **Priority Scores Must Remain Unchanged**: When a test type is missing required inputs (e.g., Playwright recordings, traces), **DO NOT**:
+   - Mark the test as "blocked" in the output
+   - Adjust or reduce the priority score
+   - Exclude it from recommendations if it ranks in the top 7
+2. DO NOT CREATE ANY .json or .md file during repository analysis, test mapping, or test recommendation.
+
 **CONTRACT TEST:**
 - Purpose: Ensures a service is properly communicating with another service
 - Requirements: At least ONE of the following combinations:

package/build/services/TestGenerationService.js

@@ -1,6 +1,5 @@
 import { SkyrampClient } from "@skyramp/skyramp";
 import { analyzeOpenAPIWithGivenEndpoint } from "../utils/analyze-openapi.js";
-import * as fs from "fs";
 import { getPathParameterValidationError, OUTPUT_DIR_FIELD_NAME, PATH_PARAMS_FIELD_NAME, QUERY_PARAMS_FIELD_NAME, FORM_PARAMS_FIELD_NAME, validateParams, validatePath, validateRequestData, TELEMETRY_entrypoint_FIELD_NAME, } from "../utils/utils.js";
 import { getLanguageSteps } from "../utils/language-helper.js";
 import { logger } from "../utils/logger.js";
@@ -27,11 +26,6 @@ export class TestGenerationService {
             if (apiAnalysisResult) {
                 return apiAnalysisResult;
             }
-            if (params.scenarioFile) {
-                //read file and convert and pass to generateOptions as `rawTrace`
-                const scenarioFile = await fs.promises.readFile(params.scenarioFile, "utf8");
-                generateOptions.rawTrace = scenarioFile;
-            }
             const result = await this.executeGeneration(generateOptions);
             const testType = this.getTestType();
             const languageSteps = getLanguageSteps({

package/build/tools/executeSkyrampTestTool.js

@@ -5,7 +5,7 @@ import { Writable } from "stream";
 import { logger } from "../utils/logger.js";
 import { stripVTControlCharacters } from "util";
 import fs from "fs";
-const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.2.28";
+const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.2.29";
 const DOCKER_PLATFORM = "linux/amd64";
 export function registerExecuteSkyrampTestTool(server) {
     server.registerTool("skyramp_execute_test", {
@@ -105,6 +105,12 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             "SKYRAMP_TEST_TOKEN=" + params.token,
             "SKYRAMP_IN_DOCKER=true",
         ];
+        if (process.env.SKYRAMP_DEBUG) {
+            env.push("SKYRAMP_DEBUG=" + process.env.SKYRAMP_DEBUG);
+        }
+        if (process.env.API_KEY) {
+            env.push("API_KEY=" + process.env.API_KEY);
+        }
         var output = "";
         class DockerStream extends Writable {
             _write(data, encode, cb) {

package/build/tools/generate-tests/generateIntegrationRestTool.js

@@ -11,12 +11,13 @@ const integrationTestSchema = z
     trace: baseTraceSchema.shape.trace.optional(),
     include: baseTraceSchema.shape.include.optional(),
     exclude: baseTraceSchema.shape.exclude.optional(),
-    endpointURL: baseTestSchema.endpointURL.default(""),
-    ...codeRefactoringSchema.shape,
     scenarioFile: z
         .string()
         .describe("Path to the scenario file to be used for test generation. This file is generated by the skyramp_scenario_test_generation tool.")
         .optional(),
+    ...codeRefactoringSchema.shape,
+    ...baseTestSchema,
+    endpointURL: baseTestSchema.endpointURL.default(""),
 })
     .omit({ method: true }).shape;
 export class IntegrationTestService extends TestGenerationService {
@@ -28,7 +29,7 @@ export class IntegrationTestService extends TestGenerationService {
             ...super.buildBaseGenerationOptions(params),
             responseData: params.responseData,
             playwrightInput: params.playwrightInput,
-            scenarioFile: params.scenarioFile,
+            trace: params.scenarioFile,
         };
     }
     async handleApiAnalysis(params, generateOptions) {

package/build/tools/test-recommendation/analyzeRepositoryTool.js

@@ -0,0 +1,96 @@
+import { z } from "zod";
+import { logger } from "../../utils/logger.js";
+import { getRepositoryAnalysisPrompt } from "../../prompts/test-recommendation/repository-analysis-prompt.js";
+/**
+ * Analyze Repository Tool
+ * MCP tool for comprehensive repository analysis
+ */
+const analyzeRepositorySchema = z.object({
+    repositoryPath: z
+        .string()
+        .describe("Absolute path to the repository to analyze (e.g., /path/to/my-repo)"),
+    scanDepth: z
+        .enum(["quick", "full"])
+        .default("full")
+        .describe("Analysis depth: 'quick' for basic info, 'full' for comprehensive analysis"),
+    focusAreas: z
+        .array(z.string())
+        .optional()
+        .describe("Optional: Specific areas to focus on (e.g., ['api', 'frontend', 'infrastructure'])"),
+});
+export function registerAnalyzeRepositoryTool(server) {
+    server.registerTool("skyramp_analyze_repository", {
+        description: `Analyze a code repository to understand its structure, technology stack, and testing readiness.
+
+This tool performs comprehensive repository analysis including:
+- Project type classification (REST API, Frontend, Full-stack, Microservices, etc.)
+- Technology stack identification (languages, frameworks, dependencies)
+- Artifact discovery (OpenAPI specs, Playwright recordings, trace files)
+- API endpoint detection and cataloging
+- Authentication mechanism analysis
+- Infrastructure configuration (Docker, Kubernetes, CI/CD)
+- Existing test coverage assessment
+
+The analysis provides structured data that can be used by skyramp_map_tests to calculate test priorities.
+
+Example usage:
+\`\`\`
+{
+  "repositoryPath": "/Users/dev/my-api",
+  "scanDepth": "full"
+}
+\`\`\`
+
+**CRITICAL RULES**:
+- DO NOT CREATE ANY .json or .md file during repository analysis.
+
+Output: Detailed RepositoryAnalysis JSON object with all repository characteristics.`,
+        inputSchema: analyzeRepositorySchema.shape,
+    }, async (params) => {
+        try {
+            logger.info("Analyze repository tool invoked", {
+                repositoryPath: params.repositoryPath,
+                scanDepth: params.scanDepth,
+            });
+            // Return prompt for LLM to execute
+            const analysisPrompt = getRepositoryAnalysisPrompt(params.repositoryPath);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `# Repository Analysis Request
+
+Please analyze the repository at: \`${params.repositoryPath}\`
+
+${params.focusAreas ? `Focus on: ${params.focusAreas.join(", ")}\n` : ""}
+
+Use the following tools to gather information:
+- \`codebase_search\` - to understand code patterns and structure
+- \`grep\` - to find specific patterns (route definitions, dependencies, etc.)
+- \`glob_file_search\` - to find files by pattern (OpenAPI specs, config files, trace files, etc.)
+- \`read_file\` - to read specific files (package.json, requirements.txt, etc.)
+- \`list_dir\` - to explore directory structure
+
+${analysisPrompt}
+
+After gathering all information, return a JSON object matching the RepositoryAnalysis structure shown in the prompt above.`,
+                    },
+                ],
+                isError: false,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error("Analyze repository tool failed", { error: errorMessage });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Error analyzing repository: ${errorMessage}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/build/tools/test-recommendation/mapTestsTool.js

@@ -0,0 +1,195 @@
+import { z } from "zod";
+import { repositoryAnalysisSchema } from "../../types/RepositoryAnalysis.js";
+import { TestType } from "../../types/TestTypes.js";
+import { ScoringEngine } from "../../utils/scoring-engine.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Map Tests Tool
+ * MCP tool for calculating test priority scores
+ */
+const mapTestsSchema = z.object({
+    analysisReport: z
+        .union([z.string(), repositoryAnalysisSchema])
+        .describe("Repository analysis result (JSON string or object from skyramp_analyze_repository)"),
+    customWeights: z
+        .record(z.number())
+        .optional()
+        .describe("Optional: Custom weight multipliers for specific test types (e.g., {'load': 1.5, 'fuzz': 1.3})"),
+    focusTestTypes: z
+        .array(z.nativeEnum(TestType))
+        .optional()
+        .describe("Optional: Only evaluate specific test types (e.g., ['integration', 'smoke'])"),
+});
+export function registerMapTestsTool(server) {
+    server.registerTool("skyramp_map_tests", {
+        description: `Calculate priority scores for Skyramp test types based on repository analysis.
+
+This tool evaluates all test types (E2E, UI, Integration, Load, Fuzz, Contract, Smoke) and calculates priority scores using:
+- Base impact scores (E2E: 100, UI: 95, Integration: 85, etc.)
+- Context multipliers based on repository characteristics
+- Feasibility assessment based on available artifacts
+
+The scoring algorithm considers:
+- Project type (full-stack, microservices, REST API, etc.)
+- Infrastructure (Kubernetes, Docker Compose, CI/CD)
+- Existing test coverage gaps
+- Available artifacts (OpenAPI specs, Playwright recordings)
+- Security requirements (authentication, sensitive data)
+
+Example usage:
+\`\`\`
+{
+  "analysisReport": "<RepositoryAnalysis JSON from skyramp_analyze_repository>",
+  "customWeights": {
+    "load": 1.5,
+    "fuzz": 1.3
+  }
+}
+\`\`\`
+
+**CRITICAL RULES**:
+- DO NOT CREATE ANY .json or .md file during test mapping.
+
+Output: TestMappingResult JSON with priority scores, feasibility, and reasoning for each test type.`,
+        inputSchema: mapTestsSchema.shape,
+    }, async (params) => {
+        try {
+            logger.info("Map tests tool invoked");
+            // Parse and validate analysis report
+            let analysis = params.analysisReport;
+            if (typeof analysis === "string") {
+                try {
+                    analysis = JSON.parse(analysis);
+                }
+                catch (error) {
+                    throw new Error("analysisReport must be valid JSON string or RepositoryAnalysis object. JSON parsing failed.");
+                }
+            }
+            // Validate the analysis object against the schema
+            const validationResult = repositoryAnalysisSchema.safeParse(analysis);
+            if (!validationResult.success) {
+                const errors = validationResult.error.errors
+                    .map((e) => `${e.path.join(".")}: ${e.message}`)
+                    .join("; ");
+                throw new Error(`analysisReport validation failed: ${errors}`);
+            }
+            analysis = validationResult.data;
+            // Determine which test types to evaluate
+            const testTypesToEvaluate = params.focusTestTypes ||
+                Object.values(TestType).filter((t) => typeof t === "string");
+            // Calculate scores for each test type
+            const priorityScores = [];
+            for (const testType of testTypesToEvaluate) {
+                const score = ScoringEngine.calculateTestScore(testType, analysis);
+                // Apply custom weights if provided
+                if (params.customWeights && params.customWeights[testType]) {
+                    score._finalScore *= params.customWeights[testType];
+                    score.contextMultiplier *= params.customWeights[testType];
+                    score.reasoning += ` (custom weight: ×${params.customWeights[testType]})`;
+                }
+                priorityScores.push(score);
+            }
+            // Sort by final score (descending)
+            priorityScores.sort((a, b) => b._finalScore - a._finalScore);
+            // Create summary
+            const highPriority = [];
+            const mediumPriority = [];
+            const lowPriority = [];
+            for (const score of priorityScores) {
+                if (score.feasibility === "not-applicable")
+                    continue;
+                if (score._finalScore >= 80) {
+                    highPriority.push(score.testType);
+                }
+                else if (score._finalScore >= 60) {
+                    mediumPriority.push(score.testType);
+                }
+                else {
+                    lowPriority.push(score.testType);
+                }
+            }
+            // Extract context factors
+            const contextFactors = { applied: [] };
+            if (analysis.infrastructure.hasKubernetes) {
+                contextFactors.applied.push({
+                    factor: "hasKubernetes",
+                    impact: "Increases load and integration test importance",
+                    multiplier: 1.2,
+                });
+            }
+            if (analysis.infrastructure.hasDockerCompose) {
+                contextFactors.applied.push({
+                    factor: "hasDockerCompose",
+                    impact: "Suggests scaled infrastructure",
+                    multiplier: 1.1,
+                });
+            }
+            if (analysis.infrastructure.hasCiCd) {
+                contextFactors.applied.push({
+                    factor: "hasCiCd",
+                    impact: "Increases smoke test importance",
+                    multiplier: 1.1,
+                });
+            }
+            const mapping = {
+                priorityScores,
+                contextFactors,
+                summary: { highPriority, mediumPriority, lowPriority },
+            };
+            // Format output
+            const output = `# Test Priority Mapping
+
+## Summary
+- **High Priority**: ${mapping.summary.highPriority.join(", ") || "None"}
+- **Medium Priority**: ${mapping.summary.mediumPriority.join(", ") || "None"}
+- **Low Priority**: ${mapping.summary.lowPriority.join(", ") || "None"}
+
+## Test Type Priorities (Ordered by Score)
+
+${mapping.priorityScores
+                .map((score) => `### ${score.testType.toUpperCase()}
+- **Feasibility**: ${score.feasibility}
+- **Available Artifacts**: ${score.requiredArtifacts.available.join(", ") || "None"}
+- **Missing Artifacts**: ${score.requiredArtifacts.missing.join(", ") || "None"}
+- **Reasoning**: ${score.reasoning}
+`)
+                .join("\n")}
+
+## Context Factors Applied
+
+${mapping.contextFactors.applied
+                .map((factor) => `- **${factor.factor}**: ${factor.impact} (×${factor.multiplier})`)
+                .join("\n")}
+
+## Complete Mapping Result (JSON)
+
+\`\`\`json
+${JSON.stringify(mapping, null, 2)}
+\`\`\`
+
+**Next Step**: Use \`skyramp_recommend_tests\` with this mapping result to get actionable test recommendations.`;
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: output,
+                    },
+                ],
+                isError: false,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error("Map tests tool failed", { error: errorMessage });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Error mapping tests: ${errorMessage}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/build/tools/test-recommendation/recommendTestsTool.js

@@ -0,0 +1,131 @@
+import { z } from "zod";
+import { testMappingResultSchema } from "../../types/TestMapping.js";
+import { repositoryAnalysisSchema } from "../../types/RepositoryAnalysis.js";
+import { getTestRecommendationPrompt } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Recommend Tests Tool
+ * MCP tool for generating actionable test recommendations
+ */
+const recommendTestsSchema = z.object({
+    priorityScores: z
+        .union([z.string(), testMappingResultSchema])
+        .describe("Test mapping result (JSON string or object from skyramp_map_tests)"),
+    analysisReport: z
+        .union([z.string(), repositoryAnalysisSchema])
+        .describe("Repository analysis result (JSON string or object from skyramp_analyze_repository)"),
+    topN: z
+        .number()
+        .default(7)
+        .describe("Number of top test types to recommend (default: 5)"),
+    minScore: z
+        .number()
+        .default(60)
+        .describe("Minimum score threshold for recommendations (default: 60)"),
+});
+export function registerRecommendTestsTool(server) {
+    server.registerTool("skyramp_recommend_tests", {
+        description: `Generate actionable test recommendations with ready-to-use generation prompts.
+
+This tool takes test priority and repository analysis to create:
+- Ordered list of recommended test types (high/medium/low priority)
+- Specific test scenarios based on actual repository endpoints and flows
+- Artifact availability status and guidance for missing artifacts
+- Step-by-step next actions
+
+SAMPLE OUTPUT:
+\`\`\`
+Recommended Tests (Prioritized)
+Priority: <PRIORITY HIGH, MEDIUM, OR LOW>
+1. <TEST TYPE> Tests [**CRITICAL: NEVER MENTION THE SCORE HERE IN THE OUTPUT**]
+Rationale: 
+Specific Tests to Create:
+Test 1: <TEST NAME>
+Description: <TEST DESCRIPTION>
+Target Flow: <TARGET FLOW>
+\`\`\`
+
+For each recommended test type, you'll get:
+- 2-3 specific tests you can create right now
+- Required vs. available artifacts. THIS SHOULD NOT CHANGE PRIORITIZATION OF THE TESTS.
+- Guidance for creating missing artifacts WITHOUT PROVIDING ANY CLI COMMANDS.
+
+**CRITICAL RULES**:
+- THE PRIORITY SHOULD ONLY BE DEFINED AS HIGH, MEDIUM, OR LOW NOTHING ELSE.
+- DO NOT SHOW ANY PRIORITY BREAKDOWN IN THE OUTPUT.
+- DON'T MARK ANY TEST BLOCKED EVEN IF REQUIRED ARTIFACTS ARE MISSING.
+- DO NOT SHOW RESULTS IN .MD OR .JSON OR ANY OTHER FILE FORMAT.
+
+Output: TestRecommendation JSON with prioritized, actionable test recommendations.`,
+        inputSchema: recommendTestsSchema.shape,
+    }, async (params) => {
+        try {
+            logger.info("Recommend tests tool invoked", {
+                topN: params.topN,
+                minScore: params.minScore,
+            });
+            // Parse inputs if they're strings
+            let mapping = params.priorityScores;
+            let analysis = params.analysisReport;
+            if (typeof mapping === "string") {
+                try {
+                    mapping = JSON.parse(mapping);
+                }
+                catch (error) {
+                    throw new Error("priorityScores must be valid JSON string or TestMappingResult object");
+                }
+            }
+            if (typeof analysis === "string") {
+                try {
+                    analysis = JSON.parse(analysis);
+                }
+                catch (error) {
+                    throw new Error("analysisReport must be valid JSON string or RepositoryAnalysis object");
+                }
+            }
+            // Generate the prompt for LLM to create dynamic recommendations
+            const prompt = getTestRecommendationPrompt(mapping, analysis, params.topN);
+            // Return prompt for LLM to execute
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `# Test Recommendation Request
+
+Please generate actionable test recommendations based on the priority scores and repository analysis.
+
+${prompt}
+
+**Important Guidelines:**
+1. **NO SCORES IN OUTPUT**: The numeric scores are for internal ranking only. DO NOT include "score" field in JSON and DO NOT mention numeric scores in any text (rationale, description, etc.). Use ONLY priority levels: "high", "medium", "low".
+2. Include guidance for missing artifacts with specific Skyramp tool commands
+3. Prioritize quick wins (tests with all artifacts available)
+4. Use actual endpoint paths and file paths from the repository analysis
+
+**MANDATORY RULES**:
+- THE PRIORITY SHOULD ONLY BE DEFINED AS HIGH, MEDIUM, OR LOW NOTHING ELSE.
+- DO NOT SHOW ANY PRIORITY BREAKDOWN IN THE OUTPUT.
+- DON'T MARK ANY TEST BLOCKED EVEN IF REQUIRED ARTIFACTS ARE MISSING.
+- DO NOT SHOW RESULTS IN .MD OR .JSON OR ANY OTHER FILE FORMAT.
+
+After analyzing the data above, return the complete JSON response following the structure defined in the prompt.`,
+                    },
+                ],
+                isError: false,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger.error("Recommend tests tool failed", { error: errorMessage });
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: `Error generating recommendations: ${errorMessage}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    });
+}

package/build/types/RepositoryAnalysis.js

@@ -0,0 +1,106 @@
+import { z } from "zod";
+// Zod schemas for validation
+export const analysisMetadataSchema = z.object({
+    repositoryName: z.string(),
+    analysisDate: z.string(),
+    scanDepth: z.enum(["quick", "full"]),
+});
+export const projectClassificationSchema = z.object({
+    projectType: z.enum([
+        "rest-api",
+        "frontend",
+        "full-stack",
+        "microservices",
+        "library",
+        "cli",
+        "other",
+    ]),
+    primaryLanguage: z.string(),
+    primaryFramework: z.string(),
+    deploymentPattern: z.enum([
+        "microservices",
+        "full-stack",
+        "containerized-monolith",
+        "traditional",
+        "unknown",
+    ]),
+});
+export const repositoryAnalysisSchema = z.object({
+    metadata: analysisMetadataSchema,
+    projectClassification: projectClassificationSchema,
+    technologyStack: z.object({
+        languages: z.array(z.string()),
+        frameworks: z.array(z.string()),
+        runtime: z.string(),
+        keyDependencies: z.array(z.object({
+            name: z.string(),
+            version: z.string(),
+            purpose: z.string(),
+        })),
+    }),
+    businessContext: z.object({
+        mainPurpose: z.string(),
+        userFlows: z.array(z.string()),
+        dataFlows: z.array(z.string()),
+        integrationPatterns: z.array(z.string()),
+    }),
+    artifacts: z.object({
+        openApiSpecs: z.array(z.object({
+            path: z.string(),
+            version: z.string(),
+            endpointCount: z.number(),
+            baseUrl: z.string(),
+            authType: z.string(),
+        })),
+        playwrightRecordings: z.array(z.object({
+            path: z.string(),
+            description: z.string(),
+        })),
+        traceFiles: z.array(z.object({
+            path: z.string(),
+            format: z.string(),
+            analyzed: z.boolean().optional(),
+            userFlows: z.array(z.string()).optional(),
+        })),
+        notFound: z.array(z.string()),
+    }),
+    apiEndpoints: z.object({
+        totalCount: z.number(),
+        baseUrl: z.string(),
+        endpoints: z.array(z.object({
+            path: z.string(),
+            method: z.string(),
+            resourceGroup: z.string(),
+            authRequired: z.boolean(),
+            sourceFile: z.string().optional(),
+        })),
+    }),
+    authentication: z.object({
+        method: z.enum(["bearer", "api-key", "oauth2", "basic", "jwt", "none"]),
+        configLocation: z.string(),
+        envVarsRequired: z.array(z.string()),
+        setupExample: z.string(),
+    }),
+    infrastructure: z.object({
+        isContainerized: z.boolean(),
+        hasDockerCompose: z.boolean(),
+        hasKubernetes: z.boolean(),
+        hasCiCd: z.boolean(),
+        ciCdPlatform: z.string().optional(),
+    }),
+    existingTests: z.object({
+        frameworks: z.array(z.string()),
+        coverage: z.object({
+            unit: z.number(),
+            integration: z.number(),
+            e2e: z.number(),
+            ui: z.number(),
+            load: z.number(),
+            contract: z.number(),
+            smoke: z.number(),
+        }),
+        testLocations: z.record(z.string()),
+        hasCoverageReports: z.boolean(),
+        estimatedCoverage: z.number().optional(),
+    }),
+});