@skyramp/mcp 0.0.63 → 0.0.64-rc.2

package/build/tools/generate-tests/generateUIRestTool.js

@@ -2,6 +2,7 @@ import { baseSchema, languageSchema, TestType, codeRefactoringSchema, baseTraceS
 import { z } from "zod";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { TestGenerationService, } from "../../services/TestGenerationService.js";
+import { normalizeLanguageParams, resolveParamAliases, } from "../../utils/normalizeParams.js";
 const TOOL_NAME = "skyramp_ui_test_generation";
 export class UITestService extends TestGenerationService {
     getTestType() {
@@ -23,7 +24,7 @@ const uiTestSchema = {
     ...languageSchema.shape,
     playwrightInput: z
         .string()
-        .describe("MUST be absolute path to the playwright input file like /path/to/playwright-ui-test.zip and MUST be a zip file captured using start_trace_collection tool"),
+        .describe("MUST be absolute path to the playwright trace zip file like /path/to/trace.zip. Capture using browser_* tools + skyramp_export_zip, or via manual trace collection with skyramp_start_trace_collection/skyramp_stop_trace_collection."),
     include: baseTraceSchema.shape.include
         .optional()
         .describe("Array of FQDN strings to only include from trace data when correlating frontend actions with backend calls"),
@@ -41,12 +42,19 @@ export function registerUITestTool(server) {
         description: `Generate a UI test using Skyramp's deterministic test generation platform.
       
 UI tests validate user interface functionality by simulating real user interactions with your web application. They test user workflows, form submissions, navigation, responsive design, and ensure that your frontend works correctly across different browsers and devices. UI tests use Playwright recordings as input to generate comprehensive test suites that replay user interactions, validate UI elements, and verify expected behaviors in browser environments.
-**CRITICAL: USE TRACE COLLECTION START/STOP TOOLS FOR PLAYWRIGHT TRACE COLLECTION.**`,
+**CRITICAL: To collect a Playwright trace, use the browser_* tools (browser_navigate, browser_click, browser_type, etc.) to interact with the application, then call skyramp_export_zip to export the trace zip. Do NOT use skyramp_start_trace_collection/skyramp_stop_trace_collection.**`,
         inputSchema: uiTestSchema,
         _meta: {
             keywords: ["ui test", "playwright"],
         },
     }, async (params) => {
+        resolveParamAliases(params, {
+            zipPath: "playwrightInput",
+            tracePath: "playwrightInput",
+            traceFile: "playwrightInput",
+            zip: "playwrightInput",
+        });
+        normalizeLanguageParams(params);
         const service = new UITestService();
         const result = await service.generateTest(params);
         AnalyticsService.pushTestGenerationToolEvent(TOOL_NAME, result, params).catch(() => {

package/build/tools/submitReportTool.js

@@ -6,14 +6,14 @@ import { AnalyticsService } from "../services/AnalyticsService.js";
 const TOOL_NAME = "skyramp_submit_report";
 const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
 const testResultSchema = z.object({
-    testType: z.string().describe("Type of test: Smoke, Contract, Integration, E2E, Load, etc."),
+    testType: z.string().describe("Type of test: Smoke, Contract, Integration, E2E, Load, etc. Do not include priority or other metadata in this field."),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     status: z.enum(["Pass", "Fail", "Skipped"]).describe("Test execution result"),
     details: z.string().describe("Execution time and test file name, e.g. '10.8s, products_smoke_test.py'"),
 });
 const newTestSchema = z.object({
-    testId: z.string().uuid().describe("UUID v4 identifier for this test. Generate a fresh UUID for each test to ensure uniqueness across runs and avoid backend conflicts."),
-    testType: z.string().describe("Type of test created: Smoke, Contract, Integration, etc."),
+    testId: z.string().describe("Human-readable kebab-case identifier, e.g. 'contract-get-products' or 'integration-users-orders-workflow'. Format: '<testType>-<method>-<resource>' for single-endpoint tests or '<testType>-<scenario-slug>' for multi-step tests. Must be unique within the report."),
+    testType: z.string().describe("Type of test created: Smoke, Contract, Integration, etc. Do not include priority or other metadata in this field."),
     category: z.enum(["security_boundary", "business_rule", "breaking_change", "data_integrity", "workflow"]).describe("Test category — critical categories (security_boundary, business_rule, data_integrity, breaking_change) get generation priority over workflow"),
     endpoint: z.string().describe("HTTP verb and path, e.g. 'GET /api/v1/products'"),
     fileName: z.string().describe("Name of the generated test file"),
@@ -35,8 +35,8 @@ const scenarioStepSchema = z.object({
     responseBody: z.record(z.any()).optional().describe("Key response fields to verify, e.g. { id: 'number', name: 'string', in_stock: 'boolean?' }"),
 });
 const additionalRecommendationSchema = z.object({
-    testId: z.string().uuid().describe("UUID v4 identifier for this recommendation. Generate a fresh UUID for each recommendation to ensure uniqueness across runs and avoid backend conflicts."),
-    testType: z.string().describe("Type of test: Integration, E2E, Contract, UI, etc."),
+    testId: z.string().describe("Human-readable kebab-case identifier, e.g. 'integration-products-orders-workflow' or 'e2e-checkout-flow'. Format: '<testType>-<scenario-slug>'. Must be unique within the report."),
+    testType: z.string().describe("Type of test: Integration, E2E, Contract, UI, etc. Do not include priority or other metadata in this field."),
     category: z.enum(["security_boundary", "business_rule", "breaking_change", "data_integrity", "workflow"]).describe("Test category — critical categories get generation priority over workflow"),
     scenarioName: z.string().describe("Name of the scenario, e.g. 'products_orders_workflow'"),
     steps: z.array(scenarioStepSchema).describe("Ordered sequence of API/UI steps in this test scenario"),
@@ -102,10 +102,29 @@ export function registerSubmitReportTool(server) {
     }, async (params) => {
         const startTime = Date.now();
         let errorResult;
+        const deduplicateById = (items) => {
+            const seen = new Set();
+            const result = [];
+            for (const item of items) {
+                let id = item.testId;
+                if (seen.has(id)) {
+                    let suffix = 2;
+                    while (seen.has(`${id}-${suffix}`))
+                        suffix++;
+                    id = `${id}-${suffix}`;
+                    item.testId = id;
+                }
+                seen.add(id);
+                result.push(item);
+            }
+            return result;
+        };
+        const dedupedNewTests = deduplicateById([...params.newTestsCreated]);
+        const dedupedRecommendations = deduplicateById([...(params.additionalRecommendations ?? [])]);
         const reportJson = JSON.stringify({
             businessCaseAnalysis: params.businessCaseAnalysis,
-            newTestsCreated: params.newTestsCreated,
-            additionalRecommendations: params.additionalRecommendations ?? [],
+            newTestsCreated: dedupedNewTests,
+            additionalRecommendations: dedupedRecommendations,
             testMaintenance: params.testMaintenance,
             testResults: params.testResults,
             issuesFound: params.issuesFound,

package/build/tools/submitReportTool.test.js

@@ -24,7 +24,7 @@ function sampleReportParams(outputFile) {
         summaryOutputFile: outputFile,
         businessCaseAnalysis: "This PR adds product search. Tests needed for filtering.",
         newTestsCreated: [
-            { testId: "550e8400-e29b-41d4-a716-446655440000", testType: "Smoke", category: "workflow", endpoint: "GET /api/v1/products/search", fileName: "search_smoke_test.py", reasoning: "Validates product search returns correct results for category filter" },
+            { testId: "smoke-get-products-search", testType: "Smoke", category: "workflow", endpoint: "GET /api/v1/products/search", fileName: "search_smoke_test.py", reasoning: "Validates product search returns correct results for category filter" },
         ],
         testMaintenance: [{ description: "Updated auth flow in existing tests" }],
         testResults: [

package/build/tools/test-management/analyzeChangesTool.js

@@ -17,7 +17,7 @@ import { scanAllRepoEndpoints, scanRelatedEndpoints, grepRouterMountingContext,
 import { detectProjectMetadata } from "../../utils/projectMetadata.js";
 import { draftScenariosFromEndpoints } from "../../utils/scenarioDrafting.js";
 import { buildAnalysisOutputText } from "../../prompts/test-recommendation/analysisOutputPrompt.js";
-import { parseTraceFile, discoverTraceFiles, } from "../../utils/trace-parser.js";
+import { parseTraceFile, discoverTraceFiles, discoverPlaywrightZips, } from "../../utils/trace-parser.js";
 import { parsePRComments } from "../../utils/pr-comment-parser.js";
 const TOOL_NAME = "skyramp_analyze_changes";
 // Must match testbot/src/constants.ts BOT_EMAIL
@@ -341,10 +341,15 @@ to produce a unified state file for the test health workflow.
                     if (svc?.api?.schemaPath)
                         wsSchemaPath = svc.api.schemaPath;
                     wsAuthType = parseWorkspaceAuthType(svc?.api?.authType);
+                    // If authHeader not explicitly set but authType implies Authorization header, infer it.
+                    // Bearer/Token/Basic all use the Authorization header by convention.
+                    if (!wsAuthHeader && wsAuthType && !["none", "apikey", "session", "cookie"].includes(wsAuthType.toLowerCase())) {
+                        wsAuthHeader = "Authorization";
+                    }
                     if (wsAuthHeader) {
                         wsAuthMethod = /cookie|session/i.test(wsAuthHeader)
                             ? "session"
-                            : "bearer";
+                            : "";
                     }
                 }
             }
@@ -486,7 +491,7 @@ to produce a unified state file for the test health workflow.
                 }
             }
             // ── Step 9: Draft scenarios ──
-            const codeInferredScenarios = draftScenariosFromEndpoints(skeletonEndpoints);
+            const codeInferredScenarios = draftScenariosFromEndpoints(skeletonEndpoints, parsedDiff?.newEndpoints ?? []);
             let allDraftedScenarios = codeInferredScenarios;
             if (traceResult && traceResult.userFlows.length > 0) {
                 const traceScenarios = traceResult.userFlows
@@ -589,7 +594,7 @@ to produce a unified state file for the test health workflow.
                 },
                 artifacts: {
                     openApiSpecs: wsSchemaPath ? [{ path: wsSchemaPath, version: "from-workspace-config", endpointCount: 0, baseUrl: wsBaseUrl, authType: wsAuthMethod }] : [],
-                    playwrightRecordings: [],
+                    playwrightRecordings: discoverPlaywrightZips(params.repositoryPath).map(p => ({ path: p, description: "" })),
                     traceFiles: traceResult ? [traceResult] : [],
                     notFound: [],
                 },

package/build/tools/trace/startTraceCollectionTool.js

@@ -9,20 +9,17 @@ import path from "path";
 const TOOL_NAME = "skyramp_start_trace_collection";
 export function registerTraceTool(server) {
     server.registerTool(TOOL_NAME, {
-        description: `Start trace collection using Skyramp's comprehensive tracing capabilities.
+        description: `Start MANUAL trace collection using Skyramp's Docker-based tracing.
 
-Trace collection monitors your application in real-time to capture user interactions, API calls, service communications, and data flows. This captured data is then used to generate comprehensive integration, load, and E2E tests that reflect real-world usage patterns.
+Use this tool ONLY when the user explicitly asks for "manual recording", "manual trace", "record myself", "I will interact", "Docker trace", or wants to interact with the browser themselves. For AI-driven browser automation (where the agent fills forms, clicks buttons, etc.), use the browser_* tools instead.
 
-COLLECTION TYPES:
-• UI-Only Tracing: Captures frontend user interactions for UI test generation
-• Backend-Only Tracing: Records API calls and service communications for integration tests
-• Full-Stack Tracing: Combines UI and backend tracing for comprehensive E2E test generation
+This tool launches a Docker proxy that captures network traffic while the USER manually interacts with the application in a browser window. The user performs clicks, form fills, and navigation themselves.
 
 WORKFLOW:
-1. Start trace collection with desired configuration
-2. Interact with your application (UI clicks, API calls, workflows)
+1. Start trace collection — opens a browser for the user
+2. USER manually interacts with the application
 3. Stop trace collection to save captured data
-4. Use traces to generate realistic test scenarios
+4. Use traces to generate test scenarios
 
 For detailed documentation visit: https://www.skyramp.dev/docs/load-test/advanced-generation#start-trace-collection`,
         inputSchema: {

package/build/tools/trace/stopTraceCollectionTool.js

@@ -9,14 +9,14 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
 const TOOL_NAME = "skyramp_stop_trace_collection";
 export function registerTraceStopTool(server) {
     server.registerTool(TOOL_NAME, {
-        description: `Stop trace collection and save captured data using Skyramp's comprehensive tracing capabilities.
+        description: `Stop MANUAL trace collection and save captured data. Use this ONLY after skyramp_start_trace_collection was used (manual/Docker-based recording where the user interacted with the browser themselves). Do NOT use this after browser_* tool interactions — use skyramp_export_zip instead.
 
 Stopping trace collection finalizes the monitoring process and exports all captured data to specified files. This includes API calls, service communications, user interactions, and network traffic that occurred during the collection period.
 
 INTEGRATION WITH TEST GENERATION:
 The generated trace and Playwright files can be used as inputs for:
 - Integration test generation (using trace files)
-- Load test generation (using trace files)  
+- Load test generation (using trace files)
 - E2E test generation (using both trace and Playwright files)
 - UI test generation (using Playwright files)
 

package/build/types/RepositoryAnalysis.js

@@ -85,6 +85,7 @@ export const draftedScenarioSchema = z.object({
         "business_rule",
         "data_integrity",
         "breaking_change",
+        "new_endpoint",
     ]),
     priority: z.enum(["high", "medium", "low"]),
     steps: z.array(scenarioStepSchema),
@@ -92,6 +93,7 @@ export const draftedScenarioSchema = z.object({
     requiresAuth: z.boolean(),
     estimatedComplexity: z.enum(["simple", "moderate", "complex"]),
     source: z.enum(["code-inferred", "trace", "documentation"]).optional(),
+    testType: z.enum(["integration", "contract", "e2e", "ui"]).optional(),
 });
 export const branchDiffContextSchema = z.object({
     currentBranch: z.string(),
@@ -188,7 +190,7 @@ export const repositoryAnalysisSchema = z.object({
         endpoints: z.array(enrichedEndpointSchema),
     }),
     authentication: z.object({
-        method: z.enum(["bearer", "api-key", "oauth2", "basic", "jwt", "none"]),
+        method: z.string(),
         configLocation: z.string(),
         envVarsRequired: z.array(z.string()),
         setupExample: z.string(),

package/build/utils/normalizeParams.js

@@ -0,0 +1,45 @@
+/**
+ * Normalize tool parameters to handle common LLM mistakes:
+ * - Case-insensitive language/framework values
+ * - Common parameter name aliases
+ */
+const LANGUAGE_MAP = {
+    typescript: "typescript",
+    javascript: "javascript",
+    python: "python",
+    java: "java",
+    ts: "typescript",
+    js: "javascript",
+    py: "python",
+};
+const FRAMEWORK_MAP = {
+    playwright: "playwright",
+    pytest: "pytest",
+    junit: "junit",
+    robot: "robot",
+};
+/**
+ * Normalize language and framework to lowercase accepted values.
+ */
+export function normalizeLanguageParams(params) {
+    if (typeof params.language === "string") {
+        const lower = params.language.toLowerCase();
+        params.language = LANGUAGE_MAP[lower] || lower;
+    }
+    if (typeof params.framework === "string") {
+        const lower = params.framework.toLowerCase();
+        params.framework = FRAMEWORK_MAP[lower] || lower;
+    }
+}
+/**
+ * Resolve common parameter name aliases that LLMs use.
+ * e.g., zipPath -> playwrightInput, testFilePath -> testFile
+ */
+export function resolveParamAliases(params, aliases) {
+    for (const [alias, canonical] of Object.entries(aliases)) {
+        if (params[alias] !== undefined && params[canonical] === undefined) {
+            params[canonical] = params[alias];
+            delete params[alias];
+        }
+    }
+}