@skyramp/mcp 0.3.2-rc.pom-4 → 0.3.2

package/build/tools/executeSkyrampTestTool.d.ts

@@ -1,2 +1,13 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { TestType } from "../types/TestTypes.js";
+export declare const CONTRACT_EXECUTION_MODES: readonly ["provider", "consumer"];
+export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
+/**
+ * Resolve the effective auth token for test execution.
+ * `unauthenticated: true` forces no token (empty string) so the container
+ * env omits SKYRAMP_TEST_TOKEN entirely — unauthenticated endpoints won't
+ * receive an empty Authorization header that triggers encoding errors (E7).
+ */
+export declare function resolveEffectiveToken(unauthenticated?: boolean, paramToken?: string, envToken?: string): string;
+export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractMode?: ContractExecutionMode): boolean;
 export declare function registerExecuteSkyrampTestTool(server: McpServer): void;

package/build/tools/executeSkyrampTestTool.js

@@ -12,6 +12,21 @@ import { TestSource } from "../types/TestAnalysis.js";
 import { logger } from "../utils/logger.js";
 import { toolError } from "../utils/utils.js";
 const TOOL_NAME = "skyramp_execute_test";
+export const CONTRACT_EXECUTION_MODES = ["provider", "consumer"];
+/**
+ * Resolve the effective auth token for test execution.
+ * `unauthenticated: true` forces no token (empty string) so the container
+ * env omits SKYRAMP_TEST_TOKEN entirely — unauthenticated endpoints won't
+ * receive an empty Authorization header that triggers encoding errors (E7).
+ */
+export function resolveEffectiveToken(unauthenticated, paramToken, envToken) {
+    if (unauthenticated)
+        return "";
+    return paramToken ?? envToken ?? "";
+}
+export function shouldInjectSkyrampBaseUrl(testType, contractMode) {
+    return testType !== TestType.CONTRACT || contractMode !== "consumer";
+}
 export function registerExecuteSkyrampTestTool(server) {
     server.registerTool(TOOL_NAME, {
         description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing. Call this once a test file exists on disk (from a skyramp_*_test_generation tool). First-time execution may take longer while Docker images download — this is expected, not a failure.`,
@@ -30,10 +45,14 @@ export function registerExecuteSkyrampTestTool(server) {
                 .describe("Programming language of the test file to execute (e.g., python, javascript, typescript, java)"),
             testType: z
                 .nativeEnum(TestType)
-                .describe("Type of the test to execute (e.g., smoke, contract, fuzz, integration, load, ui, e2e)."),
+                .describe("Type of the test to execute. Note: 'mock' is NOT a valid test type — mock files are deployed via their apply_mock() function, not executed as tests."),
             testFile: z
                 .string()
                 .describe("Absolute path to the test file to execute."),
+            contractMode: z
+                .enum(CONTRACT_EXECUTION_MODES)
+                .optional()
+                .describe("Only applies when testType is 'contract'. Use 'provider' for provider contract tests that hit the real service under test and need SKYRAMP_TEST_BASE_URL. Use 'consumer' only for consumer contract tests with inline mocks that do not hit the real service. Defaults to provider behavior when omitted."),
             token: z
                 .string()
                 .optional()
@@ -64,15 +83,17 @@ export function registerExecuteSkyrampTestTool(server) {
         },
     }, async (params, extra) => {
         let errorResult;
-        // Helper to send progress notifications to the MCP client
+        // Helper to send progress notifications to the MCP client.
         const sendProgress = makeProgressReporter(extra);
+        // Send immediate acknowledgment
+        await sendProgress(0, 100, "Starting execution...");
         // Progress callback adapter for TestExecutionService
         const onExecutionProgress = async (progress) => {
             await sendProgress(progress.percent, 100, progress.message);
         };
         const previousBaseUrl = process.env.SKYRAMP_TEST_BASE_URL;
         let didSetSkyrampBaseUrl = false;
-        let useHostNetwork = false;
+        let dockerNetwork;
         try {
             if (!path.isAbsolute(params.testFile)) {
                 errorResult = toolError(`testFile must be an absolute path, got: ${params.testFile}`);
@@ -107,18 +128,21 @@ export function registerExecuteSkyrampTestTool(server) {
                 }
             }
             // Send initial progress
-            await sendProgress(0, 100, "Starting test execution...");
+            await sendProgress(5, 100, "Starting test execution...");
             // Cross-repo run (SKYR-3819): tests delivered under the run's testsRepoDir
             // (set by skyramp_analyze_changes, run-scoped) live outside workspacePath.
             // Thread it through so the executor mounts the test repo and service
             // matching still resolves the SUT baseUrl/dockerNetwork.
             const testRepoPath = getTestsRepoDir();
-            // Always resolve workspace config for dockerNetwork (host networking)
-            // and optionally inject SKYRAMP_TEST_BASE_URL if not already set.
+            // Resolve workspace config for base URL injection and Docker network
+            // attachment. A compose dockerNetwork is not host networking: on macOS
+            // the executor still needs localhost rewritten to host.docker.internal.
             if (params.workspacePath) {
-                const { baseUrl, dockerNetwork, candidates } = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language, testRepoPath);
-                useHostNetwork = !!dockerNetwork;
-                if (!process.env.SKYRAMP_TEST_BASE_URL) {
+                const workspaceConfig = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language, testRepoPath);
+                const { baseUrl, candidates } = workspaceConfig;
+                dockerNetwork = workspaceConfig.dockerNetwork;
+                const shouldInjectBaseUrl = shouldInjectSkyrampBaseUrl(params.testType, params.contractMode);
+                if (shouldInjectBaseUrl && !process.env.SKYRAMP_TEST_BASE_URL) {
                     if (baseUrl) {
                         process.env.SKYRAMP_TEST_BASE_URL = baseUrl;
                         didSetSkyrampBaseUrl = true;
@@ -135,16 +159,16 @@ export function registerExecuteSkyrampTestTool(server) {
                 }
             }
             const executionService = new TestExecutionService();
-            // `unauthenticated: true` forces no auth token regardless of what's passed or
-            // configured, so tests asserting 401/403 unauthenticated behavior can't be broken
-            // by an ambient SKYRAMP_TEST_TOKEN — see SKYR-3874. Otherwise fall back to the
-            // environment token when the caller omits `token`; this keeps secrets server-side
-            // and avoids embedding credential values in prompts.
-            const effectiveToken = params.unauthenticated
-                ? ""
-                : (params.token ?? process.env.SKYRAMP_TEST_TOKEN ?? "");
-            // Execute test with progress callback - reports Docker cache/pull status
-            const result = await executionService.executeTest({
+            const effectiveToken = resolveEffectiveToken(params.unauthenticated, params.token, process.env.SKYRAMP_TEST_TOKEN);
+            if (!effectiveToken && !params.unauthenticated && params.token === undefined) {
+                logger.warning("No auth token available — authenticated endpoints will likely return 401. Set SKYRAMP_TEST_TOKEN or pass token/unauthenticated.");
+            }
+            // Execute test with progress callback - reports Docker cache/pull status.
+            // Retry once on transient connection errors (EOF, connection refused) —
+            // these indicate the SUT isn't fully ready yet, not a test failure.
+            // Retrying inside the tool call saves agent turns vs failing and
+            // requiring the agent to re-invoke.
+            const execOptions = {
                 testFile: params.testFile,
                 workspacePath: params.workspacePath,
                 testRepoPath,
@@ -152,8 +176,25 @@ export function registerExecuteSkyrampTestTool(server) {
                 testType: params.testType,
                 token: effectiveToken,
                 playwrightSaveStoragePath: params.playwrightSaveStoragePath,
-                useHostNetwork,
-            }, onExecutionProgress);
+                dockerNetwork,
+                useHostNetwork: false,
+            };
+            let result;
+            try {
+                result = await executionService.executeTest(execOptions, onExecutionProgress);
+            }
+            catch (firstErr) {
+                const errMsg = firstErr instanceof Error ? firstErr.message : String(firstErr);
+                if (/\bEOF\b|connection refused|ECONNREFUSED|ECONNRESET/i.test(errMsg)) {
+                    logger.info("Test execution hit transient connection error, retrying after 10s...", { error: errMsg });
+                    await sendProgress(50, 100, "SUT connection error — retrying in 10s...");
+                    await new Promise((r) => setTimeout(r, 10_000));
+                    result = await executionService.executeTest(execOptions, onExecutionProgress);
+                }
+                else {
+                    throw firstErr;
+                }
+            }
             // Update stateFile with execution results if provided. Multi-repo: write
             // into the section for `repository` (primary when omitted/matching the
             // root) of the run-scoped file.

package/build/tools/generate-tests/batchMockGenerationTool.d.ts

@@ -0,0 +1,106 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { MockProtocol } from "../../types/TestTypes.js";
+export { MockProtocol } from "../../types/TestTypes.js";
+export declare const BATCH_MOCK_PROTOCOL_EXAMPLES = "\n<example protocol=\"rest\">\nBatch REST item. Required fields: protocol=\"rest\", endpointURL, method unless apiSchema is provided.\n{\n  \"protocol\": \"rest\",\n  \"endpointURL\": \"http://identity-service:4000/api/internal/v1/users/validate\",\n  \"method\": \"POST\",\n  \"requestAware\": true,\n  \"responseData\": \"{\"valid\":true}\"\n}\n</example>\n\n<example protocol=\"rest-traffic-simulation\">\nBatch REST item with traffic simulation. minLatency/maxLatency inject a random delay per response; dropRate drops a percentage of requests.\n{\n  \"protocol\": \"rest\",\n  \"endpointURL\": \"http://payment-service:5001/payments\",\n  \"method\": \"POST\",\n  \"responseData\": \"{\"status\":\"ok\"}\",\n  \"minLatency\": 500,\n  \"maxLatency\": 1000,\n  \"dropRate\": 10\n}\n</example>\n\n<example protocol=\"rest-conditional\">\nBatch REST item with conditional response selection. Use \"when\" to match request fields.\n{\n  \"protocol\": \"rest\",\n  \"endpointURL\": \"http://payment-service:5001/api/v1/orders\",\n  \"method\": \"POST\",\n  \"when\": {\"request.body.payment.method\": \"card\"},\n  \"requestSchema\": \"{\"type\":\"object\",\"required\":[\"orderId\",\"payment\"]}\",\n  \"responseData\": \"{\"status\":\"confirmed\"}\",\n  \"responseStatusCode\": \"201\",\n  \"noMatchStatusCode\": 404,\n  \"validationFailureStatusCode\": 422\n}\n</example>\n\n<example protocol=\"grpc\">\nBatch gRPC item. Required fields: protocol=\"grpc\", endpointURL as host:port, protoPath, grpcServiceName. method is the RPC name; empty means all service methods.\n{\n  \"protocol\": \"grpc\",\n  \"endpointURL\": \"profile-service:50051\",\n  \"protoPath\": \"/repo/proto/profiles.proto\",\n  \"grpcServiceName\": \"ProfileService\",\n  \"method\": \"GetProfile\",\n  \"responseData\": \"{\"id\":\"profile_123\"}\"\n}\n</example>\n\n<example protocol=\"grpc-traffic-simulation\">\nBatch gRPC item with traffic simulation. Same latency/drop fields as REST.\n{\n  \"protocol\": \"grpc\",\n  \"endpointURL\": \"profile-service:50051\",\n  \"protoPath\": \"/repo/proto/profiles.proto\",\n  \"grpcServiceName\": \"ProfileService\",\n  \"method\": \"GetProfile\",\n  \"responseData\": \"{\"eligible\":true}\",\n  \"minLatency\": 200,\n  \"maxLatency\": 800,\n  \"dropRate\": 5\n}\n</example>\n\n<example protocol=\"grpc-status-error\">\nBatch gRPC status error item. Do not provide responseData with grpcStatusCode.\n{\n  \"protocol\": \"grpc\",\n  \"endpointURL\": \"profile-service:50051\",\n  \"protoPath\": \"/repo/proto/profiles.proto\",\n  \"grpcServiceName\": \"ProfileService\",\n  \"method\": \"GetProfile\",\n  \"grpcStatusCode\": 5,\n  \"grpcStatusMessage\": \"profile not found\"\n}\n</example>\n\n<example protocol=\"kafka-producer\">\nBatch Kafka producer validation item. Required fields: protocol=\"kafka\", endpointURL as broker host:port, kafkaTopic, and requestData. Do not pass method or responseData. Traffic simulation is NOT supported for Kafka.\n{\n  \"protocol\": \"kafka\",\n  \"endpointURL\": \"broker:9092\",\n  \"kafkaTopic\": \"orders\",\n  \"requestData\": \"{\"order_id\":\"ord_123\"}\"\n}\n</example>\n\n<example protocol=\"kafka-consumer\">\nBatch Kafka consumer message item. Required fields: protocol=\"kafka\", endpointURL as broker host:port, kafkaTopic, and responseData. Do not pass method or requestData. Traffic simulation is NOT supported for Kafka.\n{\n  \"protocol\": \"kafka\",\n  \"endpointURL\": \"broker:9092\",\n  \"kafkaTopic\": \"orders\",\n  \"responseData\": \"{\"accepted\":true}\"\n}\n</example>";
+declare const mockSpecSchema: z.ZodObject<{
+    protocol: z.ZodDefault<z.ZodNativeEnum<typeof MockProtocol>>;
+    endpointURL: z.ZodString;
+    method: z.ZodDefault<z.ZodString>;
+    responseData: z.ZodOptional<z.ZodString>;
+    responseStatusCode: z.ZodEffects<z.ZodDefault<z.ZodString>, string, string | undefined>;
+    grpcStatusCode: z.ZodOptional<z.ZodNumber>;
+    grpcStatusMessage: z.ZodOptional<z.ZodString>;
+    requestData: z.ZodDefault<z.ZodString>;
+    requestAware: z.ZodDefault<z.ZodBoolean>;
+    apiSchema: z.ZodDefault<z.ZodString>;
+    protoPath: z.ZodOptional<z.ZodString>;
+    grpcServiceName: z.ZodOptional<z.ZodString>;
+    protobufExtraFiles: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    protobufExtraImportPath: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    kafkaTopic: z.ZodOptional<z.ZodString>;
+    minLatency: z.ZodOptional<z.ZodNumber>;
+    maxLatency: z.ZodOptional<z.ZodNumber>;
+    dropRate: z.ZodOptional<z.ZodNumber>;
+    when: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    requestSchema: z.ZodOptional<z.ZodString>;
+    noMatchStatusCode: z.ZodOptional<z.ZodNumber>;
+    validationFailureStatusCode: z.ZodOptional<z.ZodNumber>;
+    output: z.ZodOptional<z.ZodString>;
+    outputDir: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    method: string;
+    endpointURL: string;
+    apiSchema: string;
+    requestData: string;
+    responseStatusCode: string;
+    protocol: MockProtocol;
+    requestAware: boolean;
+    output?: string | undefined;
+    outputDir?: string | undefined;
+    responseData?: string | undefined;
+    when?: Record<string, unknown> | undefined;
+    protoPath?: string | undefined;
+    grpcServiceName?: string | undefined;
+    protobufExtraFiles?: string[] | undefined;
+    protobufExtraImportPath?: string[] | undefined;
+    kafkaTopic?: string | undefined;
+    grpcStatusCode?: number | undefined;
+    grpcStatusMessage?: string | undefined;
+    minLatency?: number | undefined;
+    maxLatency?: number | undefined;
+    dropRate?: number | undefined;
+    requestSchema?: string | undefined;
+    noMatchStatusCode?: number | undefined;
+    validationFailureStatusCode?: number | undefined;
+}, {
+    endpointURL: string;
+    output?: string | undefined;
+    outputDir?: string | undefined;
+    method?: string | undefined;
+    apiSchema?: string | undefined;
+    requestData?: string | undefined;
+    responseStatusCode?: string | undefined;
+    responseData?: string | undefined;
+    when?: Record<string, unknown> | undefined;
+    protocol?: MockProtocol | undefined;
+    protoPath?: string | undefined;
+    grpcServiceName?: string | undefined;
+    protobufExtraFiles?: string[] | undefined;
+    protobufExtraImportPath?: string[] | undefined;
+    kafkaTopic?: string | undefined;
+    grpcStatusCode?: number | undefined;
+    grpcStatusMessage?: string | undefined;
+    requestAware?: boolean | undefined;
+    minLatency?: number | undefined;
+    maxLatency?: number | undefined;
+    dropRate?: number | undefined;
+    requestSchema?: string | undefined;
+    noMatchStatusCode?: number | undefined;
+    validationFailureStatusCode?: number | undefined;
+}>;
+export type MockSpec = z.infer<typeof mockSpecSchema>;
+export type MockSpecInput = z.input<typeof mockSpecSchema>;
+export interface BatchMockResult {
+    index: number;
+    protocol: string;
+    endpointURL: string;
+    method: string;
+    topic?: string;
+    success: boolean;
+    outputDir?: string;
+    output?: string;
+    error?: string;
+    detail?: string;
+}
+export interface BatchMockParams {
+    mocks: MockSpecInput[];
+    outputDir: string;
+    runtime?: string;
+    dockerNetwork?: string;
+    language?: string;
+    framework?: string;
+}
+export declare function executeBatchMockGeneration(params: BatchMockParams): Promise<CallToolResult>;
+export declare function registerBatchMockTool(server: McpServer): void;