mcp-server-diff 2.1.6 → 2.2.0

package/README.md

@@ -223,6 +223,46 @@ When using `configurations`, each object supports:
 | `headers` | HTTP headers for this configuration | No |
 | `env_vars` | Additional environment variables | No |
 | `custom_messages` | Config-specific custom messages | No |
+| `base_start_command` | Command for baseline comparison (skips git checkout for this config) | No |
+| `base_server_url` | URL for baseline HTTP server (used with `base_start_command`) | No |
+
+### Comparing Against External Servers
+
+When comparing against external servers (e.g., Docker images, remote services), use `base_start_command` to specify a different command for the baseline. This skips git checkout for that configuration and probes the specified server directly:
+
+```yaml
+configurations: |
+  [
+    {
+      "name": "compare-versions",
+      "transport": "stdio",
+      "start_command": "docker run -i ghcr.io/example/mcp-server:v2.0.0",
+      "base_start_command": "docker run -i ghcr.io/example/mcp-server:v1.0.0"
+    }
+  ]
+```
+
+This is useful for:
+- **Version comparison**: Compare a new release against the previous version
+- **Golden reference testing**: Compare your local code against a known-good reference
+- **Cross-implementation testing**: Compare different implementations of the same server
+- **Self-testing CI**: Verify the action detects diffs by comparing two known-different servers
+
+For HTTP transport, use `base_server_url` alongside `base_start_command`:
+
+```yaml
+configurations: |
+  [
+    {
+      "name": "http-comparison",
+      "transport": "streamable-http",
+      "start_command": "docker run -p 3000:3000 myserver:latest",
+      "server_url": "http://localhost:3000/mcp",
+      "base_start_command": "docker run -p 3001:3000 myserver:v1.0.0",
+      "base_server_url": "http://localhost:3001/mcp"
+    }
+  ]
+```
 
 ## How It Works
 
@@ -595,8 +635,8 @@ npx mcp-server-diff -c servers.json -o diff
 | `-b, --base <cmd\|url>` | Base server command (stdio) or URL (http) |
 | `-t, --target <cmd\|url>` | Target server command (stdio) or URL (http) |
 | `-H, --header <header>` | HTTP header for target (repeatable) |
-| `--base-header <header>` | HTTP header for base server (repeatable) |
-| `--target-header <header>` | HTTP header for target (same as `-H`) |
+| `-B, --base-header <header>` | HTTP header for base server (repeatable) |
+| `-T, --target-header <header>` | HTTP header for target (same as `-H`) |
 | `-c, --config <file>` | Config file with base and targets |
 | `-o, --output <format>` | Output: `diff`, `json`, `markdown`, `summary` (default) |
 | `-v, --verbose` | Verbose output |

package/dist/cli/index.js

@@ -56368,6 +56368,7 @@ function isMethodNotFound(error) {
 async function probeServer(options) {
     const result = {
         initialize: null,
+        instructions: null,
         tools: null,
         prompts: null,
         resources: null,
@@ -56426,6 +56427,12 @@ async function probeServer(options) {
             serverInfo,
             capabilities: serverCapabilities,
         };
+        // Get server instructions
+        const instructions = client.getInstructions();
+        if (instructions) {
+            result.instructions = instructions;
+            log.info(`  Got server instructions (${instructions.length} chars)`);
+        }
         // Probe tools if supported
         if (serverCapabilities?.tools) {
             try {
@@ -56623,6 +56630,9 @@ function probeResultToFiles(result) {
     if (result.initialize) {
         files.set("initialize", JSON.stringify(normalizeProbeResult(result.initialize), null, 2));
     }
+    if (result.instructions) {
+        files.set("instructions", result.instructions);
+    }
     if (result.tools) {
         files.set("tools", JSON.stringify(normalizeProbeResult(result.tools), null, 2));
     }
@@ -56870,8 +56880,8 @@ function parseCliArgs() {
             base: { type: "string", short: "b" },
             target: { type: "string", short: "t" },
             header: { type: "string", short: "H", multiple: true },
-            "base-header": { type: "string", multiple: true },
-            "target-header": { type: "string", multiple: true },
+            "base-header": { type: "string", short: "B", multiple: true },
+            "target-header": { type: "string", short: "T", multiple: true },
             config: { type: "string", short: "c" },
             output: { type: "string", short: "o", default: "summary" },
             verbose: { type: "boolean", short: "v", default: false },
@@ -56900,8 +56910,8 @@ OPTIONS:
   -b, --base <command>       Base server command (stdio) or URL (http)
   -t, --target <command>     Target server command (stdio) or URL (http)
   -H, --header <header>      HTTP header for target (repeatable)
-      --base-header <header> HTTP header for base server (repeatable)
-      --target-header <hdr>  HTTP header for target server (repeatable, same as -H)
+  -B, --base-header <header> HTTP header for base server (repeatable)
+  -T, --target-header <hdr>  HTTP header for target server (repeatable, same as -H)
                              Values support: env:VAR_NAME, secret:name, "Bearer secret:token"
   -c, --config <file>        Config file with base and targets
   -o, --output <format>      Output format: diff, json, markdown, summary (default: summary)
@@ -57060,44 +57070,46 @@ function findSecrets(headerStrings) {
  */
 async function promptSecret(prompt) {
     return new Promise((resolve) => {
-        const rl = external_readline_namespaceObject.createInterface({
-            input: process.stdin,
-            output: process.stdout,
-        });
+        process.stdout.write(`${prompt}: `);
         // Hide input by using raw mode if available
-        if (process.stdin.isTTY) {
-            process.stdout.write(`${prompt}: `);
+        if (process.stdin.isTTY && process.stdin.setRawMode) {
             process.stdin.setRawMode(true);
             process.stdin.resume();
+            process.stdin.setEncoding("utf8");
             let value = "";
             const onData = (char) => {
-                const c = char.toString();
-                if (c === "\n" || c === "\r") {
+                if (char === "\n" || char === "\r" || char === "\u0004") {
                     process.stdin.setRawMode(false);
+                    process.stdin.pause();
                     process.stdin.removeListener("data", onData);
-                    rl.close();
                     process.stdout.write("\n");
                     resolve(value);
                 }
-                else if (c === "\u0003") {
+                else if (char === "\u0003") {
                     // Ctrl+C
+                    process.stdin.setRawMode(false);
+                    process.stdout.write("\n");
                     process.exit(1);
                 }
-                else if (c === "\u007F" || c === "\b") {
+                else if (char === "\u007F" || char === "\b") {
                     // Backspace
                     if (value.length > 0) {
                         value = value.slice(0, -1);
                     }
                 }
                 else {
-                    value += c;
+                    value += char;
                 }
             };
             process.stdin.on("data", onData);
         }
         else {
-            // Non-TTY: just read the line (won't be hidden)
-            rl.question(`${prompt}: `, (answer) => {
+            // Non-TTY: use readline (won't be hidden)
+            const rl = external_readline_namespaceObject.createInterface({
+                input: process.stdin,
+                output: process.stdout,
+            });
+            rl.question("", (answer) => {
                 rl.close();
                 resolve(answer);
             });

package/dist/cli/types.d.ts

@@ -18,6 +18,14 @@ export interface TestConfiguration {
     startup_wait_ms?: number;
     /** Command to run after stopping the MCP server for this config (cleanup) */
     post_test_command?: string;
+    /**
+     * Command to use for the base/comparison server instead of checking out a git ref.
+     * When set, skips git operations and uses this command directly for comparison.
+     * Useful for comparing against external servers (e.g., docker images of previous releases).
+     */
+    base_start_command?: string;
+    /** Server URL for base comparison (for HTTP transport with base_start_command) */
+    base_server_url?: string;
 }
 export interface CustomMessage {
     id: number;
@@ -53,6 +61,7 @@ export interface ActionInputs {
 }
 export interface ProbeResult {
     initialize: InitializeInfo | null;
+    instructions: string | null;
     tools: ToolsResult | null;
     prompts: PromptsResult | null;
     resources: ResourcesResult | null;

package/dist/index.js

@@ -56618,6 +56618,7 @@ function isMethodNotFound(error) {
 async function probeServer(options) {
     const result = {
         initialize: null,
+        instructions: null,
         tools: null,
         prompts: null,
         resources: null,
@@ -56676,6 +56677,12 @@ async function probeServer(options) {
             serverInfo,
             capabilities: serverCapabilities,
         };
+        // Get server instructions
+        const instructions = client.getInstructions();
+        if (instructions) {
+            result.instructions = instructions;
+            log.info(`  Got server instructions (${instructions.length} chars)`);
+        }
         // Probe tools if supported
         if (serverCapabilities?.tools) {
             try {
@@ -56873,6 +56880,9 @@ function probeResultToFiles(result) {
     if (result.initialize) {
         files.set("initialize", JSON.stringify(normalizeProbeResult(result.initialize), null, 2));
     }
+    if (result.instructions) {
+        files.set("instructions", result.instructions);
+    }
     if (result.tools) {
         files.set("tools", JSON.stringify(normalizeProbeResult(result.tools), null, 2));
     }
@@ -57133,8 +57143,10 @@ async function runPostTestCommand(config, workDir) {
 /**
  * Probe a server with a specific configuration
  * @param useSharedServer - If true, skip starting per-config HTTP server (shared server is already running)
+ * @param overrideCommand - If provided, use this command instead of config.start_command (for base_start_command)
+ * @param overrideUrl - If provided, use this URL instead of config.server_url (for base_server_url)
  */
-async function probeWithConfig(config, workDir, globalEnvVars, globalHeaders, globalCustomMessages, useSharedServer = false) {
+async function probeWithConfig(config, workDir, globalEnvVars, globalHeaders, globalCustomMessages, useSharedServer = false, overrideCommand, overrideUrl) {
     const configEnvVars = parseEnvVars(config.env_vars);
     const envVars = { ...globalEnvVars, ...configEnvVars };
     const headers = { ...globalHeaders, ...config.headers };
@@ -57142,12 +57154,12 @@ async function probeWithConfig(config, workDir, globalEnvVars, globalHeaders, gl
     // Run pre-test command before probing
     await runPreTestCommand(config, workDir);
     if (config.transport === "stdio") {
-        const command = config.start_command || "";
+        const command = overrideCommand || config.start_command || "";
         const parts = command.split(/\s+/);
         const cmd = parts[0];
         const args = parts.slice(1);
-        // Also parse additional args if provided
-        if (config.args) {
+        // Also parse additional args if provided (only when not using override)
+        if (!overrideCommand && config.args) {
             args.push(...config.args.split(/\s+/));
         }
         return await probeServer({
@@ -57163,13 +57175,17 @@ async function probeWithConfig(config, workDir, globalEnvVars, globalHeaders, gl
         // For HTTP transport, optionally start the server if start_command is provided
         // Skip if using shared server
         let serverProcess = null;
+        const serverUrl = overrideUrl || config.server_url;
         try {
-            if (config.start_command && !useSharedServer) {
-                serverProcess = await startHttpServer(config, workDir, envVars);
+            const startCmd = overrideCommand || config.start_command;
+            if (startCmd && !useSharedServer) {
+                // Create a temporary config for the override
+                const tempConfig = { ...config, start_command: startCmd };
+                serverProcess = await startHttpServer(tempConfig, workDir, envVars);
             }
             return await probeServer({
                 transport: "streamable-http",
-                url: config.server_url,
+                url: serverUrl,
                 headers,
                 envVars,
                 customMessages,
@@ -57442,13 +57458,18 @@ async function startSharedHttpServer(command, workDir, waitMs, envVars) {
 }
 /**
  * Probe a single configuration (without comparison)
- */
-async function probeConfig(config, workDir, envVars, headers, customMessages, useSharedServer) {
-    lib_core.info(`  📋 Probing: ${config.name} (${config.transport})`);
+ * @param overrideCommand - If provided, use this command instead of config.start_command
+ * @param overrideUrl - If provided, use this URL instead of config.server_url
+ */
+async function probeConfig(config, workDir, envVars, headers, customMessages, useSharedServer, overrideCommand, overrideUrl) {
+    const displayName = overrideCommand
+        ? `${config.name} (base: ${overrideCommand.slice(0, 50)}${overrideCommand.length > 50 ? "..." : ""})`
+        : config.name;
+    lib_core.info(`  📋 Probing: ${displayName} (${config.transport})`);
     const start = Date.now();
     let result;
     try {
-        result = await probeWithConfig(config, workDir, envVars, headers, customMessages, useSharedServer);
+        result = await probeWithConfig(config, workDir, envVars, headers, customMessages, useSharedServer, overrideCommand, overrideUrl);
     }
     finally {
         await runPostTestCommand(config, workDir);
@@ -57491,6 +57512,7 @@ async function runAllTests(ctx) {
                 branchResults.set(config.name, {
                     result: {
                         initialize: null,
+                        instructions: null,
                         tools: null,
                         prompts: null,
                         resources: null,
@@ -57511,65 +57533,102 @@ async function runAllTests(ctx) {
         }
     }
     // ========================================
-    // PHASE 2: Probe all configs on base ref
+    // PHASE 2: Probe all configs for base comparison
     // ========================================
-    lib_core.info(`\n🔄 Phase 2: Testing comparison ref: ${ctx.compareRef}...`);
-    const worktreePath = external_path_.join(ctx.workDir, ".mcp-diff-base");
-    let useWorktree = false;
-    try {
-        // Set up comparison ref
-        useWorktree = await createWorktree(ctx.compareRef, worktreePath);
-        if (!useWorktree) {
-            lib_core.info("  Worktree not available, using checkout");
-            await checkout(ctx.compareRef);
-        }
-        const baseWorkDir = useWorktree ? worktreePath : ctx.workDir;
-        // Build on base
-        lib_core.info("🔨 Building on comparison ref...");
-        await runBuild(baseWorkDir, ctx.inputs);
-        let baseServerProcess = null;
-        try {
-            // Start HTTP server for base ref if needed
-            if (useSharedServer) {
-                baseServerProcess = await startSharedHttpServer(httpStartCommand, baseWorkDir, httpStartupWaitMs, globalEnvVars);
+    // Split configs: those with base_start_command use direct probing,
+    // those without need git checkout/worktree
+    const configsWithBaseCommand = ctx.inputs.configurations.filter((c) => c.base_start_command);
+    const configsNeedingGit = ctx.inputs.configurations.filter((c) => !c.base_start_command);
+    // PHASE 2a: Probe configs with base_start_command directly (no git operations)
+    if (configsWithBaseCommand.length > 0) {
+        lib_core.info(`\n🔄 Phase 2a: Testing ${configsWithBaseCommand.length} config(s) with explicit base commands...`);
+        for (const config of configsWithBaseCommand) {
+            try {
+                // Use base_start_command/base_server_url for comparison
+                const probeData = await probeConfig(config, ctx.workDir, // Use current workdir since we're not checking out
+                globalEnvVars, globalHeaders, globalCustomMessages, false, // Don't use shared server for base commands
+                config.base_start_command, config.base_server_url);
+                baseResults.set(config.name, probeData);
             }
-            for (const config of ctx.inputs.configurations) {
-                const configUsesSharedServer = useSharedServer && config.transport === "streamable-http";
-                try {
-                    const probeData = await probeConfig(config, baseWorkDir, globalEnvVars, globalHeaders, globalCustomMessages, configUsesSharedServer);
-                    baseResults.set(config.name, probeData);
+            catch (error) {
+                lib_core.error(`Failed to probe ${config.name} with base command: ${error}`);
+                baseResults.set(config.name, {
+                    result: {
+                        initialize: null,
+                        instructions: null,
+                        tools: null,
+                        prompts: null,
+                        resources: null,
+                        resourceTemplates: null,
+                        customResponses: new Map(),
+                        error: String(error),
+                    },
+                    time: 0,
+                });
+            }
+        }
+    }
+    // PHASE 2b: Probe configs needing git checkout
+    if (configsNeedingGit.length > 0) {
+        lib_core.info(`\n🔄 Phase 2b: Testing comparison ref: ${ctx.compareRef}...`);
+        const worktreePath = external_path_.join(ctx.workDir, ".mcp-diff-base");
+        let useWorktree = false;
+        try {
+            // Set up comparison ref
+            useWorktree = await createWorktree(ctx.compareRef, worktreePath);
+            if (!useWorktree) {
+                lib_core.info("  Worktree not available, using checkout");
+                await checkout(ctx.compareRef);
+            }
+            const baseWorkDir = useWorktree ? worktreePath : ctx.workDir;
+            // Build on base
+            lib_core.info("🔨 Building on comparison ref...");
+            await runBuild(baseWorkDir, ctx.inputs);
+            let baseServerProcess = null;
+            try {
+                // Start HTTP server for base ref if needed
+                if (useSharedServer) {
+                    baseServerProcess = await startSharedHttpServer(httpStartCommand, baseWorkDir, httpStartupWaitMs, globalEnvVars);
                 }
-                catch (error) {
-                    lib_core.error(`Failed to probe ${config.name} on base ref: ${error}`);
-                    baseResults.set(config.name, {
-                        result: {
-                            initialize: null,
-                            tools: null,
-                            prompts: null,
-                            resources: null,
-                            resourceTemplates: null,
-                            customResponses: new Map(),
-                            error: String(error),
-                        },
-                        time: 0,
-                    });
+                for (const config of configsNeedingGit) {
+                    const configUsesSharedServer = useSharedServer && config.transport === "streamable-http";
+                    try {
+                        const probeData = await probeConfig(config, baseWorkDir, globalEnvVars, globalHeaders, globalCustomMessages, configUsesSharedServer);
+                        baseResults.set(config.name, probeData);
+                    }
+                    catch (error) {
+                        lib_core.error(`Failed to probe ${config.name} on base ref: ${error}`);
+                        baseResults.set(config.name, {
+                            result: {
+                                initialize: null,
+                                instructions: null,
+                                tools: null,
+                                prompts: null,
+                                resources: null,
+                                resourceTemplates: null,
+                                customResponses: new Map(),
+                                error: String(error),
+                            },
+                            time: 0,
+                        });
+                    }
+                }
+            }
+            finally {
+                if (baseServerProcess) {
+                    lib_core.info("🛑 Stopping base ref HTTP server...");
+                    stopHttpServer(baseServerProcess);
                 }
             }
         }
         finally {
-            if (baseServerProcess) {
-                lib_core.info("🛑 Stopping base ref HTTP server...");
-                stopHttpServer(baseServerProcess);
+            // Clean up
+            if (useWorktree) {
+                await removeWorktree(worktreePath);
+            }
+            else {
+                await checkoutPrevious();
             }
-        }
-    }
-    finally {
-        // Clean up
-        if (useWorktree) {
-            await removeWorktree(worktreePath);
-        }
-        else {
-            await checkoutPrevious();
         }
     }
     // ========================================
@@ -57773,6 +57832,7 @@ function saveReport(report, markdown, outputDir) {
         external_fs_.appendFileSync(githubOutput, `has_differences=${report.diffCount > 0}\n`);
         external_fs_.appendFileSync(githubOutput, `passed_count=${report.passedCount}\n`);
         external_fs_.appendFileSync(githubOutput, `diff_count=${report.diffCount}\n`);
+        external_fs_.appendFileSync(githubOutput, `total_configs=${report.results.length}\n`);
     }
     // Also set via core for compatibility
     lib_core.setOutput("report_path", mdPath);
@@ -57780,6 +57840,7 @@ function saveReport(report, markdown, outputDir) {
     lib_core.setOutput("has_differences", report.diffCount > 0);
     lib_core.setOutput("passed_count", report.passedCount);
     lib_core.setOutput("diff_count", report.diffCount);
+    lib_core.setOutput("total_configs", report.results.length);
 }
 /**
  * Write a simple summary for PR comments

package/dist/types.d.ts

@@ -18,6 +18,14 @@ export interface TestConfiguration {
     startup_wait_ms?: number;
     /** Command to run after stopping the MCP server for this config (cleanup) */
     post_test_command?: string;
+    /**
+     * Command to use for the base/comparison server instead of checking out a git ref.
+     * When set, skips git operations and uses this command directly for comparison.
+     * Useful for comparing against external servers (e.g., docker images of previous releases).
+     */
+    base_start_command?: string;
+    /** Server URL for base comparison (for HTTP transport with base_start_command) */
+    base_server_url?: string;
 }
 export interface CustomMessage {
     id: number;
@@ -53,6 +61,7 @@ export interface ActionInputs {
 }
 export interface ProbeResult {
     initialize: InitializeInfo | null;
+    instructions: string | null;
     tools: ToolsResult | null;
     prompts: PromptsResult | null;
     resources: ResourcesResult | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-server-diff",
-  "version": "2.1.6",
+  "version": "2.2.0",
   "description": "Diff MCP server public interfaces - CLI tool and GitHub Action",
   "main": "dist/index.js",
   "bin": {