@bryan-thompson/inspector-assessment-cli 1.25.9 → 1.26.0

package/build/__tests__/testbed-integration.test.js

@@ -0,0 +1,355 @@
+/**
+ * Testbed A/B Comparison Integration Tests
+ *
+ * Tests the inspector's pure behavior-based detection by comparing two servers
+ * with IDENTICAL tool names but different implementations:
+ * - vulnerable-mcp: Exploitable implementations (10 vulnerable + 6 safe tools)
+ * - hardened-mcp: Safe implementations (same tool names, all safe)
+ *
+ * This proves the inspector detects vulnerabilities based on tool behavior,
+ * not name-based heuristics.
+ *
+ * Note: Tests skip gracefully when testbed servers are unavailable.
+ */
+import { describe, it, expect, beforeAll } from "@jest/globals";
+// Testbed server URLs
+const VULNERABLE_URL = "http://localhost:10900/mcp";
+const HARDENED_URL = "http://localhost:10901/mcp";
+/**
+ * Default headers required by MCP HTTP servers
+ */
+const DEFAULT_HEADERS = {
+    "Content-Type": "application/json",
+    Accept: "application/json, text/event-stream",
+};
+/**
+ * Check if a server is available by sending an initialize request
+ */
+async function checkServerAvailable(url) {
+    try {
+        const response = await fetch(url, {
+            method: "POST",
+            headers: DEFAULT_HEADERS,
+            body: JSON.stringify({
+                jsonrpc: "2.0",
+                method: "initialize",
+                params: {
+                    protocolVersion: "2024-11-05",
+                    capabilities: {},
+                    clientInfo: { name: "testbed-test", version: "1.0.0" },
+                },
+                id: 1,
+            }),
+        });
+        return response.status < 500;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Parse SSE response to extract JSON data
+ * MCP streamable HTTP returns Server-Sent Events format
+ */
+async function parseSSEResponse(response) {
+    const text = await response.text();
+    // If it's plain JSON, parse directly
+    if (text.trim().startsWith("{")) {
+        return JSON.parse(text);
+    }
+    // Parse SSE format: "event: message\ndata: {...}\n\n"
+    const lines = text.split("\n");
+    for (const line of lines) {
+        if (line.startsWith("data:")) {
+            const jsonStr = line.slice(5).trim();
+            if (jsonStr) {
+                return JSON.parse(jsonStr);
+            }
+        }
+    }
+    throw new Error(`Unable to parse SSE response: ${text.slice(0, 100)}`);
+}
+/**
+ * Send an MCP JSON-RPC request and parse response
+ */
+async function sendMcpRequest(url, method, params = {}) {
+    const response = await fetch(url, {
+        method: "POST",
+        headers: DEFAULT_HEADERS,
+        body: JSON.stringify({
+            jsonrpc: "2.0",
+            method,
+            params,
+            id: Date.now(),
+        }),
+    });
+    let data = null;
+    if (response.ok) {
+        try {
+            data = await parseSSEResponse(response.clone());
+        }
+        catch {
+            // Response might not be parseable
+        }
+    }
+    return { response, data };
+}
+/**
+ * Get tool list from server
+ */
+async function getToolList(url) {
+    const { data } = await sendMcpRequest(url, "tools/list");
+    if (!data)
+        return [];
+    const result = data.result;
+    const tools = result.tools;
+    return tools.map((tool) => tool.name);
+}
+/**
+ * Call a tool and return the response
+ */
+async function callTool(url, toolName, args) {
+    const { data } = await sendMcpRequest(url, "tools/call", {
+        name: toolName,
+        arguments: args,
+    });
+    return data;
+}
+describe("Testbed A/B Comparison", () => {
+    let bothServersAvailable = false;
+    let vulnerableAvailable = false;
+    let hardenedAvailable = false;
+    beforeAll(async () => {
+        const [v, h] = await Promise.all([
+            checkServerAvailable(VULNERABLE_URL),
+            checkServerAvailable(HARDENED_URL),
+        ]);
+        vulnerableAvailable = v;
+        hardenedAvailable = h;
+        bothServersAvailable = v && h;
+        if (!bothServersAvailable) {
+            console.log("\n⚠️  Skipping testbed A/B comparison tests - servers not available");
+            console.log("   Start servers with:");
+            console.log("   - vulnerable-mcp: http://localhost:10900/mcp");
+            console.log("   - hardened-mcp: http://localhost:10901/mcp\n");
+        }
+    });
+    describe("Health Check Tests", () => {
+        it("should connect to vulnerable-mcp server", async () => {
+            if (!vulnerableAvailable) {
+                console.log("⏩ Skipping: vulnerable-mcp not available");
+                return;
+            }
+            const { response, data } = await sendMcpRequest(VULNERABLE_URL, "initialize", {
+                protocolVersion: "2024-11-05",
+                capabilities: {},
+                clientInfo: {
+                    name: "testbed-test",
+                    version: "1.0.0",
+                },
+            });
+            expect(response.ok).toBe(true);
+            expect(data).toHaveProperty("jsonrpc", "2.0");
+            expect(data).toHaveProperty("result");
+            const result = data.result;
+            expect(result).toHaveProperty("serverInfo");
+        });
+        it("should connect to hardened-mcp server", async () => {
+            if (!hardenedAvailable) {
+                console.log("⏩ Skipping: hardened-mcp not available");
+                return;
+            }
+            const { response, data } = await sendMcpRequest(HARDENED_URL, "initialize", {
+                protocolVersion: "2024-11-05",
+                capabilities: {},
+                clientInfo: {
+                    name: "testbed-test",
+                    version: "1.0.0",
+                },
+            });
+            expect(response.ok).toBe(true);
+            expect(data).toHaveProperty("jsonrpc", "2.0");
+            expect(data).toHaveProperty("result");
+            const result = data.result;
+            expect(result).toHaveProperty("serverInfo");
+        });
+        it("should list tools on both servers", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const [vulnerableTools, hardenedTools] = await Promise.all([
+                getToolList(VULNERABLE_URL),
+                getToolList(HARDENED_URL),
+            ]);
+            // Skip if tools lists are empty (server may require session state)
+            if (vulnerableTools.length === 0 && hardenedTools.length === 0) {
+                console.log("⏩ Skipping: servers returned empty tool lists");
+                return;
+            }
+            expect(vulnerableTools.length).toBeGreaterThan(0);
+            expect(hardenedTools.length).toBeGreaterThan(0);
+        });
+    });
+    describe("Tool List Parity", () => {
+        it("should have identical tool names on both servers", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const [vulnerableTools, hardenedTools] = await Promise.all([
+                getToolList(VULNERABLE_URL),
+                getToolList(HARDENED_URL),
+            ]);
+            // Skip if tools lists are empty (server may require session state)
+            if (vulnerableTools.length === 0 || hardenedTools.length === 0) {
+                console.log("⏩ Skipping: server returned empty tool list");
+                return;
+            }
+            // Sort for comparison
+            const sortedVulnerable = [...vulnerableTools].sort();
+            const sortedHardened = [...hardenedTools].sort();
+            expect(sortedVulnerable).toEqual(sortedHardened);
+        });
+        it("should have 16 tools (10 vulnerable + 6 safe)", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const [vulnerableTools, hardenedTools] = await Promise.all([
+                getToolList(VULNERABLE_URL),
+                getToolList(HARDENED_URL),
+            ]);
+            // Skip if tools lists are empty (server may require session state)
+            if (vulnerableTools.length === 0 || hardenedTools.length === 0) {
+                console.log("⏩ Skipping: server returned empty tool list");
+                return;
+            }
+            expect(vulnerableTools.length).toBe(16);
+            expect(hardenedTools.length).toBe(16);
+        });
+        it("should include expected vulnerable tool names", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const vulnerableTools = await getToolList(VULNERABLE_URL);
+            // Skip if tools list is empty (server may require session state)
+            if (vulnerableTools.length === 0) {
+                console.log("⏩ Skipping: server returned empty tool list");
+                return;
+            }
+            const expectedTools = [
+                "vulnerable_calculator_tool",
+                "vulnerable_system_exec_tool",
+            ];
+            expectedTools.forEach((toolName) => {
+                expect(vulnerableTools).toContain(toolName);
+            });
+        });
+        it("should include expected safe tool names", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const tools = await getToolList(VULNERABLE_URL);
+            // Skip if tools list is empty (server may require session state)
+            if (tools.length === 0) {
+                console.log("⏩ Skipping: server returned empty tool list");
+                return;
+            }
+            const expectedSafeTools = [
+                "safe_storage_tool",
+                "safe_calculator_tool",
+                "safe_validation_tool",
+            ];
+            expectedSafeTools.forEach((toolName) => {
+                expect(tools).toContain(toolName);
+            });
+        });
+    });
+    describe("Tool Behavior Comparison", () => {
+        it("should show different behavior for calculator tool", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const testInput = { expression: "__import__('os').system('id')" };
+            const [vulnerableResponse, hardenedResponse] = await Promise.all([
+                callTool(VULNERABLE_URL, "vulnerable_calculator_tool", testInput),
+                callTool(HARDENED_URL, "vulnerable_calculator_tool", testInput),
+            ]);
+            // Skip if either response is null (server may require session state)
+            if (!vulnerableResponse || !hardenedResponse) {
+                console.log("⏩ Skipping: tool call returned null (session state required)");
+                return;
+            }
+            // Both should respond, but vulnerable may execute, hardened should reject
+            expect(vulnerableResponse).toBeDefined();
+            expect(hardenedResponse).toBeDefined();
+            // Vulnerable server might show exploitable behavior
+            // Hardened server should show safe behavior (error or sanitized)
+            // We're not asserting specific behavior, just that they're different
+            expect(vulnerableResponse).not.toEqual(hardenedResponse);
+        });
+        it("should show identical behavior for safe tools", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const testInput = { value: "test_data" };
+            const [vulnerableResponse, hardenedResponse] = await Promise.all([
+                callTool(VULNERABLE_URL, "safe_storage_tool", testInput),
+                callTool(HARDENED_URL, "safe_storage_tool", testInput),
+            ]);
+            // Skip if either response is null (server may require session state)
+            if (!vulnerableResponse || !hardenedResponse) {
+                console.log("⏩ Skipping: tool call returned null (session state required)");
+                return;
+            }
+            expect(vulnerableResponse).toBeDefined();
+            expect(hardenedResponse).toBeDefined();
+        });
+    });
+    describe("Session Management", () => {
+        it("should return response headers from vulnerable server", async () => {
+            if (!vulnerableAvailable) {
+                console.log("⏩ Skipping: vulnerable-mcp not available");
+                return;
+            }
+            const { response } = await sendMcpRequest(VULNERABLE_URL, "tools/list");
+            expect(response.headers.get("content-type")).toBeTruthy();
+        });
+        it("should return response headers from hardened server", async () => {
+            if (!hardenedAvailable) {
+                console.log("⏩ Skipping: hardened-mcp not available");
+                return;
+            }
+            const { response } = await sendMcpRequest(HARDENED_URL, "tools/list");
+            expect(response.headers.get("content-type")).toBeTruthy();
+        });
+        it("should handle protocol version negotiation", async () => {
+            if (!bothServersAvailable) {
+                console.log("⏩ Skipping: both servers not available");
+                return;
+            }
+            const initParams = {
+                protocolVersion: "2024-11-05",
+                capabilities: {},
+                clientInfo: { name: "testbed-test", version: "1.0.0" },
+            };
+            const [vulnerableResult, hardenedResult] = await Promise.all([
+                sendMcpRequest(VULNERABLE_URL, "initialize", initParams),
+                sendMcpRequest(HARDENED_URL, "initialize", initParams),
+            ]);
+            const vulnerableData = vulnerableResult.data;
+            const hardenedData = hardenedResult.data;
+            expect(vulnerableData).toBeDefined();
+            expect(hardenedData).toBeDefined();
+            const vulnerableResultData = vulnerableData.result;
+            const hardenedResultData = hardenedData.result;
+            expect(vulnerableResultData).toHaveProperty("protocolVersion");
+            expect(hardenedResultData).toHaveProperty("protocolVersion");
+        });
+    });
+});

package/build/assess-full.js

@@ -384,10 +384,22 @@ function buildConfig(options) {
         config.temporalInvocations = options.temporalInvocations;
     }
     if (options.claudeEnabled) {
+        // Check for HTTP transport via --claude-http flag or environment variables
+        const useHttpTransport = options.claudeHttp || process.env.INSPECTOR_CLAUDE === "true";
+        const auditorUrl = options.mcpAuditorUrl ||
+            process.env.INSPECTOR_MCP_AUDITOR_URL ||
+            "http://localhost:8085";
         config.claudeCode = {
             enabled: true,
             timeout: FULL_CLAUDE_CODE_CONFIG.timeout || 60000,
             maxRetries: FULL_CLAUDE_CODE_CONFIG.maxRetries || 2,
+            // Use HTTP transport when --claude-http flag or INSPECTOR_CLAUDE env is set
+            ...(useHttpTransport && {
+                transport: "http",
+                httpConfig: {
+                    baseUrl: auditorUrl,
+                },
+            }),
             features: {
                 intelligentTestGeneration: true,
                 aupSemanticAnalysis: true,
@@ -395,6 +407,9 @@ function buildConfig(options) {
                 documentationQuality: true,
             },
         };
+        if (useHttpTransport) {
+            console.log(`🔗 Claude Bridge HTTP transport: ${auditorUrl}`);
+        }
     }
     // Pass custom annotation pattern config path
     if (options.patternConfigPath) {
@@ -890,6 +905,32 @@ function parseArgs() {
             case "--claude-enabled":
                 options.claudeEnabled = true;
                 break;
+            case "--claude-http":
+                // Enable Claude Bridge with HTTP transport (connects to mcp-auditor)
+                options.claudeEnabled = true;
+                options.claudeHttp = true;
+                break;
+            case "--mcp-auditor-url": {
+                const urlValue = args[++i];
+                if (!urlValue || urlValue.startsWith("-")) {
+                    console.error("Error: --mcp-auditor-url requires a URL argument");
+                    setTimeout(() => process.exit(1), 10);
+                    options.helpRequested = true;
+                    return options;
+                }
+                try {
+                    new URL(urlValue); // Validate URL format
+                    options.mcpAuditorUrl = urlValue;
+                }
+                catch {
+                    console.error(`Error: Invalid URL for --mcp-auditor-url: ${urlValue}`);
+                    console.error("  Expected format: http://hostname:port or https://hostname:port");
+                    setTimeout(() => process.exit(1), 10);
+                    options.helpRequested = true;
+                    return options;
+                }
+                break;
+            }
             case "--full":
                 options.fullAssessment = true;
                 break;
@@ -1047,6 +1088,23 @@ function parseArgs() {
         options.helpRequested = true;
         return options;
     }
+    // Environment variable fallbacks (matches run-security-assessment.ts behavior)
+    // INSPECTOR_CLAUDE=true enables Claude with HTTP transport
+    if (process.env.INSPECTOR_CLAUDE === "true" && !options.claudeEnabled) {
+        options.claudeEnabled = true;
+        options.claudeHttp = true; // HTTP transport when enabled via env var
+    }
+    // INSPECTOR_MCP_AUDITOR_URL overrides default URL (only if not set via CLI)
+    if (process.env.INSPECTOR_MCP_AUDITOR_URL && !options.mcpAuditorUrl) {
+        const envUrl = process.env.INSPECTOR_MCP_AUDITOR_URL;
+        try {
+            new URL(envUrl);
+            options.mcpAuditorUrl = envUrl;
+        }
+        catch {
+            console.warn(`Warning: Invalid INSPECTOR_MCP_AUDITOR_URL: ${envUrl}, using default`);
+        }
+    }
     return options;
 }
 /**
@@ -1072,7 +1130,9 @@ Options:
   --diff-only            Output only the comparison diff (requires --compare)
   --resume               Resume from previous interrupted assessment
   --no-resume            Force fresh start, clear any existing state
-  --claude-enabled       Enable Claude Code integration for intelligent analysis
+  --claude-enabled       Enable Claude Code integration (CLI transport: requires 'claude' binary)
+  --claude-http          Enable Claude Code via HTTP transport (connects to mcp-auditor proxy)
+  --mcp-auditor-url <url>  mcp-auditor URL for HTTP transport (default: http://localhost:8085)
   --full                 Enable all assessment modules (default)
   --profile <name>       Use predefined module profile (quick, security, compliance, full)
   --temporal-invocations <n>  Number of invocations per tool for rug pull detection (default: 25)
@@ -1086,6 +1146,11 @@ Options:
                          Also supports LOG_LEVEL environment variable
   --help, -h             Show this help message
 
+Environment Variables:
+  INSPECTOR_CLAUDE=true         Enable Claude with HTTP transport (same as --claude-http)
+  INSPECTOR_MCP_AUDITOR_URL     Override default mcp-auditor URL (default: http://localhost:8085)
+  LOG_LEVEL                     Set log level (overridden by --log-level flag)
+
 ${getProfileHelpText()}
 Module Selection:
   --profile, --skip-modules, and --only-modules are mutually exclusive.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bryan-thompson/inspector-assessment-cli",
-  "version": "1.25.9",
+  "version": "1.26.0",
   "description": "CLI for the Enhanced MCP Inspector with assessment capabilities",
   "license": "MIT",
   "author": "Bryan Thompson <bryan@triepod.ai>",