@beeeeen/mcp-probe 0.1.0

package/dist/cli.js

@@ -0,0 +1,292 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync } from 'node:fs';
+import { run, exitCodeFor } from './run.js';
+import { renderTerminal } from './report/terminal.js';
+import { renderJUnit } from './report/junit.js';
+const VERSION = '0.1.0';
+const HELP = `
+  mcp-probe ${VERSION}
+  Conformance and robustness tests for MCP servers. Built for CI.
+
+  USAGE
+    mcp-probe <command> [args...]        run a stdio server and probe it
+    mcp-probe --url <url>               probe a streamable-HTTP server
+    mcp-probe --config <file> --server <name>
+
+  EXAMPLES
+    mcp-probe node build/index.js
+    mcp-probe -- npx -y @modelcontextprotocol/server-filesystem /tmp
+    mcp-probe --url http://localhost:3000/mcp
+    mcp-probe --config ~/.claude.json --server github
+    mcp-probe --junit results.xml --strict -- node server.js
+
+  OUTPUT
+    --json                  machine-readable report on stdout
+    --json-out <file>       write the JSON report to a file, keeping the
+                            human-readable report on stdout
+    --junit <file>          write JUnit XML (GitHub, GitLab, Jenkins, ...)
+    --quiet                 suppress the terminal report
+    --verbose               expand the explanation for warnings too
+
+  SELECTION
+    --only <id>             run only these checks; repeatable, prefix match
+    --skip <id>             skip these checks; repeatable, prefix match
+                            groups: protocol, schema, robustness, hygiene
+
+  BEHAVIOUR
+    --timeout <ms>          per-request timeout (default 10000)
+    --strict                treat warnings as failures for the exit code
+    --call-tools            invoke every tool with empty arguments.
+                            OFF by default: mcp-probe never triggers a side
+                            effect you did not ask for.
+    --safe-tool <name>      invoke just this tool; repeatable
+    --header <k:v>          extra HTTP header; repeatable (http only)
+    --env <k=v>             extra environment variable; repeatable (stdio only)
+
+  EXIT CODES
+    0  no failures
+    1  at least one failure (or a warning under --strict)
+    2  could not run at all
+
+  mcp-probe's own flags come first. Everything from the first non-flag argument
+  (or from a bare "--") onward is the server's command line, passed through
+  untouched, so a server flag can never collide with one of ours. Use "--"
+  whenever the command itself starts with a dash.
+`;
+function parseArgs(argv) {
+    const out = {
+        target: null,
+        options: {},
+        json: false,
+        jsonOut: null,
+        junit: null,
+        quiet: false,
+        verbose: false,
+        help: false,
+        version: false,
+    };
+    const only = [];
+    const skip = [];
+    const safeTools = [];
+    const headers = {};
+    const env = {};
+    let url = null;
+    let configPath = null;
+    let serverName = null;
+    let command = null;
+    let commandArgs = [];
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        // Once a command is known, the rest belongs to the server, untouched.
+        if (command !== null) {
+            commandArgs.push(arg);
+            continue;
+        }
+        // `--` ends our own flags: the next token is the command, whatever it
+        // looks like. Needed when the command itself begins with a dash.
+        if (arg === '--') {
+            const rest = argv.slice(i + 1);
+            if (rest.length === 0)
+                return { ...out, error: '`--` must be followed by the server command' };
+            command = rest[0];
+            commandArgs = rest.slice(1);
+            break;
+        }
+        const next = () => argv[++i];
+        switch (arg) {
+            case '-h':
+            case '--help':
+                out.help = true;
+                break;
+            case '-v':
+            case '--version':
+                out.version = true;
+                break;
+            case '--json':
+                out.json = true;
+                break;
+            case '--quiet':
+                out.quiet = true;
+                break;
+            case '--verbose':
+                out.verbose = true;
+                break;
+            case '--strict':
+                out.options.strict = true;
+                break;
+            case '--call-tools':
+                out.options.callTools = true;
+                break;
+            case '--junit':
+                out.junit = next() ?? null;
+                break;
+            case '--json-out':
+                out.jsonOut = next() ?? null;
+                break;
+            case '--url':
+                url = next() ?? null;
+                break;
+            case '--config':
+                configPath = next() ?? null;
+                break;
+            case '--server':
+                serverName = next() ?? null;
+                break;
+            case '--timeout': {
+                const v = Number(next());
+                if (!Number.isFinite(v) || v <= 0)
+                    return { ...out, error: '--timeout needs a positive number of milliseconds' };
+                out.options.timeoutMs = v;
+                break;
+            }
+            case '--only':
+                only.push(...(next() ?? '').split(',').filter(Boolean));
+                break;
+            case '--skip':
+                skip.push(...(next() ?? '').split(',').filter(Boolean));
+                break;
+            case '--safe-tool':
+                safeTools.push(...(next() ?? '').split(',').filter(Boolean));
+                break;
+            case '--header': {
+                const raw = next() ?? '';
+                const idx = raw.indexOf(':');
+                if (idx < 1)
+                    return { ...out, error: `--header expects "Name: value", got "${raw}"` };
+                headers[raw.slice(0, idx).trim()] = raw.slice(idx + 1).trim();
+                break;
+            }
+            case '--env': {
+                const raw = next() ?? '';
+                const idx = raw.indexOf('=');
+                if (idx < 1)
+                    return { ...out, error: `--env expects "KEY=value", got "${raw}"` };
+                env[raw.slice(0, idx)] = raw.slice(idx + 1);
+                break;
+            }
+            default:
+                if (arg.startsWith('-'))
+                    return { ...out, error: `Unknown option "${arg}". Try --help.` };
+                command = arg;
+        }
+    }
+    if (only.length)
+        out.options.only = only;
+    if (skip.length)
+        out.options.skip = skip;
+    if (safeTools.length)
+        out.options.safeTools = safeTools;
+    if (configPath) {
+        const resolved = resolveFromConfig(configPath, serverName);
+        if ('error' in resolved)
+            return { ...out, error: resolved.error };
+        out.target = resolved.target;
+        if (Object.keys(env).length)
+            out.target.env = { ...out.target.env, ...env };
+        return out;
+    }
+    if (url) {
+        out.target = { kind: 'http', url, headers };
+        return out;
+    }
+    if (command) {
+        out.target = { kind: 'stdio', command, args: commandArgs, env };
+        return out;
+    }
+    return out;
+}
+/**
+ * Read a server definition out of an existing MCP config file, so a user can
+ * point mcp-probe at a server they already run without retyping its command.
+ * Handles both the `mcpServers` shape (Claude Desktop, Cursor) and the
+ * `servers` shape (VS Code).
+ */
+function resolveFromConfig(path, name) {
+    let parsed;
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch (e) {
+        return { error: `Could not read ${path}: ${e.message}` };
+    }
+    const block = (parsed['mcpServers'] ?? parsed['servers']);
+    if (!block || typeof block !== 'object') {
+        return { error: `${path} has no "mcpServers" or "servers" object.` };
+    }
+    const names = Object.keys(block);
+    if (names.length === 0)
+        return { error: `${path} defines no servers.` };
+    const chosen = name ?? (names.length === 1 ? names[0] : null);
+    if (!chosen) {
+        return { error: `${path} defines ${names.length} servers. Pick one with --server <name>: ${names.join(', ')}` };
+    }
+    const entry = block[chosen];
+    if (!entry)
+        return { error: `No server named "${chosen}" in ${path}. Available: ${names.join(', ')}` };
+    if (typeof entry['url'] === 'string') {
+        return { target: { kind: 'http', url: entry['url'], headers: entry['headers'] ?? {} } };
+    }
+    if (typeof entry['command'] === 'string') {
+        return {
+            target: {
+                kind: 'stdio',
+                command: entry['command'],
+                args: Array.isArray(entry['args']) ? entry['args'] : [],
+                env: entry['env'] ?? {},
+            },
+        };
+    }
+    return { error: `Server "${chosen}" in ${path} has neither "command" nor "url".` };
+}
+async function main() {
+    const parsed = parseArgs(process.argv.slice(2));
+    if (parsed.help) {
+        process.stdout.write(HELP);
+        process.exit(0);
+    }
+    if (parsed.version) {
+        process.stdout.write(VERSION + '\n');
+        process.exit(0);
+    }
+    if (parsed.error) {
+        process.stderr.write(`mcp-probe: ${parsed.error}\n`);
+        process.exit(2);
+    }
+    if (!parsed.target) {
+        process.stderr.write(HELP);
+        process.exit(2);
+    }
+    const report = await run(parsed.target, parsed.options);
+    if (parsed.junit) {
+        try {
+            writeFileSync(parsed.junit, renderJUnit(report), 'utf8');
+        }
+        catch (e) {
+            process.stderr.write(`mcp-probe: could not write ${parsed.junit}: ${e.message}\n`);
+        }
+    }
+    if (parsed.jsonOut) {
+        try {
+            writeFileSync(parsed.jsonOut, JSON.stringify(report, null, 2), 'utf8');
+        }
+        catch (e) {
+            process.stderr.write(`mcp-probe: could not write ${parsed.jsonOut}: ${e.message}
+`);
+        }
+    }
+    if (parsed.json) {
+        process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+    }
+    else if (!parsed.quiet) {
+        process.stdout.write(renderTerminal(report, { verbose: parsed.verbose }));
+    }
+    // Nothing ran at all: distinguish "server is broken" from "could not start".
+    if (report.results.length === 1 && report.results[0]?.id === 'connect' && report.results[0].status === 'fail') {
+        process.exit(2);
+    }
+    process.exit(exitCodeFor(report, parsed.options.strict ?? false));
+}
+main().catch((e) => {
+    process.stderr.write(`mcp-probe: internal error: ${e.stack ?? String(e)}\n`);
+    process.exit(2);
+});

package/dist/client/http.d.ts

@@ -0,0 +1,37 @@
+import type { Transport } from './transport.js';
+import { type JsonRpcResponse } from './jsonrpc.js';
+export interface HttpOptions {
+    url: string;
+    headers?: Record<string, string>;
+}
+/**
+ * Streamable HTTP transport. Each request is a POST; the server may answer
+ * with `application/json` or an SSE stream, and may hand us a session id on
+ * the initialize response that must be echoed on every later call.
+ */
+export declare class HttpTransport implements Transport {
+    private opts;
+    readonly kind = "http";
+    readonly target: string;
+    readonly stdoutNoise: string[];
+    readonly stderr: string[];
+    readonly serverNotifications: JsonRpcResponse[];
+    private nextId;
+    private sessionId;
+    private closed;
+    /** Populated when a response arrives with a shape we could not read. */
+    readonly protocolNotes: string[];
+    constructor(opts: HttpOptions);
+    start(): Promise<void>;
+    private headers;
+    request(method: string, params?: unknown, timeoutMs?: number): Promise<JsonRpcResponse>;
+    requestRaw(payload: Record<string, unknown>, _id: number | string, method: string, timeoutMs?: number): Promise<JsonRpcResponse>;
+    private post;
+    /** Pull the first `data:` frame that carries a JSON-RPC reply. */
+    private parseSse;
+    notify(method: string, params?: unknown): void;
+    writeRaw(text: string): void;
+    isAlive(): boolean;
+    exitInfo(): null;
+    close(): Promise<void>;
+}

package/dist/client/http.js

@@ -0,0 +1,133 @@
+import { TimeoutError, TransportClosedError } from './jsonrpc.js';
+/**
+ * Streamable HTTP transport. Each request is a POST; the server may answer
+ * with `application/json` or an SSE stream, and may hand us a session id on
+ * the initialize response that must be echoed on every later call.
+ */
+export class HttpTransport {
+    opts;
+    kind = 'http';
+    target;
+    stdoutNoise = [];
+    stderr = [];
+    serverNotifications = [];
+    nextId = 1;
+    sessionId = null;
+    closed = false;
+    /** Populated when a response arrives with a shape we could not read. */
+    protocolNotes = [];
+    constructor(opts) {
+        this.opts = opts;
+        this.target = opts.url;
+    }
+    async start() {
+        /* Nothing to spawn; the first POST is the real connection test. */
+    }
+    headers() {
+        const h = {
+            'content-type': 'application/json',
+            accept: 'application/json, text/event-stream',
+            ...this.opts.headers,
+        };
+        if (this.sessionId)
+            h['mcp-session-id'] = this.sessionId;
+        return h;
+    }
+    request(method, params, timeoutMs = 10_000) {
+        const id = this.nextId++;
+        return this.post({ jsonrpc: '2.0', id, method, ...(params !== undefined ? { params } : {}) }, method, timeoutMs);
+    }
+    requestRaw(payload, _id, method, timeoutMs = 10_000) {
+        return this.post(payload, method, timeoutMs);
+    }
+    async post(payload, method, timeoutMs) {
+        if (this.closed)
+            throw new TransportClosedError('Transport already closed');
+        const ac = new AbortController();
+        const timer = setTimeout(() => ac.abort(), timeoutMs);
+        let res;
+        try {
+            res = await fetch(this.opts.url, {
+                method: 'POST',
+                headers: this.headers(),
+                body: JSON.stringify(payload),
+                signal: ac.signal,
+            });
+        }
+        catch (e) {
+            clearTimeout(timer);
+            if (ac.signal.aborted)
+                throw new TimeoutError(method, timeoutMs);
+            throw new TransportClosedError(`POST ${this.opts.url} failed: ${e.message}`);
+        }
+        clearTimeout(timer);
+        const sid = res.headers.get('mcp-session-id');
+        if (sid)
+            this.sessionId = sid;
+        // 202 with no body is the legal answer to a notification.
+        if (res.status === 202)
+            return {};
+        const ctype = res.headers.get('content-type') ?? '';
+        const body = await res.text();
+        if (!res.ok) {
+            throw new TransportClosedError(`HTTP ${res.status} ${res.statusText} from ${this.opts.url}: ${body.slice(0, 400)}`);
+        }
+        if (ctype.includes('text/event-stream'))
+            return this.parseSse(body, method);
+        if (!body.trim())
+            return {};
+        try {
+            return JSON.parse(body);
+        }
+        catch {
+            this.protocolNotes.push(`Non-JSON body for \`${method}\` (content-type: ${ctype || 'none'}): ${body.slice(0, 200)}`);
+            throw new TransportClosedError(`Server returned unparseable body for \`${method}\``);
+        }
+    }
+    /** Pull the first `data:` frame that carries a JSON-RPC reply. */
+    parseSse(body, method) {
+        const frames = body.split(/\n\n/);
+        let last = null;
+        for (const frame of frames) {
+            const data = frame
+                .split('\n')
+                .filter((l) => l.startsWith('data:'))
+                .map((l) => l.slice(5).trim())
+                .join('');
+            if (!data)
+                continue;
+            let msg;
+            try {
+                msg = JSON.parse(data);
+            }
+            catch {
+                this.protocolNotes.push(`Unparseable SSE frame during \`${method}\`: ${data.slice(0, 200)}`);
+                continue;
+            }
+            if (msg.id !== undefined && msg.id !== null)
+                last = msg;
+            else
+                this.serverNotifications.push(msg);
+        }
+        if (!last)
+            throw new TransportClosedError(`SSE stream for \`${method}\` carried no JSON-RPC response`);
+        return last;
+    }
+    notify(method, params) {
+        void this.post({ jsonrpc: '2.0', method, ...(params !== undefined ? { params } : {}) }, method, 5000).catch(() => {
+            /* Notifications are fire-and-forget; a failure here is not a check result. */
+        });
+    }
+    writeRaw(text) {
+        void fetch(this.opts.url, { method: 'POST', headers: this.headers(), body: text }).catch(() => { });
+    }
+    isAlive() {
+        return !this.closed;
+    }
+    exitInfo() {
+        return null;
+    }
+    async close() {
+        this.closed = true;
+    }
+}

package/dist/client/index.d.ts

@@ -0,0 +1,47 @@
+import type { Transport } from './transport.js';
+import type { JsonRpcResponse } from './jsonrpc.js';
+import type { ToolDef } from '../types.js';
+export { StdioTransport } from './stdio.js';
+export { HttpTransport } from './http.js';
+export type { Transport } from './transport.js';
+/** Versions we will negotiate, newest first. */
+export declare const SUPPORTED_PROTOCOL_VERSIONS: string[];
+export interface HandshakeResult {
+    raw: JsonRpcResponse;
+    protocolVersion: string | null;
+    serverInfo: {
+        name?: string;
+        version?: string;
+    } | null;
+    capabilities: Record<string, unknown>;
+    ms: number;
+}
+/**
+ * A thin, unopinionated MCP client. It performs the handshake and exposes the
+ * few calls the checks need -- but never normalises or repairs a response,
+ * because the checks have to see exactly what the server sent.
+ */
+export declare class McpClient {
+    readonly transport: Transport;
+    private timeoutMs;
+    constructor(transport: Transport, timeoutMs?: number);
+    get target(): string;
+    start(): Promise<void>;
+    initialize(protocolVersion?: string): Promise<HandshakeResult>;
+    /** The spec requires this notification before any other request. */
+    notifyInitialized(): void;
+    call(method: string, params?: unknown, timeoutMs?: number): Promise<JsonRpcResponse>;
+    callRaw(payload: Record<string, unknown>, id: number | string, method: string, timeoutMs?: number): Promise<JsonRpcResponse>;
+    /** Walk `nextCursor` so a paginated server does not under-report. */
+    listAll(method: 'tools/list' | 'resources/list' | 'prompts/list', key: string): Promise<{
+        items: unknown[];
+        pages: number;
+        error?: JsonRpcResponse;
+    }>;
+    listTools(): Promise<{
+        tools: ToolDef[];
+        pages: number;
+        error?: JsonRpcResponse;
+    }>;
+    close(): Promise<void>;
+}

package/dist/client/index.js

@@ -0,0 +1,79 @@
+export { StdioTransport } from './stdio.js';
+export { HttpTransport } from './http.js';
+/** Versions we will negotiate, newest first. */
+export const SUPPORTED_PROTOCOL_VERSIONS = ['2025-06-18', '2025-03-26', '2024-11-05'];
+/**
+ * A thin, unopinionated MCP client. It performs the handshake and exposes the
+ * few calls the checks need -- but never normalises or repairs a response,
+ * because the checks have to see exactly what the server sent.
+ */
+export class McpClient {
+    transport;
+    timeoutMs;
+    constructor(transport, timeoutMs = 10_000) {
+        this.transport = transport;
+        this.timeoutMs = timeoutMs;
+    }
+    get target() {
+        return this.transport.target;
+    }
+    async start() {
+        await this.transport.start();
+    }
+    async initialize(protocolVersion = SUPPORTED_PROTOCOL_VERSIONS[0]) {
+        const t0 = Date.now();
+        const raw = await this.transport.request('initialize', {
+            protocolVersion,
+            capabilities: { roots: { listChanged: true }, sampling: {}, elicitation: {} },
+            clientInfo: { name: 'mcp-probe', version: '0.1.0' },
+        }, this.timeoutMs);
+        const ms = Date.now() - t0;
+        const result = (raw.result ?? {});
+        return {
+            raw,
+            ms,
+            protocolVersion: typeof result['protocolVersion'] === 'string' ? result['protocolVersion'] : null,
+            serverInfo: result['serverInfo'] ?? null,
+            capabilities: result['capabilities'] ?? {},
+        };
+    }
+    /** The spec requires this notification before any other request. */
+    notifyInitialized() {
+        this.transport.notify('notifications/initialized');
+    }
+    call(method, params, timeoutMs) {
+        return this.transport.request(method, params, timeoutMs ?? this.timeoutMs);
+    }
+    callRaw(payload, id, method, timeoutMs) {
+        return this.transport.requestRaw(payload, id, method, timeoutMs ?? this.timeoutMs);
+    }
+    /** Walk `nextCursor` so a paginated server does not under-report. */
+    async listAll(method, key) {
+        const items = [];
+        let cursor;
+        let pages = 0;
+        for (;;) {
+            const res = await this.call(method, cursor ? { cursor } : {});
+            if (res.error)
+                return { items, pages, error: res };
+            pages++;
+            const result = (res.result ?? {});
+            const batch = result[key];
+            if (Array.isArray(batch))
+                items.push(...batch);
+            const next = result['nextCursor'];
+            if (typeof next === 'string' && next && pages < 50)
+                cursor = next;
+            else
+                break;
+        }
+        return { items, pages };
+    }
+    async listTools() {
+        const { items, pages, error } = await this.listAll('tools/list', 'tools');
+        return { tools: items, pages, error };
+    }
+    close() {
+        return this.transport.close();
+    }
+}

package/dist/client/jsonrpc.d.ts

@@ -0,0 +1,40 @@
+export interface JsonRpcRequest {
+    jsonrpc: '2.0';
+    id: number | string;
+    method: string;
+    params?: unknown;
+}
+export interface JsonRpcNotification {
+    jsonrpc: '2.0';
+    method: string;
+    params?: unknown;
+}
+export interface JsonRpcError {
+    code: number;
+    message: string;
+    data?: unknown;
+}
+export interface JsonRpcResponse {
+    jsonrpc?: string;
+    id?: number | string | null;
+    result?: unknown;
+    error?: JsonRpcError;
+}
+/** Error codes the spec pins down. Servers get these wrong constantly. */
+export declare const RPC: {
+    readonly PARSE_ERROR: -32700;
+    readonly INVALID_REQUEST: -32600;
+    readonly METHOD_NOT_FOUND: -32601;
+    readonly INVALID_PARAMS: -32602;
+    readonly INTERNAL_ERROR: -32603;
+};
+export declare class TimeoutError extends Error {
+    method: string;
+    ms: number;
+    constructor(method: string, ms: number);
+}
+export declare class TransportClosedError extends Error {
+    code?: number | null | undefined;
+    stderr?: string | undefined;
+    constructor(message: string, code?: number | null | undefined, stderr?: string | undefined);
+}

package/dist/client/jsonrpc.js

@@ -0,0 +1,28 @@
+/** Error codes the spec pins down. Servers get these wrong constantly. */
+export const RPC = {
+    PARSE_ERROR: -32700,
+    INVALID_REQUEST: -32600,
+    METHOD_NOT_FOUND: -32601,
+    INVALID_PARAMS: -32602,
+    INTERNAL_ERROR: -32603,
+};
+export class TimeoutError extends Error {
+    method;
+    ms;
+    constructor(method, ms) {
+        super(`No response to \`${method}\` within ${ms}ms`);
+        this.method = method;
+        this.ms = ms;
+        this.name = 'TimeoutError';
+    }
+}
+export class TransportClosedError extends Error {
+    code;
+    stderr;
+    constructor(message, code, stderr) {
+        super(message);
+        this.code = code;
+        this.stderr = stderr;
+        this.name = 'TransportClosedError';
+    }
+}