@beeeeen/mcp-probe 0.1.0

package/dist/checks/robustness.d.ts

@@ -0,0 +1,17 @@
+import type { Check } from '../types.js';
+export declare const unknownToolCheck: Check;
+/**
+ * Invalid-argument probes are safe to run unattended: a correct server rejects
+ * them at validation, before any side effect. That is precisely what is being
+ * measured, and it is why this runs by default while valid calls do not.
+ */
+export declare const invalidArgsCheck: Check;
+/** Garbage on the wire must not be fatal; a public server sees it constantly. */
+export declare const malformedInputCheck: Check;
+/**
+ * Tool results land in the context window verbatim. A tool that returns a
+ * whole file or an unpaginated list can consume the entire budget in one call.
+ * Only measured for tools actually invoked, so it is opt-in with the rest.
+ */
+export declare const responseSizeCheck: Check;
+export declare const robustnessChecks: Check[];

package/dist/checks/robustness.js

@@ -0,0 +1,240 @@
+import { result, preview, isPlainObject, estimateTokens } from './util.js';
+const SPEC = 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools';
+/**
+ * A tools/call that failed is reported one of two legal ways: a JSON-RPC error,
+ * or a result carrying `isError: true`. Both are fine; neither is a crash.
+ */
+function rejectedCleanly(res) {
+    if (res.error)
+        return true;
+    return isPlainObject(res.result) && res.result['isError'] === true;
+}
+export const unknownToolCheck = {
+    id: 'robustness.unknown_tool',
+    title: 'Calling a tool that does not exist is rejected',
+    severity: 'error',
+    spec: SPEC,
+    async run(ctx) {
+        const name = '__mcp_probe_no_such_tool__';
+        try {
+            const res = await ctx.client.call('tools/call', { name, arguments: {} }, Math.min(ctx.options.timeoutMs, 8000));
+            if (rejectedCleanly(res)) {
+                return [result('robustness.unknown_tool', 'Calling a tool that does not exist is rejected', 'pass', 'error')];
+            }
+            return [
+                result('robustness.unknown_tool', 'Calling a tool that does not exist is rejected', 'fail', 'error', {
+                    message: 'An unknown tool name returned a success result.',
+                    detail: `Result: ${preview(res.result)}\nModels hallucinate tool names. A server that answers them feeds the hallucination back as fact.`,
+                    spec: SPEC,
+                }),
+            ];
+        }
+        catch (e) {
+            return [
+                result('robustness.unknown_tool', 'Calling a tool that does not exist is rejected', 'fail', 'error', {
+                    message: `Server stopped responding: ${e.message}`,
+                    detail: 'An unknown tool name must not be fatal. Models routinely invent names, and this takes the server down every time.',
+                    spec: SPEC,
+                }),
+            ];
+        }
+    },
+};
+/** Build arguments that are the wrong type for every declared property. */
+function wrongTypedArgs(schema) {
+    if (!isPlainObject(schema) || !isPlainObject(schema.properties))
+        return null;
+    const args = {};
+    for (const [key, prop] of Object.entries(schema.properties)) {
+        const t = Array.isArray(prop.type) ? prop.type?.[0] : prop.type;
+        // Deliberately mismatched: a string where a number is wanted, and so on.
+        args[key] = t === 'string' ? 12345 : t === 'number' || t === 'integer' ? 'not-a-number' : t === 'boolean' ? 'maybe' : t === 'array' ? {} : [];
+    }
+    return Object.keys(args).length > 0 ? args : null;
+}
+/**
+ * Invalid-argument probes are safe to run unattended: a correct server rejects
+ * them at validation, before any side effect. That is precisely what is being
+ * measured, and it is why this runs by default while valid calls do not.
+ */
+export const invalidArgsCheck = {
+    id: 'robustness.invalid_args',
+    title: 'Invalid arguments are rejected, not executed',
+    severity: 'error',
+    spec: SPEC,
+    async run(ctx) {
+        const out = [];
+        const candidates = ctx.tools.filter((t) => t?.name && isPlainObject(t.inputSchema)).slice(0, 12);
+        if (candidates.length === 0) {
+            return [
+                result('robustness.invalid_args', 'Invalid arguments are rejected', 'skip', 'error', {
+                    message: 'No tools with an inputSchema to probe.',
+                }),
+            ];
+        }
+        for (const tool of candidates) {
+            const name = tool.name;
+            const required = Array.isArray(tool.inputSchema?.required) ? tool.inputSchema.required : [];
+            // 1. Omit every required argument.
+            if (required.length > 0) {
+                try {
+                    const res = await ctx.client.call('tools/call', { name, arguments: {} }, Math.min(ctx.options.timeoutMs, 8000));
+                    if (!rejectedCleanly(res)) {
+                        out.push(result('robustness.invalid_args.missing_required', 'Missing required arguments are rejected', 'fail', 'error', {
+                            target: name,
+                            message: `Ran with none of its ${required.length} required argument${required.length === 1 ? '' : 's'} and reported success.`,
+                            detail: `Required: ${required.join(', ')}\nResult: ${preview(res.result)}\nThe tool either silently used defaults or acted on undefined -- both produce wrong answers the model will trust.`,
+                            spec: SPEC,
+                        }));
+                    }
+                }
+                catch (e) {
+                    out.push(result('robustness.invalid_args.missing_required', 'Missing required arguments are rejected', 'fail', 'error', {
+                        target: name,
+                        message: `Server stopped responding when required arguments were omitted: ${e.message}`,
+                        detail: 'This is an unguarded property access on the argument object. Validate before you dereference.',
+                        spec: SPEC,
+                    }));
+                }
+            }
+            // 2. Send every argument with the wrong type.
+            const bad = wrongTypedArgs(tool.inputSchema);
+            if (bad) {
+                try {
+                    const res = await ctx.client.call('tools/call', { name, arguments: bad }, Math.min(ctx.options.timeoutMs, 8000));
+                    if (!rejectedCleanly(res)) {
+                        out.push(result('robustness.invalid_args.wrong_types', 'Wrongly typed arguments are rejected', 'warn', 'warn', {
+                            target: name,
+                            message: 'Accepted arguments whose types contradict its own schema.',
+                            detail: `Sent: ${preview(bad, 200)}\nResult: ${preview(res.result, 200)}\nModels do emit the wrong type. Coercing silently turns a type error into a wrong result.`,
+                            spec: SPEC,
+                        }));
+                    }
+                }
+                catch (e) {
+                    out.push(result('robustness.invalid_args.wrong_types', 'Wrongly typed arguments are rejected', 'fail', 'error', {
+                        target: name,
+                        message: `Server stopped responding on wrongly typed arguments: ${e.message}`,
+                        spec: SPEC,
+                    }));
+                }
+            }
+        }
+        if (out.length === 0) {
+            out.push(result('robustness.invalid_args', 'Invalid arguments are rejected, not executed', 'pass', 'error', {
+                message: `${candidates.length} tool${candidates.length === 1 ? '' : 's'} rejected bad input cleanly.`,
+            }));
+        }
+        return out;
+    },
+};
+/** Garbage on the wire must not be fatal; a public server sees it constantly. */
+export const malformedInputCheck = {
+    id: 'robustness.malformed_input',
+    title: 'Survives malformed JSON-RPC',
+    severity: 'error',
+    spec: 'https://www.jsonrpc.org/specification#error_object',
+    async run(ctx) {
+        const t = ctx.client.transport;
+        if (t.kind !== 'stdio') {
+            return [result('robustness.malformed_input', 'Survives malformed JSON-RPC', 'skip', 'error', { message: 'stdio only.' })];
+        }
+        t.writeRaw('this is not json at all\n');
+        t.writeRaw('{"jsonrpc":"2.0","id":  \n');
+        t.writeRaw('{"jsonrpc":"2.0","method":123,"id":"x"}\n');
+        // Give the server a moment to mishandle it, then check it is still there.
+        await new Promise((r) => setTimeout(r, 300));
+        if (!t.isAlive()) {
+            const info = t.exitInfo();
+            return [
+                result('robustness.malformed_input', 'Survives malformed JSON-RPC', 'fail', 'error', {
+                    message: `Server exited (code ${info?.code ?? 'null'}) after receiving malformed input.`,
+                    detail: `Last stderr:\n${t.stderr.slice(-8).join('\n') || '(none)'}\n\nA parse failure must produce a -32700 response, not terminate the process.`,
+                }),
+            ];
+        }
+        try {
+            const res = await ctx.client.call('tools/list', {}, Math.min(ctx.options.timeoutMs, 5000));
+            if (res.error) {
+                return [
+                    result('robustness.malformed_input', 'Survives malformed JSON-RPC', 'warn', 'warn', {
+                        message: `Still running, but tools/list now errors: ${res.error.code} ${res.error.message}`,
+                        detail: 'The garbage left the read loop in a bad state. Usually a buffer that was never reset after a parse failure.',
+                    }),
+                ];
+            }
+            return [
+                result('robustness.malformed_input', 'Survives malformed JSON-RPC', 'pass', 'error', {
+                    message: 'Still serving requests after three malformed frames.',
+                }),
+            ];
+        }
+        catch (e) {
+            return [
+                result('robustness.malformed_input', 'Survives malformed JSON-RPC', 'fail', 'error', {
+                    message: `Process alive but unresponsive after malformed input: ${e.message}`,
+                    detail: 'The read loop is wedged -- typically a partial frame left in the buffer that every later read tries to parse again.',
+                }),
+            ];
+        }
+    },
+};
+/**
+ * Tool results land in the context window verbatim. A tool that returns a
+ * whole file or an unpaginated list can consume the entire budget in one call.
+ * Only measured for tools actually invoked, so it is opt-in with the rest.
+ */
+export const responseSizeCheck = {
+    id: 'robustness.response_size',
+    title: 'Tool results fit in a context window',
+    severity: 'warn',
+    spec: SPEC,
+    async run(ctx) {
+        const allowed = new Set(ctx.options.safeTools);
+        const targets = ctx.options.callTools
+            ? ctx.tools.filter((t) => t?.name)
+            : ctx.tools.filter((t) => t?.name && allowed.has(t.name));
+        if (targets.length === 0) {
+            return [
+                result('robustness.response_size', 'Tool results fit in a context window', 'skip', 'warn', {
+                    message: 'No tools were invoked. Pass --call-tools, or --safe-tool <name>, to measure real responses.',
+                }),
+            ];
+        }
+        const out = [];
+        for (const tool of targets.slice(0, 12)) {
+            try {
+                const res = await ctx.client.call('tools/call', { name: tool.name, arguments: {} }, ctx.options.timeoutMs);
+                if (res.error)
+                    continue;
+                const tokens = estimateTokens(JSON.stringify(res.result ?? {}));
+                if (tokens > 25_000) {
+                    out.push(result('robustness.response_size.huge', 'Tool result fits in a context window', 'fail', 'error', {
+                        target: tool.name,
+                        message: `Returned ~${tokens.toLocaleString('en-US')} tokens in a single call.`,
+                        detail: 'This evicts the conversation it was meant to inform. Paginate, or return a summary with a follow-up tool to drill in.',
+                        spec: SPEC,
+                    }));
+                }
+                else if (tokens > 8_000) {
+                    out.push(result('robustness.response_size.large', 'Tool result fits in a context window', 'warn', 'warn', {
+                        target: tool.name,
+                        message: `Returned ~${tokens.toLocaleString('en-US')} tokens.`,
+                        detail: 'Large enough to crowd out earlier turns. Consider a limit/cursor argument.',
+                        spec: SPEC,
+                    }));
+                }
+            }
+            catch {
+                // Timeouts and transport faults are already covered by other checks.
+            }
+        }
+        if (out.length === 0) {
+            out.push(result('robustness.response_size', 'Tool results fit in a context window', 'pass', 'warn', {
+                message: `${Math.min(targets.length, 12)} tool response${targets.length === 1 ? '' : 's'} within budget.`,
+            }));
+        }
+        return out;
+    },
+};
+export const robustnessChecks = [unknownToolCheck, invalidArgsCheck, malformedInputCheck, responseSizeCheck];

package/dist/checks/schema.d.ts

@@ -0,0 +1,12 @@
+import type { Check } from '../types.js';
+export declare const toolNameCheck: Check;
+export declare const toolDescriptionCheck: Check;
+export declare const inputSchemaCheck: Check;
+/**
+ * Every tool definition is re-sent on every single request for the whole
+ * session. A bloated tool list is a permanent tax on the context window and
+ * shows up directly on the user's bill -- but nothing in the toolchain
+ * measures it, so it grows unnoticed.
+ */
+export declare const contextWeightCheck: Check;
+export declare const schemaChecks: Check[];

package/dist/checks/schema.js

@@ -0,0 +1,224 @@
+import { result, preview, isPlainObject, estimateTokens } from './util.js';
+const SPEC = 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools';
+/** What hosts accept today. Anything else gets rejected at registration time. */
+const NAME_PATTERN = /^[a-zA-Z0-9_-]{1,128}$/;
+/** Descriptions shorter than this cannot disambiguate a tool from its siblings. */
+const MIN_DESCRIPTION_CHARS = 20;
+const PLACEHOLDER_DESCRIPTIONS = /^(todo|tbd|fixme|xxx|n\/?a|description|test|foo|bar|\.+)$/i;
+export const toolNameCheck = {
+    id: 'schema.tool_name',
+    title: 'Tool names are valid and unique',
+    severity: 'error',
+    spec: SPEC,
+    run(ctx) {
+        const out = [];
+        const seen = new Map();
+        for (const tool of ctx.tools) {
+            const name = tool?.name;
+            if (typeof name !== 'string' || !name) {
+                out.push(result('schema.tool_name.missing', 'Tool has a name', 'fail', 'error', {
+                    message: 'A tool in tools/list has no `name`.',
+                    detail: preview(tool),
+                    spec: SPEC,
+                }));
+                continue;
+            }
+            seen.set(name, (seen.get(name) ?? 0) + 1);
+            if (!NAME_PATTERN.test(name)) {
+                out.push(result('schema.tool_name.invalid', 'Tool name is host-compatible', 'fail', 'error', {
+                    target: name,
+                    message: `"${name}" contains characters hosts reject.`,
+                    detail: 'Allowed: letters, digits, underscore, hyphen; 1-128 chars. Spaces and dots break tool routing in several hosts.',
+                    spec: SPEC,
+                }));
+            }
+        }
+        for (const [name, count] of seen) {
+            if (count > 1) {
+                out.push(result('schema.tool_name.duplicate', 'Tool names are unique', 'fail', 'error', {
+                    target: name,
+                    message: `"${name}" is listed ${count} times.`,
+                    detail: 'The model cannot address a duplicated name; whichever the host registers last silently wins.',
+                    spec: SPEC,
+                }));
+            }
+        }
+        if (out.length === 0 && ctx.tools.length > 0) {
+            out.push(result('schema.tool_name', 'Tool names are valid and unique', 'pass', 'error', {
+                message: `${ctx.tools.length} tool${ctx.tools.length === 1 ? '' : 's'} checked.`,
+            }));
+        }
+        return out;
+    },
+};
+export const toolDescriptionCheck = {
+    id: 'schema.tool_description',
+    title: 'Tools are described well enough for a model to choose between them',
+    severity: 'warn',
+    spec: SPEC,
+    run(ctx) {
+        const out = [];
+        for (const tool of ctx.tools) {
+            const name = tool?.name ?? '(unnamed)';
+            const desc = tool?.description;
+            if (!desc || !desc.trim()) {
+                out.push(result('schema.tool_description.missing', 'Tool has a description', 'fail', 'error', {
+                    target: name,
+                    message: 'No description.',
+                    detail: 'The description is the only thing the model reads when deciding whether to call this tool. Without it the tool is effectively invisible.',
+                    spec: SPEC,
+                }));
+                continue;
+            }
+            const trimmed = desc.trim();
+            if (PLACEHOLDER_DESCRIPTIONS.test(trimmed)) {
+                out.push(result('schema.tool_description.placeholder', 'Description is not a placeholder', 'fail', 'error', {
+                    target: name,
+                    message: `Description is a placeholder: "${trimmed}"`,
+                    spec: SPEC,
+                }));
+            }
+            else if (trimmed.length < MIN_DESCRIPTION_CHARS) {
+                out.push(result('schema.tool_description.short', 'Description is substantive', 'warn', 'warn', {
+                    target: name,
+                    message: `Only ${trimmed.length} chars: "${trimmed}"`,
+                    detail: 'State what it does, when to use it, and what it returns. Models pick the wrong tool when two terse descriptions overlap.',
+                    spec: SPEC,
+                }));
+            }
+        }
+        if (out.length === 0 && ctx.tools.length > 0) {
+            out.push(result('schema.tool_description', 'Tools are described', 'pass', 'warn'));
+        }
+        return out;
+    },
+};
+function walkSchema(schema, path, visit, depth = 0) {
+    if (!isPlainObject(schema) || depth > 12)
+        return;
+    visit(schema, path);
+    if (isPlainObject(schema.properties)) {
+        for (const [key, child] of Object.entries(schema.properties)) {
+            walkSchema(child, `${path}.${key}`, visit, depth + 1);
+        }
+    }
+    if (schema.items && !Array.isArray(schema.items)) {
+        walkSchema(schema.items, `${path}[]`, visit, depth + 1);
+    }
+}
+export const inputSchemaCheck = {
+    id: 'schema.input_schema',
+    title: 'inputSchema is a usable JSON Schema',
+    severity: 'error',
+    spec: SPEC,
+    run(ctx) {
+        const out = [];
+        for (const tool of ctx.tools) {
+            const name = tool?.name ?? '(unnamed)';
+            const schema = tool?.inputSchema;
+            if (!schema) {
+                out.push(result('schema.input_schema.missing', 'Tool declares an inputSchema', 'fail', 'error', {
+                    target: name,
+                    message: 'No `inputSchema`.',
+                    detail: 'Without it the model has to guess the argument shape, and most hosts refuse to register the tool at all.',
+                    spec: SPEC,
+                }));
+                continue;
+            }
+            if (!isPlainObject(schema)) {
+                out.push(result('schema.input_schema.malformed', 'inputSchema is an object', 'fail', 'error', {
+                    target: name,
+                    message: `inputSchema is ${Array.isArray(schema) ? 'an array' : typeof schema}, not an object.`,
+                    detail: preview(schema),
+                    spec: SPEC,
+                }));
+                continue;
+            }
+            if (schema.type !== 'object') {
+                out.push(result('schema.input_schema.root_type', 'inputSchema root is type object', 'fail', 'error', {
+                    target: name,
+                    message: `Root \`type\` is ${schema.type === undefined ? 'absent' : `"${String(schema.type)}"`}, expected "object".`,
+                    detail: 'Tool arguments are always a named-argument object. Hosts validate against this before dispatching.',
+                    spec: SPEC,
+                }));
+            }
+            // required entries that do not exist are the most common schema bug:
+            // the model is told to send a field that the server never reads.
+            const props = isPlainObject(schema.properties) ? schema.properties : {};
+            if (Array.isArray(schema.required)) {
+                const orphans = schema.required.filter((r) => typeof r === 'string' && !(r in props));
+                if (orphans.length > 0) {
+                    out.push(result('schema.input_schema.orphan_required', 'required fields exist in properties', 'fail', 'error', {
+                        target: name,
+                        message: `required lists ${orphans.map((o) => `"${o}"`).join(', ')}, which ${orphans.length === 1 ? 'is' : 'are'} not in properties.`,
+                        detail: 'Strict validators reject every call to this tool, because the argument can never be supplied in a valid way.',
+                        spec: SPEC,
+                    }));
+                }
+            }
+            // Undescribed parameters force the model to infer meaning from the name.
+            const undescribed = [];
+            walkSchema(schema, name, (s, p) => {
+                if (p === name)
+                    return;
+                if (!s.description && !s.enum && s.type !== 'object')
+                    undescribed.push(p.slice(name.length + 1));
+            });
+            if (undescribed.length > 0) {
+                out.push(result('schema.input_schema.undescribed_params', 'Parameters carry descriptions', 'warn', 'warn', {
+                    target: name,
+                    message: `${undescribed.length} parameter${undescribed.length === 1 ? '' : 's'} with no description: ${undescribed.slice(0, 6).join(', ')}${undescribed.length > 6 ? ', ...' : ''}`,
+                    detail: 'Parameter descriptions are how the model learns formats -- date layouts, id shapes, units, allowed ranges.',
+                    spec: SPEC,
+                }));
+            }
+            if (Object.keys(props).length === 0 && schema.type === 'object' && schema.additionalProperties !== false) {
+                out.push(result('schema.input_schema.open_object', 'Schema constrains its arguments', 'warn', 'warn', {
+                    target: name,
+                    message: 'Schema is an object with no properties and additionalProperties is not false.',
+                    detail: 'This accepts anything. If the tool truly takes no arguments, set `additionalProperties: false` to say so.',
+                    spec: SPEC,
+                }));
+            }
+        }
+        if (out.length === 0 && ctx.tools.length > 0) {
+            out.push(result('schema.input_schema', 'inputSchema is usable', 'pass', 'error'));
+        }
+        return out;
+    },
+};
+/**
+ * Every tool definition is re-sent on every single request for the whole
+ * session. A bloated tool list is a permanent tax on the context window and
+ * shows up directly on the user's bill -- but nothing in the toolchain
+ * measures it, so it grows unnoticed.
+ */
+export const contextWeightCheck = {
+    id: 'schema.context_weight',
+    title: 'Tool list does not dominate the context window',
+    severity: 'warn',
+    spec: SPEC,
+    run(ctx) {
+        if (ctx.tools.length === 0)
+            return [];
+        const perTool = ctx.tools
+            .map((tool) => ({ name: tool?.name ?? '(unnamed)', tokens: estimateTokens(JSON.stringify(tool ?? {})) }))
+            .sort((a, b) => b.tokens - a.tokens);
+        const total = perTool.reduce((sum, t) => sum + t.tokens, 0);
+        const heaviest = perTool
+            .slice(0, 5)
+            .map((t) => `  ${String(t.tokens).padStart(6)} tok  ${t.name}`)
+            .join('\n');
+        const detail = `Estimated at ~4 chars/token from the serialised tool definitions.\n\nHeaviest tools:\n${heaviest}\n\nThis cost is paid on every request for the lifetime of the session, not once.`;
+        const status = total > 10_000 ? 'fail' : total > 4_000 ? 'warn' : 'pass';
+        const severity = total > 10_000 ? 'error' : 'warn';
+        return [
+            result('schema.context_weight', 'Tool list does not dominate the context window', status, severity, {
+                message: `~${total.toLocaleString('en-US')} tokens across ${ctx.tools.length} tool${ctx.tools.length === 1 ? '' : 's'}${status === 'pass' ? '' : ` (budget: 4,000 warn / 10,000 fail)`}`,
+                detail,
+                spec: SPEC,
+            }),
+        ];
+    },
+};
+export const schemaChecks = [toolNameCheck, toolDescriptionCheck, inputSchemaCheck, contextWeightCheck];

package/dist/checks/util.d.ts

@@ -0,0 +1,7 @@
+import type { CheckResult, Severity, Status } from '../types.js';
+export declare function result(id: string, title: string, status: Status, severity: Severity, extra?: Partial<CheckResult>): CheckResult;
+/** Trim a payload for display without hiding the part that matters. */
+export declare function preview(value: unknown, max?: number): string;
+export declare function isPlainObject(v: unknown): v is Record<string, unknown>;
+/** Rough token estimate. Good enough to flag payloads that will blow context. */
+export declare function estimateTokens(text: string): number;

package/dist/checks/util.js

@@ -0,0 +1,23 @@
+export function result(id, title, status, severity, extra = {}) {
+    return { id, title, status, severity, ...extra };
+}
+/** Trim a payload for display without hiding the part that matters. */
+export function preview(value, max = 300) {
+    let s;
+    try {
+        s = typeof value === 'string' ? value : JSON.stringify(value);
+    }
+    catch {
+        s = String(value);
+    }
+    if (s === undefined)
+        return 'undefined';
+    return s.length > max ? s.slice(0, max) + ` ... (+${s.length - max} chars)` : s;
+}
+export function isPlainObject(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+/** Rough token estimate. Good enough to flag payloads that will blow context. */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};