mcp-vitals 0.1.0

package/dist/assertions/schema.d.ts

@@ -0,0 +1,147 @@
+export declare const ASSERTIONS_SCHEMA: {
+    readonly $schema: "https://json-schema.org/draft/2020-12/schema";
+    readonly $id: "https://unpkg.com/mcp-vitals/schema/assertions.schema.json";
+    readonly title: "mcp-vitals assertions";
+    readonly type: "object";
+    readonly additionalProperties: false;
+    readonly required: readonly ["server"];
+    readonly properties: {
+        readonly $schema: {
+            readonly type: "string";
+        };
+        readonly server: {
+            readonly type: "object";
+            readonly additionalProperties: false;
+            readonly properties: {
+                readonly command: {
+                    readonly type: "string";
+                };
+                readonly args: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "string";
+                    };
+                };
+                readonly env: {
+                    readonly type: "object";
+                    readonly additionalProperties: {
+                        readonly type: "string";
+                    };
+                };
+                readonly url: {
+                    readonly type: "string";
+                };
+                readonly headers: {
+                    readonly type: "object";
+                    readonly additionalProperties: {
+                        readonly type: "string";
+                    };
+                };
+                readonly transport: {
+                    readonly enum: readonly ["stdio", "http", "sse"];
+                };
+                readonly connectTimeoutMs: {
+                    readonly type: "number";
+                };
+                readonly timeoutMs: {
+                    readonly type: "number";
+                };
+            };
+        };
+        readonly defaults: {
+            readonly type: "object";
+            readonly additionalProperties: false;
+            readonly properties: {
+                readonly iterations: {
+                    readonly type: "number";
+                };
+                readonly warmup: {
+                    readonly type: "number";
+                };
+                readonly concurrency: {
+                    readonly type: "number";
+                };
+                readonly timeoutMs: {
+                    readonly type: "number";
+                };
+            };
+        };
+        readonly expect: {
+            readonly type: "object";
+            readonly additionalProperties: false;
+            readonly properties: {
+                readonly tools: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "string";
+                    };
+                };
+                readonly resources: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "string";
+                    };
+                };
+                readonly prompts: {
+                    readonly type: "array";
+                    readonly items: {
+                        readonly type: "string";
+                    };
+                };
+                readonly schemasValid: {
+                    readonly type: "boolean";
+                };
+            };
+        };
+        readonly latency: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly additionalProperties: false;
+                readonly required: readonly ["id"];
+                readonly properties: {
+                    readonly id: {
+                        readonly type: "string";
+                    };
+                    readonly tool: {
+                        readonly type: "string";
+                    };
+                    readonly probe: {
+                        readonly enum: readonly ["listTools", "listResources", "listPrompts"];
+                    };
+                    readonly args: {};
+                    readonly iterations: {
+                        readonly type: "number";
+                    };
+                    readonly warmup: {
+                        readonly type: "number";
+                    };
+                    readonly concurrency: {
+                        readonly type: "number";
+                    };
+                    readonly p50: {
+                        type: string[];
+                    };
+                    readonly p90: {
+                        type: string[];
+                    };
+                    readonly p95: {
+                        type: string[];
+                    };
+                    readonly p99: {
+                        type: string[];
+                    };
+                    readonly max: {
+                        type: string[];
+                    };
+                    readonly mean: {
+                        type: string[];
+                    };
+                    readonly errorRate: {
+                        readonly type: "number";
+                    };
+                };
+            };
+        };
+    };
+};

package/dist/assertions/schema.js

@@ -0,0 +1,72 @@
+// JSON Schema (Draft 2020-12) for mcp-vitals.{yaml,yml,json}.
+// Source of truth; schema/assertions.schema.json is a published copy of this object.
+const threshold = { type: ['string', 'number'] };
+export const ASSERTIONS_SCHEMA = {
+    $schema: 'https://json-schema.org/draft/2020-12/schema',
+    $id: 'https://unpkg.com/mcp-vitals/schema/assertions.schema.json',
+    title: 'mcp-vitals assertions',
+    type: 'object',
+    additionalProperties: false,
+    required: ['server'],
+    properties: {
+        $schema: { type: 'string' },
+        server: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                command: { type: 'string' },
+                args: { type: 'array', items: { type: 'string' } },
+                env: { type: 'object', additionalProperties: { type: 'string' } },
+                url: { type: 'string' },
+                headers: { type: 'object', additionalProperties: { type: 'string' } },
+                transport: { enum: ['stdio', 'http', 'sse'] },
+                connectTimeoutMs: { type: 'number' },
+                timeoutMs: { type: 'number' },
+            },
+        },
+        defaults: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                iterations: { type: 'number' },
+                warmup: { type: 'number' },
+                concurrency: { type: 'number' },
+                timeoutMs: { type: 'number' },
+            },
+        },
+        expect: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                tools: { type: 'array', items: { type: 'string' } },
+                resources: { type: 'array', items: { type: 'string' } },
+                prompts: { type: 'array', items: { type: 'string' } },
+                schemasValid: { type: 'boolean' },
+            },
+        },
+        latency: {
+            type: 'array',
+            items: {
+                type: 'object',
+                additionalProperties: false,
+                required: ['id'],
+                properties: {
+                    id: { type: 'string' },
+                    tool: { type: 'string' },
+                    probe: { enum: ['listTools', 'listResources', 'listPrompts'] },
+                    args: {},
+                    iterations: { type: 'number' },
+                    warmup: { type: 'number' },
+                    concurrency: { type: 'number' },
+                    p50: threshold,
+                    p90: threshold,
+                    p95: threshold,
+                    p99: threshold,
+                    max: threshold,
+                    mean: threshold,
+                    errorRate: { type: 'number' },
+                },
+            },
+        },
+    },
+};

package/dist/bench/engine.d.ts

@@ -0,0 +1,7 @@
+import type { Connection } from '../mcpClient.js';
+import type { BenchConfig, BenchResult } from '../types.js';
+export interface BenchHooks {
+    /** Called after each measured iteration with the running completed count. */
+    onTick?: (completed: number) => void;
+}
+export declare function runBench(conn: Connection, config: BenchConfig, timeoutMs: number, hooks?: BenchHooks): Promise<BenchResult>;

package/dist/bench/engine.js

@@ -0,0 +1,121 @@
+import { computeStats } from '../stats.js';
+import { UsageError } from '../errors.js';
+function buildOp(conn, config, timeoutMs) {
+    if (config.targetKind === 'tool') {
+        const name = config.targetName;
+        return async () => {
+            const r = await conn.callTool(name, config.args, timeoutMs);
+            return { toolError: r.isError };
+        };
+    }
+    const probe = config.targetName;
+    return async () => {
+        await conn.probeOnce(probe, timeoutMs);
+        return { toolError: false };
+    };
+}
+async function runOnce(op) {
+    const start = process.hrtime.bigint();
+    try {
+        const r = await op();
+        const ms = Number(process.hrtime.bigint() - start) / 1e6;
+        return { ms, threw: false, toolError: r.toolError };
+    }
+    catch {
+        const ms = Number(process.hrtime.bigint() - start) / 1e6;
+        return { ms, threw: true, toolError: false };
+    }
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** Closed model: keep `concurrency` operations in flight until the budget is exhausted. */
+async function runClosed(op, concurrency, shouldContinue, onOutcome) {
+    const worker = async () => {
+        while (shouldContinue()) {
+            onOutcome(await runOnce(op));
+        }
+    };
+    const workers = [];
+    for (let i = 0; i < concurrency; i++)
+        workers.push(worker());
+    await Promise.all(workers);
+}
+/** Open model: dispatch at a target arrival rate regardless of in-flight count. */
+async function runOpen(op, rps, shouldDispatch, onOutcome) {
+    const interval = 1000 / rps;
+    const inflight = [];
+    // Pace against an absolute clock so dispatches don't drift slow from
+    // cumulative setTimeout overshoot, and so there's no wasted trailing delay.
+    const t0 = performance.now();
+    let i = 0;
+    while (shouldDispatch()) {
+        if (i > 0)
+            await delay(Math.max(0, t0 + i * interval - performance.now()));
+        inflight.push(runOnce(op).then(onOutcome));
+        i++;
+    }
+    await Promise.all(inflight);
+}
+export async function runBench(conn, config, timeoutMs, hooks = {}) {
+    if (config.iterations !== undefined && config.durationMs !== undefined) {
+        throw new UsageError('cannot combine --iterations and --duration');
+    }
+    const op = buildOp(conn, config, timeoutMs);
+    // --- warmup (sequential, excluded from warm stats) ---
+    let coldStartMs = null;
+    for (let i = 0; i < config.warmup; i++) {
+        const o = await runOnce(op);
+        if (i === 0)
+            coldStartMs = o.ms;
+    }
+    // --- measured phase ---
+    const raw = [];
+    const latencySamples = [];
+    let toolErrors = 0;
+    let threws = 0;
+    const onOutcome = (o) => {
+        raw.push({ ms: o.ms, error: o.threw || o.toolError });
+        if (!o.threw)
+            latencySamples.push(o.ms); // throws have no meaningful latency
+        if (o.toolError)
+            toolErrors++;
+        if (o.threw)
+            threws++;
+        hooks.onTick?.(raw.length);
+    };
+    const byCount = config.iterations !== undefined;
+    const targetCount = config.iterations ?? 0;
+    const deadline = config.durationMs !== undefined ? performance.now() + config.durationMs : 0;
+    // For closed-count mode we must not over-dispatch past targetCount.
+    let dispatched = 0;
+    const wall0 = process.hrtime.bigint();
+    if (config.rps !== undefined) {
+        const shouldDispatch = byCount
+            ? () => dispatched++ < targetCount
+            : () => performance.now() < deadline;
+        await runOpen(op, config.rps, shouldDispatch, onOutcome);
+    }
+    else {
+        const shouldContinue = byCount
+            ? () => dispatched++ < targetCount
+            : () => performance.now() < deadline;
+        await runClosed(op, Math.max(1, config.concurrency), shouldContinue, onOutcome);
+    }
+    const wallMs = Number(process.hrtime.bigint() - wall0) / 1e6;
+    const completed = raw.length;
+    const errors = toolErrors + threws;
+    const throughput = {
+        rps: wallMs > 0 ? (completed / wallMs) * 1000 : 0,
+        completed,
+        errors,
+        errorRate: completed > 0 ? errors / completed : 0,
+    };
+    return {
+        coldStartMs,
+        warm: computeStats(latencySamples),
+        throughput,
+        samples: latencySamples,
+        raw,
+    };
+}

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare function buildProgram(): Command;
+export declare function main(argv: string[]): Promise<number>;

package/dist/cli.js

@@ -0,0 +1,137 @@
+import { createRequire } from 'node:module';
+import { Command, CommanderError } from 'commander';
+import { addConnectionOptions, addOutputOptions } from './context.js';
+import { CliError, EXIT_CODES, toExitCode, UsageError } from './errors.js';
+import { writeErr } from './output.js';
+import { runInspect } from './commands/inspect.js';
+import { runPing } from './commands/ping.js';
+import { runBenchCommand } from './commands/bench.js';
+import { runCall } from './commands/call.js';
+import { runCheck } from './commands/check.js';
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
+// Validating numeric parsers: a bad value throws a UsageError (exit 4) instead
+// of yielding NaN, which would otherwise produce a silent zero-sample run.
+const posInt = (label, min = 1) => (v) => {
+    const n = Number(v);
+    if (!Number.isInteger(n) || n < min) {
+        throw new UsageError(`--${label} must be an integer >= ${min} (got "${v}")`);
+    }
+    return n;
+};
+const posFloat = (label) => (v) => {
+    const n = Number(v);
+    if (!Number.isFinite(n) || n <= 0) {
+        throw new UsageError(`--${label} must be a number > 0 (got "${v}")`);
+    }
+    return n;
+};
+const collect = (v, prev) => prev.concat([v]);
+/** Wrap a command body so its numeric exit code propagates via process.exitCode. */
+function action(fn) {
+    return async (server, opts) => {
+        const code = await fn(server, opts);
+        if (code)
+            process.exitCode = code;
+    };
+}
+function withCommon(cmd) {
+    return addOutputOptions(addConnectionOptions(cmd));
+}
+export function buildProgram() {
+    const program = new Command();
+    program
+        .name('mcp-vitals')
+        .description('Vital signs for your MCP server: inspect, benchmark latency, and assert health in CI.')
+        .version(pkg.version, '-V, --version')
+        .enablePositionalOptions()
+        .showHelpAfterError();
+    withCommon(program
+        .command('inspect')
+        .description('discover capabilities, list tools/resources/prompts, validate schemas')
+        .argument('[server...]', 'stdio server launch command + argv'))
+        .passThroughOptions()
+        .option('--tools', 'show only tools')
+        .option('--resources', 'show only resources')
+        .option('--prompts', 'show only prompts')
+        .option('--schema', 'include full inputSchema in --json output')
+        .option('--no-validate-schemas', 'skip JSON Schema validation of tool inputSchemas')
+        .option('--filter <glob>', 'filter listed names by glob (* and ?)')
+        .action(action((server, opts) => runInspect(server, opts)));
+    withCommon(program
+        .command('ping')
+        .description('measure handshake/initialize latency and liveness')
+        .argument('[server...]', 'stdio server launch command + argv'))
+        .passThroughOptions()
+        .option('-n, --count <n>', 'number of independent reconnect cycles', posInt('count'), 1)
+        .option('--list', 'also time one tools/list round-trip')
+        .action(action((server, opts) => runPing(server, opts)));
+    withCommon(program
+        .command('bench')
+        .description('benchmark tool-call latency: p50/p95/p99, throughput, cold start')
+        .argument('[server...]', 'stdio server launch command + argv'))
+        .passThroughOptions()
+        .option('--tool <name>', 'tool to benchmark')
+        .option('--probe <op>', 'no-arg protocol op: listTools | listResources | listPrompts')
+        .option('--args <json>', "tool args JSON ('-' = stdin, '@file.json' = file)")
+        .option('-n, --iterations <n>', 'measured iterations (default 50)', posInt('iterations'))
+        .option('-w, --warmup <n>', 'warmup iterations, reported as cold start', posInt('warmup', 0), 1)
+        .option('-c, --concurrency <n>', 'keep N operations in flight (closed model)', posInt('concurrency'), 1)
+        .option('--rps <r>', 'drive R requests/sec arrival rate (open model)', posFloat('rps'))
+        .option('-d, --duration <ms>', 'run for a wall-clock duration instead of -n', posInt('duration'))
+        .option('--fail-on <expr>', "SLA gate, e.g. 'p95<200ms' (repeatable)", collect, [])
+        .action(action((server, opts) => runBenchCommand(server, opts)));
+    withCommon(program
+        .command('call')
+        .description('invoke one tool once and print the result + timing')
+        .argument('[server...]', 'stdio server launch command + argv'))
+        .passThroughOptions()
+        .option('--tool <name>', 'tool to call (required)')
+        .option('--args <json>', "tool args JSON ('-' = stdin, '@file.json' = file)")
+        .option('--raw', 'print only the tool result content (for piping)')
+        .option('--expect-error', 'exit 0 only if the call returns an MCP tool error')
+        .action(action((server, opts) => runCall(server, opts)));
+    withCommon(program
+        .command('check')
+        .description('run a committed mcp-vitals.yaml assertion suite (the CI gate)')
+        .argument('[server...]', 'optional stdio command override for the config server'))
+        .passThroughOptions()
+        .option('-c, --config <path>', 'assertions file (default: auto-discover mcp-vitals.{yaml,yml,json})')
+        .option('--junit <path>', "write JUnit XML to a path ('-' for stdout)")
+        .option('--only <glob>', 'run only assertions whose id matches (repeatable)', collect, [])
+        .option('--skip <glob>', 'skip assertions whose id matches (repeatable)', collect, [])
+        .option('--iterations <n>', 'override bench iterations for all latency assertions', posInt('iterations'))
+        .option('--warmup <n>', 'override bench warmup for all latency assertions', posInt('warmup', 0))
+        .option('--concurrency <n>', 'override bench concurrency for all latency assertions', posInt('concurrency'))
+        .option('--bail', 'stop at the first failed assertion', false)
+        .option('--no-latency', 'skip timed latency benches (presence + schema only)')
+        .action(action((server, opts) => runCheck(server, opts)));
+    return program;
+}
+function reportError(err) {
+    if (err instanceof CliError) {
+        writeErr(`mcp-vitals: ${err.message}`);
+        return err.exitCode;
+    }
+    writeErr(`mcp-vitals: ${err?.message ?? String(err)}`);
+    return toExitCode(err) === 0 ? 1 : toExitCode(err);
+}
+export async function main(argv) {
+    const program = buildProgram();
+    // exitOverride is per-command — apply to the program AND every subcommand so
+    // parse/usage errors throw a CommanderError instead of calling process.exit().
+    program.exitOverride();
+    for (const sub of program.commands)
+        sub.exitOverride();
+    try {
+        await program.parseAsync(argv);
+    }
+    catch (err) {
+        if (err instanceof CommanderError) {
+            // Help/version display exits 0; parse/usage errors map to our usage code.
+            return err.exitCode === 0 ? 0 : EXIT_CODES.USAGE;
+        }
+        return reportError(err);
+    }
+    return process.exitCode ? Number(process.exitCode) : 0;
+}

package/dist/commands/bench.d.ts

@@ -0,0 +1,13 @@
+import type { CommonOpts } from '../context.js';
+export interface BenchOpts extends CommonOpts {
+    tool?: string;
+    probe?: string;
+    args?: string;
+    iterations?: number;
+    warmup: number;
+    concurrency: number;
+    rps?: number;
+    duration?: number;
+    failOn: string[];
+}
+export declare function runBenchCommand(server: string[], opts: BenchOpts): Promise<number>;

package/dist/commands/bench.js

@@ -0,0 +1,129 @@
+import { buildContext, buildSpec } from '../context.js';
+import { Connection } from '../mcpClient.js';
+import { runBench } from '../bench/engine.js';
+import { resolveArgs } from '../args.js';
+import { evaluateExpr } from '../thresholds.js';
+import { AssertionFailure, UsageError } from '../errors.js';
+import { makeColors } from '../renderers/colors.js';
+import { emitJson } from '../renderers/json.js';
+import { formatMs, formatPct, renderKeyValues, renderTable } from '../renderers/table.js';
+import { writeOut } from '../output.js';
+import { Progress } from '../renderers/progress.js';
+const PROBES = ['listTools', 'listResources', 'listPrompts'];
+export async function runBenchCommand(server, opts) {
+    const ctx = buildContext(opts);
+    const spec = buildSpec(opts, server);
+    const c = makeColors(ctx.color);
+    const progress = new Progress({ quiet: ctx.quiet, json: ctx.json });
+    if (opts.tool && opts.probe) {
+        throw new UsageError('pass either --tool or --probe, not both');
+    }
+    if (opts.iterations !== undefined && opts.duration !== undefined) {
+        throw new UsageError('pass either --iterations or --duration, not both');
+    }
+    if (opts.probe && !PROBES.includes(opts.probe)) {
+        throw new UsageError(`invalid --probe "${opts.probe}" (expected ${PROBES.join(' | ')})`);
+    }
+    const targetKind = opts.tool ? 'tool' : 'probe';
+    const targetName = opts.tool ?? opts.probe ?? 'listTools';
+    const args = targetKind === 'tool' ? await resolveArgs(opts.args) : {};
+    const config = {
+        targetKind,
+        targetName,
+        args,
+        iterations: opts.duration !== undefined ? undefined : (opts.iterations ?? 50),
+        durationMs: opts.duration,
+        warmup: opts.warmup,
+        concurrency: opts.concurrency,
+        rps: opts.rps,
+        keepAlive: true,
+    };
+    progress.note(c.dim('connecting…'));
+    const conn = await Connection.connect(spec);
+    const target = conn.target;
+    const total = config.iterations;
+    const result = await runBench(conn, config, spec.requestTimeoutMs, {
+        onTick: (n) => {
+            if (total)
+                progress.status(c.dim(`bench ${n}/${total}…`));
+            else
+                progress.status(c.dim(`bench ${n}…`));
+        },
+    });
+    progress.clearStatus();
+    await conn.close();
+    const assertions = opts.failOn.map((expr) => evaluateExpr(expr, result.warm, result.throughput));
+    const anyAssertFail = assertions.some((a) => !a.pass);
+    const implicitErrorFail = opts.failOn.length === 0 && result.throughput.errors > 0;
+    const failed = anyAssertFail || implicitErrorFail;
+    if (ctx.json) {
+        emitJson({
+            ok: !failed,
+            target: { kind: targetKind, name: targetName },
+            transport: conn.kind,
+            config: {
+                iterations: config.iterations ?? null,
+                durationMs: config.durationMs ?? null,
+                concurrency: config.concurrency,
+                rps: config.rps ?? null,
+                warmup: config.warmup,
+                keepAlive: config.keepAlive,
+            },
+            coldStartMs: result.coldStartMs,
+            warm: result.warm,
+            throughput: result.throughput,
+            assertions: assertions.map((a) => ({ expr: a.expr, actual: a.actual, pass: a.pass })),
+        });
+    }
+    else {
+        const load = config.rps !== undefined ? `${config.rps} rps` : `concurrency ${config.concurrency}`;
+        const size = config.durationMs !== undefined ? `${config.durationMs} ms` : `${config.iterations} iters`;
+        writeOut(renderKeyValues([
+            ['Target', `${c.bold(targetName)} ${c.dim('(' + targetKind + ')')}`],
+            ['Server', `${target} ${c.dim('via ' + conn.kind)}`],
+            ['Load', `${size}, ${load}, warmup ${config.warmup}`],
+        ], c));
+        writeOut('');
+        if (result.coldStartMs !== null) {
+            writeOut(`${c.dim('Cold start')}  ${c.bold(formatMs(result.coldStartMs))}`);
+        }
+        if (result.warm) {
+            const w = result.warm;
+            writeOut('');
+            writeOut(renderTable([
+                { header: 'min', align: 'right' },
+                { header: 'mean', align: 'right' },
+                { header: 'p50', align: 'right' },
+                { header: 'p90', align: 'right' },
+                { header: 'p95', align: 'right' },
+                { header: 'p99', align: 'right' },
+                { header: 'max', align: 'right' },
+                { header: 'stddev', align: 'right' },
+            ], [[w.min, w.mean, w.p50, w.p90, w.p95, w.p99, w.max, w.stddev].map((n) => formatMs(n))], c));
+        }
+        else {
+            writeOut(c.red('no successful samples'));
+        }
+        const tp = result.throughput;
+        writeOut('');
+        const errPart = tp.errors > 0 ? c.red(`${tp.errors} errors (${formatPct(tp.errorRate)})`) : c.green('0 errors');
+        writeOut(c.dim(`${tp.completed} completed · ${tp.rps.toFixed(1)} req/s · `) + errPart);
+        if (assertions.length > 0) {
+            writeOut('');
+            for (const a of assertions) {
+                const mark = a.pass ? c.green('PASS') : c.red('FAIL');
+                writeOut(`  ${mark}  ${a.expr}  ${c.dim('actual ' + formatMetric(a))}`);
+            }
+        }
+    }
+    if (failed) {
+        const reason = anyAssertFail ? 'a --fail-on assertion failed' : 'requests errored';
+        throw new AssertionFailure(reason);
+    }
+    return 0;
+}
+function formatMetric(a) {
+    if (a.actual === null)
+        return '—';
+    return a.metric === 'errorRate' ? formatPct(a.actual) : formatMs(a.actual);
+}

package/dist/commands/call.d.ts

@@ -0,0 +1,8 @@
+import type { CommonOpts } from '../context.js';
+export interface CallOpts extends CommonOpts {
+    tool?: string;
+    args?: string;
+    raw?: boolean;
+    expectError?: boolean;
+}
+export declare function runCall(server: string[], opts: CallOpts): Promise<number>;