mcp-vitals 0.1.0

package/dist/thresholds.js

@@ -0,0 +1,109 @@
+import { UsageError } from './errors.js';
+export const METRICS = [
+    'p50',
+    'p90',
+    'p95',
+    'p99',
+    'max',
+    'min',
+    'mean',
+    'stddev',
+    'errorRate',
+];
+const OPS = ['<=', '>=', '!=', '==', '<', '>'];
+/**
+ * Parse a duration to milliseconds.
+ * Bare number => ms. Suffix `ms` => ms. Suffix `s` => seconds.
+ */
+export function parseDuration(input) {
+    if (typeof input === 'number')
+        return input;
+    const s = input.trim();
+    const m = /^(-?\d+(?:\.\d+)?)\s*(ms|s)?$/i.exec(s);
+    if (!m)
+        throw new UsageError(`invalid duration: "${input}"`);
+    const value = Number(m[1]);
+    const unit = (m[2] ?? 'ms').toLowerCase();
+    return unit === 's' ? value * 1000 : value;
+}
+/** errorRate is a fraction 0..1; everything else is a duration. */
+export function parseBound(metric, raw) {
+    if (metric === 'errorRate') {
+        const n = typeof raw === 'number' ? raw : Number(String(raw).trim());
+        if (!Number.isFinite(n))
+            throw new UsageError(`invalid errorRate bound: "${raw}"`);
+        return n;
+    }
+    return parseDuration(raw);
+}
+/** Parse an inline `--fail-on` expression like `p95<200ms` or `errorRate<=0`. */
+export function parseExpr(expr) {
+    const compact = expr.replace(/\s+/g, '');
+    for (const op of OPS) {
+        const i = compact.indexOf(op);
+        if (i > 0) {
+            const metric = compact.slice(0, i);
+            const rest = compact.slice(i + op.length);
+            if (!METRICS.includes(metric)) {
+                throw new UsageError(`unknown metric "${metric}" in "${expr}" (allowed: ${METRICS.join(', ')})`);
+            }
+            if (rest === '')
+                throw new UsageError(`missing bound in "${expr}"`);
+            return { metric, op, bound: parseBound(metric, rest) };
+        }
+    }
+    throw new UsageError(`invalid assertion "${expr}" (expected e.g. p95<200ms)`);
+}
+function metricValue(metric, stats, throughput) {
+    if (metric === 'errorRate')
+        return throughput.errorRate;
+    if (!stats)
+        return null;
+    switch (metric) {
+        case 'p50':
+            return stats.p50;
+        case 'p90':
+            return stats.p90;
+        case 'p95':
+            return stats.p95;
+        case 'p99':
+            return stats.p99;
+        case 'max':
+            return stats.max;
+        case 'min':
+            return stats.min;
+        case 'mean':
+            return stats.mean;
+        case 'stddev':
+            return stats.stddev;
+        default:
+            return null;
+    }
+}
+function compare(actual, op, bound) {
+    switch (op) {
+        case '<':
+            return actual < bound;
+        case '<=':
+            return actual <= bound;
+        case '>':
+            return actual > bound;
+        case '>=':
+            return actual >= bound;
+        case '==':
+            return actual === bound;
+        case '!=':
+            return actual !== bound;
+    }
+}
+/** Evaluate one parsed expression against a stats + throughput pair. */
+export function evaluate(parsed, stats, throughput) {
+    const actual = metricValue(parsed.metric, stats, throughput);
+    const exprStr = `${parsed.metric}${parsed.op}${parsed.bound}`;
+    const pass = actual === null ? false : compare(actual, parsed.op, parsed.bound);
+    return { expr: exprStr, metric: parsed.metric, op: parsed.op, bound: parsed.bound, actual, pass };
+}
+/** Convenience for inline string expressions. */
+export function evaluateExpr(expr, stats, throughput) {
+    return evaluate(parseExpr(expr), stats, throughput);
+}

package/dist/transport.d.ts

@@ -0,0 +1,8 @@
+import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
+import type { TransportKind, TransportSpec } from './types.js';
+/** Infer the transport kind from the spec when not explicitly forced. */
+export declare function inferKind(spec: TransportSpec): TransportKind;
+/** Build a fresh Transport instance for the requested kind. */
+export declare function createTransport(spec: TransportSpec, kind: TransportKind): Transport;
+/** Human label for the resolved target, for headers and JSON output. */
+export declare function describeTarget(spec: TransportSpec, kind: TransportKind): string;

package/dist/transport.js

@@ -0,0 +1,68 @@
+import { StdioClientTransport, getDefaultEnvironment } from '@modelcontextprotocol/sdk/client/stdio.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
+import { UsageError } from './errors.js';
+/** Infer the transport kind from the spec when not explicitly forced. */
+export function inferKind(spec) {
+    if (spec.forced)
+        return spec.forced;
+    if (spec.url)
+        return 'http';
+    if (spec.command)
+        return 'stdio';
+    throw new UsageError('no server specified: pass a stdio command or --url <url>');
+}
+function processEnvAsStrings() {
+    const out = {};
+    for (const [k, v] of Object.entries(process.env)) {
+        if (typeof v === 'string')
+            out[k] = v;
+    }
+    return out;
+}
+function stdioEnv(spec) {
+    return {
+        ...getDefaultEnvironment(),
+        ...(spec.inheritEnv ? processEnvAsStrings() : {}),
+        ...spec.env,
+    };
+}
+function toUrl(spec) {
+    if (!spec.url)
+        throw new UsageError('a URL is required for http/sse transports (use --url)');
+    try {
+        return new URL(spec.url);
+    }
+    catch {
+        throw new UsageError(`invalid --url: "${spec.url}"`);
+    }
+}
+/** Build a fresh Transport instance for the requested kind. */
+export function createTransport(spec, kind) {
+    switch (kind) {
+        case 'stdio': {
+            if (!spec.command)
+                throw new UsageError('no stdio command specified');
+            return new StdioClientTransport({
+                command: spec.command,
+                args: spec.args,
+                env: stdioEnv(spec),
+                stderr: 'pipe',
+            });
+        }
+        case 'http':
+            return new StreamableHTTPClientTransport(toUrl(spec), {
+                requestInit: { headers: spec.headers },
+            });
+        case 'sse':
+            return new SSEClientTransport(toUrl(spec), {
+                requestInit: { headers: spec.headers },
+            });
+    }
+}
+/** Human label for the resolved target, for headers and JSON output. */
+export function describeTarget(spec, kind) {
+    if (kind === 'stdio')
+        return [spec.command, ...spec.args].filter(Boolean).join(' ');
+    return spec.url ?? '(unknown)';
+}

package/dist/types.d.ts

@@ -0,0 +1,180 @@
+export type TransportKind = 'stdio' | 'http' | 'sse';
+/** How to reach the MCP server under test. */
+export interface TransportSpec {
+    /** Forced transport, or undefined to infer from command/url. */
+    forced?: TransportKind;
+    /** stdio: executable + argv. */
+    command?: string;
+    args: string[];
+    /** http/sse: target URL. */
+    url?: string;
+    /** http/sse: extra request headers (auth, etc). */
+    headers: Record<string, string>;
+    /** stdio: extra env vars merged over getDefaultEnvironment(). */
+    env: Record<string, string>;
+    /** stdio: spread process.env into the child before --env vars. */
+    inheritEnv: boolean;
+    /** Handshake timeout (ms). */
+    connectTimeoutMs: number;
+    /** Per-request timeout (ms). */
+    requestTimeoutMs: number;
+}
+/** Global options resolved once and shared by every command. */
+export interface RunContext {
+    json: boolean;
+    color: boolean;
+    quiet: boolean;
+    verbose: boolean;
+}
+export interface SchemaError {
+    path: string;
+    message: string;
+}
+export interface ToolInfo {
+    name: string;
+    title?: string;
+    description?: string;
+    inputSchema?: unknown;
+    outputSchema?: unknown;
+    requiredArgs: number;
+    totalArgs: number;
+    schemaValid: boolean;
+    schemaErrors: SchemaError[];
+}
+export interface ResourceInfo {
+    uri: string;
+    name?: string;
+    mimeType?: string;
+}
+export interface PromptInfo {
+    name: string;
+    description?: string;
+    arguments: {
+        name: string;
+        required?: boolean;
+    }[];
+}
+export interface ServerIdentity {
+    name: string;
+    version: string;
+    protocolVersion?: string;
+    instructions?: string;
+}
+export interface CapabilitySummary {
+    tools: boolean;
+    resources: boolean;
+    prompts: boolean;
+    logging: boolean;
+    completions: boolean;
+    raw: Record<string, unknown>;
+}
+export interface Stats {
+    count: number;
+    min: number;
+    mean: number;
+    p50: number;
+    p90: number;
+    p95: number;
+    p99: number;
+    max: number;
+    stddev: number;
+    unit: 'ms';
+}
+export interface Throughput {
+    rps: number;
+    completed: number;
+    errors: number;
+    errorRate: number;
+}
+export type ProbeOp = 'listTools' | 'listResources' | 'listPrompts';
+export interface BenchConfig {
+    targetKind: 'tool' | 'probe';
+    targetName: string;
+    args: unknown;
+    iterations?: number;
+    durationMs?: number;
+    warmup: number;
+    concurrency: number;
+    rps?: number;
+    keepAlive: boolean;
+}
+export interface BenchSample {
+    ms: number;
+    error: boolean;
+}
+export interface BenchResult {
+    coldStartMs: number | null;
+    warm: Stats | null;
+    throughput: Throughput;
+    samples: number[];
+    raw: BenchSample[];
+}
+export type CompareOp = '<' | '<=' | '>' | '>=' | '==' | '!=';
+export interface AssertionOutcome {
+    expr: string;
+    metric: string;
+    op: CompareOp;
+    bound: number;
+    actual: number | null;
+    pass: boolean;
+}
+export type SuiteKind = 'presence' | 'schema' | 'latency';
+export type CheckStatus = 'pass' | 'fail' | 'skip';
+export interface CheckRow {
+    id: string;
+    kind: SuiteKind;
+    target: string;
+    expected: string;
+    actual: string;
+    status: CheckStatus;
+}
+export interface CheckSummary {
+    passed: number;
+    failed: number;
+    skipped: number;
+    durationMs: number;
+}
+export interface ServerConfig {
+    command?: string;
+    args?: string[];
+    env?: Record<string, string>;
+    url?: string;
+    headers?: Record<string, string>;
+    transport?: TransportKind;
+    connectTimeoutMs?: number;
+    timeoutMs?: number;
+}
+export interface BenchDefaults {
+    iterations?: number;
+    warmup?: number;
+    concurrency?: number;
+    timeoutMs?: number;
+}
+export interface ExpectConfig {
+    tools?: string[];
+    resources?: string[];
+    prompts?: string[];
+    schemasValid?: boolean;
+}
+export interface LatencyAssertion {
+    id: string;
+    tool?: string;
+    probe?: ProbeOp;
+    args?: unknown;
+    iterations?: number;
+    warmup?: number;
+    concurrency?: number;
+    p50?: string | number;
+    p90?: string | number;
+    p95?: string | number;
+    p99?: string | number;
+    max?: string | number;
+    mean?: string | number;
+    errorRate?: number;
+}
+export interface AssertionsConfig {
+    server: ServerConfig;
+    defaults?: BenchDefaults;
+    expect?: ExpectConfig;
+    latency?: LatencyAssertion[];
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+// Shared types for mcp-vitals. No runtime code lives here.
+export {};

package/package.json

@@ -0,0 +1,83 @@
+{
+  "name": "mcp-vitals",
+  "version": "0.1.0",
+  "description": "Vital signs for your MCP server: inspect capabilities, benchmark tool-call latency (p50/p95/p99), and assert health in CI. The ab/k6/pytest for MCP servers — non-interactive, scriptable, LLM-free.",
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "benchmark",
+    "latency",
+    "load-test",
+    "ci",
+    "cli",
+    "inspector",
+    "anthropic",
+    "claude",
+    "devtools",
+    "p95",
+    "observability"
+  ],
+  "type": "module",
+  "bin": {
+    "mcp-vitals": "bin/mcp-vitals.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "bin",
+    "schema",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",
+    "build": "npm run clean && tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "lint": "eslint .",
+    "pretest": "npm run build",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "coverage": "vitest run --coverage",
+    "prepublishOnly": "npm run build",
+    "dev": "tsx src/cli.ts"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
+    "commander": "^12.1.0",
+    "picocolors": "^1.1.1",
+    "yaml": "^2.6.1"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.17.0",
+    "@types/node": "^20.14.0",
+    "@vitest/coverage-v8": "^2.1.8",
+    "eslint": "^9.17.0",
+    "execa": "^9.5.2",
+    "globals": "^15.15.0",
+    "tsx": "^4.19.2",
+    "typescript": "^5.6.3",
+    "typescript-eslint": "^8.18.1",
+    "vitest": "^2.1.8"
+  },
+  "author": "Shaxzodbek Sobirov <shaxzodbek@blaze.uz>",
+  "license": "MIT",
+  "homepage": "https://github.com/shaxzodbek-uzb/mcp-vitals#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shaxzodbek-uzb/mcp-vitals.git"
+  },
+  "bugs": {
+    "url": "https://github.com/shaxzodbek-uzb/mcp-vitals/issues"
+  }
+}

package/schema/assertions.schema.json

@@ -0,0 +1,177 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://unpkg.com/mcp-vitals/schema/assertions.schema.json",
+  "title": "mcp-vitals assertions",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "server"
+  ],
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "server": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "command": {
+          "type": "string"
+        },
+        "args": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "env": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
+        "url": {
+          "type": "string"
+        },
+        "headers": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "string"
+          }
+        },
+        "transport": {
+          "enum": [
+            "stdio",
+            "http",
+            "sse"
+          ]
+        },
+        "connectTimeoutMs": {
+          "type": "number"
+        },
+        "timeoutMs": {
+          "type": "number"
+        }
+      }
+    },
+    "defaults": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "iterations": {
+          "type": "number"
+        },
+        "warmup": {
+          "type": "number"
+        },
+        "concurrency": {
+          "type": "number"
+        },
+        "timeoutMs": {
+          "type": "number"
+        }
+      }
+    },
+    "expect": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "tools": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "resources": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "prompts": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "schemasValid": {
+          "type": "boolean"
+        }
+      }
+    },
+    "latency": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "id"
+        ],
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "tool": {
+            "type": "string"
+          },
+          "probe": {
+            "enum": [
+              "listTools",
+              "listResources",
+              "listPrompts"
+            ]
+          },
+          "args": {},
+          "iterations": {
+            "type": "number"
+          },
+          "warmup": {
+            "type": "number"
+          },
+          "concurrency": {
+            "type": "number"
+          },
+          "p50": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "p90": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "p95": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "p99": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "max": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "mean": {
+            "type": [
+              "string",
+              "number"
+            ]
+          },
+          "errorRate": {
+            "type": "number"
+          }
+        }
+      }
+    }
+  }
+}